docker: add initial docker host playbook and dockerfiles

ansible/README.md

@@ -42,6 +42,8 @@ These playbooks are available to you:
 
   - **jenkins/worker/upgrade-jar.yml**: Upgrades the worker jar file.
 
+  - **jenkins/docker-host.yml**: Sets up a host to run Docker workers.
+
   - **jenkins/linter.yml**: Sets up the code linters (flavour of a worker).
 
   - **create-webhost.yml**: Configures the server(s) that host nodejs.org,
@@ -67,7 +69,6 @@ If something isn't working, you will likely get a warning or error.
 Have a look at the playbooks or roles. They are well documented and should
 (hopefully) be easy to improve.
 
-
 ## Adding a new host to inventory.yml
 
 Hosts are listed as part of an yaml collection. Find the type and provider and
@@ -99,7 +100,6 @@ $type-$provider(_$optionalmeta)-$os-$architecture(_$optionalmeta)-$uid
 For more information refer to other hosts in `inventory.yml` or the
 [ansible callback that is responsible for parsing it][callback].
 
-
 ### Metadata
 
 Each host needs a bit of metadata:
@@ -130,7 +130,29 @@ ansible_python_interpreter: /usr/local/bin/python
           since that will enable the `paramiko` connection plugin, disregard
           other ssh-specific options.
 
+### Docker host configuration options
+
+When configuring a Docker host using the `jenkins/docker-host.yml` playbook,
+your host_vars file for the new host(s) will need to have a special set of
+options to configure the containers run on the host. It should look something
+like this:
+
+```yaml
+containers:
+  - { name: 'test-digitalocean-alpine34_container-x64-1', os: 'alpine34', secret: 'abc123' }
+  - { name: 'test-digitalocean-alpine35_container-x64-1', os: 'alpine35', secret: 'abc456' }
+  - { name: 'test-digitalocean-alpine36_container-x64-1', os: 'alpine36', secret: 'abc567' }
+  - { name: 'test-digitalocean-ubuntu1604_container-x64-1', os: 'ubuntu1604', secret: 'abc890' }
+```
+
+Where each item corresponds to a container to be set up and run on the host.
+
+Each `name` should exist as a node in Jenkins and the corresponding `secret`
+should be given. The `os` determines the `Dockerfile` to use to build the host.
+The templates for these can be found in `roles/docker/templates/`.
 
+Note that the Docker host itself doesn't need to be known by Jenkins, just the
+containers that are managed there.
 
 ### TODO
 

ansible/inventory.yml

@@ -96,6 +96,8 @@ hosts:
         ubuntu1404-x64-1: {ip: 45.55.252.223}
         ubuntu1404-x86-1: {ip: 159.203.115.220}
         ubuntu1604-x86-1: {ip: 159.203.77.233}
+        ubuntu1604_docker-x64-1: {ip: 128.199.198.56}
+        ubuntu1604_docker-x64-2: {ip: 138.68.241.115}
 
     - ibm:
         aix61-ppc64-1: {ip: 50.200.166.131, port: 18822}
@@ -111,6 +113,7 @@ hosts:
         smartos15-x64-2: {ip: 165.225.139.77}
         smartos16-x64-1: {ip: 72.2.115.252}
         smartos16-x64-2: {ip: 72.2.113.74}
+        ubuntu1604_docker-x64-1: {ip: 72.2.118.27, user: ubuntu}
         ubuntu1710-x64-1: {ip: 37.153.110.33, user: ubuntu}
         ubuntu1710-x64-2: {ip: 8.19.32.121, user: ubuntu}
 
@@ -222,6 +225,7 @@ hosts:
         debian8-x86-1: {ip: 169.44.16.126}
         ubuntu1404-x64-1: {ip: 50.97.245.5}
         ubuntu1404-x86-1: {ip: 50.97.245.9}
+        ubuntu1604_docker-x64-1: {ip: 130.198.66.50}
 
     - packetnet:
         centos7-arm64-1: {ip: 147.75.193.230}

ansible/playbooks/jenkins/docker-host.yaml

@@ -0,0 +1,20 @@
+---
+
+#
+# set up a jenkins worker -- muy bueno!
+#
+
+- hosts:
+  - test
+
+  roles:
+    - bootstrap
+    - package-upgrade
+    - docker
+
+  pre_tasks:
+    - name: check if containers property is properly set
+      fail:
+      failed_when: not containers
+
+  environment: '{{remote_env}}'

ansible/roles/bootstrap/tasks/partials/ubuntu1604.yml

@@ -16,4 +16,4 @@
 
 - name: install python and aptitude
   when: has_python.rc == 1 or has_aptitude.rc == 1
-  raw: apt install -y python-minimal aptitude
+  raw: apt-get update && apt install -y python-minimal aptitude

ansible/roles/docker/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+
+#
+# generic handlers for baselayout stuff
+#
+
+- name: restart sshd
+  service: name="{{ sshd_service_name }}" state=restarted

ansible/roles/docker/tasks/main.yml

@@ -0,0 +1,141 @@
+---
+
+#
+# common tasks suitable for all machines
+#
+
+- name: gather facts
+  setup:
+
+- name: set hostname
+  hostname:
+    name: "{{ inventory_hostname|replace('_', '-') }}"
+
+- name: disable sftp
+  when: not os|startswith("win")
+  notify: restart sshd
+  lineinfile:
+    state: absent
+    dest: "{{ ssh_config }}"
+    regexp: ^Subsystem(\s+)sftp
+
+- name: add os-specific repos
+  include: "{{ repos_include }}"
+  loop_control:
+    loop_var: repos_include
+  with_first_found:
+    - files:
+        - "{{ role_path }}/tasks/partials/repo/{{ os }}.yml"
+        - "{{ role_path }}/tasks/partials/repo/{{ os|stripversion }}.yml"
+      skip: true
+
+- name: install packages
+  package:
+    name: "{{ package }}"
+    state: present
+  loop_control:
+    loop_var: package
+  with_items:
+    # ansible doesn't like empty lists
+    - "{{ packages[os]|default('[]') }}"
+    - "{{ packages[os|stripversion]|default('[]') }}"
+    - "{{ common_packages|default('[]') }}"
+
+- name: remove fortune from login shells
+  when: os|stripversion == 'freebsd'
+  lineinfile:
+    dest: "/home/{{ server_user }}/{{ login_item }}"
+    state: absent
+    regexp: fortune freebsd
+  loop_control:
+    loop_var: login_item
+  with_items: [ '.login', '.profile' ]
+
+- name: set up ntp
+  include: "{{ ntp_include }}"
+  loop_control:
+    loop_var: ntp_include
+  with_first_found:
+    - files:
+        - "{{ role_path }}/../baselayout/tasks/partials/ntp/{{ os }}.yml"
+        - "{{ role_path }}/../baselayout/tasks/partials/ntp/{{ os|stripversion }}.yml"
+        - "{{ role_path }}/../baselayout/tasks/partials/ntp/{{ os|match_key(ntp_service, raise_error=False) }}.yml"
+      skip: true
+
+- name: create group
+  group:
+    name: "{{ server_user }}"
+
+- name: create user
+  user:
+    name: "{{ server_user }}"
+    group: "{{ server_user }}"
+
+- name: add ::1 to /etc/hosts for ipv6 compat
+  lineinfile:
+    dest: /etc/hosts
+    state: present
+    line: ::1 localhost.localdomain localhost
+
+- name: create worker directory
+  file:
+    path: "/home/{{ server_user }}/{{ item.name }}/"
+    state: directory
+    owner: "{{ server_user }}"
+    group: "{{ server_user }}"
+    mode: 0755
+  with_items:
+    - "{{ containers }}"
+
+- name: create NODE_TEST_DIR directory
+  file:
+    path: "/home/{{ server_user }}/{{ item.name }}/tmp"
+    state: directory
+    owner: "{{ server_user }}"
+    group: "{{ server_user }}"
+    mode: 0755
+  with_items:
+    - "{{ containers }}"
+
+- name: register {{ server_user }} GID
+  raw: "grep ^{{ server_user }} /etc/passwd | awk -F: '{print $4}'"
+  register: server_user_gid
+
+- name: register {{ server_user }} UID
+  raw: "grep ^{{ server_user }} /etc/passwd | awk -F: '{print $3}'"
+  register: server_user_uid
+
+- name: "docker : make build directory"
+  file:
+    path: /root/docker-container-{{ item.name }}
+    state: directory
+  with_items:
+    - "{{ containers }}"
+
+- name: "docker : generate Dockerfile"
+  template:
+    src: "{{ role_path }}/templates/{{ item.os }}.Dockerfile.j2"
+    dest: /root/docker-container-{{ item.name }}/Dockerfile
+    mode: "0644"
+  with_items:
+    - "{{ containers }}"
+
+- name: "docker : build image"
+  command: docker build -t node-ci:{{ item.name }} /root/docker-container-{{ item.name }}/
+  with_items:
+    - "{{ containers }}"
+
+- name: "docker : generate and copy init script"
+  template:
+    src: "{{ role_path }}/templates/jenkins.service.j2"
+    dest: "/lib/systemd/system/jenkins-{{ item.name }}.service"
+  with_items:
+    - "{{ containers }}"
+
+- name: "docker : start Jenkins"
+  service:
+    name: "jenkins-{{ item.name }}"
+    state: started
+    enabled: yes
+  with_items:
+    - "{{ containers }}"

ansible/roles/docker/tasks/partials/repo/ubuntu.yml

@@ -0,0 +1,15 @@
+---
+
+#
+# add Docker repo
+#
+
+- name: "repo : add Ubuntu Docker repo key"
+  raw: curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+
+- name: "repo : add Ubuntu Docker repo"
+  raw: add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+  register: has_updated_package_repo
+
+- name: "repo : update apt cache"
+  apt: update_cache=yes

ansible/roles/docker/templates/alpine34.Dockerfile.j2

@@ -0,0 +1,56 @@
+FROM alpine:3.4
+
+ENV LC_ALL C
+ENV USER {{ server_user }}
+ENV JOBS {{ server_jobs | default(ansible_processor_vcpus) }}
+ENV SHELL /bin/bash
+ENV HOME /home/{{ server_user }}
+ENV PATH /usr/lib/ccache/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV NODE_COMMON_PIPE /home/{{ server_user }}/test.pipe
+ENV NODE_TEST_DIR /home/{{ server_user }}/tmp
+ENV OSTYPE linux-gnu
+ENV OSVARIANT docker
+ENV DESTCPU x64
+ENV ARCH x64
+
+RUN echo 'https://dl-3.alpinelinux.org/alpine/edge/main' >> /etc/apk/repositories && \
+    echo 'https://dl-3.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
+
+RUN apk add --no-cache \
+        libstdc++ \
+    && apk add --no-cache --virtual .build-deps \
+        binutils-gold \
+        curl \
+        g++ \
+        gcc \
+        gnupg \
+        libgcc \
+        linux-headers \
+        make \
+        paxctl \
+        python \
+        tar \
+        ccache \
+        openjdk8 \
+        git \
+        procps \
+        openssh-client \
+        py2-pip \
+        bash
+
+RUN pip install tap2junit
+
+RUN addgroup -g {{ server_user_gid.stdout_lines[0] }} {{ server_user }}
+
+RUN adduser -G {{ server_user }} -D -u {{ server_user_uid.stdout_lines[0] }} {{ server_user }}
+
+VOLUME [ "/home/{{ server_user }}/" ]
+
+USER iojs:iojs
+
+CMD cd /home/iojs \
+  && curl https://ci.nodejs.org/jnlpJars/slave.jar -O \
+  && java -Xmx{{ server_ram|default('128m') }} \
+          -jar /home/{{ server_user }}/slave.jar \
+          -jnlpUrl {{ jenkins_url }}/computer/{{ item.name }}/slave-agent.jnlp \
+          -secret {{ item.secret }}

ansible/roles/docker/templates/alpine35.Dockerfile.j2

@@ -0,0 +1,57 @@
+FROM alpine:3.5
+
+ENV LC_ALL C
+ENV USER {{ server_user }}
+ENV JOBS {{ server_jobs | default(ansible_processor_vcpus) }}
+ENV SHELL /bin/bash
+ENV HOME /home/{{ server_user }}
+ENV PATH /usr/lib/ccache/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV NODE_COMMON_PIPE /home/{{ server_user }}/test.pipe
+ENV NODE_TEST_DIR /home/{{ server_user }}/tmp
+ENV OSTYPE linux-gnu
+ENV OSVARIANT docker
+ENV DESTCPU x64
+ENV ARCH x64
+
+RUN echo 'https://dl-3.alpinelinux.org/alpine/edge/main' >> /etc/apk/repositories && \
+    echo 'https://dl-3.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
+
+RUN apk add --no-cache \
+        libstdc++ \
+    && apk add --no-cache --virtual .build-deps \
+        shadow \
+        binutils-gold \
+        curl \
+        g++ \
+        gcc \
+        gnupg \
+        libgcc \
+        linux-headers \
+        make \
+        paxctl \
+        python \
+        tar \
+        ccache \
+        openjdk8 \
+        git \
+        procps \
+        openssh-client \
+        py2-pip \
+        bash
+
+RUN pip install tap2junit
+
+RUN addgroup -g {{ server_user_gid.stdout_lines[0] }} {{ server_user }}
+
+RUN adduser -G {{ server_user }} -D -u {{ server_user_uid.stdout_lines[0] }} {{ server_user }}
+
+VOLUME [ "/home/{{ server_user }}/" ]
+
+USER iojs:iojs
+
+CMD cd /home/iojs \
+  && curl https://ci.nodejs.org/jnlpJars/slave.jar -O \
+  && java -Xmx{{ server_ram|default('128m') }} \
+          -jar /home/{{ server_user }}/slave.jar \
+          -jnlpUrl {{ jenkins_url }}/computer/{{ item.name }}/slave-agent.jnlp \
+          -secret {{ item.secret }}