From 842362378958c76cf6e4cde3ebb4085c69cbbf9c Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Tue, 13 Mar 2018 15:06:45 -0400 Subject: [PATCH] Add qemu builders and update README - Adds packer qemu builders - Adds Vagrant and vagrant cloud support for building and pushing Vagrant libvirt images - Adds additional packer cleanup - Adds a Vagrantfile for doing a test of node conformance against libvirt/ubuntu vagrant cloud box Signed-off-by: Jason DeTiberus --- ansible/roles/packer-cleanup/tasks/debian.yml | 20 +++++ ansible/roles/packer-cleanup/tasks/main.yml | 15 +++- ansible/roles/packer-cleanup/tasks/redhat.yml | 1 + ansible/roles/providers/defaults/main.yml | 2 +- .../filter_plugins/provider_filters.py | 23 +++++ ansible/roles/providers/tasks/main.yml | 2 + .../roles/providers/tasks/vagrant-debian.yml | 8 ++ ansible/roles/providers/tasks/vagrant.yml | 9 ++ packer/.gitignore | 2 + packer/README.md | 90 ++++++++++++++----- packer/kickstart/centos7/ks.cfg | 66 ++++++++++++++ packer/kickstart/ubuntu-16.04/preseed.cfg | 62 +++++++++++++ packer/packer.json | 76 +++++++++++++++- .../Vagrantfile | 11 +++ .../node-conformance.sh | 7 ++ 15 files changed, 367 insertions(+), 27 deletions(-) create mode 100644 ansible/roles/packer-cleanup/tasks/debian.yml create mode 100644 ansible/roles/packer-cleanup/tasks/redhat.yml create mode 100644 ansible/roles/providers/filter_plugins/provider_filters.py create mode 100644 ansible/roles/providers/tasks/vagrant-debian.yml create mode 100644 ansible/roles/providers/tasks/vagrant.yml create mode 100644 packer/kickstart/centos7/ks.cfg create mode 100644 packer/kickstart/ubuntu-16.04/preseed.cfg create mode 100644 packer/tests/vagrant-libvirt-node-conformance/Vagrantfile create mode 100644 packer/tests/vagrant-libvirt-node-conformance/node-conformance.sh diff --git a/ansible/roles/packer-cleanup/tasks/debian.yml b/ansible/roles/packer-cleanup/tasks/debian.yml new file mode 100644 index 0000000..0436d01 --- /dev/null +++ b/ansible/roles/packer-cleanup/tasks/debian.yml @@ -0,0 +1,20 @@ +--- +- name: cleanup udev rules + file: + state: absent + path: "{{ item }}" + with_items: + - /dev/.udev + - /lib/udev/rules.d/75-persistent-net-generator.rules + +- name: cleanup dhcp leases + file: + state: absent + path: "{{ item }}" + with_items: + - /var/lib/dhcp3 + - /var/lib/dhcp + +- command: apt-get -y autoremove --purge +- command: apt-get -y clean +- command: apt-get -y autoclean \ No newline at end of file diff --git a/ansible/roles/packer-cleanup/tasks/main.yml b/ansible/roles/packer-cleanup/tasks/main.yml index e279ef5..1727aed 100644 --- a/ansible/roles/packer-cleanup/tasks/main.yml +++ b/ansible/roles/packer-cleanup/tasks/main.yml @@ -5,7 +5,6 @@ path: "{{ item }}" with_items: - /root/.ssh/authorized_keys - - "/home/{{ ansible_env.SUDO_USER }}/.ssh/authorized_keys" - /etc/machine-id - /var/lib/cloud - /var/log/cloud-init.log @@ -15,3 +14,17 @@ file: dest: /etc/machine-id state: touch + +- name: cleanup packer artifacts + file: + state: absent + path: "{{ item }}" + with_items: + - "/home/{{ ansible_env.SUDO_USER }}/.ssh/authorized_keys" + when: packer_builder_type | default(None) != 'qemu' + +- import_tasks: debian.yml + when: ansible_os_family == "Debian" + +- import_tasks: redhat.yml + when: ansible_os_family == "RedHat" \ No newline at end of file diff --git a/ansible/roles/packer-cleanup/tasks/redhat.yml b/ansible/roles/packer-cleanup/tasks/redhat.yml new file mode 100644 index 0000000..73b314f --- /dev/null +++ b/ansible/roles/packer-cleanup/tasks/redhat.yml @@ -0,0 +1 @@ +--- \ No newline at end of file diff --git a/ansible/roles/providers/defaults/main.yml b/ansible/roles/providers/defaults/main.yml index f64cb8c..75b3a1b 100644 --- a/ansible/roles/providers/defaults/main.yml +++ b/ansible/roles/providers/defaults/main.yml @@ -1,3 +1,3 @@ --- packer_builder_type: '' -provider_name: "{{ 'aws' if packer_builder_type.startswith('amazon') else '' }}" \ No newline at end of file +provider_name: "{{ packer_builder_type | provider_from_builder_type }}" \ No newline at end of file diff --git a/ansible/roles/providers/filter_plugins/provider_filters.py b/ansible/roles/providers/filter_plugins/provider_filters.py new file mode 100644 index 0000000..f6eadc7 --- /dev/null +++ b/ansible/roles/providers/filter_plugins/provider_filters.py @@ -0,0 +1,23 @@ +# Provider Filters + +from ansible import errors + + +def provider_from_builder_type(builder_type): + ''' Returns normalized provider name ''' + if builder_type.startswith('amazon'): + return 'aws' + elif builder_type == 'qemu': + return 'vagrant' + else: + raise errors.AnsibleFilterError('Unknown builder_type: {}'.format(builder_type)) + + +# ---- Ansible filters ---- +class FilterModule(object): + ''' Provider Filters ''' + + def filters(self): + return { + 'provider_from_builder_type': provider_from_builder_type + } \ No newline at end of file diff --git a/ansible/roles/providers/tasks/main.yml b/ansible/roles/providers/tasks/main.yml index 0f7b39b..47d4402 100644 --- a/ansible/roles/providers/tasks/main.yml +++ b/ansible/roles/providers/tasks/main.yml @@ -2,3 +2,5 @@ - include_tasks: aws.yml when: provider_name.lower() == 'aws' +- include_tasks: vagrant.yml + when: provider_name.lower() == 'vagrant' diff --git a/ansible/roles/providers/tasks/vagrant-debian.yml b/ansible/roles/providers/tasks/vagrant-debian.yml new file mode 100644 index 0000000..eb611f7 --- /dev/null +++ b/ansible/roles/providers/tasks/vagrant-debian.yml @@ -0,0 +1,8 @@ +--- +- name: Update network interfaces + command: sed -i "s/ens3/ens5/g" /etc/network/interfaces + +- name: Fix for vagrant reload + lineinfile: + path: /etc/network/interfaces + line: pre-up sleep 2 \ No newline at end of file diff --git a/ansible/roles/providers/tasks/vagrant.yml b/ansible/roles/providers/tasks/vagrant.yml new file mode 100644 index 0000000..bfa18f2 --- /dev/null +++ b/ansible/roles/providers/tasks/vagrant.yml @@ -0,0 +1,9 @@ +--- +- name: install vagrant ssh key + authorized_key: + user: vagrant + state: present + key: https://raw.githubusercontent.com/mitchellh/vagrant/master/keys/vagrant.pub + +- import_tasks: vagrant-debian.yml + when: ansible_os_family == "Debian" \ No newline at end of file diff --git a/packer/.gitignore b/packer/.gitignore index f650bce..e1722c3 100644 --- a/packer/.gitignore +++ b/packer/.gitignore @@ -1,2 +1,4 @@ venv *.egg-info +packer_cache +packer_output diff --git a/packer/README.md b/packer/README.md index 7bcf30f..fd60055 100644 --- a/packer/README.md +++ b/packer/README.md @@ -1,45 +1,91 @@ -building images -=============== -Building images for Kubernetes is easily accomplished with the [Packer](https://github.com/hashicorp/packer) and the templates found in this directory. +# Building Wardroom Images -aws-quickstart --------------- -This directory contains the build scripts for the AWS AMI that's used by Heptio's [AWS Quick Start](https://github.com/heptioaws-quickstart). Heptio's AMI is, in turn, built on Ubuntu 16.04 LTS. +This directory contains tooling for building base images for use as nodes in Kubernetes clusters. [Packer](https://www.packer.io/) is used for building the images -prerequisites -------------- -To build the AMI, you need: +## Prerequisites + +Prerequisites for all images: - [Packer](https://www.packer.io/docs/installation.html) - [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) version >= 2.4.0.0 + +Prerequisites for building AMIs for use in Amazon Web Services: + - An AWS account - The AWS CLI installed and configured -build the AMI's ---------------- -From this directory, simply run: +Prerequisites for building QEMU qcow2 images: + +- qemu + +## Building Images + +### Build Variables + +The following variables can be overriden when building images using the `-var` option when calling `packer build`: +| Variable | Default | Description | +|----------|---------|-------------| +| build_version | unset | A unique build version for the image | +| kubernetes_version | 1.9.3-00 | Kubernetes Version to install | +| kubernetes_cni_version | 0.6.0-00 | CNI Version to install | + +For exmaple, to build all images for use with Kubernetes 1.8.9 for build version 1: + +```bash +packer build -var kubernetes_version=1.8.9-00 -var build_version=1 --only=qemu-ubuntu-16.04 packer.json ``` -/path/to/packer build -var-file .json -var kubernetes_version= -var kubernetes_cni_version= -var build_version=`git rev-parse HEAD` packer.json + +### Limiting Images to Build + +If packer build is run without specifying which images to build, then all configured images will be built. This currently includes QEMU images and AWS AMI images for Ubuntu 16.04 and CentOS 7. The `--only` option can be specified when running `packer build` to limit the images built. + +To build only the QEMU Ubuntu image: + +```bash +packer build -var build_version=`git rev-parse HEAD` --only=qemu-ubuntu-16.04 packer.json ``` -This will build AMI images in the us-east AWS region (additional region support to follow). -You may limit which images build by adding the `-only=` flag to Packer. +To build only the QEMU CentOS image: -testing the AMIs ----------------- +```bash +packer build -var build_version=`git rev-parse HEAD` --only=qemu-centos-7.4 packer.json ``` + +To build both the Ubuntu and CentOS AWS AMIs: + +```bash +packer build -var build_version=`git rev-parse HEAD` --only=ami-centos-7.4,ami-ubuntu-16.04 packer.json +``` + +## Testing Images + +Connect remotely to an instance created from the image and run the Node Conformance tests using the following commands: + +```bash wget https://dl.k8s.io/v1.9.3/kubernetes-test.tar.gz tar -zxvf kubernetes-test.tar.gz cd kubernetes/platforms/linux/amd64 sudo ./ginkgo --nodes=8 --flakeAttempts=2 --focus="\[Conformance\]" --skip="\[Flaky\]|\[Serial\]" ./e2e_node.test -- --k8s-bin-dir=/usr/bin ``` -deployment ----------- -There is a helper script to aid in seeding built AMI's to all other AWS regions. -You can install them with `python3 setup.py install`. +## Deploying Images -``` +### AWS + +There is a helper script to aid in seeding built AMI's to all other AWS regions. This script can be installed with `python3 setup.py install`. + +```bash copy-ami -r -i [-q] ``` + +## Updating the AWS Quick Start Images + +- Build the base image + + ```bash + packer build -var-file us-east-1.json -var build_version=`git rev-parse HEAD` --only=ami-ubuntu-16.04 packer.json + ``` +- Run Node Conformance against the built image +- Deploy the image using copy-ami +- Update the [Quick Start](https://github.com/heptio/aws-quickstart) to use the new images \ No newline at end of file diff --git a/packer/kickstart/centos7/ks.cfg b/packer/kickstart/centos7/ks.cfg new file mode 100644 index 0000000..2d87f0e --- /dev/null +++ b/packer/kickstart/centos7/ks.cfg @@ -0,0 +1,66 @@ +# CentOS 7.x kickstart file - ks.cfg + +# Required settings +lang en_US.UTF-8 +keyboard us +authconfig --enableshadow --passalgo=sha256 +timezone UTC +rootpw --plaintext vagrant + +# Optional settings +install +cdrom +user --name=vagrant --plaintext --password vagrant +unsupported_hardware +network --bootproto=dhcp +firewall --disabled +selinux --permissive +bootloader --location=mbr --append="no_timer_check console=tty0 console=ttyS0,115200" +text +skipx +zerombr +clearpart --all --initlabel +autopart +firstboot --disabled +reboot + +%packages --nobase --ignoremissing --excludedocs +openssh-clients +sudo +nfs-utils +-fprintd-pam +-intltool + +# Microcode updates cannot work in a VM +-microcode_ctl +# unnecessary firmware +-aic94xx-firmware +-alsa-firmware +-alsa-tools-firmware +-atmel-firmware +-b43-openfwwf +-bfa-firmware +-ipw*-firmware +-irqbalance +-ivtv-firmware +-iwl*-firmware +-kernel-firmware +-libertas-usb8388-firmware +-ql*-firmware +-rt61pci-firmware +-rt73usb-firmware +-xorg-x11-drv-ati-firmware +-zd1211-firmware +# Don't build rescue initramfs +-dracut-config-rescue +%end + +%post +# configure vagrant user in sudoers +cat <<-EOF > /etc/sudoers.d/vagrant +%vagrant ALL=(ALL) NOPASSWD: ALL +Defaults:vagrant !requiretty +Defaults:vagrant env_keep += "HTTP_PROXY HTTPS_PROXY FTP_PROXY RSYNC_PROXY NO_PROXY" +EOF +chmod 0440 /etc/sudoers.d/vagrant +%end diff --git a/packer/kickstart/ubuntu-16.04/preseed.cfg b/packer/kickstart/ubuntu-16.04/preseed.cfg new file mode 100644 index 0000000..47862ec --- /dev/null +++ b/packer/kickstart/ubuntu-16.04/preseed.cfg @@ -0,0 +1,62 @@ +choose-mirror-bin mirror/http/proxy string + +d-i base-installer/kernel/override-image string linux-server + +d-i clock-setup/utc boolean true +d-i clock-setup/utc-auto boolean true + +d-i console-setup/ask_detect boolean false +d-i console-setup/layoutcode string us + +d-i debian-installer/framebuffer boolean false +d-i debian-installer/locale string en_US.UTF-8 +d-i debian-installer/quiet boolean false +d-i debian-installer/splash boolean false + +d-i debconf/priority select critical +d-i debconf/frontend select noninteractive + +d-i finish-install/keep-consoles boolean true +d-i finish-install/reboot_in_progress note + +d-i grub-installer/only_debian boolean true +d-i grub-installer/with_other_os boolean true + +d-i keyboard-configuration/xkb-keymap select us + +d-i partman-auto/method string regular +d-i partman/choose_partition select finish +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true +d-i partman/confirm_write_new_label boolean true +d-i partman-basicfilesystems/no_swap boolean false +d-i partman-auto/expert_recipe string flat_noswap :: 1000 50 -1 ext4 \ + $primary{ } $bootable{ } method{ format } \ + format{ } use_filesystem{ } filesystem{ ext4 } \ + mountpoint{ / } \ + . +d-i partman-auto/choose_recipe select flat_noswap + +d-i passwd/root-password-again password vagrant +d-i passwd/root-password password vagrant +d-i passwd/user-fullname string vagrant +d-i passwd/username string vagrant +d-i passwd/user-password password vagrant +d-i passwd/user-password-again password vagrant + +d-i pkgsel/include string openssh-server +d-i pkgsel/install-language-support boolean false +d-i pkgsel/update-policy select none +d-i pkgsel/upgrade select none + +d-i time/zone string UTC + +d-i user-setup/allow-password-weak boolean true +d-i user-setup/encrypt-home boolean false + +tasksel tasksel/first multiselect standard, ubuntu-server + +d-i preseed/late_command string \ + echo 'vagrant ALL=(ALL) NOPASSWD: ALL' > /target/etc/sudoers.d/vagrant; \ + echo 'Defaults:vagrant !requiretty' >> /target/etc/sudoers.d/vagrant; \ + in-target chmod 440 /etc/sudoers.d/vagrant; \ No newline at end of file diff --git a/packer/packer.json b/packer/packer.json index ef20a8d..385a76b 100644 --- a/packer/packer.json +++ b/packer/packer.json @@ -5,11 +5,14 @@ "build_version": null, "kubernetes_version": "1.9.3-00", "kubernetes_cni_version": "0.6.0-00", - "existing_ansible_ssh_args": "{{env `ANSIBLE_SSH_ARGS`}}" + "existing_ansible_ssh_args": "{{env `ANSIBLE_SSH_ARGS`}}", + "vagrant_cloud_box_version": "1.0.{{timestamp}}", + "vagrant_cloud_centos_tag": "heptio/quickstart-centos", + "vagrant_cloud_ubuntu_tag": "heptio/quickstart-ubuntu" }, "builders": [ { - "name": "ami-ubuntu-16.04", + "name": "ami-ubuntu", "type": "amazon-ebs", "instance_type": "t2.small", "source_ami": "{{user `ubuntu_16_04_ami`}}", @@ -29,7 +32,7 @@ } }, { - "name": "ami-centos-7.4", + "name": "ami-centos", "type": "amazon-ebs", "instance_type": "t2.small", "source_ami": "{{user `centos_7_4_ami`}}", @@ -47,6 +50,52 @@ "kubernetes_version": "{{user `kubernetes_version`}}", "kubernetes_cni_version": "{{user `kubernetes_cni_version`}}" } + }, + { + "name": "qemu-ubuntu", + "type": "qemu", + "iso_url": "http://releases.ubuntu.com/16.04/ubuntu-16.04.4-server-amd64.iso", + "iso_checksum_url": "http://releases.ubuntu.com/16.04/SHA256SUMS", + "iso_checksum_type": "sha256", + "http_directory": "kickstart/ubuntu-16.04", + "output_directory": "packer_output/qemu-ubuntu", + "ssh_username": "vagrant", + "ssh_password": "vagrant", + "shutdown_command": "sudo systemctl poweroff", + "boot_command": [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "/install/vmlinuz noapic ", + "initrd=/install/initrd.gz ", + "auto=true ", + "priority=critical ", + "url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg ", + "" + ] + }, + { + "name": "qemu-centos", + "type": "qemu", + "iso_url": "http://mirror.rackspace.com/CentOS/7/isos/x86_64/CentOS-7-x86_64-Minimal-1708.iso", + "iso_checksum_url": "http://mirror.rackspace.com/CentOS/7/isos/x86_64/sha256sum.txt", + "iso_checksum_type": "sha256", + "http_directory": "kickstart/centos7", + "output_directory": "packer_output/qemu-centos", + "ssh_username": "vagrant", + "ssh_password": "vagrant", + "shutdown_command": "sudo systemctl poweroff", + "boot_command": [ + " text ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/ks.cfg" + ] } ], "provisioners": [ @@ -61,5 +110,26 @@ "common_upgrade_base=true kubernetes_version={{user `kubernetes_version`}} kubernetes_cni_version={{user `kubernetes_cni_version`}}" ] } + ], + "post-processors": [ + [ + { + "type": "vagrant", + "output": "packer_output/{{ .BuildName }}.box", + "only": ["qemu-ubuntu","qemu-centos"] + }, + { + "type": "vagrant-cloud", + "box_tag": "{{user `vagrant_cloud_ubuntu_tag`}}", + "version": "{{user `vagrant_cloud_box_version`}}", + "only": ["qemu-ubuntu"] + }, + { + "type": "vagrant-cloud", + "box_tag": "{{user `vagrant_cloud_centos_tag`}}", + "version": "{{user `vagrant_cloud_box_version`}}", + "only": ["qemu-centos"] + } + ] ] } diff --git a/packer/tests/vagrant-libvirt-node-conformance/Vagrantfile b/packer/tests/vagrant-libvirt-node-conformance/Vagrantfile new file mode 100644 index 0000000..4a77b31 --- /dev/null +++ b/packer/tests/vagrant-libvirt-node-conformance/Vagrantfile @@ -0,0 +1,11 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + config.vm.box = "heptio/quickstart-ubuntu" + config.vm.provider "libvirt" do |libvirt| + libvirt.memory = 4096 + end + + config.vm.provision "shell", path: "node-conformance.sh" +end diff --git a/packer/tests/vagrant-libvirt-node-conformance/node-conformance.sh b/packer/tests/vagrant-libvirt-node-conformance/node-conformance.sh new file mode 100644 index 0000000..b4c1e70 --- /dev/null +++ b/packer/tests/vagrant-libvirt-node-conformance/node-conformance.sh @@ -0,0 +1,7 @@ +#! /bin/bash -xe + +K8S_VERSION=$(< /etc/kubernetes_community_ami_version) +wget -Nq https://dl.k8s.io/${K8S_VERSION}/kubernetes-test.tar.gz +tar -zxf kubernetes-test.tar.gz kubernetes/hack kubernetes/test kubernetes/platforms/linux/amd64 +cd kubernetes/platforms/linux/amd64 +sudo ./ginkgo --nodes=8 --flakeAttempts=2 --focus="\[Conformance\]" --skip="\[Flaky\]|\[Serial\]|\[sig-network\]|Container Lifecycle Hook" ./e2e_node.test -- --k8s-bin-dir=/usr/bin \ No newline at end of file