diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 00000000..785428ae
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,36 @@
+version: 2
+jobs:
+ build:
+ # Use 'machine' type so we can run the lint step with directory mounted
+ machine:
+ docker_layer_caching: true
+ working_directory: /home/circleci/src/github.com/weaveworks/build-tools
+ environment:
+ GOPATH: /home/circleci/
+ steps:
+ - checkout
+ - run: cd build; make
+ - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c ./lint .
+ - run: cd cover; make
+ # Socks makefile needs to overwrite Go std library
+ - run: sudo chmod a+wr --recursive /usr/local/go/pkg
+ - run: cd socks; make
+ - run: cd runner; make
+
+ - deploy:
+ command: |
+ if [ "${CIRCLE_BRANCH}" == "master" ]; then
+ cd build
+ docker login -u $DOCKER_REGISTRY_USER -p $DOCKER_REGISTRY_PASSWORD
+ for image in $(make images); do
+ # Push all tags - latest and git-tag
+ docker push "${image}"
+
+ # Tag the built images with something derived from the base images in
+ # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a
+ # base image would lead to a tag of "1.8.0-stretch"
+ IMG_TAG=$(make "imagetag-${image#weaveworks/build-}")
+ docker tag "${image}:latest" "${image}:${IMG_TAG}"
+ docker push "${image}:${IMG_TAG}"
+ done
+ fi
diff --git a/COPYING.LGPL-3 b/COPYING.LGPL-3
new file mode 100644
index 00000000..f01171d4
--- /dev/null
+++ b/COPYING.LGPL-3
@@ -0,0 +1,175 @@
+./integration/assert.sh is a copy of
+
+ https://github.com/lehmannro/assert.sh/blob/master/assert.sh
+
+Since it was added to this codebase, it has only received cosmetic
+modifications. As it is licensed under the LGPL-3, here's the license
+text in its entirety:
+
+
+
+ GNU LESSER GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+ This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+ 0. Additional Definitions.
+
+ As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+ "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+ An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+ A "Combined Work" is a work produced by combining or linking an
+Application with the Library. The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+ The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+ The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+ 1. Exception to Section 3 of the GNU GPL.
+
+ You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+ 2. Conveying Modified Versions.
+
+ If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+ a) under this License, provided that you make a good faith effort to
+ ensure that, in the event an Application does not supply the
+ function or data, the facility still operates, and performs
+ whatever part of its purpose remains meaningful, or
+
+ b) under the GNU GPL, with none of the additional permissions of
+ this License applicable to that copy.
+
+ 3. Object Code Incorporating Material from Library Header Files.
+
+ The object code form of an Application may incorporate material from
+a header file that is part of the Library. You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+ a) Give prominent notice with each copy of the object code that the
+ Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the object code with a copy of the GNU GPL and this license
+ document.
+
+ 4. Combined Works.
+
+ You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+ a) Give prominent notice with each copy of the Combined Work that
+ the Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the Combined Work with a copy of the GNU GPL and this license
+ document.
+
+ c) For a Combined Work that displays copyright notices during
+ execution, include the copyright notice for the Library among
+ these notices, as well as a reference directing the user to the
+ copies of the GNU GPL and this license document.
+
+ d) Do one of the following:
+
+ 0) Convey the Minimal Corresponding Source under the terms of this
+ License, and the Corresponding Application Code in a form
+ suitable for, and under terms that permit, the user to
+ recombine or relink the Application with a modified version of
+ the Linked Version to produce a modified Combined Work, in the
+ manner specified by section 6 of the GNU GPL for conveying
+ Corresponding Source.
+
+ 1) Use a suitable shared library mechanism for linking with the
+ Library. A suitable mechanism is one that (a) uses at run time
+ a copy of the Library already present on the user's computer
+ system, and (b) will operate properly with a modified version
+ of the Library that is interface-compatible with the Linked
+ Version.
+
+ e) Provide Installation Information, but only if you would otherwise
+ be required to provide such information under section 6 of the
+ GNU GPL, and only to the extent that such information is
+ necessary to install and execute a modified version of the
+ Combined Work produced by recombining or relinking the
+ Application with a modified version of the Linked Version. (If
+ you use option 4d0, the Installation Information must accompany
+ the Minimal Corresponding Source and Corresponding Application
+ Code. If you use option 4d1, you must provide the Installation
+ Information in the manner specified by section 6 of the GNU GPL
+ for conveying Corresponding Source.)
+
+ 5. Combined Libraries.
+
+ You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+ a) Accompany the combined library with a copy of the same work based
+ on the Library, uncombined with any other library facilities,
+ conveyed under the terms of this License.
+
+ b) Give prominent notice with the combined library that part of it
+ is a work based on the Library, and explaining where to find the
+ accompanying uncombined form of the same work.
+
+ 6. Revised Versions of the GNU Lesser General Public License.
+
+ The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+ If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 00000000..9cd1640b
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,13 @@
+Copyright 2018 Weaveworks. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/README.md b/README.md
index 9092b8e2..389e4980 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,7 @@
Included in this repo are tools shared by weave.git and scope.git. They include
+- ```bazel-rules```: Bazel build rules used in our projects
- ```build```: a set of docker base-images for building weave
projects. These should be used instead of giving each project its
own build image.
@@ -32,7 +33,11 @@ Included in this repo are tools shared by weave.git and scope.git. They include
## Requirements
- ```lint``` requires shfmt to lint sh files; get shfmt with
- ```go get -u gopkg.in/mvdan/sh.v1/cmd/shfmt```
+```
+curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64
+chmod +x shfmt
+```
+ (we pin that version, and it doesn't build from the source repo any more)
## Using build-tools.git
@@ -50,3 +55,13 @@ To update the code in build-tools.git, the process is therefore:
- PR into build-tools.git, go through normal review process etc.
- Do `git subtree pull --prefix tools https://github.com/weaveworks/build-tools.git master --squash`
in your repo, and PR that.
+
+## Getting Help
+
+If you have any questions about, feedback for or problems with `build-tools`:
+
+- Invite yourself to the Weave Users Slack.
+- Ask a question on the [#general](https://weave-community.slack.com/messages/general/) slack channel.
+- [File an issue](https://github.com/weaveworks/build-tools/issues/new).
+
+Your feedback is always welcome!
diff --git a/bazel-rules/BUILD.bazel b/bazel-rules/BUILD.bazel
new file mode 100644
index 00000000..751b3707
--- /dev/null
+++ b/bazel-rules/BUILD.bazel
@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//proto:compiler.bzl", "go_proto_compiler")
+
+go_proto_compiler(
+ name = "gogo_proto",
+ deps = [
+ "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library",
+ "//vendor/github.com/gogo/protobuf/proto:go_default_library",
+ "//vendor/github.com/gogo/protobuf/sortkeys:go_default_library",
+ ],
+ plugin = "@com_github_gogo_protobuf//protoc-gen-gogoslick",
+ visibility = ["//visibility:public"],
+)
+
+go_proto_compiler(
+ name = "gogo_grpc",
+ deps = [
+ "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library",
+ "//vendor/github.com/gogo/protobuf/proto:go_default_library",
+ "//vendor/github.com/gogo/protobuf/sortkeys:go_default_library",
+ "//vendor/google.golang.org/grpc:go_default_library",
+ "//vendor/golang.org/x/net/context:go_default_library",
+ ],
+ plugin = "@com_github_gogo_protobuf//protoc-gen-gogoslick",
+ options = ["plugins=grpc"],
+ visibility = ["//visibility:public"],
+)
diff --git a/bazel-rules/gogo.bzl b/bazel-rules/gogo.bzl
new file mode 100644
index 00000000..82f24461
--- /dev/null
+++ b/bazel-rules/gogo.bzl
@@ -0,0 +1,36 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_repository")
+
+
+_BUILD_FILE = """
+proto_library(
+ name = "gogoproto",
+ srcs = ["gogo.proto"],
+ deps = [
+ "@com_google_protobuf//:descriptor_proto",
+ ],
+ visibility = ["//visibility:public"],
+)
+"""
+
+def _go_repository_impl(ctx):
+ ctx.file("BUILD.bazel", content="")
+ ctx.file("github.com/gogo/protobuf/gogoproto/BUILD.bazel", content=_BUILD_FILE)
+ ctx.template("github.com/gogo/protobuf/gogoproto/gogo.proto", ctx.attr._proto)
+
+_gogo_proto_repository = repository_rule(
+ implementation = _go_repository_impl,
+ attrs = {
+ "_proto": attr.label(default="//vendor/github.com/gogo/protobuf/gogoproto:gogo.proto"),
+ },
+)
+
+def gogo_dependencies():
+ go_repository(
+ name = "com_github_gogo_protobuf",
+ importpath = "github.com/gogo/protobuf",
+ urls = ["https://codeload.github.com/ianthehat/protobuf/zip/2adc21fd136931e0388e278825291678e1d98309"],
+ strip_prefix = "protobuf-2adc21fd136931e0388e278825291678e1d98309",
+ type = "zip",
+ build_file_proto_mode="disable",
+ )
+ _gogo_proto_repository(name = "internal_gogo_proto_repository")
diff --git a/build/Makefile b/build/Makefile
index cea049be..5c5db8b5 100644
--- a/build/Makefile
+++ b/build/Makefile
@@ -3,15 +3,16 @@
# Boiler plate for bulding Docker containers.
# All this must go at top of file I'm afraid.
-IMAGE_PREFIX := quay.io/weaveworks/build-
+IMAGE_PREFIX := weaveworks/build-
IMAGE_TAG := $(shell ../image-tag)
+GIT_REVISION := $(shell git rev-parse HEAD)
UPTODATE := .uptodate
# Every directory with a Dockerfile in it builds an image called
# $(IMAGE_PREFIX). Dependencies (i.e. things that go in the image)
# still need to be explicitly declared.
%/$(UPTODATE): %/Dockerfile %/*
- $(SUDO) docker build -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/
+ $(SUDO) docker build --build-arg=revision=$(GIT_REVISION) -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/
$(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D)) $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG)
touch $@
diff --git a/build/golang/Dockerfile b/build/golang/Dockerfile
index 6936eec2..e20873a4 100644
--- a/build/golang/Dockerfile
+++ b/build/golang/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.8.0-stretch
+FROM golang:1.12.1-stretch
RUN apt-get update && \
apt-get install -y \
curl \
@@ -11,9 +11,14 @@ RUN apt-get update && \
python-pip \
python-requests \
python-yaml \
+ shellcheck \
unzip && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-RUN pip install attrs
+RUN pip install attrs pyhcl yapf==0.16.2 flake8==3.3.0
+RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+ echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2 shfmt" | sha256sum -c && \
+ chmod +x shfmt && \
+ mv shfmt /usr/bin
RUN go clean -i net && \
go install -tags netgo std && \
go install -race -tags netgo std
@@ -25,11 +30,10 @@ RUN go get -tags netgo \
github.com/gogo/protobuf/gogoproto \
github.com/gogo/protobuf/protoc-gen-gogoslick \
github.com/golang/dep/... \
- github.com/golang/lint/golint \
+ golang.org/x/lint/golint \
github.com/golang/protobuf/protoc-gen-go \
github.com/kisielk/errcheck \
github.com/mjibson/esc \
- github.com/mvdan/sh/cmd/shfmt \
github.com/prometheus/prometheus/cmd/promtool && \
rm -rf /go/pkg /go/src
RUN mkdir protoc && \
@@ -44,3 +48,10 @@ RUN mkdir -p /var/run/secrets/kubernetes.io/serviceaccount && \
touch /var/run/secrets/kubernetes.io/serviceaccount/token
COPY build.sh /
ENTRYPOINT ["/build.sh"]
+
+ARG revision
+LABEL maintainer="Weaveworks " \
+ org.opencontainers.image.title="golang" \
+ org.opencontainers.image.source="https://github.com/weaveworks/build-tools/tree/master/build/golang" \
+ org.opencontainers.image.revision="${revision}" \
+ org.opencontainers.image.vendor="Weaveworks"
diff --git a/build/golang/build.sh b/build/golang/build.sh
index e8f9df58..cf70e1c5 100755
--- a/build/golang/build.sh
+++ b/build/golang/build.sh
@@ -13,8 +13,8 @@ fi
# will have awkward ownership. So we switch to a user with the
# same user and group IDs as source directory. We have to set a
# few things up so that sudo works without complaining later on.
-uid=$(stat --format="%u" $SRC_PATH)
-gid=$(stat --format="%g" $SRC_PATH)
+uid=$(stat --format="%u" "$SRC_PATH")
+gid=$(stat --format="%g" "$SRC_PATH")
echo "weave:x:$uid:$gid::$SRC_PATH:/bin/sh" >>/etc/passwd
echo "weave:*:::::::" >>/etc/shadow
echo "weave ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers
diff --git a/build/haskell/Dockerfile b/build/haskell/Dockerfile
index 8d40c662..79f34a80 100644
--- a/build/haskell/Dockerfile
+++ b/build/haskell/Dockerfile
@@ -2,3 +2,10 @@ FROM fpco/stack-build:lts-8.9
COPY build.sh /
COPY copy-libraries /usr/local/bin/
ENTRYPOINT ["/build.sh"]
+
+ARG revision
+LABEL maintainer="Weaveworks " \
+ org.opencontainers.image.title="haskell" \
+ org.opencontainers.image.source="https://github.com/weaveworks/build-tools/tree/master/build/haskell" \
+ org.opencontainers.image.revision="${revision}" \
+ org.opencontainers.image.vendor="Weaveworks"
diff --git a/build/haskell/build.sh b/build/haskell/build.sh
index bd529053..e80d2abb 100755
--- a/build/haskell/build.sh
+++ b/build/haskell/build.sh
@@ -9,4 +9,4 @@ if [ -z "${SRC_PATH:-}" ]; then
exit 1
fi
-make -C $SRC_PATH BUILD_IN_CONTAINER=false $*
+make -C "$SRC_PATH" BUILD_IN_CONTAINER=false "$@"
diff --git a/circle.yml b/circle.yml
deleted file mode 100644
index 57686d69..00000000
--- a/circle.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-machine:
- services:
- - docker
- environment:
- GOPATH: /home/ubuntu
- SRCDIR: /home/ubuntu/src/github.com/weaveworks/tools
- PATH: $PATH:$HOME/bin
-
-dependencies:
- post:
- - sudo chmod a+wr --recursive /usr/local/go/pkg
- - go clean -i net
- - go install -tags netgo std
- - mkdir -p $(dirname $SRCDIR)
- - cp -r $(pwd)/ $SRCDIR
- - |
- cd $SRCDIR;
- go get \
- github.com/fzipp/gocyclo \
- github.com/golang/lint/golint \
- github.com/kisielk/errcheck \
- github.com/fatih/hclfmt \
- gopkg.in/mvdan/sh.v1/cmd/shfmt
-
-test:
- override:
- - cd $SRCDIR; ./lint .
- - cd $SRCDIR/cover; make
- - cd $SRCDIR/socks; make
- - cd $SRCDIR/runner; make
- - cd $SRCDIR/build; make
-
-deployment:
- snapshot:
- branch: master
- commands:
- - docker login -e "$DOCKER_REGISTRY_EMAIL" -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASS" "$DOCKER_REGISTRY_URL"
- - |
- cd $SRCDIR/build;
- for image in $(make images); do
- # Tag the built images with the revision of this repo.
- docker push "${image}:${GIT_TAG}"
-
- # Tag the built images with something derived from the base images in
- # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a
- # base image would lead to a tag of "1.8.0-stretch"
- IMG_TAG=$(make "imagetag-${image#quay.io/weaveworks/build-}")
- docker tag "${image}:latest" "${image}:${IMG_TAG}"
- docker push "${image}:${IMG_TAG}"
- done
diff --git a/config_management/README.md b/config_management/README.md
index e98b9b4e..bf1f6f65 100644
--- a/config_management/README.md
+++ b/config_management/README.md
@@ -113,6 +113,28 @@ N.B.: `--ssh-extra-args` is used to provide:
* `StrictHostKeyChecking=no`: as VMs come and go, the same IP can be used by a different machine, so checking the host's SSH key may fail. Note that this introduces a risk of a man-in-the-middle attack.
* `UserKnownHostsFile=/dev/null`: if you previously connected a VM with the same IP but a different public key, and added it to `~/.ssh/known_hosts`, SSH may still fail to connect, hence we use `/dev/null` instead of `~/.ssh/known_hosts`.
+
+### Docker installation role
+
+Various ways to install Docker are provided:
+
+- `docker-from-docker-ce-repo`
+- `docker-from-docker-repo`
+- `docker-from-get.docker.com`
+- `docker-from-tarball`
+
+each producing a slightly different outcome, which can be useful for testing various setup scenarios.
+
+The `docker-install` role selects one of the above ways to install Docker based on the `docker_install_role` variable.
+The default value for this variable is configured in `group_vars/all`.
+You can however override it with whichever role you would want to run by passing the name of the role as a key-value pair in `extra-vars`, e.g.:
+
+```
+ansible-playbook .yml \
+ --extra-vars "docker_install_role=docker-from-docker-ce-repo"
+```
+
+
## Resources
* [https://www.vagrantup.com/docs/provisioning/ansible.html](https://www.vagrantup.com/docs/provisioning/ansible.html)
diff --git a/config_management/group_vars/all b/config_management/group_vars/all
index 24ac08cf..d728cce8 100644
--- a/config_management/group_vars/all
+++ b/config_management/group_vars/all
@@ -1,8 +1,8 @@
---
-go_version: 1.7.4
+go_version: 1.8.1
terraform_version: 0.8.5
-docker_version: 1.11.2
-docker_install_role: 'docker-from-get.docker.com'
+docker_version: 17.06
+docker_install_role: 'docker-from-docker-ce-repo'
kubernetes_version: 1.6.1
kubernetes_cni_version: 0.5.1
kubernetes_token: '123456.0123456789123456'
diff --git a/config_management/roles/docker-configuration/files/docker.conf b/config_management/roles/docker-configuration/files/docker.conf
index 6d02b55e..626d8022 100644
--- a/config_management/roles/docker-configuration/files/docker.conf
+++ b/config_management/roles/docker-configuration/files/docker.conf
@@ -1,3 +1,3 @@
[Service]
ExecStart=
-ExecStart=/usr/bin/docker daemon -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay --insecure-registry "weave-ci-registry:5000"
+ExecStart=/usr/bin/dockerd -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay --insecure-registry "weave-ci-registry:5000"
diff --git a/config_management/roles/docker-from-docker-ce-repo/tasks/debian.yml b/config_management/roles/docker-from-docker-ce-repo/tasks/debian.yml
new file mode 100644
index 00000000..3e2ae127
--- /dev/null
+++ b/config_management/roles/docker-from-docker-ce-repo/tasks/debian.yml
@@ -0,0 +1,35 @@
+---
+# Debian / Ubuntu specific:
+
+- name: install dependencies for docker repository
+ package:
+ name: "{{ item }}"
+ state: present
+ with_items:
+ - apt-transport-https
+ - ca-certificates
+
+- name: add apt key for the docker repository
+ apt_key:
+ keyserver: hkp://ha.pool.sks-keyservers.net:80
+ id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
+ state: present
+ register: apt_key_docker_repo
+
+- name: add docker's apt repository ({{ ansible_distribution | lower }}-{{ ansible_distribution_release }})
+ apt_repository:
+ repo: deb https://download.docker.com/linux/ubuntu {{ ansible_lsb.codename|lower }} stable
+ state: present
+ register: apt_docker_repo
+
+- name: update apt's cache
+ apt:
+ update_cache: yes
+ when: apt_key_docker_repo.changed or apt_docker_repo.changed
+
+- name: install docker-engine
+ package:
+ name: "{{ item }}"
+ state: present
+ with_items:
+ - docker-ce={{ docker_version }}*
diff --git a/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml b/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml
index ea9a3fa4..0acb6d8c 100644
--- a/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml
+++ b/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml
@@ -1,29 +1,10 @@
-# Docker installation from Docker's CentOS Community Edition
-# See also: https://docs.docker.com/engine/installation/linux/centos/
+---
+# Set up Docker
+# See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install
-- name: remove all potentially pre existing packages
- yum:
- name: '{{ item }}'
- state: absent
- with_items:
- - docker
- - docker-common
- - container-selinux
- - docker-selinux
- - docker-engine
+# Distribution-specific tasks:
+- include: debian.yml
+ when: ansible_os_family == "Debian"
-- name: install yum-utils
- yum:
- name: yum-utils
- state: present
-
-- name: add docker ce repo
- command: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
-
-# Note that Docker CE versions do not follow regular Docker versions, but look
-# like, for example: "17.03.0.el7"
-- name: install docker
- yum:
- name: 'docker-ce-{{ docker_version }}'
- update_cache: yes
- state: present
+- include: redhat.yml
+ when: ansible_os_family == "RedHat"
diff --git a/config_management/roles/docker-from-docker-ce-repo/tasks/redhat.yml b/config_management/roles/docker-from-docker-ce-repo/tasks/redhat.yml
new file mode 100644
index 00000000..ea9a3fa4
--- /dev/null
+++ b/config_management/roles/docker-from-docker-ce-repo/tasks/redhat.yml
@@ -0,0 +1,29 @@
+# Docker installation from Docker's CentOS Community Edition
+# See also: https://docs.docker.com/engine/installation/linux/centos/
+
+- name: remove all potentially pre existing packages
+ yum:
+ name: '{{ item }}'
+ state: absent
+ with_items:
+ - docker
+ - docker-common
+ - container-selinux
+ - docker-selinux
+ - docker-engine
+
+- name: install yum-utils
+ yum:
+ name: yum-utils
+ state: present
+
+- name: add docker ce repo
+ command: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+
+# Note that Docker CE versions do not follow regular Docker versions, but look
+# like, for example: "17.03.0.el7"
+- name: install docker
+ yum:
+ name: 'docker-ce-{{ docker_version }}'
+ update_cache: yes
+ state: present
diff --git a/config_management/roles/docker-from-get.docker.com/tasks/debian.yml b/config_management/roles/docker-from-get.docker.com/tasks/debian.yml
index 97b5b7a3..7444194e 100644
--- a/config_management/roles/docker-from-get.docker.com/tasks/debian.yml
+++ b/config_management/roles/docker-from-get.docker.com/tasks/debian.yml
@@ -5,4 +5,4 @@
shell: curl -sSL https://get.docker.com/gpg | sudo apt-key add -
- name: install docker
- shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine={{ docker_version }}*/ | sh'
+ shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine={{ docker_version }}*/ -e s/docker-ce/docker-ce={{ docker_version }}*/ | sh'
diff --git a/config_management/roles/setup-apt/files/apt-daily.timer.conf b/config_management/roles/setup-apt/files/apt-daily.timer.conf
new file mode 100644
index 00000000..bd19c61f
--- /dev/null
+++ b/config_management/roles/setup-apt/files/apt-daily.timer.conf
@@ -0,0 +1,2 @@
+[Timer]
+Persistent=false
diff --git a/config_management/roles/setup-apt/tasks/main.yml b/config_management/roles/setup-apt/tasks/main.yml
new file mode 100644
index 00000000..3593cf70
--- /dev/null
+++ b/config_management/roles/setup-apt/tasks/main.yml
@@ -0,0 +1,10 @@
+---
+# Set up apt
+
+# Ubuntu runs an apt update process that will run on first boot from image.
+# This is of questionable value when the machines are only going to live for a few minutes.
+# If you leave them on they will run the process daily.
+# Also we have seen the update process create a 'defunct' process which then throws off Weave Net smoke-test checks.
+# So, we override the 'persistent' setting so it will still run at the scheduled time but will not try to catch up on first boot.
+- name: copy apt daily override
+ copy: src=apt-daily.timer.conf dest=/etc/systemd/system/apt-daily.timer.d/
diff --git a/config_management/roles/weave-net-utilities/tasks/main.yml b/config_management/roles/weave-net-utilities/tasks/main.yml
index 89038597..6883d23a 100644
--- a/config_management/roles/weave-net-utilities/tasks/main.yml
+++ b/config_management/roles/weave-net-utilities/tasks/main.yml
@@ -45,3 +45,12 @@
- alpine
- aanand/docker-dnsutils
- weaveworks/hello-world
+
+- name: docker pull docker-py which is used by tests
+ docker_image:
+ name: joffrey/docker-py
+ tag: '{{ item }}'
+ state: present
+ with_items:
+ - '1.8.1'
+ - '1.9.0-rc2'
diff --git a/config_management/setup_bare_docker.yml b/config_management/setup_bare_docker.yml
new file mode 100644
index 00000000..fac8405f
--- /dev/null
+++ b/config_management/setup_bare_docker.yml
@@ -0,0 +1,16 @@
+---
+################################################################################
+# Install Docker from Docker's official repository
+################################################################################
+
+- name: install docker
+ hosts: all
+ gather_facts: false # required in case Python is not available on the host
+ become: true
+ become_user: root
+
+ pre_tasks:
+ - include: library/setup_ansible_dependencies.yml
+
+ roles:
+ - docker-install
diff --git a/config_management/setup_weave-net_dev.yml b/config_management/setup_weave-net_dev.yml
index bdfa08e9..1923d011 100644
--- a/config_management/setup_weave-net_dev.yml
+++ b/config_management/setup_weave-net_dev.yml
@@ -13,6 +13,7 @@
- include: library/setup_ansible_dependencies.yml
roles:
+ - setup-apt
- dev-tools
- golang-from-tarball
- docker-install
diff --git a/config_management/setup_weave-net_test.yml b/config_management/setup_weave-net_test.yml
index fbd155df..7125d054 100644
--- a/config_management/setup_weave-net_test.yml
+++ b/config_management/setup_weave-net_test.yml
@@ -13,6 +13,7 @@
- include: library/setup_ansible_dependencies.yml
roles:
+ - setup-apt
- docker-install
- weave-net-utilities
- kubernetes-install
diff --git a/dependencies/cross_versions.py b/dependencies/cross_versions.py
index bc93cf31..dd920f0e 100755
--- a/dependencies/cross_versions.py
+++ b/dependencies/cross_versions.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
# Generate the cross product of latest versions of Weave Net's dependencies:
# - Go
@@ -20,74 +20,72 @@
from itertools import product
# See also: /usr/include/sysexits.h
-_ERROR_RUNTIME=1
-_ERROR_ILLEGAL_ARGS=64
+_ERROR_RUNTIME = 1
+_ERROR_ILLEGAL_ARGS = 64
+
def _usage(error_message=None):
- if error_message:
- stderr.write('ERROR: ' + error_message + linesep)
- stdout.write(linesep.join([
- 'Usage:',
- ' cross_versions.py [OPTION]...',
- 'Examples:',
- ' cross_versions.py',
- ' cross_versions.py -r',
- ' cross_versions.py --rc',
- ' cross_versions.py -l',
- ' cross_versions.py --latest',
- 'Options:',
- '-l/--latest Include only the latest version of each major and minor versions sub-tree.',
- '-r/--rc Include release candidate versions.',
- '-h/--help Prints this!',
- ''
- ]))
+ if error_message:
+ stderr.write('ERROR: ' + error_message + linesep)
+ stdout.write(
+ linesep.join([
+ 'Usage:', ' cross_versions.py [OPTION]...', 'Examples:',
+ ' cross_versions.py', ' cross_versions.py -r',
+ ' cross_versions.py --rc', ' cross_versions.py -l',
+ ' cross_versions.py --latest', 'Options:',
+ '-l/--latest Include only the latest version of each major and'
+ ' minor versions sub-tree.',
+ '-r/--rc Include release candidate versions.',
+ '-h/--help Prints this!', ''
+ ]))
+
def _validate_input(argv):
- try:
- config = {
- 'rc': False,
- 'latest': False
- }
- opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc'])
- for opt, value in opts:
- if opt in ('-h', '--help'):
- _usage()
- exit()
- if opt in ('-l', '--latest'):
- config['latest'] = True
- if opt in ('-r', '--rc'):
- config['rc'] = True
- if len(args) != 0:
- raise ValueError('Unsupported argument(s): %s.' % args)
- return config
- except GetoptError as e:
- _usage(str(e))
- exit(_ERROR_ILLEGAL_ARGS)
- except ValueError as e:
- _usage(str(e))
- exit(_ERROR_ILLEGAL_ARGS)
+ try:
+ config = {'rc': False, 'latest': False}
+ opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc'])
+ for opt, value in opts:
+ if opt in ('-h', '--help'):
+ _usage()
+ exit()
+ if opt in ('-l', '--latest'):
+ config['latest'] = True
+ if opt in ('-r', '--rc'):
+ config['rc'] = True
+ if len(args) != 0:
+ raise ValueError('Unsupported argument(s): %s.' % args)
+ return config
+ except GetoptError as e:
+ _usage(str(e))
+ exit(_ERROR_ILLEGAL_ARGS)
+ except ValueError as e:
+ _usage(str(e))
+ exit(_ERROR_ILLEGAL_ARGS)
+
def _versions(dependency, config):
- return map(str,
- filter_versions(
- get_versions_from(DEPS[dependency]['url'], DEPS[dependency]['re']),
- DEPS[dependency]['min'],
- **config
- )
- )
+ return map(str,
+ filter_versions(
+ get_versions_from(DEPS[dependency]['url'],
+ DEPS[dependency]['re']),
+ DEPS[dependency]['min'], **config))
+
def cross_versions(config):
- docker_versions = _versions('docker', config)
- k8s_versions = _versions('kubernetes', config)
- return product(docker_versions, k8s_versions)
+ docker_versions = _versions('docker', config)
+ k8s_versions = _versions('kubernetes', config)
+ return product(docker_versions, k8s_versions)
+
def main(argv):
- try:
- config = _validate_input(argv)
- print(linesep.join('\t'.join(triple) for triple in cross_versions(config)))
- except Exception as e:
- print(str(e))
- exit(_ERROR_RUNTIME)
+ try:
+ config = _validate_input(argv)
+ print(linesep.join('\t'.join(triple)
+ for triple in cross_versions(config)))
+ except Exception as e:
+ print(str(e))
+ exit(_ERROR_RUNTIME)
+
if __name__ == '__main__':
- main(argv[1:])
+ main(argv[1:])
diff --git a/dependencies/list_os_images.sh b/dependencies/list_os_images.sh
index 7d2495da..139a0814 100755
--- a/dependencies/list_os_images.sh
+++ b/dependencies/list_os_images.sh
@@ -8,11 +8,11 @@ Dependencies:
- gcloud, Google Cloud Platform's CLI
- aws,
Usage:
- $ ./$(basename "$0") PROVIDER OS
+ \$ ./$(basename "$0") PROVIDER OS
PROVIDER={gcp}
OS={ubuntu|debian|centos}
Example:
- $ ./$(basename "$0") gcp ubuntu
+ \$ ./$(basename "$0") gcp ubuntu
ubuntu-os-cloud/ubuntu-1204-lts
ubuntu-os-cloud/ubuntu-1404-lts
ubuntu-os-cloud/ubuntu-1604-lts
@@ -57,7 +57,7 @@ fi
case "$1" in
'gcp')
- gcloud compute images list --standard-images --regexp=".*?$2.*" \
+ gcloud compute images list --standard-images --filter="name~'.*?$2.*'" \
--format="csv[no-heading][separator=/](selfLink.map().scope(projects).segment(0),family)" \
| sort -d
;;
diff --git a/dependencies/list_versions.py b/dependencies/list_versions.py
index 3b756cd1..e008ecfe 100755
--- a/dependencies/list_versions.py
+++ b/dependencies/list_versions.py
@@ -1,17 +1,32 @@
-#!/usr/bin/python
+#!/usr/bin/env python
# List all available versions of Weave Net's dependencies:
# - Go
# - Docker
# - Kubernetes
#
-# Depending on the parameters passed, it can gather the equivalent of the below bash one-liners:
-# git ls-remote --tags https://github.com/golang/go | grep -oP '(?<=refs/tags/go)[\.\d]+$' | sort --version-sort
-# git ls-remote --tags https://github.com/golang/go | grep -oP '(?<=refs/tags/go)[\.\d]+rc\d+$' | sort --version-sort | tail -n 1
-# git ls-remote --tags https://github.com/docker/docker | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+$' | sort --version-sort
-# git ls-remote --tags https://github.com/docker/docker | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+\-rc\d*$' | sort --version-sort | tail -n 1
-# git ls-remote --tags https://github.com/kubernetes/kubernetes | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+$' | sort --version-sort
-# git ls-remote --tags https://github.com/kubernetes/kubernetes | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+\-beta\.\d+$' | sort --version-sort | tail -n 1
+# Depending on the parameters passed, it can gather the equivalent of the below
+# bash one-liners:
+# git ls-remote --tags https://github.com/golang/go \
+# | grep -oP '(?<=refs/tags/go)[\.\d]+$' \
+# | sort --version-sort
+# git ls-remote --tags https://github.com/golang/go \
+# | grep -oP '(?<=refs/tags/go)[\.\d]+rc\d+$' \
+# | sort --version-sort \
+# | tail -n 1
+# git ls-remote --tags https://github.com/docker/docker \
+# | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+$' \
+# | sort --version-sort
+# git ls-remote --tags https://github.com/docker/docker \
+# | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+\-rc\d*$' \
+# | sort --version-sort \
+# | tail -n 1
+# git ls-remote --tags https://github.com/kubernetes/kubernetes \
+# | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+$' \
+# | sort --version-sort
+# git ls-remote --tags https://github.com/kubernetes/kubernetes \
+# | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+\-beta\.\d+$' \
+# | sort --version-sort | tail -n 1
#
# Dependencies:
# - python
@@ -23,7 +38,7 @@
from os import linesep, path
from sys import argv, exit, stdout, stderr
from getopt import getopt, GetoptError
-from subprocess import Popen, PIPE, STDOUT
+from subprocess import Popen, PIPE
from pkg_resources import parse_version
from itertools import groupby
from six.moves import filter
@@ -31,236 +46,298 @@
import re
# See also: /usr/include/sysexits.h
-_ERROR_RUNTIME=1
-_ERROR_ILLEGAL_ARGS=64
-
-_TAG_REGEX='^[0-9a-f]{40}\s+refs/tags/%s$'
-_VERSION='version'
-DEPS={
- 'go': {
- 'url': 'https://github.com/golang/go',
- 're': 'go(?P<%s>[\d\.]+(?:rc\d)*)' % _VERSION,
- 'min': None
- },
- 'docker': {
- 'url': 'https://github.com/docker/docker',
- 're': 'v(?P<%s>\d+\.\d+\.\d+(?:\-rc\d)*)' % _VERSION,
- # Weave Net only works with Docker from 1.10.0 onwards, so we ignore all previous versions:
- 'min': '1.10.0'
- },
- 'kubernetes': {
- 'url': 'https://github.com/kubernetes/kubernetes',
- 're': 'v(?P<%s>\d+\.\d+\.\d+(?:\-beta\.\d)*)' % _VERSION,
- # Weave Kube requires Kubernetes 1.4.2+, so we ignore all previous versions:
- 'min': '1.4.2'
- }
+_ERROR_RUNTIME = 1
+_ERROR_ILLEGAL_ARGS = 64
+
+_TAG_REGEX = '^[0-9a-f]{40}\s+refs/tags/%s$'
+_VERSION = 'version'
+DEPS = {
+ 'go': {
+ 'url': 'https://github.com/golang/go',
+ 're': 'go(?P<%s>[\d\.]+(?:rc\d)*)' % _VERSION,
+ 'min': None
+ },
+ 'docker': {
+ 'url': 'https://github.com/docker/docker',
+ 're': 'v(?P<%s>\d+\.\d+\.\d+(?:\-rc\d)*)' % _VERSION,
+ # Weave Net only works with Docker from 1.10.0 onwards, so we ignore
+ # all previous versions:
+ 'min': '1.10.0',
+ },
+ 'kubernetes': {
+ 'url': 'https://github.com/kubernetes/kubernetes',
+ 're': 'v(?P<%s>\d+\.\d+\.\d+(?:\-beta\.\d)*)' % _VERSION,
+ # Weave Kube requires Kubernetes 1.4.2+, so we ignore all previous
+ # versions:
+ 'min': '1.4.2',
+ }
}
+
class Version(object):
- ''' Helper class to parse and manipulate (sort, filter, group) software versions. '''
- def __init__(self, version):
- self.version = version
- self.digits = [int(x) if x else 0 for x in re.match('(\d*)\.?(\d*)\.?(\d*).*?', version).groups()]
- self.major, self.minor, self.patch = self.digits
- self.__parsed = parse_version(version)
- self.is_rc = self.__parsed.is_prerelease
- def __lt__ (self, other):
- return self.__parsed.__lt__(other.__parsed)
- def __gt__ (self, other):
- return self.__parsed.__gt__(other.__parsed)
- def __le__ (self, other):
- return self.__parsed.__le__(other.__parsed)
- def __ge__ (self, other):
- return self.__parsed.__ge__(other.__parsed)
- def __eq__ (self, other):
- return self.__parsed.__eq__(other.__parsed)
- def __ne__ (self, other):
- return self.__parsed.__ne__(other.__parsed)
- def __str__(self):
- return self.version
- def __repr__(self):
- return self.version
+ ''' Helper class to parse and manipulate (sort, filter, group) software
+ versions. '''
+
+ def __init__(self, version):
+ self.version = version
+ self.digits = [
+ int(x) if x else 0
+ for x in re.match('(\d*)\.?(\d*)\.?(\d*).*?', version).groups()
+ ]
+ self.major, self.minor, self.patch = self.digits
+ self.__parsed = parse_version(version)
+ self.is_rc = self.__parsed.is_prerelease
+
+ def __lt__(self, other):
+ return self.__parsed.__lt__(other.__parsed)
+
+ def __gt__(self, other):
+ return self.__parsed.__gt__(other.__parsed)
+
+ def __le__(self, other):
+ return self.__parsed.__le__(other.__parsed)
+
+ def __ge__(self, other):
+ return self.__parsed.__ge__(other.__parsed)
+
+ def __eq__(self, other):
+ return self.__parsed.__eq__(other.__parsed)
+
+ def __ne__(self, other):
+ return self.__parsed.__ne__(other.__parsed)
+
+ def __str__(self):
+ return self.version
+
+ def __repr__(self):
+ return self.version
+
def _read_go_version_from_dockerfile():
- # Read Go version from weave/build/Dockerfile
- dockerfile_path = path.join(path.dirname(path.dirname(path.dirname(path.realpath(__file__)))), 'build', 'Dockerfile')
- with open(dockerfile_path, 'r') as f:
- for line in f:
- m = re.match('^FROM golang:(\S*)$', line)
- if m:
- return m.group(1)
- raise RuntimeError("Failed to read Go version from weave/build/Dockerfile. You may be running this script from somewhere else than weave/tools.")
+ # Read Go version from weave/build/Dockerfile
+ dockerfile_path = path.join(
+ path.dirname(path.dirname(path.dirname(path.realpath(__file__)))),
+ 'build', 'Dockerfile')
+ with open(dockerfile_path, 'r') as f:
+ for line in f:
+ m = re.match('^FROM golang:(\S*)$', line)
+ if m:
+ return m.group(1)
+ raise RuntimeError(
+ "Failed to read Go version from weave/build/Dockerfile."
+ " You may be running this script from somewhere else than weave/tools."
+ )
+
def _try_set_min_go_version():
- ''' Set the current version of Go used to build Weave Net's containers as the minimum version. '''
- try:
- DEPS['go']['min'] = _read_go_version_from_dockerfile()
- except IOError as e:
- stderr.write('WARNING: No minimum Go version set. Root cause: %s%s' % (e, linesep))
+ ''' Set the current version of Go used to build Weave Net's containers as
+ the minimum version. '''
+ try:
+ DEPS['go']['min'] = _read_go_version_from_dockerfile()
+ except IOError as e:
+ stderr.write('WARNING: No minimum Go version set. Root cause: %s%s' %
+ (e, linesep))
+
def _sanitize(out):
- return out.decode('ascii').strip().split(linesep)
+ return out.decode('ascii').strip().split(linesep)
+
def _parse_tag(tag, version_pattern, debug=False):
- ''' Parse Git tag output's line using the provided `version_pattern`, e.g.:
- >>> _parse_tag('915b77eb4efd68916427caf8c7f0b53218c5ea4a refs/tags/v1.4.6', 'v(?P\d+\.\d+\.\d+(?:\-beta\.\d)*)')
- '1.4.6'
- '''
- pattern = _TAG_REGEX % version_pattern
- m = re.match(pattern, tag)
- if m:
- return m.group(_VERSION)
- elif debug:
- stderr.write('ERROR: Failed to parse version out of tag [%s] using [%s].%s' % (tag, pattern, linesep))
+ ''' Parse Git tag output's line using the provided `version_pattern`, e.g.:
+ >>> _parse_tag(
+ '915b77eb4efd68916427caf8c7f0b53218c5ea4a refs/tags/v1.4.6',
+ 'v(?P\d+\.\d+\.\d+(?:\-beta\.\d)*)')
+ '1.4.6'
+ '''
+ pattern = _TAG_REGEX % version_pattern
+ m = re.match(pattern, tag)
+ if m:
+ return m.group(_VERSION)
+ elif debug:
+ stderr.write(
+ 'ERROR: Failed to parse version out of tag [%s] using [%s].%s' %
+ (tag, pattern, linesep))
+
def get_versions_from(git_repo_url, version_pattern):
- ''' Get release and release candidates' versions from the provided Git repository. '''
- git = Popen(shlex.split('git ls-remote --tags %s' % git_repo_url), stdout=PIPE)
- out, err = git.communicate()
- status_code = git.returncode
- if status_code != 0:
- raise RuntimeError('Failed to retrieve git tags from %s. Status code: %s. Output: %s. Error: %s' % (git_repo_url, status_code, out, err))
- return list(filter(None, (_parse_tag(line, version_pattern) for line in _sanitize(out))))
+ ''' Get release and release candidates' versions from the provided Git
+ repository. '''
+ git = Popen(
+ shlex.split('git ls-remote --tags %s' % git_repo_url), stdout=PIPE)
+ out, err = git.communicate()
+ status_code = git.returncode
+ if status_code != 0:
+ raise RuntimeError('Failed to retrieve git tags from %s. '
+ 'Status code: %s. Output: %s. Error: %s' %
+ (git_repo_url, status_code, out, err))
+ return list(
+ filter(None, (_parse_tag(line, version_pattern)
+ for line in _sanitize(out))))
+
def _tree(versions, level=0):
- ''' Group versions by major, minor and patch version digits. '''
- if not versions or level >= len(versions[0].digits):
- return # Empty versions or no more digits to group by.
- versions_tree = []
- for _, versions_group in groupby(versions, lambda v: v.digits[level]):
- subtree = _tree(list(versions_group), level+1)
- if subtree:
- versions_tree.append(subtree)
- # Return the current subtree if non-empty, or the list of "leaf" versions:
- return versions_tree if versions_tree else versions
+ ''' Group versions by major, minor and patch version digits. '''
+ if not versions or level >= len(versions[0].digits):
+ return # Empty versions or no more digits to group by.
+ versions_tree = []
+ for _, versions_group in groupby(versions, lambda v: v.digits[level]):
+ subtree = _tree(list(versions_group), level + 1)
+ if subtree:
+ versions_tree.append(subtree)
+ # Return the current subtree if non-empty, or the list of "leaf" versions:
+ return versions_tree if versions_tree else versions
+
def _is_iterable(obj):
- '''
- Check if the provided object is an iterable collection, i.e. not a string, e.g. a list, a generator:
- >>> _is_iterable('string')
- False
- >>> _is_iterable([1, 2, 3])
- True
- >>> _is_iterable((x for x in [1, 2, 3]))
- True
- '''
- return hasattr(obj, '__iter__') and not isinstance(obj, str)
+ '''
+ Check if the provided object is an iterable collection, i.e. not a string,
+ e.g. a list, a generator:
+ >>> _is_iterable('string')
+ False
+ >>> _is_iterable([1, 2, 3])
+ True
+ >>> _is_iterable((x for x in [1, 2, 3]))
+ True
+ '''
+ return hasattr(obj, '__iter__') and not isinstance(obj, str)
+
def _leaf_versions(tree, rc):
- '''
- Recursively traverse the versions tree in a depth-first fashion,
- and collect the last node of each branch, i.e. leaf versions.
- '''
- versions = []
- if _is_iterable(tree):
- for subtree in tree:
- versions.extend(_leaf_versions(subtree, rc))
- if not versions:
- if rc:
- last_rc = next(filter(lambda v: v.is_rc, reversed(tree)), None)
- last_prod = next(filter(lambda v: not v.is_rc, reversed(tree)), None)
- if last_rc and last_prod and (last_prod < last_rc):
- versions.extend([last_prod, last_rc])
- elif not last_prod:
- versions.append(last_rc)
- else:
- # Either there is no RC, or we ignore the RC as older than the latest production version:
- versions.append(last_prod)
- else:
- versions.append(tree[-1])
- return versions
+ '''
+ Recursively traverse the versions tree in a depth-first fashion,
+ and collect the last node of each branch, i.e. leaf versions.
+ '''
+ versions = []
+ if _is_iterable(tree):
+ for subtree in tree:
+ versions.extend(_leaf_versions(subtree, rc))
+ if not versions:
+ if rc:
+ last_rc = next(filter(lambda v: v.is_rc, reversed(tree)), None)
+ last_prod = next(
+ filter(lambda v: not v.is_rc, reversed(tree)), None)
+ if last_rc and last_prod and (last_prod < last_rc):
+ versions.extend([last_prod, last_rc])
+ elif not last_prod:
+ versions.append(last_rc)
+ else:
+ # Either there is no RC, or we ignore the RC as older than
+ # the latest production version:
+ versions.append(last_prod)
+ else:
+ versions.append(tree[-1])
+ return versions
+
def filter_versions(versions, min_version=None, rc=False, latest=False):
- ''' Filter provided versions
-
- >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version=None, latest=False, rc=False)
- [1.0.0, 1.0.1, 1.1.1, 2.0.0]
-
- >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version=None, latest=True, rc=False)
- [1.0.1, 1.1.1, 2.0.0]
-
- >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version=None, latest=False, rc=True)
- [1.0.0-beta.1, 1.0.0, 1.0.1, 1.1.1, 1.1.2-rc1, 2.0.0]
-
- >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version='1.1.0', latest=False, rc=True)
- [1.1.1, 1.1.2-rc1, 2.0.0]
-
- >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version=None, latest=True, rc=True)
- [1.0.1, 1.1.1, 1.1.2-rc1, 2.0.0]
-
- >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version='1.1.0', latest=True, rc=True)
- [1.1.1, 1.1.2-rc1, 2.0.0]
- '''
- versions = sorted([Version(v) for v in versions])
- if min_version:
- min_version = Version(min_version)
- versions = [v for v in versions if v >= min_version]
- if not rc:
- versions = [v for v in versions if not v.is_rc]
- if latest:
- versions_tree = _tree(versions)
- return _leaf_versions(versions_tree, rc)
- else:
- return versions
+ ''' Filter provided versions
+
+ >>> filter_versions(
+ ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'],
+ min_version=None, latest=False, rc=False)
+ [1.0.0, 1.0.1, 1.1.1, 2.0.0]
+
+ >>> filter_versions(
+ ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'],
+ min_version=None, latest=True, rc=False)
+ [1.0.1, 1.1.1, 2.0.0]
+
+ >>> filter_versions(
+ ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'],
+ min_version=None, latest=False, rc=True)
+ [1.0.0-beta.1, 1.0.0, 1.0.1, 1.1.1, 1.1.2-rc1, 2.0.0]
+
+ >>> filter_versions(
+ ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'],
+ min_version='1.1.0', latest=False, rc=True)
+ [1.1.1, 1.1.2-rc1, 2.0.0]
+
+ >>> filter_versions(
+ ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'],
+ min_version=None, latest=True, rc=True)
+ [1.0.1, 1.1.1, 1.1.2-rc1, 2.0.0]
+
+ >>> filter_versions(
+ ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'],
+ min_version='1.1.0', latest=True, rc=True)
+ [1.1.1, 1.1.2-rc1, 2.0.0]
+ '''
+ versions = sorted([Version(v) for v in versions])
+ if min_version:
+ min_version = Version(min_version)
+ versions = [v for v in versions if v >= min_version]
+ if not rc:
+ versions = [v for v in versions if not v.is_rc]
+ if latest:
+ versions_tree = _tree(versions)
+ return _leaf_versions(versions_tree, rc)
+ else:
+ return versions
+
def _usage(error_message=None):
- if error_message:
- stderr.write('ERROR: ' + error_message + linesep)
- stdout.write(linesep.join([
- 'Usage:',
- ' list_versions.py [OPTION]... [DEPENDENCY]',
- 'Examples:',
- ' list_versions.py go',
- ' list_versions.py -r docker',
- ' list_versions.py --rc docker',
- ' list_versions.py -l kubernetes',
- ' list_versions.py --latest kubernetes',
- 'Options:',
- '-l/--latest Include only the latest version of each major and minor versions sub-tree.',
- '-r/--rc Include release candidate versions.',
- '-h/--help Prints this!',
- ''
- ]))
+ if error_message:
+ stderr.write('ERROR: ' + error_message + linesep)
+ stdout.write(
+ linesep.join([
+ 'Usage:', ' list_versions.py [OPTION]... [DEPENDENCY]',
+ 'Examples:', ' list_versions.py go',
+ ' list_versions.py -r docker',
+ ' list_versions.py --rc docker',
+ ' list_versions.py -l kubernetes',
+ ' list_versions.py --latest kubernetes', 'Options:',
+ '-l/--latest Include only the latest version of each major and'
+ ' minor versions sub-tree.',
+ '-r/--rc Include release candidate versions.',
+ '-h/--help Prints this!', ''
+ ]))
+
def _validate_input(argv):
- try:
- config = {
- 'rc': False,
- 'latest': False
- }
- opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc'])
- for opt, value in opts:
- if opt in ('-h', '--help'):
- _usage()
- exit()
- if opt in ('-l', '--latest'):
- config['latest'] = True
- if opt in ('-r', '--rc'):
- config['rc'] = True
- if len(args) != 1:
- raise ValueError('Please provide a dependency to get versions of. Expected 1 argument but got %s: %s.' % (len(args), args))
- dependency=args[0].lower()
- if dependency not in DEPS.keys():
- raise ValueError('Please provide a valid dependency. Supported one dependency among {%s} but got: %s.' % (', '.join(DEPS.keys()), dependency))
- return dependency, config
- except GetoptError as e:
- _usage(str(e))
- exit(_ERROR_ILLEGAL_ARGS)
- except ValueError as e:
- _usage(str(e))
- exit(_ERROR_ILLEGAL_ARGS)
+ try:
+ config = {'rc': False, 'latest': False}
+ opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc'])
+ for opt, value in opts:
+ if opt in ('-h', '--help'):
+ _usage()
+ exit()
+ if opt in ('-l', '--latest'):
+ config['latest'] = True
+ if opt in ('-r', '--rc'):
+ config['rc'] = True
+ if len(args) != 1:
+ raise ValueError('Please provide a dependency to get versions of.'
+ ' Expected 1 argument but got %s: %s.' %
+ (len(args), args))
+ dependency = args[0].lower()
+ if dependency not in DEPS.keys():
+ raise ValueError(
+ 'Please provide a valid dependency.'
+ ' Supported one dependency among {%s} but got: %s.' %
+ (', '.join(DEPS.keys()), dependency))
+ return dependency, config
+ except GetoptError as e:
+ _usage(str(e))
+ exit(_ERROR_ILLEGAL_ARGS)
+ except ValueError as e:
+ _usage(str(e))
+ exit(_ERROR_ILLEGAL_ARGS)
+
def main(argv):
- try:
- dependency, config = _validate_input(argv)
- if dependency == 'go':
- _try_set_min_go_version()
- versions = get_versions_from(DEPS[dependency]['url'], DEPS[dependency]['re'])
- versions = filter_versions(versions, DEPS[dependency]['min'], **config)
- print(linesep.join(map(str, versions)))
- except Exception as e:
- print(str(e))
- exit(_ERROR_RUNTIME)
+ try:
+ dependency, config = _validate_input(argv)
+ if dependency == 'go':
+ _try_set_min_go_version()
+ versions = get_versions_from(DEPS[dependency]['url'],
+ DEPS[dependency]['re'])
+ versions = filter_versions(versions, DEPS[dependency]['min'], **config)
+ print(linesep.join(map(str, versions)))
+ except Exception as e:
+ print(str(e))
+ exit(_ERROR_RUNTIME)
+
if __name__ == '__main__':
- main(argv[1:])
+ main(argv[1:])
diff --git a/image-tag b/image-tag
index d1fd2f72..6a11cb24 100755
--- a/image-tag
+++ b/image-tag
@@ -4,6 +4,9 @@ set -o errexit
set -o nounset
set -o pipefail
-WORKING_SUFFIX=$(if ! git diff --exit-code --quiet HEAD >&2; then echo "-WIP"; else echo ""; fi)
+WORKING_SUFFIX=$(if git status --porcelain | grep -qE '^(?:[^?][^ ]|[^ ][^?])\s'; then echo "-WIP"; else echo ""; fi)
BRANCH_PREFIX=$(git rev-parse --abbrev-ref HEAD)
-echo "${BRANCH_PREFIX//\//-}-$(git rev-parse --short HEAD)$WORKING_SUFFIX"
+
+# Fix the object name prefix length to 8 characters to have it consistent across the system.
+# See https://git-scm.com/docs/git-rev-parse#Documentation/git-rev-parse.txt---shortlength
+echo "${BRANCH_PREFIX//\//-}-$(git rev-parse --short=8 HEAD)$WORKING_SUFFIX"
diff --git a/integration/config.sh b/integration/config.sh
index 6bf20860..54192192 100644
--- a/integration/config.sh
+++ b/integration/config.sh
@@ -115,11 +115,8 @@ rm_containers() {
start_suite() {
for host in $HOSTS; do
[ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave"
- PLUGIN_ID=$(docker_on "$host" ps -aq --filter=name=weaveplugin)
- PLUGIN_FILTER="cat"
- [ -n "$PLUGIN_ID" ] && PLUGIN_FILTER="grep -v $PLUGIN_ID"
# shellcheck disable=SC2046
- rm_containers "$host" $(docker_on "$host" ps -aq 2>/dev/null | $PLUGIN_FILTER)
+ rm_containers "$host" $(docker_on "$host" ps -aq 2>/dev/null)
run_on "$host" "docker network ls | grep -q ' weave ' && docker network rm weave" || true
weave_on "$host" reset 2>/dev/null
done
diff --git a/lint b/lint
index 1589bd9e..40ad4e7f 100755
--- a/lint
+++ b/lint
@@ -6,7 +6,8 @@
#
# For shell files, it runs shfmt. If you don't have that installed, you can get
# it with:
-# go get -u gopkg.in/mvdan/sh.v1/cmd/shfmt
+# curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64
+# chmod +x shfmt
#
# With no arguments, it lints the current files staged
# for git commit. Or you can pass it explicit filenames
@@ -50,14 +51,6 @@ spell_check() {
local filename="$1"
local lint_result=0
- # we don't want to spell check tar balls, binaries, Makefile and json files
- if file "$filename" | grep executable >/dev/null 2>&1; then
- return $lint_result
- fi
- if [[ $filename == *".tar" || $filename == *".gz" || $filename == *".json" || $(basename "$filename") == "Makefile" ]]; then
- return $lint_result
- fi
-
# misspell is completely optional. If you don't like it
# don't have it installed.
if ! type misspell >/dev/null 2>&1; then
@@ -72,6 +65,7 @@ spell_check() {
}
lint_go() {
+ # This function is called on a whole directory containing Go files
local filename="$1"
local lint_result=0
@@ -80,7 +74,7 @@ lint_go() {
echo "${filename}: run gofmt -s -w ${filename}"
fi
- go tool vet "${filename}" || lint_result=$?
+ go vet "${filename}" || lint_result=$?
# golint is completely optional. If you don't like it
# don't have it installed.
@@ -113,9 +107,12 @@ lint_sh() {
local filename="$1"
local lint_result=0
- if ! diff <(shfmt -i 4 "${filename}") "${filename}" >/dev/null; then
- lint_result=1
- echo "${filename}: run shfmt -i 4 -w ${filename}"
+ # Skip shfmt validation, if not installed
+ if type shfmt >/dev/null 2>&1; then
+ if ! diff -u "${filename}" <(shfmt -i 4 "${filename}"); then
+ lint_result=1
+ echo "${filename}: run shfmt -i 4 -w ${filename}"
+ fi
fi
# the shellcheck is completely optional. If you don't like it
@@ -131,7 +128,7 @@ lint_tf() {
local filename="$1"
local lint_result=0
- if ! diff <(hclfmt "${filename}") "${filename}" >/dev/null; then
+ if ! diff -u <(hclfmt "${filename}") "${filename}"; then
lint_result=1
echo "${filename}: run hclfmt -w ${filename}"
fi
@@ -153,6 +150,21 @@ lint_md() {
return $lint_result
}
+lint_py() {
+ local filename="$1"
+ local lint_result=0
+
+ if yapf --diff "${filename}" | grep -qE '^[+-]'; then
+ lint_result=1
+ echo "${filename} needs reformatting. Run: yapf --in-place ${filename}"
+ else
+ # Only run flake8 if yapf passes, since they pick up a lot of similar issues
+ flake8 "${filename}" || lint_result=1
+ fi
+
+ return $lint_result
+}
+
lint() {
filename="$1"
ext="${filename##*\.}"
@@ -170,18 +182,24 @@ lint() {
*.pb.go) return ;;
esac
- if [[ "$(file --mime-type "${filename}" | awk '{print $2}')" == "text/x-shellscript" ]]; then
- ext="sh"
- fi
+ mimetype=$(file --mime-type "${filename}" | awk '{print $2}')
- case "$ext" in
- go) lint_go "${filename}" || lint_result=1 ;;
- sh) lint_sh "${filename}" || lint_result=1 ;;
- tf) lint_tf "${filename}" || lint_result=1 ;;
- md) lint_md "${filename}" || lint_result=1 ;;
+ case "$mimetype.$ext" in
+ text/x-shellscript.*) lint_sh "${filename}" || lint_result=1 ;;
+ *.go) ;; # done at directory level
+ *.tf) lint_tf "${filename}" || lint_result=1 ;;
+ *.md) lint_md "${filename}" || lint_result=1 ;;
+ *.py) lint_py "${filename}" || lint_result=1 ;;
esac
- spell_check "${filename}" || lint_result=1
+ # we don't want to spell check tar balls, binaries, Makefile and json files
+ case "$mimetype.$ext" in
+ *.tar | *.gz | *.json) ;;
+ *.req | *.key | *.pem | *.crt) ;;
+ application/x-executable.*) ;;
+ text/x-makefile.*) ;;
+ *) spell_check "${filename}" || lint_result=1 ;;
+ esac
return $lint_result
}
@@ -191,7 +209,7 @@ lint_files() {
while read -r filename; do
lint "${filename}" || lint_result=1
done
- exit $lint_result
+ return $lint_result
}
matches_any() {
@@ -222,18 +240,33 @@ filter_out() {
fi
}
-list_files() {
+lint_directory() {
+ local dirname="$1"
+ local lint_result=0
+ # This test is just checking if there are any Go files in the directory
+ if compgen -G "$dirname/*.go" >/dev/null; then
+ lint_go "${dirname}" || lint_result=1
+ fi
+ ls $dirname/* | filter_out "$LINT_IGNORE_FILE" | lint_files
+ return $lint_result
+}
+
+lint_directories() {
+ local lint_result=0
+ while read -r dirname; do
+ lint_directory "${dirname}" || lint_result=1
+ done
+ exit $lint_result
+}
+
+list_directories() {
if [ $# -gt 0 ]; then
- find "$@" | grep -vE '(^|/)vendor/'
- else
- git ls-files --exclude-standard | grep -vE '(^|/)vendor/'
+ find "$@" \( -name vendor -o -name .git -o -name .cache -o -name .pkg \) -prune -o -type d
fi
}
if [ $# = 1 ] && [ -f "$1" ]; then
lint "$1"
-elif [ -n "$PARALLEL" ]; then
- list_files "$@" | filter_out "$LINT_IGNORE_FILE" | xargs -n1 -P16 "$0"
else
- list_files "$@" | filter_out "$LINT_IGNORE_FILE" | lint_files
+ list_directories "$@" | lint_directories
fi
diff --git a/provisioning/README.md b/provisioning/README.md
index 627bb42e..6ff739ca 100755
--- a/provisioning/README.md
+++ b/provisioning/README.md
@@ -16,16 +16,15 @@ You can then use these machines as is or run various Ansible playbooks from `../
* On macOS: `brew install vagrant`
* On Linux (via Aptitude): `sudo apt install vagrant`
- * If you need a specific version:
-
- curl -fsS https://releases.hashicorp.com/terraform/x.y.z/terraform_x.y.z_linux_amd64.zip | gunzip > terraform && chmod +x terraform && sudo mv terraform /usr/bin
-
* For other platforms or more details, see [here](https://www.vagrantup.com/docs/installation/)
* You will need [Terraform](https://www.terraform.io) installed on your machine and added to your `PATH` in order to be able to provision cloud-hosted machines automatically.
* On macOS: `brew install terraform`
* On Linux (via Aptitude): `sudo apt install terraform`
+ * If you need a specific version:
+
+ curl -fsS https://releases.hashicorp.com/terraform/x.y.z/terraform_x.y.z_linux_amd64.zip | gunzip > terraform && chmod +x terraform && sudo mv terraform /usr/bin
* For other platforms or more details, see [here](https://www.terraform.io/intro/getting-started/install.html)
* Depending on the cloud provider, you may have to create an account, manually onboard, create and register SSH keys, etc.
diff --git a/provisioning/aws/variables.tf b/provisioning/aws/variables.tf
index ed5b8b4b..5f4b4628 100755
--- a/provisioning/aws/variables.tf
+++ b/provisioning/aws/variables.tf
@@ -43,9 +43,11 @@ variable "aws_amis" {
"eu-west-2" = "ami-23d0da47"
# Red Hat Enterprise Linux 7.3 (HVM), SSD Volume Type:
+
#"us-east-1" = "ami-b63769a1"
# CentOS 7 (x86_64) - with Updates HVM
+
#"us-east-1" = "ami-6d1c2007"
}
}
diff --git a/provisioning/gcp/main.tf b/provisioning/gcp/main.tf
index abfddb7d..af5a22eb 100755
--- a/provisioning/gcp/main.tf
+++ b/provisioning/gcp/main.tf
@@ -77,3 +77,17 @@ resource "google_compute_firewall" "fw-allow-esp" {
source_ranges = ["${var.gcp_network_global_cidr}"]
}
+
+# Required for WKS Kubernetes API server access
+resource "google_compute_firewall" "fw-allow-kube-apiserver" {
+ name = "${var.name}-allow-kube-apiserver"
+ network = "${var.gcp_network}"
+ target_tags = ["${var.name}"]
+
+ allow {
+ protocol = "tcp"
+ ports = ["6443"]
+ }
+
+ source_ranges = ["${var.client_ip}"]
+}
diff --git a/provisioning/gcp/outputs.tf b/provisioning/gcp/outputs.tf
index 9aa1e33e..210398ba 100755
--- a/provisioning/gcp/outputs.tf
+++ b/provisioning/gcp/outputs.tf
@@ -6,6 +6,10 @@ output "public_ips" {
value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip}"]
}
+output "private_ips" {
+ value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.address}"]
+}
+
output "hostnames" {
value = "${join("\n",
"${formatlist("%v.%v.%v",
diff --git a/provisioning/setup.sh b/provisioning/setup.sh
index 2eb67170..965ee28f 100755
--- a/provisioning/setup.sh
+++ b/provisioning/setup.sh
@@ -18,7 +18,8 @@ function decrypt() {
echo >&2 "Failed to decode and decrypt $2: no secret key was provided."
return 1
fi
- echo "$3" | openssl base64 -d | openssl enc -d -aes256 -pass "pass:$1"
+ # Set md5 because existing keys were encrypted that way and openssl default changed
+ echo "$3" | openssl base64 -d | openssl enc -md md5 -d -aes256 -pass "pass:$1"
}
function ssh_private_key() {
@@ -304,11 +305,11 @@ function tf_ssh_usage() {
ERROR: $1
Usage:
- $ tf_ssh [OPTION]...
+ \$ tf_ssh [OPTION]...
Examples:
- $ tf_ssh 1
- $ tf_ssh 1 -o LogLevel VERBOSE
- $ tf_ssh 1 -i ~/.ssh/custom_private_key_id_rsa
+ \$ tf_ssh 1
+ \$ tf_ssh 1 -o LogLevel VERBOSE
+ \$ tf_ssh 1 -i ~/.ssh/custom_private_key_id_rsa
Available machines:
EOF
cat -n >&2 <<<"$(terraform output public_etc_hosts)"
@@ -330,12 +331,12 @@ function tf_ansi_usage() {
ERROR: $1
Usage:
- $ tf_ansi [OPTION]...
+ \$ tf_ansi [OPTION]...
Examples:
- $ tf_ansi setup_weave-net_dev
- $ tf_ansi 1
- $ tf_ansi 1 -vvv --private-key=~/.ssh/custom_private_key_id_rsa
- $ tf_ansi setup_weave-kube --extra-vars "docker_version=1.12.6 kubernetes_version=1.5.6"
+ \$ tf_ansi setup_weave-net_dev
+ \$ tf_ansi 1
+ \$ tf_ansi 1 -vvv --private-key=~/.ssh/custom_private_key_id_rsa
+ \$ tf_ansi setup_weave-kube --extra-vars "docker_version=1.12.6 kubernetes_version=1.5.6"
Available playbooks:
EOF
cat -n >&2 <<<"$(for file in "$(dirname "${BASH_SOURCE[0]}")"/../../config_management/*.yml; do basename "$file" | sed 's/.yml//'; done)"
@@ -348,7 +349,7 @@ function tf_ansi() {
shift # Drop the first argument to allow passing other arguments to Ansible using "$@" -- see below.
if [[ "$id" =~ ^[0-9]+$ ]]; then
local playbooks=(../../config_management/*.yml)
- local path="${playbooks[(($id-1))]}" # Select the ith entry in the list of playbooks (0-based).
+ local path="${playbooks[(($id - 1))]}" # Select the ith entry in the list of playbooks (0-based).
else
local path="$(dirname "${BASH_SOURCE[0]}")/../../config_management/$id.yml"
fi
diff --git a/push-images b/push-images
index 9f1f16b1..913a8c31 100755
--- a/push-images
+++ b/push-images
@@ -26,25 +26,28 @@ while [ $# -gt 0 ]; do
esac
done
-push_image() {
- local image="$1"
- docker push ${image}:${IMAGE_TAG}
-}
-
+pids=""
for image in ${IMAGES}; do
if [[ "$image" == *"build"* ]]; then
continue
fi
echo "Will push ${image}:${IMAGE_TAG}"
- push_image "${image}" &
+ docker push "${image}:${IMAGE_TAG}" &
+ pids="$pids $!"
- if [ -z "NO_DOCKER_HUB" ]; then
+ if [ -z "$NO_DOCKER_HUB" ]; then
# remove the quey prefix and push to docker hub
docker_hub_image=${image#$QUAY_PREFIX}
- docker tag ${image}:${IMAGE_TAG} ${docker_hub_image}:${IMAGE_TAG}
+ docker tag "${image}:${IMAGE_TAG}" "${docker_hub_image}:${IMAGE_TAG}"
echo "Will push ${docker_hub_image}:${IMAGE_TAG}"
- docker push ${docker_hub_image}:${IMAGE_TAG}
+ docker push "${docker_hub_image}:${IMAGE_TAG}" &
+ pids="$pids $!"
fi
done
+# Wait individually for tasks so we fail-exit on any non-zero return code
+for p in $pids; do
+ wait "$p"
+done
+
wait
diff --git a/rebuild-image b/rebuild-image
index 1f0bb109..cfa4ced8 100755
--- a/rebuild-image
+++ b/rebuild-image
@@ -9,6 +9,7 @@ IMAGENAME=$1
SAVEDNAME=$(echo "$IMAGENAME" | sed "s/[\/\-]/\./g")
IMAGEDIR=$2
shift 2
+GIT_REVISION="$(git rev-parse HEAD)"
INPUTFILES=("$@")
CACHEDIR=$HOME/docker/
@@ -17,7 +18,7 @@ CACHEDIR=$HOME/docker/
rebuild() {
mkdir -p "$CACHEDIR"
rm "$CACHEDIR/$SAVEDNAME"* || true
- docker build -t "$IMAGENAME" "$IMAGEDIR"
+ docker build --build-arg=revision="$GIT_REVISION" -t "$IMAGENAME" "$IMAGEDIR"
docker save "$IMAGENAME:latest" | gzip - >"$CACHEDIR/$SAVEDNAME-$CIRCLE_SHA1.gz"
}
diff --git a/sched b/sched
index 72eeee65..179c650a 100755
--- a/sched
+++ b/sched
@@ -1,16 +1,31 @@
-#!/usr/bin/python
+#!/usr/bin/env python
import sys, string, urllib
import requests
+from requests.packages.urllib3.util.retry import Retry
+from requests.adapters import HTTPAdapter
import optparse
+session = requests.Session()
+adapter = HTTPAdapter(
+ max_retries=Retry(
+ connect=5,
+ status=5,
+ backoff_factor=0.1,
+ status_forcelist=[500, 502, 503, 504]
+ )
+)
+session.mount('http://', adapter)
+session.mount('https://', adapter)
+
+
def test_time(target, test_name, runtime):
- r = requests.post(target + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime))
+ r = session.post(target + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime))
print r.text.encode('utf-8')
assert r.status_code == 204
def test_sched(target, test_run, shard_count, shard_id):
tests = {'tests': string.split(sys.stdin.read())}
- r = requests.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), json=tests)
+ r = session.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), json=tests)
assert r.status_code == 200
result = r.json()
for test in sorted(result['tests']):
diff --git a/scheduler/README.md b/scheduler/README.md
index 8489d787..d9c4aa41 100644
--- a/scheduler/README.md
+++ b/scheduler/README.md
@@ -1,6 +1,64 @@
-To upload newer version:
+# scheduler
+## Development
+
+### Dependencies
+
+Download and install:
+
+- the [original App Engine SDK for Python](https://cloud.google.com/appengine/docs/standard/python/download) ([Linux](https://storage.googleapis.com/appengine-sdks/featured/google_appengine_1.9.77.zip), [macOS](https://storage.googleapis.com/appengine-sdks/featured/GoogleAppEngineLauncher-1.9.77.dmg)) -- this should add `appcfg.py` to your `PATH`
+- `python` 2.7.x
+- `pip`
+
+### Setup
+
+```console
+$ pip install -U virtualenv
+$ virtualenv --python=$(which python2.7) $TMPDIR/scheduler
+$ source $TMPDIR/scheduler/bin/activate
+$ pip install -r requirements.txt -t lib
```
-pip install -r requirements.txt -t lib
-appcfg.py update .
-```
+
+## Deployment
+
+- Run:
+ ```console
+ $ appcfg.py --version $(date '+%Y%m%dt%H%M%S') update .
+ XX:XX PM Application: positive-cocoa-90213; version: 1
+ XX:XX PM Host: appengine.google.com
+ XX:XX PM Starting update of app: positive-cocoa-90213, version: 1
+ XX:XX PM Getting current resource limits.
+ Your browser has been opened to visit:
+
+ https://accounts.google.com/o/oauth2/auth?scope=...
+
+ If your browser is on a different machine then exit and re-run this
+ application with the command-line parameter
+
+ --noauth_local_webserver
+
+ Authentication successful.
+ XX:XX PM Scanning files on local disk.
+ XX:XX PM Scanned 500 files.
+ XX:XX PM Scanned 1000 files.
+ XX:XX PM Cloning 1220 application files.
+ XX:XX PM Uploading 28 files and blobs.
+ XX:XX PM Uploaded 28 files and blobs.
+ XX:XX PM Compilation starting.
+ XX:XX PM Compilation completed.
+ XX:XX PM Starting deployment.
+ XX:XX PM Checking if deployment succeeded.
+ XX:XX PM Will check again in 1 seconds.
+ XX:XX PM Checking if deployment succeeded.
+ XX:XX PM Will check again in 2 seconds.
+ XX:XX PM Checking if deployment succeeded.
+ XX:XX PM Will check again in 4 seconds.
+ XX:XX PM Checking if deployment succeeded.
+ XX:XX PM Deployment successful.
+ XX:XX PM Checking if updated app version is serving.
+ XX:XX PM Completed update of app: positive-cocoa-90213, version: 1
+ XX:XX PM Uploading cron entries.
+ ```
+
+- Go to [console.cloud.google.com](https://console.cloud.google.com) > Weave Integration Tests (`positive-cocoa-90213`) > AppEngine > Versions and ensure traffic is being directed to the newly deployed version.
+- Click on Tools > Logs, and ensure the application is behaving well.
diff --git a/scheduler/main.py b/scheduler/main.py
index 4ed88756..733de807 100644
--- a/scheduler/main.py
+++ b/scheduler/main.py
@@ -19,157 +19,211 @@
# observations faster.
alpha = 0.3
+
class Test(ndb.Model):
- total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA
- total_runs = ndb.IntegerProperty(default=0)
-
- def parallelism(self):
- name = self.key.string_id()
- m = re.search('(\d+)_test.sh$', name)
- if m is None:
- return 1
- else:
- return int(m.group(1))
+ total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA
+ total_runs = ndb.IntegerProperty(default=0)
+
+ def parallelism(self):
+ name = self.key.string_id()
+ m = re.search('(\d+)_test.sh$', name)
+ if m is None:
+ return 1
+ else:
+ return int(m.group(1))
+
+ def cost(self):
+ p = self.parallelism()
+ logging.info("Test %s has parallelism %d and avg run time %s",
+ self.key.string_id(), p, self.total_run_time)
+ return self.parallelism() * self.total_run_time
- def cost(self):
- p = self.parallelism()
- logging.info("Test %s has parallelism %d and avg run time %s", self.key.string_id(), p, self.total_run_time)
- return self.parallelism() * self.total_run_time
class Schedule(ndb.Model):
- shards = ndb.JsonProperty()
+ shards = ndb.JsonProperty()
+
@app.route('/record//', methods=['POST'])
@ndb.transactional
def record(test_name, runtime):
- test = Test.get_by_id(test_name)
- if test is None:
- test = Test(id=test_name)
- test.total_run_time = (test.total_run_time * (1-alpha)) + (float(runtime) * alpha)
- test.total_runs += 1
- test.put()
- return ('', 204)
-
-@app.route('/schedule///', methods=['POST'])
+ test = Test.get_by_id(test_name)
+ if test is None:
+ test = Test(id=test_name)
+ test.total_run_time = (test.total_run_time *
+ (1 - alpha)) + (float(runtime) * alpha)
+ test.total_runs += 1
+ test.put()
+ return ('', 204)
+
+
+@app.route(
+ '/schedule///', methods=['POST'])
def schedule(test_run, shard_count, shard):
- # read tests from body
- test_names = flask.request.get_json(force=True)['tests']
-
- # first see if we have a scedule already
- schedule_id = "%s-%d" % (test_run, shard_count)
- schedule = Schedule.get_by_id(schedule_id)
- if schedule is not None:
+ # read tests from body
+ test_names = flask.request.get_json(force=True)['tests']
+
+ # first see if we have a scedule already
+ schedule_id = "%s-%d" % (test_run, shard_count)
+ schedule = Schedule.get_by_id(schedule_id)
+ if schedule is not None:
+ return flask.json.jsonify(tests=schedule.shards[str(shard)])
+
+ # if not, do simple greedy algorithm
+ test_times = ndb.get_multi(
+ ndb.Key(Test, test_name) for test_name in test_names)
+
+ def avg(test):
+ if test is not None:
+ return test.cost()
+ return 1
+
+ test_times = [(test_name, avg(test))
+ for test_name, test in zip(test_names, test_times)]
+ test_times_dict = dict(test_times)
+ test_times.sort(key=operator.itemgetter(1))
+
+ shards = {i: [] for i in xrange(shard_count)}
+ while test_times:
+ test_name, time = test_times.pop()
+
+ # find shortest shard and put it in that
+ s, _ = min(
+ ((i, sum(test_times_dict[t] for t in shards[i]))
+ for i in xrange(shard_count)),
+ key=operator.itemgetter(1))
+
+ shards[s].append(test_name)
+
+ # atomically insert or retrieve existing schedule
+ schedule = Schedule.get_or_insert(schedule_id, shards=shards)
return flask.json.jsonify(tests=schedule.shards[str(shard)])
- # if not, do simple greedy algorithm
- test_times = ndb.get_multi(ndb.Key(Test, test_name) for test_name in test_names)
- def avg(test):
- if test is not None:
- return test.cost()
- return 1
- test_times = [(test_name, avg(test)) for test_name, test in zip(test_names, test_times)]
- test_times_dict = dict(test_times)
- test_times.sort(key=operator.itemgetter(1))
-
- shards = {i: [] for i in xrange(shard_count)}
- while test_times:
- test_name, time = test_times.pop()
-
- # find shortest shard and put it in that
- s, _ = min(((i, sum(test_times_dict[t] for t in shards[i]))
- for i in xrange(shard_count)), key=operator.itemgetter(1))
-
- shards[s].append(test_name)
-
- # atomically insert or retrieve existing schedule
- schedule = Schedule.get_or_insert(schedule_id, shards=shards)
- return flask.json.jsonify(tests=schedule.shards[str(shard)])
FIREWALL_REGEXES = [
- re.compile(r'^(?P\w+)-allow-(?P\w+)-(?P\d+)-(?P\d+)$'),
- re.compile(r'^(?P\w+)-(?P\d+)-(?P\d+)-allow-(?P[\w\-]+)$'),
+ re.compile(
+ r'^(?P\w+)-allow-(?P\w+)-(?P\d+)-(?P\d+)$'
+ ),
+ re.compile(r'^(?P\w+)-(?P\d+)-(?P\d+)-allow-'
+ r'(?P[\w\-]+)$'),
]
NAME_REGEXES = [
- re.compile(r'^host(?P\d+)-(?P\d+)-(?P\d+)$'),
- re.compile(r'^test-(?P\d+)-(?P\d+)-(?P\d+)$'),
+ re.compile(pat)
+ for pat in (
+ r'^host(?P\d+)-(?P\d+)-(?P\d+)$',
+ r'^host(?P\d+)-(?P[a-zA-Z0-9-]+)-(?P\d+)'
+ r'-(?P\d+)$',
+ r'^test-(?P\d+)-(?P\d+)-(?P\d+)$', )
]
+
def _matches_any_regex(name, regexes):
- for regex in regexes:
- matches = regex.match(name)
- if matches:
- return matches
+ for regex in regexes:
+ matches = regex.match(name)
+ if matches:
+ return matches
+
+# See also: https://circleci.com/account/api
+CIRCLE_CI_API_TOKEN = 'cffb83afd920cfa109cbd3e9eecb7511a2d18bb9'
+
+# N.B.: When adding a project below, please ensure:
+# - its CircleCI project is either public, or is followed by the user attached
+# to the above API token
+# - user positive-cocoa-90213@appspot.gserviceaccount.com has "Compute Admin"
+# access to its GCP project (or any other role including
+# compute.instances.list/delete and compute.firewalls.list/delete)
PROJECTS = [
- ('weaveworks/weave', 'weave-net-tests', 'us-central1-a', True),
- ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True),
- ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False),
+ ('weaveworks/weave', 'weave-net-tests', 'us-central1-a', True, None),
+ ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True, None),
+ ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False,
+ None),
+ ('weaveworks/wks', 'wks-tests', 'us-central1-a', True,
+ CIRCLE_CI_API_TOKEN),
]
+
@app.route('/tasks/gc')
def gc():
- # Get list of running VMs, pick build id out of VM name
- credentials = GoogleCredentials.get_application_default()
- compute = discovery.build('compute', 'v1', credentials=credentials)
-
- for repo, project, zone, gc_fw in PROJECTS:
- gc_project(compute, repo, project, zone, gc_fw)
-
- return "Done"
-
-def gc_project(compute, repo, project, zone, gc_fw):
- logging.info("GCing %s, %s, %s", repo, project, zone)
- # Get list of builds, filter down to running builds:
- running = _get_running_builds(repo)
- # Stop VMs for builds that aren't running:
- _gc_compute_engine_instances(compute, project, zone, running)
- # Remove firewall rules for builds that aren't running:
- if gc_fw:
- _gc_firewall_rules(compute, project, running)
-
-def _get_running_builds(repo):
- result = urlfetch.fetch('https://circleci.com/api/v1/project/%s' % repo,
- headers={'Accept': 'application/json'})
- assert result.status_code == 200
- builds = json.loads(result.content)
- running = {build['build_num'] for build in builds if not build.get('stop_time')}
- logging.info("Runnings builds: %r", running)
- return running
+ # Get list of running VMs, pick build id out of VM name
+ credentials = GoogleCredentials.get_application_default()
+ compute = discovery.build(
+ 'compute', 'v1', credentials=credentials, cache_discovery=False)
+
+ for repo, project, zone, gc_fw, circleci_api_token in PROJECTS:
+ gc_project(compute, repo, project, zone, gc_fw, circleci_api_token)
+
+ return "Done"
+
+
+def gc_project(compute, repo, project, zone, gc_fw, circleci_api_token):
+ logging.info("GCing %s, %s, %s", repo, project, zone)
+ # Get list of builds, filter down to running builds:
+ running = _get_running_builds(repo, circleci_api_token)
+ # Stop VMs for builds that aren't running:
+ _gc_compute_engine_instances(compute, project, zone, running)
+ # Remove firewall rules for builds that aren't running:
+ if gc_fw:
+ _gc_firewall_rules(compute, project, running)
+
+
+def _get_running_builds(repo, circleci_api_token):
+ if circleci_api_token:
+ url = 'https://circleci.com/api/v1/project/%s?circle-token=%s' % (
+ repo, circleci_api_token)
+ else:
+ url = 'https://circleci.com/api/v1/project/%s' % repo
+ result = urlfetch.fetch(url, headers={'Accept': 'application/json'})
+ if result.status_code != 200:
+ raise RuntimeError(
+ 'Failed to get builds for "%s". URL: %s, Status: %s. Response: %s'
+ % (repo, url, result.status_code, result.content))
+ builds = json.loads(result.content)
+ running = {
+ build['build_num']
+ for build in builds if not build.get('stop_time')
+ }
+ logging.info("Runnings builds: %r", running)
+ return running
+
def _get_hosts_by_build(instances):
- host_by_build = collections.defaultdict(list)
- for instance in instances['items']:
- matches = _matches_any_regex(instance['name'], NAME_REGEXES)
- if not matches:
- continue
- host_by_build[int(matches.group('build'))].append(instance['name'])
- logging.info("Running VMs by build: %r", host_by_build)
- return host_by_build
+ host_by_build = collections.defaultdict(list)
+ for instance in instances['items']:
+ matches = _matches_any_regex(instance['name'], NAME_REGEXES)
+ if not matches:
+ continue
+ host_by_build[int(matches.group('build'))].append(instance['name'])
+ logging.info("Running VMs by build: %r", host_by_build)
+ return host_by_build
+
def _gc_compute_engine_instances(compute, project, zone, running):
- instances = compute.instances().list(project=project, zone=zone).execute()
- if 'items' not in instances:
- return
- host_by_build = _get_hosts_by_build(instances)
- stopped = []
- for build, names in host_by_build.iteritems():
- if build in running:
- continue
- for name in names:
- stopped.append(name)
- logging.info("Stopping VM %s", name)
- compute.instances().delete(project=project, zone=zone, instance=name).execute()
- return stopped
+ instances = compute.instances().list(project=project, zone=zone).execute()
+ if 'items' not in instances:
+ return
+ host_by_build = _get_hosts_by_build(instances)
+ stopped = []
+ for build, names in host_by_build.iteritems():
+ if build in running:
+ continue
+ for name in names:
+ stopped.append(name)
+ logging.info("Stopping VM %s", name)
+ compute.instances().delete(
+ project=project, zone=zone, instance=name).execute()
+ return stopped
+
def _gc_firewall_rules(compute, project, running):
- firewalls = compute.firewalls().list(project=project).execute()
- if 'items' not in firewalls:
- return
- for firewall in firewalls['items']:
- matches = _matches_any_regex(firewall['name'], FIREWALL_REGEXES)
- if not matches:
- continue
- if int(matches.group('build')) in running:
- continue
- logging.info("Deleting firewall rule %s", firewall['name'])
- compute.firewalls().delete(project=project, firewall=firewall['name']).execute()
+ firewalls = compute.firewalls().list(project=project).execute()
+ if 'items' not in firewalls:
+ return
+ for firewall in firewalls['items']:
+ matches = _matches_any_regex(firewall['name'], FIREWALL_REGEXES)
+ if not matches:
+ continue
+ if int(matches.group('build')) in running:
+ continue
+ logging.info("Deleting firewall rule %s", firewall['name'])
+ compute.firewalls().delete(
+ project=project, firewall=firewall['name']).execute()
diff --git a/scheduler/requirements.txt b/scheduler/requirements.txt
index d4d47e6e..872a7c83 100644
--- a/scheduler/requirements.txt
+++ b/scheduler/requirements.txt
@@ -1,2 +1,2 @@
-flask
-google-api-python-client
+flask==0.12.4
+google-api-python-client==1.6.7
diff --git a/socks/Dockerfile b/socks/Dockerfile
index 867cd6bc..ad0b8938 100644
--- a/socks/Dockerfile
+++ b/socks/Dockerfile
@@ -1,7 +1,13 @@
FROM gliderlabs/alpine
-MAINTAINER Weaveworks Inc
WORKDIR /
COPY proxy /
EXPOSE 8000
EXPOSE 8080
ENTRYPOINT ["/proxy"]
+
+ARG revision
+LABEL maintainer="Weaveworks " \
+ org.opencontainers.image.title="socks" \
+ org.opencontainers.image.source="https://github.com/weaveworks/build-tools/tree/master/socks" \
+ org.opencontainers.image.revision="${revision}" \
+ org.opencontainers.image.vendor="Weaveworks"
diff --git a/socks/Makefile b/socks/Makefile
index 2daeda64..b3358649 100644
--- a/socks/Makefile
+++ b/socks/Makefile
@@ -2,6 +2,7 @@
IMAGE_TAR=image.tar
IMAGE_NAME=weaveworks/socksproxy
+GIT_REVISION := $(shell git rev-parse HEAD)
PROXY_EXE=proxy
NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
rm $@; \
@@ -15,7 +16,7 @@ NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
all: $(IMAGE_TAR)
$(IMAGE_TAR): Dockerfile $(PROXY_EXE)
- docker build -t $(IMAGE_NAME) .
+ docker build --build-arg=revision=$(GIT_REVISION) -t $(IMAGE_NAME) .
docker save $(IMAGE_NAME):latest > $@
$(PROXY_EXE): *.go
diff --git a/socks/main.go b/socks/main.go
index 7cd8c708..2f16d167 100644
--- a/socks/main.go
+++ b/socks/main.go
@@ -1,6 +1,7 @@
package main
import (
+ "context"
"fmt"
"net"
"net/http"
@@ -11,23 +12,23 @@ import (
socks5 "github.com/armon/go-socks5"
"github.com/weaveworks/common/mflag"
"github.com/weaveworks/common/mflagext"
- "golang.org/x/net/context"
)
type pacFileParameters struct {
- HostMatch string
- Aliases map[string]string
+ HostMatch string
+ SocksDestination string
+ Aliases map[string]string
}
const (
pacfile = `
function FindProxyForURL(url, host) {
if(shExpMatch(host, "{{.HostMatch}}")) {
- return "SOCKS5 localhost:8000";
+ return "SOCKS5 {{.SocksDestination}}";
}
{{range $key, $value := .Aliases}}
if (host == "{{$key}}") {
- return "SOCKS5 localhost:8000";
+ return "SOCKS5 {{.SocksDestination}}";
}
{{end}}
return "DIRECT";
@@ -37,11 +38,13 @@ function FindProxyForURL(url, host) {
func main() {
var (
- as []string
- hostMatch string
+ as []string
+ hostMatch string
+ socksDestination string
)
mflagext.ListVar(&as, []string{"a", "-alias"}, []string{}, "Specify hostname aliases in the form alias:hostname. Can be repeated.")
mflag.StringVar(&hostMatch, []string{"h", "-host-match"}, "*.weave.local", "Specify main host shExpMatch expression in pacfile")
+ mflag.StringVar(&socksDestination, []string{"d", "-socks-destination"}, "localhost:8000", "Specify destination host:port in pacfile")
mflag.Parse()
var aliases = map[string]string{}
@@ -60,7 +63,7 @@ func main() {
t := template.Must(template.New("pacfile").Parse(pacfile))
http.HandleFunc("/proxy.pac", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/x-ns-proxy-autoconfig")
- t.Execute(w, pacFileParameters{hostMatch, aliases})
+ t.Execute(w, pacFileParameters{hostMatch, socksDestination, aliases})
})
if err := http.ListenAndServe(":8080", nil); err != nil {
diff --git a/test b/test
index 31805164..c284e494 100755
--- a/test
+++ b/test
@@ -7,7 +7,9 @@ SLOW=
NO_GO_GET=true
TAGS=
PARALLEL=
+RACE="-race -covermode=atomic"
TIMEOUT=1m
+VERBOSE=
usage() {
echo "$0 [-slow] [-in-container foo] [-netgo] [-(no-)go-get] [-timeout 1m]"
@@ -15,10 +17,18 @@ usage() {
while [ $# -gt 0 ]; do
case "$1" in
+ "-v")
+ VERBOSE="-v"
+ shift 1
+ ;;
"-slow")
SLOW=true
shift 1
;;
+ "-no-race")
+ RACE=
+ shift 1
+ ;;
"-no-go-get")
NO_GO_GET=true
shift 1
@@ -28,9 +38,13 @@ while [ $# -gt 0 ]; do
shift 1
;;
"-netgo")
- TAGS="-tags netgo"
+ TAGS="netgo"
shift 1
;;
+ "-tags")
+ TAGS="$2"
+ shift 2
+ ;;
"-p")
PARALLEL=true
shift 1
@@ -46,14 +60,14 @@ while [ $# -gt 0 ]; do
esac
done
-GO_TEST_ARGS=($TAGS -cpu 4 -timeout $TIMEOUT)
+GO_TEST_ARGS=(-tags "${TAGS[@]}" -cpu 4 -timeout $TIMEOUT $VERBOSE)
if [ -n "$SLOW" ] || [ -n "$CIRCLECI" ]; then
SLOW=true
fi
if [ -n "$SLOW" ]; then
- GO_TEST_ARGS=("${GO_TEST_ARGS[@]}" -race -covermode=atomic)
+ GO_TEST_ARGS=("${GO_TEST_ARGS[@]}" ${RACE})
# shellcheck disable=SC2153
if [ -n "$COVERDIR" ]; then
@@ -69,7 +83,7 @@ fail=0
if [ -z "$TESTDIRS" ]; then
# NB: Relies on paths being prefixed with './'.
- TESTDIRS=($(git ls-files -- '*_test.go' | grep -vE '^(vendor|prog|experimental)/' | xargs -n1 dirname | sort -u | sed -e 's|^|./|'))
+ TESTDIRS=($(git ls-files -- '*_test.go' | grep -vE '^(vendor|experimental)/' | xargs -n1 dirname | sort -u | sed -e 's|^|./|'))
else
# TESTDIRS on the right side is not really an array variable, it
# is just a string with spaces, but it is written like that to
@@ -92,7 +106,7 @@ go test -i "${GO_TEST_ARGS[@]}" "${TESTDIRS[@]}"
run_test() {
local dir=$1
if [ -z "$NO_GO_GET" ]; then
- go get -t "$TAGS" "$dir"
+ go get -t -tags "${TAGS[@]}" "$dir"
fi
local GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}")