diff --git a/tools/README.md b/tools/README.md
index 377d14a9fd..389e498072 100644
--- a/tools/README.md
+++ b/tools/README.md
@@ -60,8 +60,8 @@ To update the code in build-tools.git, the process is therefore:
If you have any questions about, feedback for or problems with `build-tools`:
-- Invite yourself to the #weave-community slack channel.
-- Ask a question on the #weave-community slack channel.
-- File an issue.
+- Invite yourself to the Weave Users Slack.
+- Ask a question on the [#general](https://weave-community.slack.com/messages/general/) slack channel.
+- [File an issue](https://github.com/weaveworks/build-tools/issues/new).
Your feedback is always welcome!
diff --git a/tools/build/golang/Dockerfile b/tools/build/golang/Dockerfile
index 23fb6ac838..17c19ed3fd 100644
--- a/tools/build/golang/Dockerfile
+++ b/tools/build/golang/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.10.0-stretch
+FROM golang:1.11.1-stretch
RUN apt-get update && \
apt-get install -y \
curl \
@@ -30,7 +30,7 @@ RUN go get -tags netgo \
github.com/gogo/protobuf/gogoproto \
github.com/gogo/protobuf/protoc-gen-gogoslick \
github.com/golang/dep/... \
- github.com/golang/lint/golint \
+ golang.org/x/lint/golint \
github.com/golang/protobuf/protoc-gen-go \
github.com/kisielk/errcheck \
github.com/mjibson/esc \
diff --git a/tools/scheduler/README.md b/tools/scheduler/README.md
index 8489d78706..d9c4aa41eb 100644
--- a/tools/scheduler/README.md
+++ b/tools/scheduler/README.md
@@ -1,6 +1,64 @@
-To upload newer version:
+# scheduler
+## Development
+
+### Dependencies
+
+Download and install:
+
+- the [original App Engine SDK for Python](https://cloud.google.com/appengine/docs/standard/python/download) ([Linux](https://storage.googleapis.com/appengine-sdks/featured/google_appengine_1.9.77.zip), [macOS](https://storage.googleapis.com/appengine-sdks/featured/GoogleAppEngineLauncher-1.9.77.dmg)) -- this should add `appcfg.py` to your `PATH`
+- `python` 2.7.x
+- `pip`
+
+### Setup
+
+```console
+$ pip install -U virtualenv
+$ virtualenv --python=$(which python2.7) $TMPDIR/scheduler
+$ source $TMPDIR/scheduler/bin/activate
+$ pip install -r requirements.txt -t lib
```
-pip install -r requirements.txt -t lib
-appcfg.py update .
-```
+
+## Deployment
+
+- Run:
+ ```console
+ $ appcfg.py --version $(date '+%Y%m%dt%H%M%S') update .
+ XX:XX PM Application: positive-cocoa-90213; version: 1
+ XX:XX PM Host: appengine.google.com
+ XX:XX PM Starting update of app: positive-cocoa-90213, version: 1
+ XX:XX PM Getting current resource limits.
+ Your browser has been opened to visit:
+
+ https://accounts.google.com/o/oauth2/auth?scope=...
+
+ If your browser is on a different machine then exit and re-run this
+ application with the command-line parameter
+
+ --noauth_local_webserver
+
+ Authentication successful.
+ XX:XX PM Scanning files on local disk.
+ XX:XX PM Scanned 500 files.
+ XX:XX PM Scanned 1000 files.
+ XX:XX PM Cloning 1220 application files.
+ XX:XX PM Uploading 28 files and blobs.
+ XX:XX PM Uploaded 28 files and blobs.
+ XX:XX PM Compilation starting.
+ XX:XX PM Compilation completed.
+ XX:XX PM Starting deployment.
+ XX:XX PM Checking if deployment succeeded.
+ XX:XX PM Will check again in 1 seconds.
+ XX:XX PM Checking if deployment succeeded.
+ XX:XX PM Will check again in 2 seconds.
+ XX:XX PM Checking if deployment succeeded.
+ XX:XX PM Will check again in 4 seconds.
+ XX:XX PM Checking if deployment succeeded.
+ XX:XX PM Deployment successful.
+ XX:XX PM Checking if updated app version is serving.
+ XX:XX PM Completed update of app: positive-cocoa-90213, version: 1
+ XX:XX PM Uploading cron entries.
+ ```
+
+- Go to [console.cloud.google.com](https://console.cloud.google.com) > Weave Integration Tests (`positive-cocoa-90213`) > AppEngine > Versions and ensure traffic is being directed to the newly deployed version.
+- Click on Tools > Logs, and ensure the application is behaving well.
diff --git a/tools/scheduler/main.py b/tools/scheduler/main.py
index de3a1288d7..733de80779 100644
--- a/tools/scheduler/main.py
+++ b/tools/scheduler/main.py
@@ -123,10 +123,22 @@ def _matches_any_regex(name, regexes):
return matches
+# See also: https://circleci.com/account/api
+CIRCLE_CI_API_TOKEN = 'cffb83afd920cfa109cbd3e9eecb7511a2d18bb9'
+
+# N.B.: When adding a project below, please ensure:
+# - its CircleCI project is either public, or is followed by the user attached
+# to the above API token
+# - user positive-cocoa-90213@appspot.gserviceaccount.com has "Compute Admin"
+# access to its GCP project (or any other role including
+# compute.instances.list/delete and compute.firewalls.list/delete)
PROJECTS = [
- ('weaveworks/weave', 'weave-net-tests', 'us-central1-a', True),
- ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True),
- ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False),
+ ('weaveworks/weave', 'weave-net-tests', 'us-central1-a', True, None),
+ ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True, None),
+ ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False,
+ None),
+ ('weaveworks/wks', 'wks-tests', 'us-central1-a', True,
+ CIRCLE_CI_API_TOKEN),
]
@@ -134,18 +146,19 @@ def _matches_any_regex(name, regexes):
def gc():
# Get list of running VMs, pick build id out of VM name
credentials = GoogleCredentials.get_application_default()
- compute = discovery.build('compute', 'v1', credentials=credentials)
+ compute = discovery.build(
+ 'compute', 'v1', credentials=credentials, cache_discovery=False)
- for repo, project, zone, gc_fw in PROJECTS:
- gc_project(compute, repo, project, zone, gc_fw)
+ for repo, project, zone, gc_fw, circleci_api_token in PROJECTS:
+ gc_project(compute, repo, project, zone, gc_fw, circleci_api_token)
return "Done"
-def gc_project(compute, repo, project, zone, gc_fw):
+def gc_project(compute, repo, project, zone, gc_fw, circleci_api_token):
logging.info("GCing %s, %s, %s", repo, project, zone)
# Get list of builds, filter down to running builds:
- running = _get_running_builds(repo)
+ running = _get_running_builds(repo, circleci_api_token)
# Stop VMs for builds that aren't running:
_gc_compute_engine_instances(compute, project, zone, running)
# Remove firewall rules for builds that aren't running:
@@ -153,11 +166,17 @@ def gc_project(compute, repo, project, zone, gc_fw):
_gc_firewall_rules(compute, project, running)
-def _get_running_builds(repo):
- result = urlfetch.fetch(
- 'https://circleci.com/api/v1/project/%s' % repo,
- headers={'Accept': 'application/json'})
- assert result.status_code == 200
+def _get_running_builds(repo, circleci_api_token):
+ if circleci_api_token:
+ url = 'https://circleci.com/api/v1/project/%s?circle-token=%s' % (
+ repo, circleci_api_token)
+ else:
+ url = 'https://circleci.com/api/v1/project/%s' % repo
+ result = urlfetch.fetch(url, headers={'Accept': 'application/json'})
+ if result.status_code != 200:
+ raise RuntimeError(
+ 'Failed to get builds for "%s". URL: %s, Status: %s. Response: %s'
+ % (repo, url, result.status_code, result.content))
builds = json.loads(result.content)
running = {
build['build_num']
diff --git a/tools/scheduler/requirements.txt b/tools/scheduler/requirements.txt
index d4d47e6eb6..872a7c8346 100644
--- a/tools/scheduler/requirements.txt
+++ b/tools/scheduler/requirements.txt
@@ -1,2 +1,2 @@
-flask
-google-api-python-client
+flask==0.12.4
+google-api-python-client==1.6.7
diff --git a/tools/socks/main.go b/tools/socks/main.go
index ff56a0c88f..2f16d16705 100644
--- a/tools/socks/main.go
+++ b/tools/socks/main.go
@@ -1,6 +1,7 @@
package main
import (
+ "context"
"fmt"
"net"
"net/http"
@@ -11,7 +12,6 @@ import (
socks5 "github.com/armon/go-socks5"
"github.com/weaveworks/common/mflag"
"github.com/weaveworks/common/mflagext"
- "golang.org/x/net/context"
)
type pacFileParameters struct {