From 2220971b4cdc8ad9af3a6f5fdc54f6b2dfa1a8d2 Mon Sep 17 00:00:00 2001 From: Daniel Holbach Date: Tue, 8 Jan 2019 08:10:04 +0530 Subject: [PATCH 1/2] revert change, pull in subtree update in next step --- tools/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/README.md b/tools/README.md index 377d14a9fd..8154aa988c 100644 --- a/tools/README.md +++ b/tools/README.md @@ -62,6 +62,7 @@ If you have any questions about, feedback for or problems with `build-tools`: - Invite yourself to the #weave-community slack channel. - Ask a question on the #weave-community slack channel. +- Send an email to weave-users@weave.works - File an issue. Your feedback is always welcome! From 5e3c8ab35850163674a069eb8f02cb0e7509b163 Mon Sep 17 00:00:00 2001 From: Daniel Holbach Date: Tue, 8 Jan 2019 08:10:12 +0530 Subject: [PATCH 2/2] Squashed 'tools/' changes from 2bbc9a08..ec369f58 ec369f58 Merge pull request #153 from dholbach/drop-email ef7418d3 weave-users mailing list is closed: https://groups.google.com/a/weave.works/forum/#!topic/weave-users/0QXWGOPdBfY 6954a578 Merge pull request #144 from weaveworks/golang-1.11.1 9649eed1 Upgrade build image from golang:1.10.0-strech to 1.11.1-strech 59263a71 Merge pull request #141 from weaveworks/update-context e235c9b0 Merge pull request #143 from weaveworks/gc-wks-test-vms c865b4cc scheduler: please lint/flake8 da61568d scheduler: please lint/yapf ce9d78e3 scheduler: do not cache discovery doc e4b78733 scheduler: add comment about GCP projects' IAM roles needed to list/delete instances and firewall rules ff7ec8e6 scheduler: add comment about CircleCI projects' access via the API 2477d98c scheduler: deploy command now sets the current datetime as the version 5fcd8809 scheduler: pass CircleCI API token in for private projects 6b8c3231 scheduler: more details in case of failure to get running builds from CircleCI 0871aff8 scheduler: downgrade google-api-python-client from 1.7.4 to 1.6.7 b631e7f1 scheduler: add GC of WKS test VMs and firewall rules a923a320 scheduler: document setup and deployment 013f5082 scheduler: lock dependencies' versions 6965a4a4 Merge pull request #142 from weaveworks/fix-build 23298c6f Fix golint expects import golang.org/x/lint/golint 482f4cd0 Context is now part of the Go standard library git-subtree-dir: tools git-subtree-split: ec369f58d7ac85375bad7d0f02a1462cdad290f9 --- README.md | 7 ++-- build/golang/Dockerfile | 4 +-- scheduler/README.md | 66 +++++++++++++++++++++++++++++++++++--- scheduler/main.py | 45 ++++++++++++++++++-------- scheduler/requirements.txt | 4 +-- socks/main.go | 2 +- 6 files changed, 102 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 8154aa988c..389e498072 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,8 @@ To update the code in build-tools.git, the process is therefore: If you have any questions about, feedback for or problems with `build-tools`: -- Invite yourself to the #weave-community slack channel. -- Ask a question on the #weave-community slack channel. -- Send an email to weave-users@weave.works -- File an issue. +- Invite yourself to the Weave Users Slack. +- Ask a question on the [#general](https://weave-community.slack.com/messages/general/) slack channel. +- [File an issue](https://github.com/weaveworks/build-tools/issues/new). Your feedback is always welcome! diff --git a/build/golang/Dockerfile b/build/golang/Dockerfile index 23fb6ac838..17c19ed3fd 100644 --- a/build/golang/Dockerfile +++ b/build/golang/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.10.0-stretch +FROM golang:1.11.1-stretch RUN apt-get update && \ apt-get install -y \ curl \ @@ -30,7 +30,7 @@ RUN go get -tags netgo \ github.com/gogo/protobuf/gogoproto \ github.com/gogo/protobuf/protoc-gen-gogoslick \ github.com/golang/dep/... \ - github.com/golang/lint/golint \ + golang.org/x/lint/golint \ github.com/golang/protobuf/protoc-gen-go \ github.com/kisielk/errcheck \ github.com/mjibson/esc \ diff --git a/scheduler/README.md b/scheduler/README.md index 8489d78706..d9c4aa41eb 100644 --- a/scheduler/README.md +++ b/scheduler/README.md @@ -1,6 +1,64 @@ -To upload newer version: +# scheduler +## Development + +### Dependencies + +Download and install: + +- the [original App Engine SDK for Python](https://cloud.google.com/appengine/docs/standard/python/download) ([Linux](https://storage.googleapis.com/appengine-sdks/featured/google_appengine_1.9.77.zip), [macOS](https://storage.googleapis.com/appengine-sdks/featured/GoogleAppEngineLauncher-1.9.77.dmg)) -- this should add `appcfg.py` to your `PATH` +- `python` 2.7.x +- `pip` + +### Setup + +```console +$ pip install -U virtualenv +$ virtualenv --python=$(which python2.7) $TMPDIR/scheduler +$ source $TMPDIR/scheduler/bin/activate +$ pip install -r requirements.txt -t lib ``` -pip install -r requirements.txt -t lib -appcfg.py update . -``` + +## Deployment + +- Run: + ```console + $ appcfg.py --version $(date '+%Y%m%dt%H%M%S') update . + XX:XX PM Application: positive-cocoa-90213; version: 1 + XX:XX PM Host: appengine.google.com + XX:XX PM Starting update of app: positive-cocoa-90213, version: 1 + XX:XX PM Getting current resource limits. + Your browser has been opened to visit: + + https://accounts.google.com/o/oauth2/auth?scope=... + + If your browser is on a different machine then exit and re-run this + application with the command-line parameter + + --noauth_local_webserver + + Authentication successful. + XX:XX PM Scanning files on local disk. + XX:XX PM Scanned 500 files. + XX:XX PM Scanned 1000 files. + XX:XX PM Cloning 1220 application files. + XX:XX PM Uploading 28 files and blobs. + XX:XX PM Uploaded 28 files and blobs. + XX:XX PM Compilation starting. + XX:XX PM Compilation completed. + XX:XX PM Starting deployment. + XX:XX PM Checking if deployment succeeded. + XX:XX PM Will check again in 1 seconds. + XX:XX PM Checking if deployment succeeded. + XX:XX PM Will check again in 2 seconds. + XX:XX PM Checking if deployment succeeded. + XX:XX PM Will check again in 4 seconds. + XX:XX PM Checking if deployment succeeded. + XX:XX PM Deployment successful. + XX:XX PM Checking if updated app version is serving. + XX:XX PM Completed update of app: positive-cocoa-90213, version: 1 + XX:XX PM Uploading cron entries. + ``` + +- Go to [console.cloud.google.com](https://console.cloud.google.com) > Weave Integration Tests (`positive-cocoa-90213`) > AppEngine > Versions and ensure traffic is being directed to the newly deployed version. +- Click on Tools > Logs, and ensure the application is behaving well. diff --git a/scheduler/main.py b/scheduler/main.py index de3a1288d7..733de80779 100644 --- a/scheduler/main.py +++ b/scheduler/main.py @@ -123,10 +123,22 @@ def _matches_any_regex(name, regexes): return matches +# See also: https://circleci.com/account/api +CIRCLE_CI_API_TOKEN = 'cffb83afd920cfa109cbd3e9eecb7511a2d18bb9' + +# N.B.: When adding a project below, please ensure: +# - its CircleCI project is either public, or is followed by the user attached +# to the above API token +# - user positive-cocoa-90213@appspot.gserviceaccount.com has "Compute Admin" +# access to its GCP project (or any other role including +# compute.instances.list/delete and compute.firewalls.list/delete) PROJECTS = [ - ('weaveworks/weave', 'weave-net-tests', 'us-central1-a', True), - ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True), - ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False), + ('weaveworks/weave', 'weave-net-tests', 'us-central1-a', True, None), + ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True, None), + ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False, + None), + ('weaveworks/wks', 'wks-tests', 'us-central1-a', True, + CIRCLE_CI_API_TOKEN), ] @@ -134,18 +146,19 @@ def _matches_any_regex(name, regexes): def gc(): # Get list of running VMs, pick build id out of VM name credentials = GoogleCredentials.get_application_default() - compute = discovery.build('compute', 'v1', credentials=credentials) + compute = discovery.build( + 'compute', 'v1', credentials=credentials, cache_discovery=False) - for repo, project, zone, gc_fw in PROJECTS: - gc_project(compute, repo, project, zone, gc_fw) + for repo, project, zone, gc_fw, circleci_api_token in PROJECTS: + gc_project(compute, repo, project, zone, gc_fw, circleci_api_token) return "Done" -def gc_project(compute, repo, project, zone, gc_fw): +def gc_project(compute, repo, project, zone, gc_fw, circleci_api_token): logging.info("GCing %s, %s, %s", repo, project, zone) # Get list of builds, filter down to running builds: - running = _get_running_builds(repo) + running = _get_running_builds(repo, circleci_api_token) # Stop VMs for builds that aren't running: _gc_compute_engine_instances(compute, project, zone, running) # Remove firewall rules for builds that aren't running: @@ -153,11 +166,17 @@ def gc_project(compute, repo, project, zone, gc_fw): _gc_firewall_rules(compute, project, running) -def _get_running_builds(repo): - result = urlfetch.fetch( - 'https://circleci.com/api/v1/project/%s' % repo, - headers={'Accept': 'application/json'}) - assert result.status_code == 200 +def _get_running_builds(repo, circleci_api_token): + if circleci_api_token: + url = 'https://circleci.com/api/v1/project/%s?circle-token=%s' % ( + repo, circleci_api_token) + else: + url = 'https://circleci.com/api/v1/project/%s' % repo + result = urlfetch.fetch(url, headers={'Accept': 'application/json'}) + if result.status_code != 200: + raise RuntimeError( + 'Failed to get builds for "%s". URL: %s, Status: %s. Response: %s' + % (repo, url, result.status_code, result.content)) builds = json.loads(result.content) running = { build['build_num'] diff --git a/scheduler/requirements.txt b/scheduler/requirements.txt index d4d47e6eb6..872a7c8346 100644 --- a/scheduler/requirements.txt +++ b/scheduler/requirements.txt @@ -1,2 +1,2 @@ -flask -google-api-python-client +flask==0.12.4 +google-api-python-client==1.6.7 diff --git a/socks/main.go b/socks/main.go index ff56a0c88f..2f16d16705 100644 --- a/socks/main.go +++ b/socks/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "net" "net/http" @@ -11,7 +12,6 @@ import ( socks5 "github.com/armon/go-socks5" "github.com/weaveworks/common/mflag" "github.com/weaveworks/common/mflagext" - "golang.org/x/net/context" ) type pacFileParameters struct {