diff --git a/integration/100_launch_test.sh b/integration/100_launch_test.sh index 8c80c6c23d..45dba69fa0 100755 --- a/integration/100_launch_test.sh +++ b/integration/100_launch_test.sh @@ -1,7 +1,7 @@ #! /bin/bash # shellcheck disable=SC1091 -./config.sh +. ./config.sh start_suite "Launch scope and check it boots" diff --git a/integration/config.sh b/integration/config.sh index 05586e0115..3014243c2e 100644 --- a/integration/config.sh +++ b/integration/config.sh @@ -71,8 +71,8 @@ has_connection_by_id() { for i in $(seq "$timeout"); do local nodes local edge - edge=$(echo "$nodes" | jq -r ".nodes[\"$from_id\"].adjacency | contains([\"$to_id\"])" 2>/dev/null) - nodes="$(curl -s "http://$host:4040/api/topology/${view}?system=show")" + edge=$(echo "$nodes" | (jq -r ".nodes[\"$from_id\"].adjacency | contains([\"$to_id\"])" || true) 2>/dev/null) + nodes=$(curl -s "http://$host:4040/api/topology/${view}?system=show" || true) if [ "$edge" = "true" ]; then echo "Found edge $from -> $to after $i secs" assert "curl -s http://$host:4040/api/topology/${view}?system=show | jq -r '.nodes[\"$from_id\"].adjacency | contains([\"$to_id\"])'" true @@ -109,7 +109,7 @@ wait_for() { for i in $(seq "${timeout}"); do local nodes local found=0 - nodes="$(curl -s "http://$host:4040/api/topology/${view}?system=show")" + nodes=$(curl -s "http://$host:4040/api/topology/${view}?system=show" || true) for name in "$@"; do local count count=$(echo "${nodes}" | jq -r "[.nodes[] | select(.label == \"${name}\")] | length") diff --git a/tools/.gitignore b/tools/.gitignore index 635dff1f1c..b6ea60f8fb 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -2,6 +2,5 @@ cover/cover socks/proxy socks/image.tar runner/runner -cmd/wcloud/wcloud *.pyc *~ diff --git a/tools/circle.yml b/tools/circle.yml index 84d02dc1ab..6751c66c70 100644 --- a/tools/circle.yml +++ b/tools/circle.yml @@ -27,5 +27,4 @@ test: - cd $SRCDIR/cover; make - cd $SRCDIR/socks; make - cd $SRCDIR/runner; make - - cd $SRCDIR/cmd/wcloud; make diff --git a/tools/cmd/wcloud/Makefile b/tools/cmd/wcloud/Makefile deleted file mode 100644 index 86b0df3800..0000000000 --- a/tools/cmd/wcloud/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -.PHONY: all clean - -all: wcloud - -wcloud: *.go - go get ./$(@D) - go build -o $@ ./$(@D) - -clean: - rm -rf wcloud - go clean ./... diff --git a/tools/cmd/wcloud/cli.go b/tools/cmd/wcloud/cli.go deleted file mode 100644 index ba3c355fd1..0000000000 --- a/tools/cmd/wcloud/cli.go +++ /dev/null @@ -1,238 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "os" - "os/user" - "path/filepath" - "strings" - "time" - - "github.com/olekukonko/tablewriter" - "gopkg.in/yaml.v2" -) - -// ArrayFlags allows you to collect repeated flags -type ArrayFlags []string - -func (a *ArrayFlags) String() string { - return strings.Join(*a, ",") -} - -// Set implements flags.Value -func (a *ArrayFlags) Set(value string) error { - *a = append(*a, value) - return nil -} - -func env(key, def string) string { - if val, ok := os.LookupEnv(key); ok { - return val - } - return def -} - -var ( - token = env("SERVICE_TOKEN", "") - baseURL = env("BASE_URL", "https://cloud.weave.works") -) - -func usage() { - fmt.Println(`Usage: - deploy : Deploy image to your configured env - list List recent deployments - config () Get (or set) the configured env - logs Show lots for the given deployment`) -} - -func main() { - if len(os.Args) <= 1 { - usage() - os.Exit(1) - } - - c := NewClient(token, baseURL) - - switch os.Args[1] { - case "deploy": - deploy(c, os.Args[2:]) - case "list": - list(c, os.Args[2:]) - case "config": - config(c, os.Args[2:]) - case "logs": - logs(c, os.Args[2:]) - case "events": - events(c, os.Args[2:]) - case "help": - usage() - default: - usage() - } -} - -func deploy(c Client, args []string) { - var ( - flags = flag.NewFlagSet("", flag.ContinueOnError) - username = flags.String("u", "", "Username to report to deploy service (default with be current user)") - services ArrayFlags - ) - flags.Var(&services, "service", "Service to update (can be repeated)") - if err := flags.Parse(args); err != nil { - usage() - return - } - args = flags.Args() - if len(args) != 1 { - usage() - return - } - parts := strings.SplitN(args[0], ":", 2) - if len(parts) < 2 { - usage() - return - } - if *username == "" { - user, err := user.Current() - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - *username = user.Username - } - deployment := Deployment{ - ImageName: parts[0], - Version: parts[1], - TriggeringUser: *username, - IntendedServices: services, - } - if err := c.Deploy(deployment); err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } -} - -func list(c Client, args []string) { - var ( - flags = flag.NewFlagSet("", flag.ContinueOnError) - since = flags.Duration("since", 7*24*time.Hour, "How far back to fetch results") - ) - if err := flags.Parse(args); err != nil { - usage() - return - } - through := time.Now() - from := through.Add(-*since) - deployments, err := c.GetDeployments(from.Unix(), through.Unix()) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - - table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"Created", "ID", "Image", "Version", "State"}) - table.SetBorder(false) - table.SetColumnSeparator(" ") - for _, deployment := range deployments { - table.Append([]string{ - deployment.CreatedAt.Format(time.RFC822), - deployment.ID, - deployment.ImageName, - deployment.Version, - deployment.State, - }) - } - table.Render() -} - -func events(c Client, args []string) { - var ( - flags = flag.NewFlagSet("", flag.ContinueOnError) - since = flags.Duration("since", 7*24*time.Hour, "How far back to fetch results") - ) - if err := flags.Parse(args); err != nil { - usage() - return - } - through := time.Now() - from := through.Add(-*since) - events, err := c.GetEvents(from.Unix(), through.Unix()) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - - fmt.Println("events: ", string(events)) -} - -func loadConfig(filename string) (*Config, error) { - extension := filepath.Ext(filename) - var config Config - buf, err := ioutil.ReadFile(filename) - if err != nil { - return nil, err - } - if extension == ".yaml" || extension == ".yml" { - if err := yaml.Unmarshal(buf, &config); err != nil { - return nil, err - } - } else { - if err := json.NewDecoder(bytes.NewReader(buf)).Decode(&config); err != nil { - return nil, err - } - } - return &config, nil -} - -func config(c Client, args []string) { - if len(args) > 1 { - usage() - return - } - - if len(args) == 1 { - config, err := loadConfig(args[0]) - if err != nil { - fmt.Println("Error reading config:", err) - os.Exit(1) - } - - if err := c.SetConfig(config); err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - } else { - config, err := c.GetConfig() - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - - buf, err := yaml.Marshal(config) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - - fmt.Println(string(buf)) - } -} - -func logs(c Client, args []string) { - if len(args) != 1 { - usage() - return - } - - output, err := c.GetLogs(args[0]) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - - fmt.Println(string(output)) -} diff --git a/tools/cmd/wcloud/client.go b/tools/cmd/wcloud/client.go deleted file mode 100644 index 02cbbaa747..0000000000 --- a/tools/cmd/wcloud/client.go +++ /dev/null @@ -1,150 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" -) - -// Client for the deployment service -type Client struct { - token string - baseURL string -} - -// NewClient makes a new Client -func NewClient(token, baseURL string) Client { - return Client{ - token: token, - baseURL: baseURL, - } -} - -func (c Client) newRequest(method, path string, body io.Reader) (*http.Request, error) { - req, err := http.NewRequest(method, c.baseURL+path, body) - if err != nil { - return nil, err - } - req.Header.Add("Authorization", fmt.Sprintf("Scope-Probe token=%s", c.token)) - return req, nil -} - -// Deploy notifies the deployment service about a new deployment -func (c Client) Deploy(deployment Deployment) error { - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(deployment); err != nil { - return err - } - req, err := c.newRequest("POST", "/api/deploy/deploy", &buf) - if err != nil { - return err - } - res, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - if res.StatusCode != 204 { - return fmt.Errorf("error making request: %s", res.Status) - } - return nil -} - -// GetDeployments returns a list of deployments -func (c Client) GetDeployments(from, through int64) ([]Deployment, error) { - req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/deploy?from=%d&through=%d", from, through), nil) - if err != nil { - return nil, err - } - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - if res.StatusCode != 200 { - return nil, fmt.Errorf("error making request: %s", res.Status) - } - var response struct { - Deployments []Deployment `json:"deployments"` - } - if err := json.NewDecoder(res.Body).Decode(&response); err != nil { - return nil, err - } - return response.Deployments, nil -} - -// GetEvents returns the raw events. -func (c Client) GetEvents(from, through int64) ([]byte, error) { - req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/event?from=%d&through=%d", from, through), nil) - if err != nil { - return nil, err - } - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - if res.StatusCode != 200 { - return nil, fmt.Errorf("error making request: %s", res.Status) - } - return ioutil.ReadAll(res.Body) -} - -// GetConfig returns the current Config -func (c Client) GetConfig() (*Config, error) { - req, err := c.newRequest("GET", "/api/config/deploy", nil) - if err != nil { - return nil, err - } - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - if res.StatusCode == 404 { - return nil, fmt.Errorf("no configuration uploaded yet") - } - if res.StatusCode != 200 { - return nil, fmt.Errorf("error making request: %s", res.Status) - } - var config Config - if err := json.NewDecoder(res.Body).Decode(&config); err != nil { - return nil, err - } - return &config, nil -} - -// SetConfig sets the current Config -func (c Client) SetConfig(config *Config) error { - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(config); err != nil { - return err - } - req, err := c.newRequest("POST", "/api/config/deploy", &buf) - if err != nil { - return err - } - res, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - if res.StatusCode != 204 { - return fmt.Errorf("error making request: %s", res.Status) - } - return nil -} - -// GetLogs returns the logs for a given deployment. -func (c Client) GetLogs(deployID string) ([]byte, error) { - req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/deploy/%s/log", deployID), nil) - if err != nil { - return nil, err - } - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - if res.StatusCode != 200 { - return nil, fmt.Errorf("error making request: %s", res.Status) - } - return ioutil.ReadAll(res.Body) -} diff --git a/tools/cmd/wcloud/types.go b/tools/cmd/wcloud/types.go deleted file mode 100644 index 530e76dd2e..0000000000 --- a/tools/cmd/wcloud/types.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "time" -) - -// Deployment describes a deployment -type Deployment struct { - ID string `json:"id"` - CreatedAt time.Time `json:"created_at"` - ImageName string `json:"image_name"` - Version string `json:"version"` - Priority int `json:"priority"` - State string `json:"status"` - - TriggeringUser string `json:"triggering_user"` - IntendedServices []string `json:"intended_services"` -} - -// Config for the deployment system for a user. -type Config struct { - RepoURL string `json:"repo_url" yaml:"repo_url"` - RepoBranch string `json:"repo_branch" yaml:"repo_branch"` - RepoPath string `json:"repo_path" yaml:"repo_path"` - RepoBranch string `json:"repo_branch" yaml:"repo_branch"` - RepoKey string `json:"repo_key" yaml:"repo_key"` - KubeconfigPath string `json:"kubeconfig_path" yaml:"kubeconfig_path"` - AutoApply bool `json:"auto_apply" yaml:"auto_apply"` - - Notifications []NotificationConfig `json:"notifications" yaml:"notifications"` - - // Globs of files not to change, relative to the route of the repo - ConfigFileBlackList []string `json:"config_file_black_list" yaml:"config_file_black_list"` - - CommitMessageTemplate string `json:"commit_message_template" yaml:"commit_message_template"` // See https://golang.org/pkg/text/template/ -} - -// NotificationConfig describes how to send notifications -type NotificationConfig struct { - SlackWebhookURL string `json:"slack_webhook_url" yaml:"slack_webhook_url"` - SlackUsername string `json:"slack_username" yaml:"slack_username"` - MessageTemplate string `json:"message_template" yaml:"message_template"` - ApplyMessageTemplate string `json:"apply_message_template" yaml:"apply_message_template"` -} diff --git a/tools/integration/config.sh b/tools/integration/config.sh index 9bf47ea44e..3dfa3cac42 100644 --- a/tools/integration/config.sh +++ b/tools/integration/config.sh @@ -81,7 +81,8 @@ run_on() { host=$1 shift 1 [ -z "$DEBUG" ] || greyly echo "Running on $host:" "$@" >&2 - remote "$host" "$SSH" "$host" "$@" + # shellcheck disable=SC2086 + remote "$host" $SSH "$host" "$@" } docker_on() { @@ -117,7 +118,8 @@ start_suite() { PLUGIN_ID=$(docker_on "$host" ps -aq --filter=name=weaveplugin) PLUGIN_FILTER="cat" [ -n "$PLUGIN_ID" ] && PLUGIN_FILTER="grep -v $PLUGIN_ID" - rm_containers "$host" "$(docker_on "$host" ps -aq 2>/dev/null | "$PLUGIN_FILTER")" + # shellcheck disable=SC2046 + rm_containers "$host" $(docker_on "$host" ps -aq 2>/dev/null | $PLUGIN_FILTER) run_on "$host" "docker network ls | grep -q ' weave ' && docker network rm weave" || true weave_on "$host" reset 2>/dev/null done diff --git a/tools/lint b/tools/lint index b48a5c0b42..25f191a535 100755 --- a/tools/lint +++ b/tools/lint @@ -18,7 +18,6 @@ set -e IGNORE_LINT_COMMENT= -IGNORE_TEST_PACKAGES= IGNORE_SPELLINGS= while true; do case "$1" in @@ -27,7 +26,7 @@ while true; do shift 1 ;; -notestpackage) - IGNORE_TEST_PACKAGES=1 + # NOOP, still accepted for backwards compatibility shift 1 ;; -ignorespelling) @@ -65,31 +64,6 @@ spell_check() { return $lint_result } -test_mismatch() { - local filename="$1" - local lint_result=0 - local package - package=$(grep '^package ' "$filename" | awk '{print $2}') - - if [[ $package == "main" ]]; then - return # in package main, all bets are off - fi - - if [[ $filename == *"_internal_test.go" ]]; then - if [[ $package == *"_test" ]]; then - lint_result=1 - echo "${filename}: should not be part of a _test package" - fi - else - if [[ ! $package == *"_test" ]]; then - lint_result=1 - echo "${filename}: should be part of a _test package" - fi - fi - - return $lint_result -} - lint_go() { local filename="$1" local lint_result=0 @@ -184,12 +158,6 @@ lint() { tf) lint_tf "${filename}" || lint_result=1 ;; esac - if [ -z "$IGNORE_TEST_PACKAGES" ]; then - if [[ "$filename" == *"_test.go" ]]; then - test_mismatch "${filename}" || lint_result=1 - fi - fi - spell_check "${filename}" || lint_result=1 return $lint_result diff --git a/tools/scheduler/main.py b/tools/scheduler/main.py index 8907e202de..4ed887563a 100644 --- a/tools/scheduler/main.py +++ b/tools/scheduler/main.py @@ -85,11 +85,25 @@ def avg(test): schedule = Schedule.get_or_insert(schedule_id, shards=shards) return flask.json.jsonify(tests=schedule.shards[str(shard)]) -NAME_RE = re.compile(r'^host(?P\d+)-(?P\d+)-(?P\d+)$') +FIREWALL_REGEXES = [ + re.compile(r'^(?P\w+)-allow-(?P\w+)-(?P\d+)-(?P\d+)$'), + re.compile(r'^(?P\w+)-(?P\d+)-(?P\d+)-allow-(?P[\w\-]+)$'), +] +NAME_REGEXES = [ + re.compile(r'^host(?P\d+)-(?P\d+)-(?P\d+)$'), + re.compile(r'^test-(?P\d+)-(?P\d+)-(?P\d+)$'), +] + +def _matches_any_regex(name, regexes): + for regex in regexes: + matches = regex.match(name) + if matches: + return matches PROJECTS = [ - ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a'), - ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a'), + ('weaveworks/weave', 'weave-net-tests', 'us-central1-a', True), + ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True), + ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False), ] @app.route('/tasks/gc') @@ -98,34 +112,45 @@ def gc(): credentials = GoogleCredentials.get_application_default() compute = discovery.build('compute', 'v1', credentials=credentials) - for repo, project, zone in PROJECTS: - gc_project(compute, repo, project, zone) + for repo, project, zone, gc_fw in PROJECTS: + gc_project(compute, repo, project, zone, gc_fw) return "Done" -def gc_project(compute, repo, project, zone): +def gc_project(compute, repo, project, zone, gc_fw): logging.info("GCing %s, %s, %s", repo, project, zone) - instances = compute.instances().list(project=project, zone=zone).execute() - if 'items' not in instances: - return - - host_by_build = collections.defaultdict(list) - for instance in instances['items']: - matches = NAME_RE.match(instance['name']) - if matches is None: - continue - host_by_build[int(matches.group('build'))].append(instance['name']) - logging.info("Running VMs by build: %r", host_by_build) - - # Get list of builds, filter down to runnning builds + # Get list of builds, filter down to running builds: + running = _get_running_builds(repo) + # Stop VMs for builds that aren't running: + _gc_compute_engine_instances(compute, project, zone, running) + # Remove firewall rules for builds that aren't running: + if gc_fw: + _gc_firewall_rules(compute, project, running) + +def _get_running_builds(repo): result = urlfetch.fetch('https://circleci.com/api/v1/project/%s' % repo, headers={'Accept': 'application/json'}) assert result.status_code == 200 builds = json.loads(result.content) running = {build['build_num'] for build in builds if not build.get('stop_time')} logging.info("Runnings builds: %r", running) + return running - # Stop VMs for builds that aren't running +def _get_hosts_by_build(instances): + host_by_build = collections.defaultdict(list) + for instance in instances['items']: + matches = _matches_any_regex(instance['name'], NAME_REGEXES) + if not matches: + continue + host_by_build[int(matches.group('build'))].append(instance['name']) + logging.info("Running VMs by build: %r", host_by_build) + return host_by_build + +def _gc_compute_engine_instances(compute, project, zone, running): + instances = compute.instances().list(project=project, zone=zone).execute() + if 'items' not in instances: + return + host_by_build = _get_hosts_by_build(instances) stopped = [] for build, names in host_by_build.iteritems(): if build in running: @@ -134,5 +159,17 @@ def gc_project(compute, repo, project, zone): stopped.append(name) logging.info("Stopping VM %s", name) compute.instances().delete(project=project, zone=zone, instance=name).execute() + return stopped - return +def _gc_firewall_rules(compute, project, running): + firewalls = compute.firewalls().list(project=project).execute() + if 'items' not in firewalls: + return + for firewall in firewalls['items']: + matches = _matches_any_regex(firewall['name'], FIREWALL_REGEXES) + if not matches: + continue + if int(matches.group('build')) in running: + continue + logging.info("Deleting firewall rule %s", firewall['name']) + compute.firewalls().delete(project=project, firewall=firewall['name']).execute()