From 7947621b6881bbd8ac4fc99c8cb05c3b3ad90205 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Tue, 18 Jan 2022 12:17:37 -0600 Subject: [PATCH] build: upgrade and speedup circleci configuration This PR upgrades our CI images and fixes some affected tests. - upgrade go-machine-image to premade latest ubuntu LTS (ubuntu-2004:202111-02) - eliminate go-machine-recent-image (no longer necessary) - eliminate setting GOMAXPROCS=1 (build tools were also affected by this setting) - upgrade resource type for all imanges to large (2C -> 4C) - manage GOPATH in GNUMakefile (see https://discuss.circleci.com/t/gopath-is-set-to-multiple-directories/7174) - fix tcp dial error check (message seems to be OS specific) - spot check values measured instead of specifically 'RSS' (rss no longer reported in cgroups v2) --- .circleci/config.yml | 21 ++++----------------- GNUmakefile | 10 ++++++++-- command/metrics_test.go | 2 +- drivers/exec/driver_test.go | 2 +- drivers/shared/executor/executor_linux.go | 1 + drivers/shared/executor/executor_test.go | 19 +++++++++++-------- 6 files changed, 26 insertions(+), 29 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bf2f74f9e428..c31744f961ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,15 +3,12 @@ version: 2.1 references: # environment specific references - aim to avoid conflicts go-machine-image: &go_machine_image - circleci/classic:201808-01 - go-machine-recent-image: &go_machine_recent_image - ubuntu-1604:201903-01 + ubuntu-2004:202111-02 go-windows-image: &go_windows_image windows-server-2019-vs2019:stable # common references common_envs: &common_envs - GOMAXPROCS: 1 NOMAD_SLOW_TEST: 1 GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json @@ -289,7 +286,7 @@ jobs: default: "" executor: type: string - default: "go-machine-recent" + default: "go-machine" goarch: type: string default: "amd64" @@ -513,6 +510,7 @@ executors: working_directory: /go/src/github.com/hashicorp/nomad docker: - image: docker.mirror.hashicorp.services/golang:1.17.5 + resource_class: large environment: <<: *common_envs GOPATH: /go @@ -521,19 +519,11 @@ executors: working_directory: ~/go/src/github.com/hashicorp/nomad machine: image: *go_machine_image + resource_class: large environment: &machine_env <<: *common_envs - GOPATH: /home/circleci/go GOLANG_VERSION: 1.17.5 - # uses a more recent image with unattended upgrades disabled properly - # but seems to break docker builds - go-machine-recent: - working_directory: ~/go/src/github.com/hashicorp/nomad - machine: - image: *go_machine_recent_image - environment: *machine_env - go-macos: working_directory: ~/go/src/github.com/hashicorp/nomad macos: @@ -636,9 +626,6 @@ workflows: - test-machine: name: "test-docker" test_packages: "./drivers/docker" - # docker is misbehaving in docker-machine-recent image - # and we get unexpected failures - # e.g. https://circleci.com/gh/hashicorp/nomad/3854 executor: go-machine filters: *backend_test_branches_filter - test-machine: diff --git a/GNUmakefile b/GNUmakefile index 84a02085e0a6..1620218323ac 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -8,6 +8,13 @@ GIT_DIRTY := $(if $(shell git status --porcelain),+CHANGES) GO_LDFLAGS := "-X github.com/hashicorp/nomad/version.GitCommit=$(GIT_COMMIT)$(GIT_DIRTY)" +ifneq (MSYS_NT,$(THIS_OS)) +# GOPATH supports PATH style multi-paths; assume the first entry is favorable. +# Necessary because new Circle images override GOPATH with multiple values. +# See: https://discuss.circleci.com/t/gopath-is-set-to-multiple-directories/7174 +GOPATH=$(shell go env GOPATH | cut -d: -f1) +endif + GO_TAGS ?= ifeq ($(CI),true) @@ -241,7 +248,6 @@ tidy: .PHONY: dev dev: GOOS=$(shell go env GOOS) dev: GOARCH=$(shell go env GOARCH) -dev: GOPATH=$(shell go env GOPATH) dev: DEV_TARGET=pkg/$(GOOS)_$(GOARCH)/nomad dev: hclfmt ## Build for the current development platform @echo "==> Removing old development build..." @@ -297,7 +303,7 @@ test-nomad: dev ## Run Nomad test suites .PHONY: test-nomad-module test-nomad-module: dev ## Run Nomad test suites on a sub-module - @echo "==> Running Nomad test suites on sub-module:" + @echo "==> Running Nomad test suites on sub-module $(GOTEST_MOD)" @cd $(GOTEST_MOD) && $(if $(ENABLE_RACE),GORACE="strip_path_prefix=$(GOPATH)/src") $(GO_TEST_CMD) \ $(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \ -cover \ diff --git a/command/metrics_test.go b/command/metrics_test.go index cdf18c334d87..a537f3997371 100644 --- a/command/metrics_test.go +++ b/command/metrics_test.go @@ -72,7 +72,7 @@ func TestCommand_Metrics_Cases(t *testing.T) { []string{"-address=http://foo"}, 1, "", - "no such host", + "dial tcp: lookup foo: Temporary failure in name resolution", }, } diff --git a/drivers/exec/driver_test.go b/drivers/exec/driver_test.go index 8651eda20d21..2188dc8bed1a 100644 --- a/drivers/exec/driver_test.go +++ b/drivers/exec/driver_test.go @@ -425,7 +425,7 @@ func TestExecDriver_Stats(t *testing.T) { require.NoError(err) select { case stats := <-statsCh: - require.NotZero(stats.ResourceUsage.MemoryStats.RSS) + require.NotEmpty(stats.ResourceUsage.MemoryStats.Measured) require.NotZero(stats.Timestamp) require.WithinDuration(time.Now(), time.Unix(0, stats.Timestamp), time.Second) case <-time.After(time.Second): diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index f40863a7438f..989c4a062c4c 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -71,6 +71,7 @@ type LibcontainerExecutor struct { } func NewExecutorWithIsolation(logger hclog.Logger) Executor { + logger = logger.Named("isolated_executor") if err := shelpers.Init(); err != nil { logger.Error("unable to initialize stats", "error", err) diff --git a/drivers/shared/executor/executor_test.go b/drivers/shared/executor/executor_test.go index 6a396ea6a8f2..0b7e8b000226 100644 --- a/drivers/shared/executor/executor_test.go +++ b/drivers/shared/executor/executor_test.go @@ -254,7 +254,6 @@ func TestExecutor_WaitExitSignal(pt *testing.T) { pt.Parallel() for name, factory := range executorFactories { pt.Run(name, func(t *testing.T) { - require := require.New(t) testExecCmd := testExecutorCommand(t) execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir execCmd.Cmd = "/bin/sleep" @@ -266,8 +265,8 @@ func TestExecutor_WaitExitSignal(pt *testing.T) { executor := factory.new(testlog.HCLogger(t)) defer executor.Shutdown("", 0) - ps, err := executor.Launch(execCmd) - require.NoError(err) + pState, err := executor.Launch(execCmd) + require.NoError(t, err) go func() { tu.WaitForResult(func() (bool, error) { @@ -280,10 +279,14 @@ func TestExecutor_WaitExitSignal(pt *testing.T) { return false, fmt.Errorf("stats failed to send on interval") case ru := <-ch: assert.NotEmpty(t, ru.Pids, "no pids recorded in stats") - assert.NotZero(t, ru.ResourceUsage.MemoryStats.RSS) + + // just checking we measured something; each executor type has its own abilities, + // and e.g. cgroup v2 provides different information than cgroup v1 + assert.NotEmpty(t, ru.ResourceUsage.MemoryStats.Measured) + assert.WithinDuration(t, time.Now(), time.Unix(0, ru.Timestamp), time.Second) } - proc, err := os.FindProcess(ps.Pid) + proc, err := os.FindProcess(pState.Pid) if err != nil { return false, err } @@ -298,9 +301,9 @@ func TestExecutor_WaitExitSignal(pt *testing.T) { }) }() - ps, err = executor.Wait(context.Background()) - require.NoError(err) - require.Equal(ps.Signal, int(syscall.SIGKILL)) + pState, err = executor.Wait(context.Background()) + require.NoError(t, err) + require.Equal(t, pState.Signal, int(syscall.SIGKILL)) }) } }