Merge branch 'main' into f-1.3-boogie-nights

.changelog/10808.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+cli: Added `nomad operator api` command to ease querying Nomad's HTTP API.
+```

.changelog/12129.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+csi: Added support for storage topology
+```

.changelog/12138.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+namespaces: Allow adding custom metadata to namespaces.
+```

.changelog/12144.txt

@@ -0,0 +1,7 @@
+```release-note:improvement
+api: CSI secrets for list and delete snapshots are now passed in HTTP headers
+```
+
+```release-note:improvement
+cli: CSI secrets argument for `volume snapshot list` has been made consistent with `volume snapshot delete`
+```

.changelog/12150.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+cli: Return a redacted value for mount flags in the `volume status` command, instead of `<none>`
+```

.changelog/12153.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+csi: Respect the verbose flag in the output of `volume status`
+```

.changelog/12154.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+csi: Sort allocations in `plugin status` output
+```

.changelog/12167.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+csi: Allow volumes to be re-registered to be updated while not in use
+```

.changelog/12178.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+csi: Fixed a bug where fields were missing from the Read Volume API response
+```

.changelog/12184.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+api: Apply prefix filter when querying CSI volumes in all namespaces
+```

.changelog/12186.txt

@@ -0,0 +1,7 @@
+```release-note:improvement
+api: Add support for filtering, sorting, and pagination to the ACL tokens and allocations list endpoint
+```
+
+```release-note:improvement
+api: Add support for filtering and pagination to the jobs and volumes list endpoint
+```

.changelog/12193.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+csi: Add pagination parameters to `volume snapshot list` command
+```

.changelog/12194.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+csi: Fixed a bug where `plugin status` commands could choose the incorrect plugin if a plugin with a name that matched the same prefix existed.
+```

.changelog/12195.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+csi: Fixed a bug where creating snapshots required a plugin ID instead of falling back to the volume's plugin ID
+```

.changelog/12197.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+csi: Fixed a bug where `volume snapshot list` did not correctly filter by plugin IDs. The `-plugin` parameter is required.
+```

.changelog/12293.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+server: Transfer Raft leadership in case the Nomad server fails to establish leadership
+```

.changelog/12305.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+api: Add `related` query parameter to the Evaluation details endpoint
+```

.changelog/12317.txt

@@ -0,0 +1,7 @@
+```release-note:improvement
+cli: display the Raft version instead of the Serf protocol in the `nomad server members` command
+```
+
+```release-note:improvement
+cli: rename the `nomad server members` `-detailed` flag to `-verbose` so it matches other commands
+```

.circleci/config.yml

@@ -9,7 +9,6 @@ references:
 
   # common references
   common_envs: &common_envs
-    GOMAXPROCS: 1
     NOMAD_SLOW_TEST: 1
     GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml
     GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json
@@ -520,7 +519,7 @@ executors:
     working_directory: ~/go/src/github.com/hashicorp/nomad
     machine:
       image: *go_machine_image
-      resource_class: medium
+      resource_class: large
     environment: &machine_env
       <<: *common_envs
       GOLANG_VERSION: 1.17.5
@@ -545,7 +544,7 @@ executors:
       GOBIN: c:\gopath\bin
       GOTESTSUM_PATH: c:\tmp\test-reports
       GOLANG_VERSION: 1.17.5
-      GOTESTSUM_VERSION: 0.4.2
+      GOTESTSUM_VERSION: 1.7.0
       VAULT_VERSION: 1.4.1
 
 workflows:

.github/workflows/test-core.yaml

@@ -0,0 +1,140 @@
+name: Core CI Tests
+on:
+  push:
+    branches-ignore:
+      - main
+      - release-**
+    paths-ignore:
+      - 'README.md'
+      - 'CHANGELOG.md'
+      - '.changelog/*'
+      - '.tours/*'
+      - 'contributing/*'
+      - 'demo/*'
+      - 'dev/*'
+      - 'e2e/terraform/*'
+      - 'integrations/*'
+      - 'pkg/*'
+      - 'scripts/*'
+      - 'terraform/*'
+      - 'ui/*'
+      - 'website/*'
+env:
+  GO_VERSION: 1.17.7
+  GOBIN: /usr/local/bin
+  GOTESTARCH: amd64
+  CONSUL_VERSION: 1.11.3
+  VAULT_VERSION: 1.9.3
+  NOMAD_SLOW_TEST: 0
+  NOMAD_TEST_LOG_LEVEL: ERROR
+jobs:
+  checks:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # needs tags for checkproto
+      - uses: magnetikonline/action-golang-cache@v1
+        with:
+          go-version: ${{env.GO_VERSION}}
+          cache-key-suffix: -checks
+      - name: Run make check
+        run: |
+          make bootstrap
+          make check
+  compile:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, macos-11, windows-2019]
+    runs-on: ${{matrix.os}}
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+      - uses: magnetikonline/action-golang-cache@v1
+        with:
+          go-version: ${{env.GO_VERSION}}
+          cache-key-suffix: -compile
+      - name: Run make dev
+        env:
+          GOBIN: ${{env.GOROOT}}/bin # windows kludge
+        run: |
+          make bootstrap
+          make dev
+  tests-api:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v2
+      - uses: magnetikonline/action-golang-cache@v1
+        with:
+          go-version: ${{env.GO_VERSION}}
+          cache-key-suffix: -api
+      - name: Run API tests
+        env:
+          GOTEST_MOD: api
+        run: |
+          make bootstrap
+          make generate-all
+          make test-nomad-module
+  tests-pkgs:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        pkg:
+          - acl
+          - client
+          - client/allocdir
+          - client/allochealth
+          - client/allocrunner
+          - client/allocwatcher
+          - client/config
+          - client/consul
+          - client/devicemanager
+          - client/dynamicplugins
+          - client/fingerprint
+          # - client/lib/...
+          - client/logmon
+          - client/pluginmanager
+          - client/state
+          - client/stats
+          - client/structs
+          - client/taskenv
+          - command
+          - command/agent
+          # - drivers/docker
+          # - drivers/exec
+          - drivers/java
+          - drivers/rawexec
+          - helper/...
+          - internal/...
+          - jobspec/...
+          - lib/...
+          - nomad
+          - nomad/deploymentwatcher
+          - nomad/stream
+          - nomad/structs
+          - nomad/volumewatcher
+          - plugins/...
+          - scheduler/...
+          - testutil
+    steps:
+      - uses: actions/checkout@v2
+      - uses: magnetikonline/action-golang-cache@v1
+        with:
+          go-version: ${{env.GO_VERSION}}
+          cache-key-suffix: -pkgs
+      - name: Run Matrix Tests
+        env:
+          GOTEST_PKGS: ./${{matrix.pkg}}
+        run: |
+          make bootstrap
+          make generate-all
+          hc-install vault ${{env.VAULT_VERSION}}
+          hc-install consul ${{env.CONSUL_VERSION}}
+          sudo sed -i 's!Defaults!#Defaults!g' /etc/sudoers
+          sudo -E env "PATH=$PATH" make test-nomad
+

.semgrep/go_tests.yml

@@ -13,6 +13,7 @@ rules:
     # TODO(luiz): figure out how to do a 'delete line' fix.
     fix: " "
     paths:
+      exclude: ["*"]
       include:
         - "*_test.go"
 
@@ -43,6 +44,7 @@ rules:
     severity: "WARNING"
     fix: "assert.$FUNC($T, $...ARGS)"
     paths:
+      exclude: ["*"]
       include:
         - "*_test.go"
   - id: "tests-no-assert-without-t-nested"
@@ -64,6 +66,7 @@ rules:
     severity: "WARNING"
     fix: "assert.$FUNC($T, $...ARGS)"
     paths:
+      exclude: ["*"]
       include:
         - "*_test.go"
   - id: "tests-no-require-without-t"
@@ -91,6 +94,7 @@ rules:
     severity: "WARNING"
     fix: "require.$FUNC($T, $...ARGS)"
     paths:
+      exclude: ["*"]
       include:
         - "*_test.go"
   - id: "tests-no-require-without-t-nested"
@@ -111,6 +115,18 @@ rules:
       - "go"
     severity: "WARNING"
     fix: "require.$FUNC($T, $...ARGS)"
+    paths:
+      exclude: ["*"]
+      include:
+        - "*_test.go"
+  - id: "tests-no-parallel"
+    patterns:
+      - pattern: "t.Parallel()"
+    message: "Use ci.Parallel(t) instead of t.Parallel()"
+    languages:
+      - "go"
+    severity: "WARNING"
+    fix: "ci.Parallel(t)"
     paths:
       include:
         - "*_test.go"

GNUmakefile

@@ -32,7 +32,11 @@ ifndef NOMAD_NO_UI
 GO_TAGS := ui $(GO_TAGS)
 endif
 
+ifeq ($(CIRCLECI),true)
 GO_TEST_CMD = $(if $(shell command -v gotestsum 2>/dev/null),gotestsum --,go test)
+else
+GO_TEST_CMD = go test
+endif
 
 ifeq ($(origin GOTEST_PKGS_EXCLUDE), undefined)
 GOTEST_PKGS ?= "./..."
@@ -49,12 +53,6 @@ LAST_RELEASE ?= v1.2.6
 
 default: help
 
-ifeq ($(CI),true)
-	$(info Running in a CI environment, verbose mode is disabled)
-else
-	VERBOSE="true"
-endif
-
 ifeq (Linux,$(THIS_OS))
 ALL_TARGETS = linux_386 \
 	linux_amd64 \
@@ -131,13 +129,14 @@ deps:  ## Install build and development dependencies
 	go install github.com/hashicorp/go-bindata/go-bindata@bf7910af899725e4938903fb32048c7c0b15f12e
 	go install github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs@234c15e7648ff35458026de92b34c637bae5e6f7
 	go install github.com/a8m/tree/cmd/tree@fce18e2a750ea4e7f53ee706b1c3d9cbb22de79c
-	go install gotest.tools/gotestsum@v0.4.2
+	go install gotest.tools/gotestsum@v1.7.0
 	go install github.com/hashicorp/hcl/v2/cmd/hclfmt@v2.5.1
 	go install github.com/golang/protobuf/protoc-gen-go@v1.3.4
 	go install github.com/hashicorp/go-msgpack/codec/codecgen@v1.1.5
 	go install github.com/bufbuild/buf/cmd/buf@v0.36.0
 	go install github.com/hashicorp/go-changelog/cmd/changelog-build@latest
 	go install golang.org/x/tools/cmd/stringer@v0.1.8
+	go install gophers.dev/cmds/hc-install/cmd/hc-install@v1.0.1
 
 .PHONY: lint-deps
 lint-deps: ## Install linter dependencies
@@ -300,7 +299,7 @@ test-nomad: dev ## Run Nomad test suites
 	$(if $(ENABLE_RACE),GORACE="strip_path_prefix=$(GOPATH)/src") $(GO_TEST_CMD) \
 		$(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \
 		-cover \
-		-timeout=15m \
+		-timeout=20m \
 		-tags "$(GO_TAGS)" \
 		$(GOTEST_PKGS) $(if $(VERBOSE), >test.log ; echo $$? > exit-code)
 	@if [ $(VERBOSE) ] ; then \
@@ -313,7 +312,7 @@ test-nomad-module: dev ## Run Nomad test suites on a sub-module
 	@cd $(GOTEST_MOD) && $(if $(ENABLE_RACE),GORACE="strip_path_prefix=$(GOPATH)/src") $(GO_TEST_CMD) \
 		$(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \
 		-cover \
-		-timeout=15m \
+		-timeout=20m \
 		-tags "$(GO_TAGS)" \
 		./... $(if $(VERBOSE), >test.log ; echo $$? > exit-code)
 	@if [ $(VERBOSE) ] ; then \