deps: Switch to Go modules for dependency management

This PR switches the Nomad repository from using govendor to Go modules
for managing dependencies. Aspects of the Nomad workflow remain pretty
much the same. The usual Makefile targets should continue to work as
they always did. The API submodule simply defers to the parent Nomad
version on the repository, keeping the semantics of API versioning that
currently exists.

.circleci/config/commands/run-tests.yml

@@ -16,4 +16,9 @@ steps:
 
         mkdir -p /tmp/test-reports
         sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs
-        sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad
+
+        if [ ! -z $GOTEST_MOD ]; then
+          sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module
+        else
+          sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad
+        fi

.circleci/config/jobs/build-darwin-binaries.yml

@@ -10,7 +10,7 @@ steps:
       target_directory: ~/goinstall
   - run: source ${BASH_ENV} && make deps
   - run: brew install protobuf
-  - run: sudo -E PATH="$GOPATH/bin:${HOME}/goinstall/go/bin:$PATH" make generate-structs
+  - run: PATH="$GOPATH/bin:${HOME}/goinstall/go/bin:$PATH" make generate-structs
   - run: source ${BASH_ENV} && make pkg/darwin_amd64.zip
   - store_artifacts:
       path: pkg/darwin_amd64.zip

.circleci/config/jobs/test-machine.yml

@@ -3,6 +3,9 @@ parameters:
   test_packages:
     type: string
     default: ""
+  test_module:
+    type: string
+    default: ""
   exclude_packages:
     type: string
     default: ""
@@ -15,6 +18,7 @@ parameters:
 environment:
   GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>"
   GOTEST_PKGS: "<< parameters.test_packages >>"
+  GOTEST_MOD: "<< parameters.test_module >>"
   GOTESTARCH: "<< parameters.goarch >>"
 steps:
   - checkout

.circleci/config/workflows/build-test.yml

@@ -32,7 +32,7 @@ jobs:
       # API Tests run in a VM rather than container due to the FS tests
       # requiring `mount` priviliges.
       name: "test-api"
-      test_packages: "./api/..."
+      test_module: "api"
       filters: *backend_branches_filter
   - test-container:
       name: "test-devices"

.gitignore

@@ -60,6 +60,7 @@ nomad_linux_amd64
 nomad_darwin_amd64
 TODO.md
 codecgen-*.generated.go
+**/structs.generated.go
 GNUMakefile.local
 
 .terraform
@@ -104,3 +105,6 @@ azure-hashistack.pem
 
 # generated keys for e2e tests
 e2e/terraform/keys/
+
+# darwin CGO files copied for build
+vendor/github.com/shirou/gopsutil/host/include/

GNUmakefile

@@ -44,6 +44,8 @@ endif
 # On MacOS, we only build for MacOS
 ifeq (Darwin,$(THIS_OS))
 ALL_TARGETS += darwin_amd64
+# Copy CGO files for darwin into place
+$(shell cp -R lib/darwin/include vendor/github.com/shirou/gopsutil/host)
 endif
 
 # On FreeBSD, we only build for FreeBSD
@@ -166,22 +168,22 @@ bootstrap: deps lint-deps git-hooks # Install all dependencies
 
 .PHONY: deps
 deps:  ## Install build and development dependencies
+## Keep versions in sync with tools/go.mod for now (see https://github.com/golang/go/issues/30515)
 	@echo "==> Updating build dependencies..."
-	GO111MODULE=on go get -u github.com/kardianos/govendor
-	GO111MODULE=on go get -u github.com/hashicorp/go-bindata/go-bindata@master
-	GO111MODULE=on go get -u github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs@master
-	GO111MODULE=on go get -u github.com/a8m/tree/cmd/tree
-	GO111MODULE=on go get -u github.com/magiconair/vendorfmt/cmd/vendorfmt
-	GO111MODULE=on go get -u gotest.tools/gotestsum
-	GO111MODULE=on go get -u github.com/hashicorp/hcl/v2/cmd/hclfmt@v2.5.1
-	GO111MODULE=on go get -u github.com/golang/protobuf/protoc-gen-go@v1.3.4
-	GO111MODULE=on go get -u github.com/hashicorp/go-msgpack/codec/codecgen@v1.1.5
+	GO111MODULE=on cd tools && go get github.com/hashicorp/go-bindata/go-bindata@v3.0.7+incompatible
+	GO111MODULE=on cd tools && go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs@v1.0.0
+	GO111MODULE=on cd tools && go get github.com/a8m/tree/cmd/tree
+	GO111MODULE=on cd tools && go get gotest.tools/gotestsum@v0.4.2
+	GO111MODULE=on cd tools && go get github.com/hashicorp/hcl/v2/cmd/hclfmt@v2.5.1
+	GO111MODULE=on cd tools && go get github.com/golang/protobuf/protoc-gen-go@v1.3.4
+	GO111MODULE=on cd tools && go get github.com/hashicorp/go-msgpack/codec/codecgen@v1.1.5
 
 .PHONY: lint-deps
 lint-deps: ## Install linter dependencies
+## Keep versions in sync with tools/go.mod (see https://github.com/golang/go/issues/30515)
 	@echo "==> Updating linter dependencies..."
-	GO111MODULE=on go get -u github.com/golangci/golangci-lint/cmd/golangci-lint@v1.24.0
-	GO111MODULE=on go get -u github.com/client9/misspell/cmd/misspell
+	GO111MODULE=on cd tools && go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.24.0
+	GO111MODULE=on cd tools && go get github.com/client9/misspell/cmd/misspell@v0.3.4
 
 .PHONY: git-hooks
 git-dir = $(shell git rev-parse --git-dir)
@@ -207,10 +209,7 @@ check: ## Lint the source code
 	@if (git status -s | grep -q -e '\.hcl$$' -e '\.nomad$$'); then echo the following HCL files are out of sync; git status -s | grep -e '\.hcl$$' -e '\.nomad$$'; exit 1; fi
 
 	@echo "==> Check API package is isolated from rest"
-	@if go list --test -f '{{ join .Deps "\n" }}' ./api | grep github.com/hashicorp/nomad/ | grep -v -e /vendor/ -e /nomad/api/ -e nomad/api.test; then echo "  /api package depends the ^^ above internal nomad packages.  Remove such dependency"; exit 1; fi
-
-	@echo "==> Check non-vendored packages"
-	@if go list --test -tags "$(GO_TAGS)" -f '{{join .Deps "\n"}}' . | grep -v github.com/hashicorp/nomad.test | xargs go list -tags "$(GO_TAGS)" -f '{{if not .Standard}}{{.ImportPath}}{{end}}' | grep -v -e github.com/hashicorp/nomad; then echo "  found referenced packages ^^ that are not vendored"; exit 1; fi
+	@cd ./api && if go list --test -f '{{ join .Deps "\n" }}' . | grep github.com/hashicorp/nomad/ | grep -v -e /vendor/ -e /nomad/api/ -e nomad/api.test; then echo "  /api package depends the ^^ above internal nomad packages.  Remove such dependency"; exit 1; fi
 
 .PHONY: checkscripts
 checkscripts: ## Lint shell scripts
@@ -239,11 +238,6 @@ generate-examples: command/job_init.bindata_assetfs.go
 command/job_init.bindata_assetfs.go: command/assets/*
 	go-bindata-assetfs -pkg command -o command/job_init.bindata_assetfs.go ./command/assets/...
 
-.PHONY: vendorfmt
-vendorfmt:
-	@echo "--> Formatting vendor/vendor.json"
-	vendorfmt
-
 .PHONY: changelogfmt
 changelogfmt:
 	@echo "--> Making [GH-xxxx] references clickable..."
@@ -257,12 +251,18 @@ hclfmt:
 	@find . -path ./terraform -prune -o -name 'upstart.nomad' -prune -o \( -name '*.nomad' -o -name '*.hcl' \) -exec \
 sh -c 'hclfmt -w {} || echo in path {}' ';'
 
+.PHONY: sync
+sync:
+	@echo "--> Sync vendor directory and tidy up"
+	@go mod vendor
+	@go mod tidy
+
 .PHONY: dev
 dev: GOOS=$(shell go env GOOS)
 dev: GOARCH=$(shell go env GOARCH)
 dev: GOPATH=$(shell go env GOPATH)
 dev: DEV_TARGET=pkg/$(GOOS)_$(GOARCH)/nomad
-dev: vendorfmt changelogfmt hclfmt ## Build for the current development platform
+dev: sync changelogfmt hclfmt ## Build for the current development platform
 	@echo "==> Removing old development build..."
 	@rm -f $(PROJECT_ROOT)/$(DEV_TARGET)
 	@rm -f $(PROJECT_ROOT)/bin/nomad
@@ -313,6 +313,19 @@ test-nomad: dev ## Run Nomad test suites
 		bash -C "$(PROJECT_ROOT)/scripts/test_check.sh" ; \
 	fi
 
+.PHONY: test-nomad-module
+test-nomad-module: dev ## Run Nomad test suites on a sub-module
+	@echo "==> Running Nomad test suites on sub-module:"
+	@cd $(GOTEST_MOD) && $(if $(ENABLE_RACE),GORACE="strip_path_prefix=$(GOPATH)/src") $(GO_TEST_CMD) \
+		$(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \
+		-cover \
+		-timeout=15m \
+		-tags "$(GO_TAGS)" \
+		./... $(if $(VERBOSE), >test.log ; echo $$? > exit-code)
+	@if [ $(VERBOSE) ] ; then \
+		bash -C "$(PROJECT_ROOT)/scripts/test_check.sh" ; \
+	fi
+
 .PHONY: e2e-test
 e2e-test: dev ## Run the Nomad e2e test suite
 	@echo "==> Running Nomad E2E test suites:"