From cc0098880400694f7e7eb37c7f840c5a85c66065 Mon Sep 17 00:00:00 2001 From: Richard Case Date: Thu, 20 Sep 2018 14:14:46 +0100 Subject: [PATCH] Generation of static AMI list The list of AMIs to use for the static resolver is now generated via go:generate. The list is queried from AWS and output to a code file. A separate makefile target called 'generate-ami' has been added that runs go generate. The reason for a separate target is the fact that AWS credentials are required for the code generation and this would break the current workflow for the 'generate' target. Another makefile target called 'ami-check' has been added. The intended purpose of this by an automated scheduled process that will run to check if new images are available. Issue #49 --- .editorconfig | 19 +++++ Makefile | 64 +++++++++++----- pkg/ami/generate/main.go | 131 +++++++++++++++++++++++++++++++++ pkg/ami/static_resolver.go | 21 +----- pkg/ami/static_resolver_ami.go | 21 ++++++ 5 files changed, 219 insertions(+), 37 deletions(-) create mode 100644 .editorconfig create mode 100644 pkg/ami/generate/main.go create mode 100644 pkg/ami/static_resolver_ami.go diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..a1f8b87aad --- /dev/null +++ b/.editorconfig @@ -0,0 +1,19 @@ +; http://editorconfig.org/ + +root = true + +[*] +charset = utf-8 + +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +indent_size = 4 +indent_style = space + +[*.go] +indent_style = tab + +[Makefile] +indent_style = tab diff --git a/Makefile b/Makefile index 6a7492b93f..04999de834 100644 --- a/Makefile +++ b/Makefile @@ -4,31 +4,39 @@ git_commit := $(shell git describe --dirty --always) EKSCTL_BUILD_IMAGE ?= weaveworks/eksctl:build EKSCTL_IMAGE ?= weaveworks/eksctl:latest -.PHONY: build -build: - @go build -ldflags "-X main.gitCommit=$(git_commit) -X main.builtAt=$(built_at)" ./cmd/eksctl +.DEFAULT_GOAL:=help + +##@ Dependencies .PHONY: install-build-deps -install-build-deps: +install-build-deps: ## Install dependencies (packages and tools) @cd build && dep ensure && ./install.sh +##@ Build + +.PHONY: build +build: ## Build eksctl + @go build -ldflags "-X main.gitCommit=$(git_commit) -X main.builtAt=$(built_at)" ./cmd/eksctl + +##@ Testing & CI + .PHONY: test -test: generate +test: generate ## Run unit tests @git diff --exit-code pkg/nodebootstrap/assets.go > /dev/null || (git --no-pager diff; exit 1) @git diff --exit-code ./pkg/eks/mocks > /dev/null || (git --no-pager diff; exit 1) @go test -v -covermode=count -coverprofile=coverage.out ./pkg/... ./cmd/... @test -z $(COVERALLS_TOKEN) || $(GOPATH)/bin/goveralls -coverprofile=coverage.out -service=circle-ci .PHONY: lint -lint: +lint: ## Run GoMetalinter over the codebase @$(GOPATH)/bin/gometalinter ./... .PHONY: ci -ci: test lint +ci: test lint ## Target for CI system to invoke to run tests and linting TEST_CLUSTER ?= integration-test-dev .PHONY: integration-test-dev -integration-test-dev: build +integration-test-dev: build ## Run the integration tests without cluster teardown. For use when developing integration tests. @./eksctl utils write-kubeconfig \ --auto-kubeconfig \ --name=$(TEST_CLUSTER) @@ -39,23 +47,35 @@ integration-test-dev: build -eksctl.delete=false \ -eksctl.kubeconfig=$(HOME)/.kube/eksctl/clusters/$(TEST_CLUSTER) -create-integration-test-dev-cluster: build +create-integration-test-dev-cluster: build ## Create a test cluster for use when developing integration tests @./eksctl create cluster --name=integration-test-dev --auto-kubeconfig -delete-integration-test-dev-cluster: build +delete-integration-test-dev-cluster: build ## Delete the test cluster for use when developing integration tests @./eksctl delete cluster --name=integration-test-dev --auto-kubeconfig .PHONY: integration-test -integration-test: build +integration-test: build ## Run the integration tests (with cluster creation and cleanup) @go test -tags integration -v -timeout 21m ./tests/integration/... +##@ Code Generation + .PHONY: generate -generate: +generate: ## Generate code @chmod g-w ./pkg/nodebootstrap/assets/* @go generate ./pkg/nodebootstrap ./pkg/eks/mocks +.PHONY: generate-ami +generate-ami: ## Generate the list of AMIs for use with static resolver. Queries AWS. + @go generate ./pkg/ami + +.PHONY: ami-check +ami-check: generate-ami ## Check whether the AMIs have been updated and fail if they have. Designed for a automated test + @git diff --exit-code pkg/ami/static_resolver_ami.go > /dev/null || (git --no-pager diff; exit 1) + +##@ Docker + .PHONY: eksctl-build-image -eksctl-build-image: +eksctl-build-image: ## Create the the eksctl build docker image @-docker pull $(EKSCTL_BUILD_IMAGE) @docker build --tag=$(EKSCTL_BUILD_IMAGE) --cache-from=$(EKSCTL_BUILD_IMAGE) ./build @@ -65,11 +85,13 @@ EKSCTL_IMAGE_BUILD_ARGS += --build-arg=COVERALLS_TOKEN=$(COVERALLS_TOKEN) endif .PHONY: eksctl-image -eksctl-image: eksctl-build-image +eksctl-image: eksctl-build-image ## Create the eksctl image @docker build --tag=$(EKSCTL_IMAGE) $(EKSCTL_IMAGE_BUILD_ARGS) ./ +##@ Release + .PHONY: release -release: eksctl-build-image +release: eksctl-build-image ## Create a new eksctl release @docker run \ --env=GITHUB_TOKEN \ --env=CIRCLE_TAG \ @@ -85,12 +107,20 @@ JEKYLL := docker run --tty --rm \ --publish="4000:4000" \ starefossen/github-pages +##@ Site + .PHONY: server-pages -serve-pages: +serve-pages: ## Serve the site locally @-docker rm -f eksctl-jekyll @$(JEKYLL) jekyll serve -d /_site --watch --force_polling -H 0.0.0.0 -P 4000 .PHONY: build-page -build-pages: +build-pages: ## Generate the site using jekyll @-docker rm -f eksctl-jekyll @$(JEKYLL) jekyll build --verbose + +##@ Utility + +.PHONY: help +help: ## Display this help. Thanks to https://suva.sh/posts/well-documented-makefiles/ + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/pkg/ami/generate/main.go b/pkg/ami/generate/main.go new file mode 100644 index 0000000000..d9980368bd --- /dev/null +++ b/pkg/ami/generate/main.go @@ -0,0 +1,131 @@ +// +build ignore + +package main + +import ( + "fmt" + "os" + "sort" + "text/template" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + amiapi "github.com/weaveworks/eksctl/pkg/ami" + "github.com/weaveworks/eksctl/pkg/eks/api" +) + +type imageDetail struct { + Region string + ImageId string +} + +type images struct { + General []imageDetail + Gpu []imageDetail + Timestamp time.Time +} + +var ( + supportedRegions = []string{api.EKS_REGION_US_WEST_2, api.EKS_REGION_EU_WEST_1, api.EKS_REGION_US_EAST_1} + amiTemplate = template.Must(template.New("").Parse(` +// Code generated by go generate; DO NOT EDIT. + +package ami + +// StaticImages is a map that holds the list of amis to be used by +// for static ami resolution +var StaticImages = map[string]map[int]map[string]string{ + ImageFamilyAmazonLinux2: { + ImageClassGeneral: { + {{- range .General }} + "{{ .Region}}": "{{ .ImageId}}", + {{- end }} + }, + ImageClassGPU: { + {{- range .Gpu }} + "{{ .Region}}": "{{ .ImageId}}", + {{- end }} + }, + }, +}`)) +) + +func main() { + fmt.Println("generating list of AMIs for the static resolvers") + + amiImages := images{} + for _, region := range supportedRegions { + general, gpu, err := getAmis(region) + if err != nil { + fmt.Println(err) + return + } + + fmt.Printf("region %s, general class = %s, gpu class = %s\n", region, general, gpu) + + amiImages.General = append(amiImages.General, imageDetail{ + ImageId: general, + Region: region, + }) + + amiImages.Gpu = append(amiImages.Gpu, imageDetail{ + ImageId: gpu, + Region: region, + }) + } + + // Sort on region + sort.Slice(amiImages.General, func(i, j int) bool { + return amiImages.General[i].Region < amiImages.General[j].Region + }) + sort.Slice(amiImages.Gpu, func(i, j int) bool { + return amiImages.Gpu[i].Region < amiImages.Gpu[j].Region + }) + + f, err := os.Create("./static_resolver_ami.go") + if err != nil { + fmt.Println(err) + return + } + defer f.Close() + + err = amiTemplate.Execute(f, amiImages) + if err != nil { + fmt.Println(err) + return + } +} + +func getAmis(region string) (normal string, gpu string, reterr error) { + s := newSession(region) + e := ec2.New(s) + + normal, err := amiapi.FindImage(e, "amazon-eks-node-*") + if err != nil { + return "", "", err + } + gpu, err = amiapi.FindImage(e, "amazon-eks-gpu-node-*") + if err != nil { + return "", "", err + } + + return +} + +func newSession(region string) *session.Session { + config := aws.NewConfig() + config = config.WithRegion(region) + config = config.WithCredentialsChainVerboseErrors(true) + + // Create the options for the session + opts := session.Options{ + Config: *config, + SharedConfigState: session.SharedConfigEnable, + AssumeRoleTokenProvider: stscreds.StdinTokenProvider, + } + + return session.Must(session.NewSessionWithOptions(opts)) +} diff --git a/pkg/ami/static_resolver.go b/pkg/ami/static_resolver.go index 23ec03b1a6..fa8fb95059 100644 --- a/pkg/ami/static_resolver.go +++ b/pkg/ami/static_resolver.go @@ -4,29 +4,10 @@ import ( "fmt" "github.com/kubicorn/kubicorn/pkg/logger" - "github.com/weaveworks/eksctl/pkg/eks/api" "github.com/weaveworks/eksctl/pkg/utils" ) -// TODO: This file will be go generated in the future -// https://github.com/weaveworks/eksctl/issues/49 - -// StaticImages is a map of images in each region by -// image OS family and by class -var StaticImages = map[string]map[int]map[string]string{ - ImageFamilyAmazonLinux2: { - ImageClassGeneral: { - api.EKS_REGION_US_WEST_2: "ami-0a54c984b9f908c81", - api.EKS_REGION_US_EAST_1: "ami-0440e4f6b9713faf6", - api.EKS_REGION_EU_WEST_1: "ami-0c7a4976cb6fafd3a", - }, - ImageClassGPU: { - api.EKS_REGION_US_WEST_2: "ami-0731694d53ef9604b", - api.EKS_REGION_US_EAST_1: "ami-058bfb8c236caae89", - api.EKS_REGION_EU_WEST_1: "ami-0706dc8a5eed2eed9", - }, - }, -} +//go:generate go run generate/main.go // StaticDefaultResolver resolves the AMI to the defaults for the region type StaticDefaultResolver struct { diff --git a/pkg/ami/static_resolver_ami.go b/pkg/ami/static_resolver_ami.go new file mode 100644 index 0000000000..ee20f78d0a --- /dev/null +++ b/pkg/ami/static_resolver_ami.go @@ -0,0 +1,21 @@ + +// Code generated by go generate; DO NOT EDIT. + +package ami + +// StaticImages is a map that holds the list of amis to be used by +// for static ami resolution +var StaticImages = map[string]map[int]map[string]string{ + ImageFamilyAmazonLinux2: { + ImageClassGeneral: { + "eu-west-1": "ami-0c7a4976cb6fafd3a", + "us-east-1": "ami-0440e4f6b9713faf6", + "us-west-2": "ami-0a54c984b9f908c81", + }, + ImageClassGPU: { + "eu-west-1": "ami-0706dc8a5eed2eed9", + "us-east-1": "ami-058bfb8c236caae89", + "us-west-2": "ami-0731694d53ef9604b", + }, + }, +} \ No newline at end of file