From 076790f85e536319bf5cd25006c700fe805a6922 Mon Sep 17 00:00:00 2001 From: whg517 Date: Mon, 15 Apr 2024 00:38:47 +0800 Subject: [PATCH] chore: add test config --- .chainsaw.yaml | 14 +++++ .dockerignore | 2 + .editorconfig | 35 ++++++++++++ .github/workflows/e2e.yml | 36 ++++++++++++ .gitignore | 2 + .markdownlint.yml | 3 +- Dockerfile | 2 +- Makefile | 93 +++++++++++++++++++++++++++++-- config/manager/kustomization.yaml | 2 +- config/manager/manager.yaml | 4 +- internal/controller/suite_test.go | 52 +++++++++++++++-- test/e2e/kind-config.yaml | 18 ++++++ test/e2e/sample/00-assert.yaml | 41 ++++++++++++++ test/e2e/sample/00-trino.yaml | 77 +++++++++++++++++++++++++ 14 files changed, 364 insertions(+), 17 deletions(-) create mode 100644 .chainsaw.yaml create mode 100644 .editorconfig create mode 100644 .github/workflows/e2e.yml create mode 100644 test/e2e/kind-config.yaml create mode 100644 test/e2e/sample/00-assert.yaml create mode 100644 test/e2e/sample/00-trino.yaml diff --git a/.chainsaw.yaml b/.chainsaw.yaml new file mode 100644 index 0000000..bd96e15 --- /dev/null +++ b/.chainsaw.yaml @@ -0,0 +1,14 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Configuration +metadata: + name: custom-config +spec: + timeouts: + apply: 120s + assert: 120s + cleanup: 120s + delete: 120s + error: 10s + exec: 45s + skipDelete: false + failFast: true diff --git a/.dockerignore b/.dockerignore index ecc3784..aa27574 100644 --- a/.dockerignore +++ b/.dockerignore @@ -120,3 +120,5 @@ Temporary Items .apdisk bundle.Dockerfile + +kind-kubeconfig* \ No newline at end of file diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..c5d7bcd --- /dev/null +++ b/.editorconfig @@ -0,0 +1,35 @@ +root = true + +[*] +charset = utf-8 +max_line_length = 120 + +[*.proto] +indent_size = 2 +tab_width = 2 + +[{*.bash,*.sh,*.zsh}] +indent_size = 2 +tab_width = 2 + +[{*.go,*.go2}] +indent_style = tab + +[{*.har,*.jsb2,*.jsb3,*.json,.babelrc,.eslintrc,.prettierrc,.stylelintrc,bowerrc,jest.config}] +indent_size = 2 + +[{*.hcl,*.nomad}] +indent_size = 2 + +[{*.http,*.rest}] +indent_size = 0 + +[{*.pb,*.textproto}] +indent_size = 2 +tab_width = 2 + +[{*.tf,*.tfvars}] +indent_size = 2 + +[{*.yaml,*.yml}] +indent_size = 2 \ No newline at end of file diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..bd45a13 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,36 @@ +# e2e test + +name: e2e-test + +on: ['push', 'pull_request'] + +jobs: + chainsaw-test: + name: Chainsaw Test + runs-on: ubuntu-22.04 + strategy: + matrix: + k8s-version: ['1.26.14', '1.27.11'] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.21' + cache: false + + - name: Create KinD cluster + env: + KUBECONFIG: kind-kubeconfig-${{ matrix.k8s-version }} + KIND_KUBECONFIG: kind-kubeconfig-${{ matrix.k8s-version }} + ENVTEST_K8S_VERSION: ${{ matrix.k8s-version}} + run: make kind-create + - name: Chainsaw test setup + env: + KUBECONFIG: kind-kubeconfig-${{ matrix.k8s-version }} + KIND_KUBECONFIG: kind-kubeconfig-${{ matrix.k8s-version }} + run: make chainsaw-setup + - name: Test with Chainsaw + env: + KUBECONFIG: kind-kubeconfig-${{ matrix.k8s-version }} + KIND_KUBECONFIG: kind-kubeconfig-${{ matrix.k8s-version }} + run: make chainsaw-test diff --git a/.gitignore b/.gitignore index 738e841..0b5d030 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,5 @@ Temporary Items # olm bundle bundle.Dockerfile + +kind-kubeconfig* \ No newline at end of file diff --git a/.markdownlint.yml b/.markdownlint.yml index 45d588e..fcd1dad 100644 --- a/.markdownlint.yml +++ b/.markdownlint.yml @@ -9,7 +9,6 @@ MD013: code_blocks: true tables: true headings: true - headers: true strict: false stern: false @@ -17,4 +16,4 @@ MD012: maximum: 20 MD046: false -MD051: false \ No newline at end of file +MD051: false diff --git a/Dockerfile b/Dockerfile index b948430..81f428b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN go mod download # Copy the go source COPY cmd/main.go cmd/main.go COPY api/ api/ -COPY internal/controller/ internal/controller/ +COPY internal/ internal/ # Build # the GOARCH has not a default value to allow the binary be built according to the host where the command diff --git a/Makefile b/Makefile index ae35d1e..c7bed52 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.0.1-alpha +VERSION ?= 0.0.1 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") @@ -25,13 +25,14 @@ endif BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) RETISTRY ?= quay.io/zncdata +PROJECT_NAME = trino-operator # IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. # This variable is used to construct full image tags for bundle and catalog images. # # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both # zncdata.dev/trino-operator-bundle:$VERSION and zncdata.dev/trino-operator-catalog:$VERSION. -IMAGE_TAG_BASE ?= $(RETISTRY)/trino-operator +IMAGE_TAG_BASE ?= $(RETISTRY)/$(PROJECT_NAME) # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) @@ -53,9 +54,10 @@ endif OPERATOR_SDK_VERSION ?= v1.33.0 # Image URL to use all building/pushing image targets -IMG ?= $(RETISTRY)/trino-operator:$(VERSION) +IMG ?= $(RETISTRY)/trino-operator:v$(VERSION) # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.26.0 +# ref: https://github.com/kubernetes-sigs/kubebuilder/releases in v3.11.0-v3.14.1 ENVTEST_K8S_VERSION support 1.26.1 and 1.27.1 +ENVTEST_K8S_VERSION ?= 1.26.1 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -290,4 +292,85 @@ catalog-build: opm ## Build a catalog image. # Push the catalog image. .PHONY: catalog-push catalog-push: ## Push a catalog image. - $(MAKE) docker-push IMG=$(CATALOG_IMG) \ No newline at end of file + $(MAKE) docker-push IMG=$(CATALOG_IMG) + + +##@ E2E + +# kind +KIND_VERSION ?= v0.22.0 + +KINDTEST_K8S_VERSION ?= 1.26.14 + +KIND_IMAGE ?= kindest/node:v${KINDTEST_K8S_VERSION} + +KIND_KUBECONFIG ?= ./kind-kubeconfig-$(KINDTEST_K8S_VERSION) +KIND_CLUSTER_NAME ?= ${PROJECT_NAME}-$(KINDTEST_K8S_VERSION) + +.PHONY: kind +KIND = $(LOCALBIN)/kind +kind: ## Download kind locally if necessary. +ifeq (,$(shell which $(KIND))) +ifeq (,$(shell which kind 2>/dev/null)) + @{ \ + set -e ;\ + go install sigs.k8s.io/kind@$(KIND_VERSION) ;\ + } +KIND = $(GOBIN)/bin/kind +else +KIND = $(shell which kind) +endif +endif + +OLM_VERSION ?= v0.27.0 + +# Create a kind cluster, install ingress-nginx, and wait for it to be available. +.PHONY: kind-create +kind-create: kind ## Create a kind cluster. + $(KIND) create cluster --config test/e2e/kind-config.yaml --image $(KIND_IMAGE) --name $(KIND_CLUSTER_NAME) --kubeconfig $(KIND_KUBECONFIG) --wait 120s + make kind-setup KUBECONFIG=$(KIND_KUBECONFIG) + +.PHONY: kind-setup +kind-setup: kind ## setup kind cluster base environment + @echo "\nSetup kind cluster base environment, install ingress-nginx and OLM" + kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml + kubectl -n ingress-nginx wait deployment ingress-nginx-controller --for=condition=available --timeout=300s + # curl -sSL https://github.com/operator-framework/operator-lifecycle-manager/releases/download/$(OLM_VERSION)/install.sh | bash -s $(OLM_VERSION) + +.PHONY: kind-delete +kind-delete: kind ## Delete a kind cluster. + $(KIND) delete cluster --name $(KIND_CLUSTER_NAME) + +# chainsaw + +CHAINSAW_VERSION ?= v0.1.8 + +.PHONY: chainsaw +CHAINSAW = $(LOCALBIN)/chainsaw +chainsaw: ## Download chainsaw locally if necessary. +ifeq (,$(shell which $(CHAINSAW))) +ifeq (,$(shell which chainsaw 2>/dev/null)) + @{ \ + set -e ;\ + go install github.com/kyverno/chainsaw@$(CHAINSAW_VERSION) ;\ + } +CHAINSAW = $(GOBIN)/chainsaw +else +CHAINSAW = $(shell which chainsaw) +endif +endif + +.PHONY: chainsaw-setup +chainsaw-setup: manifests kustomize ## Run the chainsaw setup + @echo "\nSetup chainsaw test environment" + make docker-build + $(KIND) --name $(KIND_CLUSTER_NAME) load docker-image $(IMG) + KUBECONFIG=$(KIND_KUBECONFIG) make deploy + +.PHONY: chainsaw-test +chainsaw-test: chainsaw ## Run the chainsaw test + $(CHAINSAW) test --cluster cluster-1=$(KIND_KUBECONFIG) --test-dir ./test/e2e + +.PHONY: chainsaw-cleanup +chainsaw-cleanup: manifests kustomize ## Run the chainsaw cleanup + KUBECONFIG=$(KIND_KUBECONFIG) make undeploy diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 2dc71ce..65134d2 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: quay.io/zncdata/trino-operator - newTag: 0.0.1-alpha + newTag: v0.0.1 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 8493739..ee444c7 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -77,13 +77,13 @@ spec: args: - --leader-elect image: controller:latest - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: manager securityContext: allowPrivilegeEscalation: false capabilities: drop: - - "ALL" + - "ALL" livenessProbe: httpGet: path: /healthz diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index fe6d635..ee9b63d 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -17,7 +17,11 @@ limitations under the License. package controller import ( + "context" + "fmt" + "os" "path/filepath" + "runtime" "testing" . "github.com/onsi/ginkgo/v2" @@ -25,6 +29,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -37,11 +42,23 @@ import ( // These tests use Ginkgo (BDD-style Go testing framework). Refer to // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. -var cfg *rest.Config -var k8sClient client.Client -var testEnv *envtest.Environment +var ( + cfg *rest.Config + k8sClient client.Client + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc + testK8sVersion string +) func TestAPIs(t *testing.T) { + testK8sVersion = os.Getenv("ENVTEST_K8S_VERSION") + + if testK8sVersion == "" { + logf.Log.Info("ENVTEST_K8S_VERSION not set, using default version 1.26.1") + testK8sVersion = "1.26.1" + } + RegisterFailHandler(Fail) RunSpecs(t, "Controller Suite") @@ -49,8 +66,14 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - + ctx, cancel = context.WithCancel(context.TODO()) By("bootstrapping test environment") + + Expect(os.Setenv( + "KUBEBUILDER_ASSETS", + fmt.Sprintf("../../bin/k8s/%s-%s-%s", testK8sVersion, runtime.GOOS, runtime.GOARCH), + )).To(Succeed()) + testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, @@ -71,10 +94,27 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + }) + Expect(err).ToNot(HaveOccurred()) + + err = (&TrinoReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + }).SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + + //go func() { + // defer GinkgoRecover() + // err = k8sManager.Start(ctx) + // Expect(err).ToNot(HaveOccurred(), "failed to run manager") + //}() }) var _ = AfterSuite(func() { + cancel() By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) + Expect(testEnv.Stop()).To(Succeed()) + Expect(os.Unsetenv("KUBEBUILDER_ASSETS")).To(Succeed()) }) diff --git a/test/e2e/kind-config.yaml b/test/e2e/kind-config.yaml new file mode 100644 index 0000000..2bd73fd --- /dev/null +++ b/test/e2e/kind-config.yaml @@ -0,0 +1,18 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + image: kindest/node:v1.26.14@sha256:5d548739ddef37b9318c70cb977f57bf3e5015e4552be4e27e57280a8cbb8e4f + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + hostPort: 18080 + protocol: TCP + - containerPort: 443 + hostPort: 18443 + protocol: TCP diff --git a/test/e2e/sample/00-assert.yaml b/test/e2e/sample/00-assert.yaml new file mode 100644 index 0000000..4f8eac9 --- /dev/null +++ b/test/e2e/sample/00-assert.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trinocluster-sample-default-coordinator +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trinocluster-sample-default-worker +status: + availableReplicas: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: trinocluster-sample-default-coordinator +spec: + type: ClusterIP +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: trinocluster-sample-default +spec: + rules: + - host: trino.example.com + http: + paths: + - backend: + service: + name: trinocluster-sample-default-coordinator + port: + number: 18080 + path: / + pathType: Prefix +status: + loadBalancer: + ingress: + - hostname: localhost diff --git a/test/e2e/sample/00-trino.yaml b/test/e2e/sample/00-trino.yaml new file mode 100644 index 0000000..e2683f9 --- /dev/null +++ b/test/e2e/sample/00-trino.yaml @@ -0,0 +1,77 @@ +apiVersion: trino.zncdata.dev/v1alpha1 +kind: TrinoCluster +metadata: + labels: + app.kubernetes.io/name: trinocluster + app.kubernetes.io/instance: trinocluster-sample + app.kubernetes.io/part-of: trino-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: trino-operator + name: trinocluster-sample +spec: + image: + repository: trinodb/trino + tag: "423" + pullPolicy: IfNotPresent + clusterConfig: + service: + port: 18080 + type: ClusterIP + ingress: + enabled: false + annotations: + kubernetes.io/ingress.class: traefik + nginx.ingress.kubernetes.io/rewrite-target: / + host: trino.example.com + clusterMode: true + coordinator: + config: + nodeProperties: + environment: "production" + dataDir: /data/trino + pluginDir: /usr/lib/trino/plugin + configProperties: + queryMaxMemory: "4GB" + path: /etc/trino + https: + enabled: false + exchangeManager: + name: "filesystem" + jvmProperties: + maxHeapSize: "4G" + gcMethodType: "UseG1GC" + gcHeapRegionSize: "32M" + roleGroups: + default: + replicas: 1 + config: + resources: + cpu: + min: "1" + max: "1.5" + memory: + limit: 1Gi + nodeSelector: + app: default-coordinator + worker: + config: + nodeProperties: + environment: "production" + dataDir: /data/trino + pluginDir: /usr/lib/trino/plugin + jvmProperties: + maxHeapSize: "8G" + gcMethodType: "UseG1GC" + gcHeapRegionSize: "32M" + configProperties: + memoryHeapHeadroomPerNode: "" + queryMaxMemoryPerNode: "1GB" + roleGroups: + default: + replicas: 1 + config: + nodeSelector: + app: default-worker + securityContext: + runAsUser: 1000 + fsGroup: 1000