From c936ecd32b4a80dffba91ab3b883e7b7495632d0 Mon Sep 17 00:00:00 2001 From: Yu-Lang Chu Date: Wed, 12 Apr 2023 23:55:30 +0200 Subject: [PATCH 1/9] refactor controller-setup.sh --- scripts/controller-setup.sh | 43 ++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/scripts/controller-setup.sh b/scripts/controller-setup.sh index 74d50673..c2f7b23c 100755 --- a/scripts/controller-setup.sh +++ b/scripts/controller-setup.sh @@ -38,27 +38,11 @@ build_and_install_controller() { _install_deployment "$__controller_namespace" "$__img_name" } -_build_controller_image() { - local __img_name=$1 - - local local_build="$(get_is_local_build)" - LOCAL_MODULES="$local_build" AWS_SERVICE_DOCKER_IMG="$__img_name" ${CODE_GENERATOR_SCRIPTS_DIR}/build-controller-image.sh ${AWS_SERVICE} 1>/dev/null -} - -_load_controller_image() { - local __cluster_name=$1 - local __img_name=$2 - - kind load docker-image --name "$__cluster_name" --nodes="$__cluster_name"-worker,"$__cluster_name"-control-plane "$__img_name" -} - -_install_deployment() { +install_crd_and_rbac(){ local __controller_namespace=$1 - local __img_name=$2 local service_controller_source_dir="$ROOT_DIR/../$AWS_SERVICE-controller" local service_config_dir="$service_controller_source_dir/config" - local test_config_dir="$ROOT_DIR/build/clusters/$cluster_name/config/test" # Register the ACK service controller's CRDs in the target k8s cluster debug_msg "Loading CRD manifests for $AWS_SERVICE into the cluster ... " @@ -76,6 +60,31 @@ _install_deployment() { debug_msg "Loading RBAC manifests for $AWS_SERVICE into the cluster ... " kustomize build "$service_config_dir"/rbac | kubectl apply -f - 1>/dev/null +} + +_build_controller_image() { + local __img_name=$1 + + local local_build="$(get_is_local_build)" + LOCAL_MODULES="$local_build" AWS_SERVICE_DOCKER_IMG="$__img_name" ${CODE_GENERATOR_SCRIPTS_DIR}/build-controller-image.sh ${AWS_SERVICE} 1>/dev/null +} + +_load_controller_image() { + local __cluster_name=$1 + local __img_name=$2 + + kind load docker-image --name "$__cluster_name" --nodes="$__cluster_name"-worker,"$__cluster_name"-control-plane "$__img_name" +} + +_install_deployment() { + local __controller_namespace=$1 + local __img_name=$2 + + install_crd_and_rbac "$__controller_namespace" + + local service_controller_source_dir="$ROOT_DIR/../$AWS_SERVICE-controller" + local service_config_dir="$service_controller_source_dir/config" + local test_config_dir="$ROOT_DIR/build/clusters/$cluster_name/config/test" # Create the ACK service controller Deployment in the target k8s cluster mkdir -p "$test_config_dir" From 98293727cde6baf5be471cb596b0a01fda6e9db6 Mon Sep 17 00:00:00 2001 From: Yu-Lang Chu Date: Wed, 12 Apr 2023 23:55:43 +0200 Subject: [PATCH 2/9] add kind-dev cmd --- Makefile | 3 +++ scripts/run-ack-dev.sh | 60 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100755 scripts/run-ack-dev.sh diff --git a/Makefile b/Makefile index 0e6cd2fa..500e38cf 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,9 @@ kind-test: ## Run functional tests for SERVICE kind-helm-test: ## Run the Helm tests for SERVICE @AWS_SERVICE=$(AWS_SERVICE) ./scripts/run-helm-tests.sh +kind-dev: ## Run a dev cluster for SERVICE + @AWS_SERVICE=$(AWS_SERVICE) ./scripts/run-ack-dev.sh + test-recommended-policy: @AWS_SERVICE=$(AWS_SERVICE) source ./scripts/iam-policy-test-runner.sh && assert_iam_policies diff --git a/scripts/run-ack-dev.sh b/scripts/run-ack-dev.sh new file mode 100755 index 00000000..f5e6ab93 --- /dev/null +++ b/scripts/run-ack-dev.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# ./scripts/run-e2e-tests.sh is the entrypoint for ACK integration testing. It +# ensures that a K8s cluster is acsessible, then configures the ACK controller +# under test onto the cluster before finally running the Python tests. + +set -Eeo pipefail + +SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR="$SCRIPTS_DIR/.." + +AWS_SERVICE=$(echo "${AWS_SERVICE:-""}" | tr '[:upper:]' '[:lower:]') + +CONTROLLER_NAMESPACE=${CONTROLLER_NAMESPACE:-"ack-system"} + +source "$SCRIPTS_DIR/lib/aws.sh" +source "$SCRIPTS_DIR/lib/common.sh" +source "$SCRIPTS_DIR/lib/config.sh" +source "$SCRIPTS_DIR/lib/logging.sh" + +source "$SCRIPTS_DIR/controller-setup.sh" +source "$SCRIPTS_DIR/kind.sh" + +ensure_cluster() { + local cluster_create="$(get_cluster_create)" + if [[ "$cluster_create" == true ]]; then + local cluster_name="ack-dev-cluster-$AWS_SERVICE" + + info_msg "Creating KIND cluster ..." + setup_kind_cluster "$cluster_name" "$CONTROLLER_NAMESPACE" + + info_msg "Installing CRDs and RBAC " + install_crd_and_rbac "$CONTROLLER_NAMESPACE" + else + info_msg "Testing connection to existing cluster ..." + _ensure_existing_context + fi +} + +run() { + ensure_aws_credentials + ensure_cluster + exit $? +} + +ensure_inputs() { + [[ -z "$AWS_SERVICE" ]] && { error_msg "Expected \`AWS_SERVICE\` to be defined"; exit 1; } || : +} + +ensure_binaries() { + check_is_installed "kubectl" +} + +ensure_inputs +ensure_binaries + +# The purpose of the `return` subshell command in this script is to determine +# whether the script was sourced, or whether it is being executed directly. +# https://stackoverflow.com/a/28776166 +(return 0 2>/dev/null) || run From c55529eb42338ea5db41212f34bd3a2d83ccea3d Mon Sep 17 00:00:00 2001 From: Yu-Lang Chu Date: Thu, 13 Apr 2023 17:10:02 +0200 Subject: [PATCH 3/9] add info to run controller --- scripts/run-ack-dev.sh | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/scripts/run-ack-dev.sh b/scripts/run-ack-dev.sh index f5e6ab93..f8ab50ec 100755 --- a/scripts/run-ack-dev.sh +++ b/scripts/run-ack-dev.sh @@ -13,6 +13,8 @@ AWS_SERVICE=$(echo "${AWS_SERVICE:-""}" | tr '[:upper:]' '[:lower:]') CONTROLLER_NAMESPACE=${CONTROLLER_NAMESPACE:-"ack-system"} +CLUSTER_NAME="ack-dev-cluster-$AWS_SERVICE" + source "$SCRIPTS_DIR/lib/aws.sh" source "$SCRIPTS_DIR/lib/common.sh" source "$SCRIPTS_DIR/lib/config.sh" @@ -22,24 +24,25 @@ source "$SCRIPTS_DIR/controller-setup.sh" source "$SCRIPTS_DIR/kind.sh" ensure_cluster() { - local cluster_create="$(get_cluster_create)" - if [[ "$cluster_create" == true ]]; then - local cluster_name="ack-dev-cluster-$AWS_SERVICE" - - info_msg "Creating KIND cluster ..." - setup_kind_cluster "$cluster_name" "$CONTROLLER_NAMESPACE" - - info_msg "Installing CRDs and RBAC " - install_crd_and_rbac "$CONTROLLER_NAMESPACE" - else - info_msg "Testing connection to existing cluster ..." - _ensure_existing_context - fi + info_msg "Creating KIND cluster ..." + setup_kind_cluster "$CLUSTER_NAME" "$CONTROLLER_NAMESPACE" + + info_msg "Installing CRDs , common and RBAC manifest..." + install_crd_and_rbac "$CONTROLLER_NAMESPACE" } run() { ensure_aws_credentials ensure_cluster + local kubeconfig_path="$ROOT_DIR/build/clusters/$CLUSTER_NAME/kubeconfig" + info_msg "Before running the controller, you need kubeconfig and aws credentials." + info_msg "After executing the above source command to set the kubeconfig to environment, " + info_msg "run the following command to start the controller from the controller repo:" + echo "" + echo "go run ./cmd/controller/main.go --aws-region eu-central-1 --log-level debug --enable-development-logging" + echo "" + info_msg "if you run the controller from your code editor/IDE, you can set the following environment variables:" + echo "KUBECONFIG=$kubeconfig_path" exit $? } From 7140ad391f90959c1b22e91c305ad9e08e64177a Mon Sep 17 00:00:00 2001 From: Yu-Lang Chu Date: Thu, 13 Apr 2023 17:19:36 +0200 Subject: [PATCH 4/9] update comment in run-dev.sh --- Makefile | 2 +- scripts/{run-ack-dev.sh => run-dev.sh} | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename scripts/{run-ack-dev.sh => run-dev.sh} (89%) diff --git a/Makefile b/Makefile index 500e38cf..92950990 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ kind-helm-test: ## Run the Helm tests for SERVICE @AWS_SERVICE=$(AWS_SERVICE) ./scripts/run-helm-tests.sh kind-dev: ## Run a dev cluster for SERVICE - @AWS_SERVICE=$(AWS_SERVICE) ./scripts/run-ack-dev.sh + @AWS_SERVICE=$(AWS_SERVICE) ./scripts/run-dev.sh test-recommended-policy: @AWS_SERVICE=$(AWS_SERVICE) source ./scripts/iam-policy-test-runner.sh && assert_iam_policies diff --git a/scripts/run-ack-dev.sh b/scripts/run-dev.sh similarity index 89% rename from scripts/run-ack-dev.sh rename to scripts/run-dev.sh index f8ab50ec..f77c2cad 100755 --- a/scripts/run-ack-dev.sh +++ b/scripts/run-dev.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# ./scripts/run-e2e-tests.sh is the entrypoint for ACK integration testing. It -# ensures that a K8s cluster is acsessible, then configures the ACK controller -# under test onto the cluster before finally running the Python tests. +# ./scripts/run-dev.sh quickly setup cluster for ACK development. It +# ensures that a K8s cluster is accessible, then configures the ACK CRD, RBAC +# but not install the controller set -Eeo pipefail From 177a044a9a14e15081b3b637eada300916cced39 Mon Sep 17 00:00:00 2001 From: Yu-Lang Chu Date: Fri, 14 Apr 2023 22:19:13 +0200 Subject: [PATCH 5/9] rename to create-dev-cluster.sh --- Makefile | 2 +- scripts/{run-dev.sh => create-dev-cluster.sh} | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) rename scripts/{run-dev.sh => create-dev-cluster.sh} (87%) diff --git a/Makefile b/Makefile index 92950990..01e8de41 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ kind-helm-test: ## Run the Helm tests for SERVICE @AWS_SERVICE=$(AWS_SERVICE) ./scripts/run-helm-tests.sh kind-dev: ## Run a dev cluster for SERVICE - @AWS_SERVICE=$(AWS_SERVICE) ./scripts/run-dev.sh + @AWS_SERVICE=$(AWS_SERVICE) ./scripts/create-dev-cluster.sh test-recommended-policy: @AWS_SERVICE=$(AWS_SERVICE) source ./scripts/iam-policy-test-runner.sh && assert_iam_policies diff --git a/scripts/run-dev.sh b/scripts/create-dev-cluster.sh similarity index 87% rename from scripts/run-dev.sh rename to scripts/create-dev-cluster.sh index f77c2cad..7b102030 100755 --- a/scripts/run-dev.sh +++ b/scripts/create-dev-cluster.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# ./scripts/run-dev.sh quickly setup cluster for ACK development. It +# ./scripts/create-dev-cluster.sh quickly setup cluster for ACK development. It # ensures that a K8s cluster is accessible, then configures the ACK CRD, RBAC # but not install the controller @@ -35,11 +35,10 @@ run() { ensure_aws_credentials ensure_cluster local kubeconfig_path="$ROOT_DIR/build/clusters/$CLUSTER_NAME/kubeconfig" - info_msg "Before running the controller, you need kubeconfig and aws credentials." info_msg "After executing the above source command to set the kubeconfig to environment, " info_msg "run the following command to start the controller from the controller repo:" echo "" - echo "go run ./cmd/controller/main.go --aws-region eu-central-1 --log-level debug --enable-development-logging" + echo "go run ./cmd/controller/main.go --aws-region $(get_aws_region) --log-level debug --enable-development-logging" echo "" info_msg "if you run the controller from your code editor/IDE, you can set the following environment variables:" echo "KUBECONFIG=$kubeconfig_path" From 7779b282dfe056b97e0c4549dddf60a6166a1958 Mon Sep 17 00:00:00 2001 From: Yu-Lang Chu Date: Fri, 14 Apr 2023 23:06:40 +0200 Subject: [PATCH 6/9] refactor ensure_cluster --- scripts/controller-setup.sh | 11 +++++------ scripts/create-dev-cluster.sh | 17 +++++++++-------- scripts/run-e2e-tests.sh | 27 ++++++++++++++++++++------- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/scripts/controller-setup.sh b/scripts/controller-setup.sh index c2f7b23c..e6be9b86 100755 --- a/scripts/controller-setup.sh +++ b/scripts/controller-setup.sh @@ -35,7 +35,7 @@ build_and_install_controller() { _load_controller_image "$__cluster_name" "$__img_name" info_msg "Installing controller deployment ... " - _install_deployment "$__controller_namespace" "$__img_name" + _install_deployment "$__cluster_name" "$__controller_namespace" "$__img_name" } install_crd_and_rbac(){ @@ -77,14 +77,13 @@ _load_controller_image() { } _install_deployment() { - local __controller_namespace=$1 - local __img_name=$2 - - install_crd_and_rbac "$__controller_namespace" + local __cluster_name=$1 + local __controller_namespace=$2 + local __img_name=$3 local service_controller_source_dir="$ROOT_DIR/../$AWS_SERVICE-controller" local service_config_dir="$service_controller_source_dir/config" - local test_config_dir="$ROOT_DIR/build/clusters/$cluster_name/config/test" + local test_config_dir="$ROOT_DIR/build/clusters/$__cluster_name/config/test" # Create the ACK service controller Deployment in the target k8s cluster mkdir -p "$test_config_dir" diff --git a/scripts/create-dev-cluster.sh b/scripts/create-dev-cluster.sh index 7b102030..993b96df 100755 --- a/scripts/create-dev-cluster.sh +++ b/scripts/create-dev-cluster.sh @@ -22,18 +22,19 @@ source "$SCRIPTS_DIR/lib/logging.sh" source "$SCRIPTS_DIR/controller-setup.sh" source "$SCRIPTS_DIR/kind.sh" +source "$SCRIPTS_DIR/run-e2e-tests.sh" -ensure_cluster() { - info_msg "Creating KIND cluster ..." - setup_kind_cluster "$CLUSTER_NAME" "$CONTROLLER_NAMESPACE" - - info_msg "Installing CRDs , common and RBAC manifest..." - install_crd_and_rbac "$CONTROLLER_NAMESPACE" -} +#ensure_cluster() { +# info_msg "Creating KIND cluster ..." +# setup_kind_cluster "$CLUSTER_NAME" "$CONTROLLER_NAMESPACE" +# +# info_msg "Installing CRDs , common and RBAC manifest..." +# install_crd_and_rbac "$CONTROLLER_NAMESPACE" +#} run() { ensure_aws_credentials - ensure_cluster + ensure_cluster "$CLUSTER_NAME" false local kubeconfig_path="$ROOT_DIR/build/clusters/$CLUSTER_NAME/kubeconfig" info_msg "After executing the above source command to set the kubeconfig to environment, " info_msg "run the following command to start the controller from the controller repo:" diff --git a/scripts/run-e2e-tests.sh b/scripts/run-e2e-tests.sh index 0f5ad26c..fd277f0d 100755 --- a/scripts/run-e2e-tests.sh +++ b/scripts/run-e2e-tests.sh @@ -30,16 +30,29 @@ source "$SCRIPTS_DIR/pytest-image-runner.sh" ensure_cluster() { local cluster_create="$(get_cluster_create)" - if [[ "$cluster_create" == true ]]; then - local cluster_name=$(_get_kind_cluster_name) + + if [[ "$cluster_create" != true ]]; then + info_msg "Testing connection to existing cluster ..." + _ensure_existing_context + else + local cluster_name=$(get_cluster_name) + local controller_install=${2:-true} + # naming cluster in the following order: + # 1. cluster_name is set in config file + # 2. cluster_name is passed as function argument. using this for dev cluster + # 3. generate cluster_name with random prefix + [ -z "$cluster_name" ] && cluster_name=$1 + [ -z "$cluster_name" ] && cluster_name=$(_get_kind_cluster_name) info_msg "Creating KIND cluster ..." setup_kind_cluster "$cluster_name" "$CONTROLLER_NAMESPACE" - build_and_install_controller "$cluster_name" "$CONTROLLER_NAMESPACE" "$CONTROLLER_IMAGE_TAG" - else - info_msg "Testing connection to existing cluster ..." - _ensure_existing_context + info_msg "Installing CRDs , common and RBAC manifest..." + install_crd_and_rbac "$CONTROLLER_NAMESPACE" + + if [[ "$controller_install" == true ]]; then + build_and_install_controller "$cluster_name" "$CONTROLLER_NAMESPACE" "$CONTROLLER_IMAGE_TAG" + fi fi } @@ -92,7 +105,7 @@ _ensure_existing_context() { run() { ensure_aws_credentials - ensure_cluster + ensure_cluster "" build_and_run_tests exit $? From 6e7926d5a63c26bbd5d4ea8dab78264cef669179 Mon Sep 17 00:00:00 2001 From: Yu-Lang Chu Date: Sat, 15 Apr 2023 22:15:25 +0200 Subject: [PATCH 7/9] clean up --- scripts/create-dev-cluster.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/scripts/create-dev-cluster.sh b/scripts/create-dev-cluster.sh index 993b96df..13d92767 100755 --- a/scripts/create-dev-cluster.sh +++ b/scripts/create-dev-cluster.sh @@ -24,14 +24,6 @@ source "$SCRIPTS_DIR/controller-setup.sh" source "$SCRIPTS_DIR/kind.sh" source "$SCRIPTS_DIR/run-e2e-tests.sh" -#ensure_cluster() { -# info_msg "Creating KIND cluster ..." -# setup_kind_cluster "$CLUSTER_NAME" "$CONTROLLER_NAMESPACE" -# -# info_msg "Installing CRDs , common and RBAC manifest..." -# install_crd_and_rbac "$CONTROLLER_NAMESPACE" -#} - run() { ensure_aws_credentials ensure_cluster "$CLUSTER_NAME" false @@ -41,8 +33,6 @@ run() { echo "" echo "go run ./cmd/controller/main.go --aws-region $(get_aws_region) --log-level debug --enable-development-logging" echo "" - info_msg "if you run the controller from your code editor/IDE, you can set the following environment variables:" - echo "KUBECONFIG=$kubeconfig_path" exit $? } From 4c321c266b567a611a20ab7fa07da60b49e85785 Mon Sep 17 00:00:00 2001 From: Yu-Lang Chu Date: Sat, 22 Apr 2023 22:41:27 +0200 Subject: [PATCH 8/9] move args into top of function --- scripts/run-e2e-tests.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/run-e2e-tests.sh b/scripts/run-e2e-tests.sh index fd277f0d..8d489e5a 100755 --- a/scripts/run-e2e-tests.sh +++ b/scripts/run-e2e-tests.sh @@ -29,19 +29,20 @@ source "$SCRIPTS_DIR/kind.sh" source "$SCRIPTS_DIR/pytest-image-runner.sh" ensure_cluster() { + local cluster_name_via_arg=${1:-""} + local controller_install=${2:-true} local cluster_create="$(get_cluster_create)" + local cluster_name=$(get_cluster_name) if [[ "$cluster_create" != true ]]; then info_msg "Testing connection to existing cluster ..." _ensure_existing_context else - local cluster_name=$(get_cluster_name) - local controller_install=${2:-true} # naming cluster in the following order: # 1. cluster_name is set in config file # 2. cluster_name is passed as function argument. using this for dev cluster # 3. generate cluster_name with random prefix - [ -z "$cluster_name" ] && cluster_name=$1 + [ -z "$cluster_name" ] && cluster_name=$cluster_name_via_arg [ -z "$cluster_name" ] && cluster_name=$(_get_kind_cluster_name) info_msg "Creating KIND cluster ..." @@ -105,7 +106,7 @@ _ensure_existing_context() { run() { ensure_aws_credentials - ensure_cluster "" + ensure_cluster build_and_run_tests exit $? From 4ed9f07797baa1045da81faae0bd02f10f29738f Mon Sep 17 00:00:00 2001 From: Yu-Lang Chu Date: Sat, 22 Apr 2023 22:48:24 +0200 Subject: [PATCH 9/9] fix message --- scripts/run-e2e-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-e2e-tests.sh b/scripts/run-e2e-tests.sh index 8d489e5a..97acd40a 100755 --- a/scripts/run-e2e-tests.sh +++ b/scripts/run-e2e-tests.sh @@ -48,7 +48,7 @@ ensure_cluster() { info_msg "Creating KIND cluster ..." setup_kind_cluster "$cluster_name" "$CONTROLLER_NAMESPACE" - info_msg "Installing CRDs , common and RBAC manifest..." + info_msg "Installing CRD and RBAC manifests..." install_crd_and_rbac "$CONTROLLER_NAMESPACE" if [[ "$controller_install" == true ]]; then