install karmada with latest tag images in ci chart-testing

Signed-off-by: chaosi-zju <chaosi@zju.edu.cn>
karmada-io · Dec 5, 2024 · 1d7f59b · 1d7f59b
1 parent 616a95e
commit 1d7f59b
Show file tree

Hide file tree

Showing 2 changed files with 175 additions and 18 deletions.
diff --git a/.github/workflows/lint-chart.yaml → ...orkflows/chart-lint-and-installation.yaml b/.github/workflows/lint-chart.yaml → ...orkflows/chart-lint-and-installation.yaml
@@ -1,12 +1,9 @@
 # validate any chart changes under charts directory
-name: Chart Lint
+name: Chart Lint and Installation
 
 env:
   HELM_VERSION: v3.11.2
   KUSTOMIZE_VERSION: 5.4.3
-  KIND_VERSION: v0.22.0
-  KIND_NODE_IMAGE: kindest/node:v1.29.0
-  K8S_VERSION: v1.29.0
 
 on:
   push:
@@ -15,14 +12,12 @@ on:
     branches-ignore:
       - 'dependabot/**'
   pull_request:
-    paths:
-      - "charts/**"
 
 permissions:
   contents: read
 
 jobs:
-  chart-lint-test:
+  chart-lint-and-installation-test:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
@@ -99,15 +94,9 @@ jobs:
         if: steps.list-changed.outputs.changed == 'true'
         run: ct lint --debug --check-version-increment=false
 
-      - name: Create kind cluster
-        uses: helm/kind-action@v1.10.0
-        if: steps.list-changed.outputs.changed == 'true'
-        with:
-          wait: 120s
-          version: ${{ env.KIND_VERSION }}
-          node_image: ${{ env.KIND_NODE_IMAGE }}
-          kubectl_version: ${{ env.K8S_VERSION }}
-
       - name: Run chart-testing (install)
-        if: steps.list-changed.outputs.changed == 'true'
-        run: ct install --debug --helm-extra-args "--timeout 800s"
+        run: |
+          hack/setup-dev-base.sh
+          export KUBECONFIG=~/.kube/karmada.config
+          ct install --charts charts/karmada --debug --helm-extra-set-args '--set components={search,metricsAdapter,descheduler},apiServer.hostNetwork=true' --helm-extra-args "--timeout 800s" --skip-clean-up
+          kubectl get pods -A
diff --git a/hack/setup-dev-base.sh b/hack/setup-dev-base.sh
@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+# Copyright 2024 The Karmada Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script works for both linux and macOS.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# This script sets up a base develop environment prepared for installing Karmada locally,
+# including a local Karmada host cluster pre-loaded with Karmada component images built from the latest code,
+# and a certain number of clusters to be joined/registered.
+# This script depends on utils in: ${REPO_ROOT}/hack/util.sh
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${REPO_ROOT}"/hack/util.sh
+
+# variable define
+KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}
+MAIN_KUBECONFIG=${MAIN_KUBECONFIG:-"${KUBECONFIG_PATH}/karmada.config"}
+HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"karmada-host"}
+MEMBER_CLUSTER_KUBECONFIG=${MEMBER_CLUSTER_KUBECONFIG:-"${KUBECONFIG_PATH}/members.config"}
+MEMBER_CLUSTER_1_NAME=${MEMBER_CLUSTER_1_NAME:-"member1"}
+MEMBER_CLUSTER_2_NAME=${MEMBER_CLUSTER_2_NAME:-"member2"}
+PULL_MODE_CLUSTER_NAME=${PULL_MODE_CLUSTER_NAME:-"member3"}
+MEMBER_TMP_CONFIG_PREFIX="member-tmp"
+MEMBER_CLUSTER_1_TMP_CONFIG="${KUBECONFIG_PATH}/${MEMBER_TMP_CONFIG_PREFIX}-${MEMBER_CLUSTER_1_NAME}.config"
+MEMBER_CLUSTER_2_TMP_CONFIG="${KUBECONFIG_PATH}/${MEMBER_TMP_CONFIG_PREFIX}-${MEMBER_CLUSTER_2_NAME}.config"
+PULL_MODE_CLUSTER_TMP_CONFIG="${KUBECONFIG_PATH}/${MEMBER_TMP_CONFIG_PREFIX}-${PULL_MODE_CLUSTER_NAME}.config"
+HOST_IPADDRESS=${HOST_IPADDRESS:-}
+BUILD_FROM_SOURCE=${BUILD_FROM_SOURCE:-"true"}
+EXTRA_IMAGES_LOAD_TO_HOST_CLUSTER=${EXTRA_IMAGES_LOAD_TO_HOST_CLUSTER:-""}
+
+CLUSTER_VERSION=${CLUSTER_VERSION:-"${DEFAULT_CLUSTER_VERSION}"}
+KIND_LOG_FILE=${KIND_LOG_FILE:-"/tmp/karmada"}
+
+#step0: prepare
+# proxy setting in China mainland
+if [[ -n ${CHINA_MAINLAND:-} ]]; then
+  util::set_mirror_registry_for_china_mainland ${REPO_ROOT}
+fi
+
+# make sure go exists and the go version is a viable version.
+util::cmd_must_exist "go"
+util::verify_go_version
+
+# make sure docker exists
+util::cmd_must_exist "docker"
+
+# install kind and kubectl
+kind_version=v0.25.0
+echo -n "Preparing: 'kind' existence check - "
+if util::cmd_exist kind; then
+  echo "passed"
+else
+  echo "not pass"
+  util::install_tools "sigs.k8s.io/kind" $kind_version
+fi
+
+# get arch name and os name in bootstrap
+BS_ARCH=$(go env GOARCH)
+BS_OS=$(go env GOOS)
+# check arch and os name before installing
+util::install_environment_check "${BS_ARCH}" "${BS_OS}"
+echo -n "Preparing: 'kubectl' existence check - "
+if util::cmd_exist kubectl; then
+  echo "passed"
+else
+  echo "not pass"
+  util::install_kubectl "" "${BS_ARCH}" "${BS_OS}"
+fi
+
+#step1. create host cluster and member clusters in parallel
+# host IP address: script parameter ahead of macOS IP
+if [[ -z "${HOST_IPADDRESS}" ]]; then
+  util::get_macos_ipaddress # Adapt for macOS
+  HOST_IPADDRESS=${MAC_NIC_IPADDRESS:-}
+fi
+#prepare for kindClusterConfig
+TEMP_PATH=$(mktemp -d)
+trap '{ rm -rf ${TEMP_PATH}; }' EXIT
+echo -e "Preparing kindClusterConfig in path: ${TEMP_PATH}"
+cp -rf "${REPO_ROOT}"/artifacts/kindClusterConfig/member1.yaml "${TEMP_PATH}"/member1.yaml
+cp -rf "${REPO_ROOT}"/artifacts/kindClusterConfig/member2.yaml "${TEMP_PATH}"/member2.yaml
+cp -rf "${REPO_ROOT}"/artifacts/kindClusterConfig/member3.yaml "${TEMP_PATH}"/member3.yaml
+
+util::delete_necessary_resources "${MAIN_KUBECONFIG},${MEMBER_CLUSTER_KUBECONFIG}" "${HOST_CLUSTER_NAME},${MEMBER_CLUSTER_1_NAME},${MEMBER_CLUSTER_2_NAME},${PULL_MODE_CLUSTER_NAME}" "${KIND_LOG_FILE}"
+
+if [[ -n "${HOST_IPADDRESS}" ]]; then # If bind the port of clusters(karmada-host, member1 and member2) to the host IP
+  util::verify_ip_address "${HOST_IPADDRESS}"
+  cp -rf "${REPO_ROOT}"/artifacts/kindClusterConfig/karmada-host.yaml "${TEMP_PATH}"/karmada-host.yaml
+  sed -i'' -e "s/{{host_ipaddress}}/${HOST_IPADDRESS}/g" "${TEMP_PATH}"/karmada-host.yaml
+  sed -i'' -e 's/networking:/&\'$'\n''  apiServerAddress: "'${HOST_IPADDRESS}'"/' "${TEMP_PATH}"/member1.yaml
+  sed -i'' -e 's/networking:/&\'$'\n''  apiServerAddress: "'${HOST_IPADDRESS}'"/' "${TEMP_PATH}"/member2.yaml
+  sed -i'' -e 's/networking:/&\'$'\n''  apiServerAddress: "'${HOST_IPADDRESS}'"/' "${TEMP_PATH}"/member3.yaml
+  util::create_cluster "${HOST_CLUSTER_NAME}" "${MAIN_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${TEMP_PATH}"/karmada-host.yaml
+else
+  util::create_cluster "${HOST_CLUSTER_NAME}" "${MAIN_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}"
+fi
+util::create_cluster "${MEMBER_CLUSTER_1_NAME}" "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${TEMP_PATH}"/member1.yaml
+util::create_cluster "${MEMBER_CLUSTER_2_NAME}" "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${TEMP_PATH}"/member2.yaml
+util::create_cluster "${PULL_MODE_CLUSTER_NAME}" "${PULL_MODE_CLUSTER_TMP_CONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${TEMP_PATH}"/member3.yaml
+
+#step2. make images and get karmadactl
+export VERSION="latest"
+export REGISTRY="docker.io/karmada"
+if [[ "${BUILD_FROM_SOURCE}" == "true" ]]; then
+  export KARMADA_IMAGE_LABEL_VALUE="May_be_pruned_in_local_up_environment"
+  export DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS:-} --label=image.karmada.io=${KARMADA_IMAGE_LABEL_VALUE}"
+  make images GOOS="linux" --directory="${REPO_ROOT}"
+  #clean up dangling images
+  docker image prune --force --filter "label=image.karmada.io=${KARMADA_IMAGE_LABEL_VALUE}"
+fi
+GO111MODULE=on go install "github.com/karmada-io/karmada/cmd/karmadactl"
+
+#step3. wait until clusters ready
+echo "Waiting for the clusters to be ready..."
+util::check_clusters_ready "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}"
+util::check_clusters_ready "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${MEMBER_CLUSTER_1_NAME}"
+util::check_clusters_ready "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${MEMBER_CLUSTER_2_NAME}"
+util::check_clusters_ready "${PULL_MODE_CLUSTER_TMP_CONFIG}" "${PULL_MODE_CLUSTER_NAME}"
+
+#step4. load components images to kind cluster
+if [[ "${BUILD_FROM_SOURCE}" == "true" ]]; then
+  # host cluster
+  kind load docker-image "${REGISTRY}/karmada-controller-manager:${VERSION}" --name="${HOST_CLUSTER_NAME}"
+  kind load docker-image "${REGISTRY}/karmada-scheduler:${VERSION}" --name="${HOST_CLUSTER_NAME}"
+  kind load docker-image "${REGISTRY}/karmada-descheduler:${VERSION}" --name="${HOST_CLUSTER_NAME}"
+  kind load docker-image "${REGISTRY}/karmada-webhook:${VERSION}" --name="${HOST_CLUSTER_NAME}"
+  kind load docker-image "${REGISTRY}/karmada-scheduler-estimator:${VERSION}" --name="${HOST_CLUSTER_NAME}"
+  kind load docker-image "${REGISTRY}/karmada-aggregated-apiserver:${VERSION}" --name="${HOST_CLUSTER_NAME}"
+  kind load docker-image "${REGISTRY}/karmada-search:${VERSION}" --name="${HOST_CLUSTER_NAME}"
+  kind load docker-image "${REGISTRY}/karmada-metrics-adapter:${VERSION}" --name="${HOST_CLUSTER_NAME}"
+  for img in ${EXTRA_IMAGES_LOAD_TO_HOST_CLUSTER//,/ }; do
+    kind load docker-image "$img" --name="${HOST_CLUSTER_NAME}"
+  done
+  # pull mode member cluster
+  kind load docker-image "${REGISTRY}/karmada-agent:${VERSION}" --name="${PULL_MODE_CLUSTER_NAME}"
+fi
+
+#step5. connecting networks between karmada-host, member1 and member2 clusters
+echo "connecting cluster networks..."
+util::add_routes "${MEMBER_CLUSTER_1_NAME}" "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${MEMBER_CLUSTER_2_NAME}"
+util::add_routes "${MEMBER_CLUSTER_2_NAME}" "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${MEMBER_CLUSTER_1_NAME}"
+
+util::add_routes "${HOST_CLUSTER_NAME}" "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${MEMBER_CLUSTER_1_NAME}"
+util::add_routes "${MEMBER_CLUSTER_1_NAME}" "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}"
+
+util::add_routes "${HOST_CLUSTER_NAME}" "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${MEMBER_CLUSTER_2_NAME}"
+util::add_routes "${MEMBER_CLUSTER_2_NAME}" "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}"
+echo "cluster networks connected"
+
+#step6. merge temporary kubeconfig of member clusters by kubectl
+export KUBECONFIG=$(find ${KUBECONFIG_PATH} -maxdepth 1 -type f | grep ${MEMBER_TMP_CONFIG_PREFIX} | tr '\n' ':')
+kubectl config view --flatten > ${MEMBER_CLUSTER_KUBECONFIG}
+rm $(find ${KUBECONFIG_PATH} -maxdepth 1 -type f | grep ${MEMBER_TMP_CONFIG_PREFIX})