diff --git a/.github/workflows/operator_test.yaml b/.github/workflows/operator_test.yaml new file mode 100644 index 000000000000..0162d373925b --- /dev/null +++ b/.github/workflows/operator_test.yaml @@ -0,0 +1,72 @@ +name: Operator Test +on: + # Run this workflow every time a new commit pushed to upstream/fork repository. + # Run workflow on fork repository will help contributors find and resolve issues before sending a PR. + push: + # Exclude branches created by Dependabot to avoid triggering current workflow + # for PRs initiated by Dependabot. + branches-ignore: + - 'dependabot/**' + pull_request: +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency +concurrency: + group: ${{ github.workflow }}-${{ github.actor }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true +permissions: + contents: read # for actions/checkout to fetch code +jobs: + set-up-operator-test-environment: + name: Set up operator test environment + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + # Here support the latest three minor releases of Kubernetes, this can be considered to be roughly + # the same as the End of Life of the Kubernetes release: https://kubernetes.io/releases/ + # Please remember to update the CI Schedule Workflow when we add a new version. + k8s: [ v1.29.0, v1.30.0, v1.31.0 ] + # prevent job running from forked repository + if: ${{ github.repository == 'karmada-io/karmada' }} + steps: + # Free up disk space on Ubuntu + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, if set to "true" but frees about 6 GB + tool-cache: false + # all of these default to true, but feel free to set to "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: false + docker-images: false + swap-storage: false + - name: checkout code + uses: actions/checkout@v4 + - name: install Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: setup operator test environment + run: | + export CLUSTER_VERSION=kindest/node:${{ matrix.k8s }} + hack/local-up-karmada-by-operator.sh + - name: run operator test + run: | + # run a single e2e + export KUBECONFIG=${HOME}/.kube/karmada.config + kubectl config use-context karmada-apiserver + GO111MODULE=on go install github.com/onsi/ginkgo/v2/ginkgo + ginkgo -v --race --trace -p --focus="[BasicPropagation] propagation testing deployment propagation testing" ./test/e2e/ + - name: upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: karmada_operator_e2e_log_${{ matrix.k8s }} + path: ${{ github.workspace }}/karmada-operator-e2e-logs/${{ matrix.k8s }}/ + - name: upload kind logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: karmada_kind_log_${{ matrix.k8s }} + path: /tmp/karmada/ diff --git a/hack/deploy-karmada-by-operator.sh b/hack/deploy-karmada-by-operator.sh new file mode 100755 index 000000000000..6f0f70566ec3 --- /dev/null +++ b/hack/deploy-karmada-by-operator.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Copyright 2024 The Karmada Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -o errexit +set -o nounset + +# This script deploy karmada instance to any cluster you want via karmada-operator. +# This script depends on utils in: ${REPO_ROOT}/hack/util.sh + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source ${REPO_ROOT}/hack/util.sh + +KARMADA_SYSTEM_NAMESPACE="karmada-system" +KARMADA_INSTANCE_NAME=${KARMADA_INSTANCE_NAME:-"karmada-demo"} +KARMADA_INSTANCE_NAMESPACE=${KARMADA_INSTANCE_NAMESPACE:-"test"} + +CERT_DIR=${CERT_DIR:-"${HOME}/.karmada"} +mkdir -p "${CERT_DIR}" &>/dev/null || mkdir -p "${CERT_DIR}" +rm -f "${CERT_DIR}/*" &>/dev/null || rm -f "${CERT_DIR}/*" + +function usage() { + echo "This script deploys karmada instance to a given cluster via karmada-operator." + echo "Note: This script is an internal script and is not intended used by end-users." + echo "Usage: hack/deploy-karmada-by-operator.sh " + echo "Example: hack/deploy-karmada-by-operator.sh ~/.kube/members.config member1 karmada-apiserver v1.11.0 true https://github.com/karmada-io/karmada/releases/download/v1.11.0/crds.tar.gz" + echo -e "Parameters:\n\tKUBECONFIG\t\tYour cluster's kubeconfig that you want to install to" + echo -e "\tCONTEXT_NAME\t\tThe name of context in 'kubeconfig'" + echo -e "\tKARMADA_CONTEXT_NAME\t\tThe context name of karmada instance, and different Karmada instances must have unique contexts to avoid being overwritten.'" + echo -e "\tKARMADA_IMAGE_TAG\t\tThe tag of image'" + echo -e "\tADDON_NEEDED\t\tWhether you need to install addons(KarmadaSearch&karmadaDescheduler), optional, defaults to false." + echo -e "\tCRD_DOWNLOAD_URL\t\tThe download url for CRDs, optional.'" +} + +if [[ $# -le 4 ]]; then + usage + exit 1 +fi + +# check config file existence +HOST_CLUSTER_KUBECONFIG=$1 +if [[ ! -f "${HOST_CLUSTER_KUBECONFIG}" ]]; then + echo -e "ERROR: failed to get kubernetes config file: '${HOST_CLUSTER_KUBECONFIG}', not existed.\n" + usage + exit 1 +fi + +# check context existence +CONTEXT_NAME=$2 +if ! kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" config get-contexts "${CONTEXT_NAME}" > /dev/null 2>&1; +then + echo -e "ERROR: failed to get context: '${CONTEXT_NAME}' not in ${HOST_CLUSTER_KUBECONFIG}. \n" + usage + exit 1 +fi + +# check for duplicate karmada context name. +KARMADA_CONTEXT_NAME=$3 +if kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" config get-contexts "${KARMADA_CONTEXT_NAME}" > /dev/null 2>&1; +then + echo -e "ERROR: context: '${KARMADA_CONTEXT_NAME}' already exists in ${HOST_CLUSTER_KUBECONFIG}. \n" + usage + exit 1 +fi + +TEMP_PATH_BOOTSTRAP=$(mktemp -d) +trap '{ rm -rf ${TEMP_PATH_BOOTSTRAP}; }' EXIT +cp -rf "${REPO_ROOT}"/operator/config/samples/karmada-sample.yaml "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml + +if kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" get namespace ${KARMADA_INSTANCE_NAMESPACE} > /dev/null 2>&1; then + echo "Namespace '${KARMADA_INSTANCE_NAMESPACE}' already exists." +else + echo "Namespace '${KARMADA_INSTANCE_NAMESPACE}' does not exist. Creating now..." + kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" create ns ${KARMADA_INSTANCE_NAMESPACE} +fi + +# modify `karmada-sample.yaml` based on custom configuration. +ADDON_NEEDED=${5:-false} +# if choosing install addons, append karmadaSearch and karmadaDescheduler to 'karmada-sample.yaml' +if [ ${ADDON_NEEDED} ]; then + echo -e ' karmadaDescheduler:\n imageRepository: docker.io/karmada/karmada-descheduler\n imageTag: {{image_tag}}\n replicas: 1' >> "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml + echo -e ' karmadaSearch:\n imageRepository: docker.io/karmada/karmada-search\n imageTag: {{image_tag}}\n replicas: 1' >> "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml +fi + +IMAGE_TAG=$4 +sed -i'' -e "s/{{image_tag}}/${IMAGE_TAG}/g" "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml +sed -i'' -e "s/{{karmada_instance_name}}/${KARMADA_INSTANCE_NAME}/g" "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml +sed -i'' -e "s/{{karmada_instance_namespace}}/${KARMADA_INSTANCE_NAMESPACE}/g" "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml + +CRD_DOWNLOAD_URL=${6:-""} +if [[ -z ${CRD_DOWNLOAD_URL} ]]; then + sed -i'' -e "s/{{crd_tarball}}/""/g" "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml +else + CRD_TAR_BALL="\n httpSource:\n url: ${CRD_DOWNLOAD_URL}" + awk -v pattern="{{crd_tarball}}" -v replacement="${CRD_TAR_BALL}" '{ gsub(pattern, replacement); print }' "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml > "${TEMP_PATH_BOOTSTRAP}"/temp && mv "${TEMP_PATH_BOOTSTRAP}"/temp "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml +fi + +# create and wait for karmada instance to be ready +kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml +kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" wait --for=condition=Ready --timeout=1000s karmada karmada-demo -n ${KARMADA_INSTANCE_NAMESPACE} + +# generate kubeconfig for karmada instance +kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" get secret -n ${KARMADA_INSTANCE_NAMESPACE} karmada-demo-admin-config -o jsonpath={.data.kubeconfig} | base64 -d > ~/.kube/karmada-tmp-apiserver.config +cat ~/.kube/karmada-tmp-apiserver.config| grep "client-certificate-data"| awk '{print $2}'| base64 -d > ${CERT_DIR}/karmada.crt +cat ~/.kube/karmada-tmp-apiserver.config| grep "client-key-data"| awk '{print $2}'| base64 -d > ${CERT_DIR}/karmada.key +KARMADA_APISERVER=$(cat ~/.kube/karmada-tmp-apiserver.config| grep "server:"| awk '{print $2}') + +# write karmada api server config to kubeconfig file +util::append_client_kubeconfig_with_server "${HOST_CLUSTER_KUBECONFIG}" "${CERT_DIR}/karmada.crt" "${CERT_DIR}/karmada.key" "${KARMADA_APISERVER}" ${KARMADA_CONTEXT_NAME} +rm ~/.kube/karmada-tmp-apiserver.config diff --git a/hack/deploy-karmada-operator.sh b/hack/deploy-karmada-operator.sh new file mode 100755 index 000000000000..ec7487efe740 --- /dev/null +++ b/hack/deploy-karmada-operator.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Copyright 2024 The Karmada Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -o errexit +set -o nounset + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source ${REPO_ROOT}/hack/util.sh +KARMADA_SYSTEM_NAMESPACE="karmada-system" + +function usage() { + echo "This script will deploy karmada-operator on the specified cluster" + echo "Usage: hack/deploy-karmada-operator.sh " + echo "Example: hack/deploy-karmada-operator.sh ~/.kube/config karmada-host" +} + +if [[ $# -ne 2 ]]; then + usage + exit 1 +fi + +# check kube config file existence +if [[ ! -f "${1}" ]]; then + echo -e "ERROR: failed to get kubernetes config file: '${1}', not existed.\n" + usage + exit 1 +fi +KUBECONFIG=$1 + +# check context existence +if ! kubectl config get-contexts "${2}" --kubeconfig="${KUBECONFIG}" > /dev/null 2>&1; +then + echo -e "ERROR: failed to get context: '${2}' not in ${KUBECONFIG}. \n" + usage + exit 1 +fi +CONTEXT_NAME=$2 + +# make images +export VERSION="latest" +export REGISTRY="docker.io/karmada" +make image-karmada-operator GOOS="linux" --directory=. + +# load the karmada-operator images +kind load docker-image "${REGISTRY}/karmada-controller-manager:${VERSION}" --name="${CONTEXT_NAME}" + +# create namespace `karmada-system` +kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/namespace.yaml" + +# create secret my-kubeconfig +kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" create secret generic my-kubeconfig --from-file=config="${KUBECONFIG}" --namespace ${KARMADA_SYSTEM_NAMESPACE} + +# install Karmada operator crds +kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f operator/config/crds/ + +# deploy karmada-operator +kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REPO_ROOT}/operator/config/deploy/karmada-operator.yaml" + +# wait karmada-operator ready +kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" wait --for=condition=Ready --timeout=30s pods -l karmada-app=karmada-operator -n ${KARMADA_SYSTEM_NAMESPACE} diff --git a/hack/local-up-karmada-by-operator.sh b/hack/local-up-karmada-by-operator.sh new file mode 100755 index 000000000000..51aaa18f7ee0 --- /dev/null +++ b/hack/local-up-karmada-by-operator.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +# Copyright 2024 The Karmada Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# This script is used in workflow to validate karmada installation by operator. +# It starts a local karmada control plane based on current codebase via karmada-operator and with a certain number of clusters joined. +# This script depends on utils in: ${REPO_ROOT}/hack/util.sh +# 1. used by developer to setup develop environment quickly. +# 2. used by e2e testing to test if the operator installs correctly. + +function usage() { + echo "Usage:" + echo " hack/verify-operator-installed.sh [-h]" + echo " h: print help information" +} + +function getCrdsDir() { + local path=$1 + local url=$2 + local key=$(echo "$url" | xargs) # Trim whitespace using xargs + local hash=$(echo -n "$key" | sha256sum | awk '{print $1}') # Calculate SHA256 hash + local hashedKey=${hash:0:64} # Take the first 64 characters of the hash + echo "${path}/cache/${hashedKey}" +} + +while getopts 'h' OPT; do + case $OPT in + h) + usage + exit 0 + ;; + ?) + usage + exit 1 + ;; + esac +done + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${REPO_ROOT}"/hack/util.sh + +# variable define +KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"} +MAIN_KUBECONFIG=${MAIN_KUBECONFIG:-"${KUBECONFIG_PATH}/karmada.config"} +HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"karmada-host"} +KARMADA_APISERVER_CLUSTER_NAME=${KARMADA_APISERVER_CLUSTER_NAME:-"karmada-apiserver"} +MEMBER_CLUSTER_KUBECONFIG=${MEMBER_CLUSTER_KUBECONFIG:-"${KUBECONFIG_PATH}/members.config"} +MEMBER_CLUSTER_1_NAME=${MEMBER_CLUSTER_1_NAME:-"member1"} +MEMBER_CLUSTER_2_NAME=${MEMBER_CLUSTER_2_NAME:-"member2"} +PULL_MODE_CLUSTER_NAME=${PULL_MODE_CLUSTER_NAME:-"member3"} +MEMBER_TMP_CONFIG_PREFIX="member-tmp" +MEMBER_CLUSTER_1_TMP_CONFIG="${KUBECONFIG_PATH}/${MEMBER_TMP_CONFIG_PREFIX}-${MEMBER_CLUSTER_1_NAME}.config" +MEMBER_CLUSTER_2_TMP_CONFIG="${KUBECONFIG_PATH}/${MEMBER_TMP_CONFIG_PREFIX}-${MEMBER_CLUSTER_2_NAME}.config" +PULL_MODE_CLUSTER_TMP_CONFIG="${KUBECONFIG_PATH}/${MEMBER_TMP_CONFIG_PREFIX}-${PULL_MODE_CLUSTER_NAME}.config" +CLUSTER_VERSION=${CLUSTER_VERSION:-"${DEFAULT_CLUSTER_VERSION}"} +KIND_LOG_FILE=${KIND_LOG_FILE:-"/tmp/karmada"} +KARMADA_SYSTEM_NAMESPACE="karmada-system" +KARMADA_INSTANCE_NAME=${KARMADA_INSTANCE_NAME:-"karmada-demo"} +KARMADA_INSTANCE_NAMESPACE=${KARMADA_INSTANCE_NAMESPACE:-"test"} + +#step0: prepare +# proxy setting in China mainland +if [[ -n ${CHINA_MAINLAND:-} ]]; then + util::set_mirror_registry_for_china_mainland ${REPO_ROOT} +fi + +# make sure go exists and the go version is a viable version. +util::cmd_must_exist "go" +util::verify_go_version + +# make sure docker exists +util::cmd_must_exist "docker" + +# install kind and kubectl +kind_version=v0.24.0 +echo -n "Preparing: 'kind' existence check - " +if util::cmd_exist kind; then + echo "passed" +else + echo "not pass" + util::install_tools "sigs.k8s.io/kind" $kind_version +fi + +# get arch name and os name in bootstrap +BS_ARCH=$(go env GOARCH) +BS_OS=$(go env GOOS) +# check arch and os name before installing +util::install_environment_check "${BS_ARCH}" "${BS_OS}" +echo -n "Preparing: 'kubectl' existence check - " +if util::cmd_exist kubectl; then + echo "passed" +else + echo "not pass" + util::install_kubectl "" "${BS_ARCH}" "${BS_OS}" +fi + +#step1. create host cluster and member clusters in parallel +#prepare for kindClusterConfig +util::delete_necessary_resources "${MAIN_KUBECONFIG},${MEMBER_CLUSTER_KUBECONFIG}" "${HOST_CLUSTER_NAME},${MEMBER_CLUSTER_1_NAME},${MEMBER_CLUSTER_2_NAME},${PULL_MODE_CLUSTER_NAME}" "${KIND_LOG_FILE}" + +util::create_cluster "${HOST_CLUSTER_NAME}" "${MAIN_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" +util::create_cluster "${MEMBER_CLUSTER_1_NAME}" "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${REPO_ROOT}"/artifacts/kindClusterConfig/member1.yaml +util::create_cluster "${MEMBER_CLUSTER_2_NAME}" "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${REPO_ROOT}"/artifacts/kindClusterConfig/member2.yaml +util::create_cluster "${PULL_MODE_CLUSTER_NAME}" "${PULL_MODE_CLUSTER_TMP_CONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${REPO_ROOT}"/artifacts/kindClusterConfig/member3.yaml + +#step2. make images and get karmadactl +export VERSION="latest" +export REGISTRY="docker.io/karmada" +export KARMADA_IMAGE_LABEL_VALUE="May_be_pruned_in_local-up-karmada_by_operator.sh" +export DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS:-} --label=image.karmada.io=${KARMADA_IMAGE_LABEL_VALUE}" +make images GOOS="linux" --directory="${REPO_ROOT}" +#clean up dangling images +docker image prune --force --filter "label=image.karmada.io=${KARMADA_IMAGE_LABEL_VALUE}" + +GO111MODULE=on go install "github.com/karmada-io/karmada/cmd/karmadactl" +GOPATH=$(go env GOPATH | awk -F ':' '{print $1}') +KARMADACTL_BIN="${GOPATH}/bin/karmadactl" + +#step3. wait until the host cluster ready +echo "Waiting for the host clusters to be ready..." +util::check_clusters_ready "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}" + +# load components images to host cluster +kind load docker-image "${REGISTRY}/karmada-controller-manager:${VERSION}" --name="${HOST_CLUSTER_NAME}" +kind load docker-image "${REGISTRY}/karmada-scheduler:${VERSION}" --name="${HOST_CLUSTER_NAME}" +kind load docker-image "${REGISTRY}/karmada-descheduler:${VERSION}" --name="${HOST_CLUSTER_NAME}" +kind load docker-image "${REGISTRY}/karmada-webhook:${VERSION}" --name="${HOST_CLUSTER_NAME}" +kind load docker-image "${REGISTRY}/karmada-aggregated-apiserver:${VERSION}" --name="${HOST_CLUSTER_NAME}" +kind load docker-image "${REGISTRY}/karmada-search:${VERSION}" --name="${HOST_CLUSTER_NAME}" +kind load docker-image "${REGISTRY}/karmada-metrics-adapter:${VERSION}" --name="${HOST_CLUSTER_NAME}" + +#step4. deploy karmada-operator +"${REPO_ROOT}"/hack/deploy-karmada-operator.sh "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}" + +#step5. install karmada instance by karmada-operator +# prepare the local crds +echo "Prepare the local crds" +cd ${REPO_ROOT}/charts/karmada/ +cp -r _crds crds +tar -zcvf ../../crds.tar.gz crds +cd - + +# copy the local crds.tar.gz file to the specified path of the karmada-operator, so that the karmada-operator will skip the step of downloading CRDs. +CRDTARBALL_URL="local" +DATA_DIR="/var/lib/karmada" +CRD_CACHE_DIR=$(getCrdsDir "${DATA_DIR}" "${CRDTARBALL_URL}") +OPERATOR_POD_NAME=$(kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" get pods -n ${KARMADA_SYSTEM_NAMESPACE} -l karmada-app=karmada-operator -o custom-columns=NAME:.metadata.name --no-headers) +kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" exec -i ${OPERATOR_POD_NAME} -n ${KARMADA_SYSTEM_NAMESPACE} -- mkdir -p ${CRD_CACHE_DIR} +kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" cp ${REPO_ROOT}/crds.tar.gz ${KARMADA_SYSTEM_NAMESPACE}/${OPERATOR_POD_NAME}:${CRD_CACHE_DIR} + +# install karmada instance +"${REPO_ROOT}"/hack/deploy-karmada-by-operator.sh "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}" "${KARMADA_APISERVER_CLUSTER_NAME}" "${VERSION}" true "${CRDTARBALL_URL}" + +#step6. join member clusters +# wait until member clusters ready +util::check_clusters_ready "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${MEMBER_CLUSTER_1_NAME}" +util::check_clusters_ready "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${MEMBER_CLUSTER_2_NAME}" +# connecting networks between karmada-host, member1 and member2 clusters +echo "connecting cluster networks..." +util::add_routes "${MEMBER_CLUSTER_1_NAME}" "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${MEMBER_CLUSTER_2_NAME}" +util::add_routes "${MEMBER_CLUSTER_2_NAME}" "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${MEMBER_CLUSTER_1_NAME}" + +util::add_routes "${HOST_CLUSTER_NAME}" "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${MEMBER_CLUSTER_1_NAME}" +util::add_routes "${MEMBER_CLUSTER_1_NAME}" "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}" + +util::add_routes "${HOST_CLUSTER_NAME}" "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${MEMBER_CLUSTER_2_NAME}" +util::add_routes "${MEMBER_CLUSTER_2_NAME}" "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}" +echo "cluster networks connected" + +#join push mode member clusters +export KUBECONFIG="${MAIN_KUBECONFIG}" +${KARMADACTL_BIN} join --karmada-context="${KARMADA_APISERVER_CLUSTER_NAME}" ${MEMBER_CLUSTER_1_NAME} --cluster-kubeconfig="${MEMBER_CLUSTER_1_TMP_CONFIG}" --cluster-context="${MEMBER_CLUSTER_1_NAME}" +${KARMADACTL_BIN} join --karmada-context="${KARMADA_APISERVER_CLUSTER_NAME}" ${MEMBER_CLUSTER_2_NAME} --cluster-kubeconfig="${MEMBER_CLUSTER_2_TMP_CONFIG}" --cluster-context="${MEMBER_CLUSTER_2_NAME}" + +# wait until the pull mode cluster ready +util::check_clusters_ready "${PULL_MODE_CLUSTER_TMP_CONFIG}" "${PULL_MODE_CLUSTER_NAME}" +kind load docker-image "${REGISTRY}/karmada-agent:${VERSION}" --name="${PULL_MODE_CLUSTER_NAME}" + +#step7. deploy karmada agent in pull mode member clusters +"${REPO_ROOT}"/hack/deploy-karmada-agent.sh "${MAIN_KUBECONFIG}" "${KARMADA_APISERVER_CLUSTER_NAME}" "${PULL_MODE_CLUSTER_TMP_CONFIG}" "${PULL_MODE_CLUSTER_NAME}" + +#step8. deploy metrics-server in member clusters +"${REPO_ROOT}"/hack/deploy-k8s-metrics-server.sh "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${MEMBER_CLUSTER_1_NAME}" +"${REPO_ROOT}"/hack/deploy-k8s-metrics-server.sh "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${MEMBER_CLUSTER_2_NAME}" +"${REPO_ROOT}"/hack/deploy-k8s-metrics-server.sh "${PULL_MODE_CLUSTER_TMP_CONFIG}" "${PULL_MODE_CLUSTER_NAME}" + +# wait all of clusters member1, member2 and member3 status is ready +util:wait_cluster_ready "${KARMADA_APISERVER_CLUSTER_NAME}" "${MEMBER_CLUSTER_1_NAME}" +util:wait_cluster_ready "${KARMADA_APISERVER_CLUSTER_NAME}" "${MEMBER_CLUSTER_2_NAME}" +util:wait_cluster_ready "${KARMADA_APISERVER_CLUSTER_NAME}" "${PULL_MODE_CLUSTER_NAME}" + +#step9. merge temporary kubeconfig of member clusters by kubectl +export KUBECONFIG=$(find ${KUBECONFIG_PATH} -maxdepth 1 -type f | grep ${MEMBER_TMP_CONFIG_PREFIX} | tr '\n' ':') +kubectl config view --flatten > ${MEMBER_CLUSTER_KUBECONFIG} +rm $(find ${KUBECONFIG_PATH} -maxdepth 1 -type f | grep ${MEMBER_TMP_CONFIG_PREFIX}) + +function print_success() { + echo -e "$KARMADA_GREETING" + echo "Local Karmada is running." + echo -e "\nTo start using your karmada, run:" + echo -e " export KUBECONFIG=${MAIN_KUBECONFIG}" + echo "Please use 'kubectl config use-context ${HOST_CLUSTER_NAME}/${KARMADA_APISERVER_CLUSTER_NAME}' to switch the host and control plane cluster." + echo -e "\nTo manage your member clusters, run:" + echo -e " export KUBECONFIG=${MEMBER_CLUSTER_KUBECONFIG}" + echo "Please use 'kubectl config use-context ${MEMBER_CLUSTER_1_NAME}/${MEMBER_CLUSTER_2_NAME}/${PULL_MODE_CLUSTER_NAME}' to switch to the different member cluster." +} + +print_success diff --git a/hack/util.sh b/hack/util.sh index ebc1ecd931d4..27f7f726630f 100755 --- a/hack/util.sh +++ b/hack/util.sh @@ -257,6 +257,19 @@ function util::append_client_kubeconfig { kubectl config set-context "${client_id}" --cluster="${client_id}" --user="${client_id}" --kubeconfig="${kubeconfig_path}" } +# util::append_client_kubeconfig_with_server creates a new context including a cluster and a user to the existed kubeconfig file +function util::append_client_kubeconfig_with_server { + local kubeconfig_path=$1 + local client_certificate_file=$2 + local client_key_file=$3 + local server=$4 + local client_id=$5 + local token=${6:-} + kubectl config set-cluster "${client_id}" --server="${server}" --insecure-skip-tls-verify=true --kubeconfig="${kubeconfig_path}" + kubectl config set-credentials "${client_id}" --token="${token}" --client-certificate="${client_certificate_file}" --client-key="${client_key_file}" --embed-certs=true --kubeconfig="${kubeconfig_path}" + kubectl config set-context "${client_id}" --cluster="${client_id}" --user="${client_id}" --kubeconfig="${kubeconfig_path}" +} + # util::write_client_kubeconfig creates a self-contained kubeconfig: args are sudo, dest-dir, client certificate data, client key data, host, port, client id, token(optional) function util::write_client_kubeconfig { local sudo=$1 diff --git a/operator/config/samples/karmada-sample.yaml b/operator/config/samples/karmada-sample.yaml new file mode 100644 index 000000000000..ce720307460c --- /dev/null +++ b/operator/config/samples/karmada-sample.yaml @@ -0,0 +1,60 @@ +# used by hack/deploy-karmada-by-operator.sh to deploy a karmada instance +apiVersion: operator.karmada.io/v1alpha1 +kind: Karmada +metadata: + name: {{karmada_instance_name}} + namespace: {{karmada_instance_namespace}} +spec: + hostCluster: + networking: + dnsDomain: cluster.local + crdTarball: {{crd_tarball}} + components: + etcd: + local: + imageRepository: registry.k8s.io/etcd + imageTag: 3.5.13-0 + replicas: 1 + volumeData: + # hostPath: + # type: DirectoryOrCreate + # path: /var/lib/karmada/etcd/karmada-demo + volumeClaim: + metadata: + name: etcd-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 3Gi + karmadaAPIServer: + imageRepository: registry.k8s.io/kube-apiserver + imageTag: v1.30.4 + replicas: 1 + serviceType: NodePort + serviceSubnet: 10.96.0.0/12 + karmadaAggregatedAPIServer: + imageRepository: docker.io/karmada/karmada-aggregated-apiserver + imageTag: {{image_tag}} + replicas: 1 + karmadaControllerManager: + imageRepository: docker.io/karmada/karmada-controller-manager + imageTag: {{image_tag}} + replicas: 1 + karmadaScheduler: + imageRepository: docker.io/karmada/karmada-scheduler + imageTag: {{image_tag}} + replicas: 1 + karmadaWebhook: + imageRepository: docker.io/karmada/karmada-webhook + imageTag: {{image_tag}} + replicas: 1 + kubeControllerManager: + imageRepository: registry.k8s.io/kube-controller-manager + imageTag: v1.30.4 + replicas: 1 + karmadaMetricsAdapter: + imageRepository: docker.io/karmada/karmada-metrics-adapter + imageTag: {{image_tag}} + replicas: 2