e2e-1.24.yaml

name: E2E-1.24

on:
  push:
    branches:
      - master
      - release-*
  pull_request: {}
  workflow_dispatch: {}

# Declare default permissions as read only.
permissions: read-all

env:
  # Common versions
  GO_VERSION: '1.20'
  KIND_ACTION_VERSION: 'v1.3.0'
  KIND_VERSION: 'v0.14.0'
  KIND_IMAGE: 'kindest/node:v1.24.6'
  KIND_CLUSTER_NAME: 'ci-testing'

jobs:

  astatefulset:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
      - name: Setup Kind Cluster
        uses: helm/kind-action@v1.10.0
        with:
          node_image: ${{ env.KIND_IMAGE }}
          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
          config: ./test/kind-conf.yaml
          version: ${{ env.KIND_VERSION }}
      - name: Build image
        run: |
          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
          docker build --pull --no-cache . -t $IMAGE
          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
      - name: Install Kruise
        run: |
          set -ex
          kubectl cluster-info
          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
          NODES=$(kubectl get node | wc -l)
          for ((i=1;i<10;i++));
          do
            set +e
            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
            set -e
            if [ "$PODS" -eq "$NODES" ]; then
              break
            fi
            sleep 3
          done
          set +e
          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
          kubectl get node -o yaml
          kubectl get all -n kruise-system -o yaml
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
          set -e
          if [ "$PODS" -eq "$NODES" ]; then
            echo "Wait for kruise-manager and kruise-daemon ready successfully"
          else
            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
            exit 1
          fi
      - name: Run E2E Tests
        run: |
          export KUBECONFIG=/home/runner/.kube/config
          make ginkgo
          set +e
          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] StatefulSet' test/e2e
          retVal=$?
          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
          if [ "${restartCount}" -eq "0" ];then
              echo "Kruise-manager has not restarted"
          else
              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
              echo "Kruise-manager has restarted, abort!!!"
              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
              exit 1
          fi
          if [ "$retVal" -ne 0 ];then
              echo "test fail, dump kruise-manager logs"
              while read pod; do 
                   kubectl logs -n kruise-system $pod
              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
              echo "test fail, dump kruise-daemon logs"
              while read pod; do 
                   kubectl logs -n kruise-system $pod
              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')             
          fi
          exit $retVal

  pullimages-containerrecreate:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
      - name: Setup Kind Cluster
        uses: helm/kind-action@v1.10.0
        with:
          node_image: ${{ env.KIND_IMAGE }}
          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
          config: ./test/kind-conf.yaml
          version: ${{ env.KIND_VERSION }}
      - name: Build image
        run: |
          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
          docker build --pull --no-cache . -t $IMAGE
          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
      - name: Install Kruise
        run: |
          set -ex
          kubectl cluster-info
          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
          NODES=$(kubectl get node | wc -l)
          for ((i=1;i<10;i++));
          do
            set +e
            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
            set -e
            if [ "$PODS" -eq "$NODES" ]; then
              break
            fi
            sleep 3
          done
          set +e
          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
          kubectl get node -o yaml
          kubectl get all -n kruise-system -o yaml
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
          set -e
          if [ "$PODS" -eq "$NODES" ]; then
            echo "Wait for kruise-manager and kruise-daemon ready successfully"
          else
            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
            exit 1
          fi
      - name: Run E2E Tests
        run: |
          export KUBECONFIG=/home/runner/.kube/config
          make ginkgo
          set +e
          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] (PullImage|ContainerRecreateRequest|PullImages)' test/e2e
          retVal=$?
          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
          if [ "${restartCount}" -eq "0" ];then
              echo "Kruise-manager has not restarted"
          else
              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
              echo "Kruise-manager has restarted, abort!!!"
              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
              exit 1
          fi
          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
          do
            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
            if [ "${restartCount}" -eq "0" ];then
                echo "Kruise-daemon has not restarted"
            else
                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
                echo "Kruise-daemon has restarted, abort!!!"
                kubectl logs -p -n ${ns} ${name}
                exit 1
            fi
          done
          if [ "$retVal" -ne 0 ];then
              echo "test fail, dump kruise-manager logs"
              while read pod; do 
                   kubectl logs -n kruise-system $pod
              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
              echo "test fail, dump kruise-daemon logs"
              while read pod; do 
                   kubectl logs -n kruise-system $pod
              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')            
          fi
          
          exit $retVal

  advanced-daemonset:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
      - name: Setup Kind Cluster
        uses: helm/kind-action@v1.10.0
        with:
          node_image: ${{ env.KIND_IMAGE }}
          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
          config: ./test/kind-conf.yaml
          version: ${{ env.KIND_VERSION }}
      - name: Build image
        run: |
          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
          docker build --pull --no-cache . -t $IMAGE
          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
      - name: Install Kruise
        run: |
          set -ex
          kubectl cluster-info
          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
          NODES=$(kubectl get node | wc -l)
          for ((i=1;i<10;i++));
          do
            set +e
            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
            set -e
            if [ "$PODS" -eq "$NODES" ]; then
              break
            fi
            sleep 3
          done
          set +e
          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
          kubectl get node -o yaml
          kubectl get all -n kruise-system -o yaml
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
          set -e
          if [ "$PODS" -eq "$NODES" ]; then
            echo "Wait for kruise-manager and kruise-daemon ready successfully"
          else
            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
            exit 1
          fi
      - name: Run E2E Tests
        run: |
          export KUBECONFIG=/home/runner/.kube/config
          make ginkgo
          set +e
          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] DaemonSet' test/e2e
          retVal=$?
          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
          if [ "${restartCount}" -eq "0" ];then
              echo "Kruise-manager has not restarted"
          else
              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
              echo "Kruise-manager has restarted, abort!!!"
              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
              exit 1
          fi
          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
          do
            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
            if [ "${restartCount}" -eq "0" ];then
                echo "Kruise-daemon has not restarted"
            else
                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
                echo "Kruise-daemon has restarted, abort!!!"
                kubectl logs -p -n ${ns} ${name}
                exit 1
            fi
          done
          
          if [ "$retVal" -ne 0 ];then
              echo "test fail, dump kruise-manager logs"
              while read pod; do 
                   kubectl logs -n kruise-system $pod
              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
              echo "test fail, dump kruise-daemon logs"
              while read pod; do 
                   kubectl logs -n kruise-system $pod
              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')          
          fi
          exit $retVal

  sidecarset:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
      - name: Setup Kind Cluster
        uses: helm/kind-action@v1.10.0
        with:
          node_image: ${{ env.KIND_IMAGE }}
          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
          config: ./test/kind-conf.yaml
          version: ${{ env.KIND_VERSION }}
      - name: Build image
        run: |
          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
          docker build --pull --no-cache . -t $IMAGE
          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
      - name: Install Kruise
        run: |
          set -ex
          kubectl cluster-info
          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
          NODES=$(kubectl get node | wc -l)
          for ((i=1;i<10;i++));
          do
            set +e
            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
            set -e
            if [ "$PODS" -eq "$NODES" ]; then
              break
            fi
            sleep 3
          done
          set +e
          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
          kubectl get node -o yaml
          kubectl get all -n kruise-system -o yaml
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
          set -e
          if [ "$PODS" -eq "$NODES" ]; then
            echo "Wait for kruise-manager and kruise-daemon ready successfully"
          else
            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
            exit 1
          fi
      - name: Run E2E Tests
        run: |
          export KUBECONFIG=/home/runner/.kube/config
          make ginkgo
          set +e
          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] SidecarSet' test/e2e
          retVal=$?
          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
          if [ "${restartCount}" -eq "0" ];then
              echo "Kruise-manager has not restarted"
          else
              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
              echo "Kruise-manager has restarted, abort!!!"
              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
              exit 1
          fi
          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
          do
            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
            if [ "${restartCount}" -eq "0" ];then
                echo "Kruise-daemon has not restarted"
            else
                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
                echo "Kruise-daemon has restarted, abort!!!"
                kubectl logs -p -n ${ns} ${name}
                exit 1
            fi
          done
          
          if [ "$retVal" -ne 0 ];then
              echo "test fail, dump kruise-manager logs"
              while read pod; do 
                   kubectl logs -n kruise-system $pod
              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
              echo "test fail, dump kruise-daemon logs"
              while read pod; do 
                   kubectl logs -n kruise-system $pod
              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')             
          fi
          exit $retVal

  ephemeraljob:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
      - name: Setup Kind Cluster
        uses: helm/kind-action@v1.10.0
        with:
          node_image: ${{ env.KIND_IMAGE }}
          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
          config: ./test/kind-conf.yaml
          version: ${{ env.KIND_VERSION }}
      - name: Build image
        run: |
          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
          docker build --pull --no-cache . -t $IMAGE
          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
      - name: Install Kruise
        run: |
          set -ex
          kubectl cluster-info
          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
          NODES=$(kubectl get node | wc -l)
          for ((i=1;i<10;i++));
          do
            set +e
            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
            set -e
            if [ "$PODS" -eq "$NODES" ]; then
              break
            fi
            sleep 3
          done
          set +e
          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
          kubectl get node -o yaml
          kubectl get all -n kruise-system -o yaml
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
          set -e
          if [ "$PODS" -eq "$NODES" ]; then
            echo "Wait for kruise-manager and kruise-daemon ready successfully"
          else
            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
            exit 1
          fi
      - name: Run E2E Tests
        run: |
          export KUBECONFIG=/home/runner/.kube/config
          make ginkgo
          set +e
          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] EphemeralJob' test/e2e
          retVal=$?
          if [ "$retVal" -ne 0 ];then
              echo "test fail, dump kruise-manager logs"
              while read pod; do 
                   kubectl logs -n kruise-system $pod
              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
              echo "test fail, dump kruise-daemon logs"
              while read pod; do 
                   kubectl logs -n kruise-system $pod
              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')           
          fi
          exit $retVal

  podUnavailableBudget:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
      - name: Setup Kind Cluster
        uses: helm/kind-action@v1.10.0
        with:
          node_image: ${{ env.KIND_IMAGE }}
          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
          config: ./test/kind-conf.yaml
          version: ${{ env.KIND_VERSION }}
      - name: Build image
        run: |
          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
          docker build --pull --no-cache . -t $IMAGE
          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
      - name: Install Kruise
        run: |
          set -ex
          kubectl cluster-info
          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
          NODES=$(kubectl get node | wc -l)
          for ((i=1;i<10;i++));
          do
            set +e
            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
            set -e
            if [ "$PODS" -eq "$NODES" ]; then
              break
            fi
            sleep 3
          done
          set +e
          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
          kubectl get node -o yaml
          kubectl get all -n kruise-system -o yaml
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
          set -e
          if [ "$PODS" -eq "$NODES" ]; then
            echo "Wait for kruise-manager and kruise-daemon ready successfully"
          else
            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
            exit 1
          fi
      - name: Run E2E Tests
        run: |
          export KUBECONFIG=/home/runner/.kube/config
          make ginkgo
          set +e
          ./bin/ginkgo -timeout 60m -v --focus='\[policy\] PodUnavailableBudget' test/e2e
          retVal=$?
          if [ "$retVal" -ne 0 ];then
              echo "test fail, dump kruise-manager logs"
              while read pod; do 
                   kubectl logs -n kruise-system $pod
              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}')             
          fi
          exit $retVal
  other:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
      - name: Setup Kind Cluster
        uses: helm/kind-action@v1.10.0
        with:
          node_image: ${{ env.KIND_IMAGE }}
          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
          config: ./test/kind-conf.yaml
          version: ${{ env.KIND_VERSION }}
      - name: Build image
        run: |
          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
          docker build --pull --no-cache . -t $IMAGE
          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
      - name: Install Kruise
        run: |
          set -ex
          kubectl cluster-info
          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
          NODES=$(kubectl get node | wc -l)
          for ((i=1;i<10;i++));
          do
            set +e
            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
            set -e
            if [ "$PODS" -eq "$NODES" ]; then
              break
            fi
            sleep 3
          done
          set +e
          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
          kubectl get node -o yaml
          kubectl get all -n kruise-system -o yaml
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
          set -e
          if [ "$PODS" -eq "$NODES" ]; then
            echo "Wait for kruise-manager and kruise-daemon ready successfully"
          else
            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
            exit 1
          fi
      - name: Run E2E Tests
        run: |
          export KUBECONFIG=/home/runner/.kube/config
          make ginkgo
          set +e
          ./bin/ginkgo -timeout 90m -v --skip='\[apps\] (StatefulSet|PullImage|PullImages|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' --skip='\[policy\] PodUnavailableBudget' test/e2e
          retVal=$?
          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
          if [ "${restartCount}" -eq "0" ];then
              echo "Kruise-manager has not restarted"
          else
              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
              echo "Kruise-manager has restarted, abort!!!"
              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
              exit 1
          fi
          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
          do
            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
            if [ "${restartCount}" -eq "0" ];then
                echo "Kruise-daemon has not restarted"
            else
                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
                echo "Kruise-daemon has restarted, abort!!!"
                kubectl logs -p -n ${ns} ${name}
                exit 1
            fi
          done
          if [ "${retVal}" -ne 0 ];then
              echo "test fail, dump kruise-manager logs"
              while read pod; do 
                   kubectl logs -n kruise-system $pod
              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
              echo "test fail, dump kruise-daemon logs"
              while read pod; do 
                   kubectl logs -n kruise-system $pod
              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')           
          fi

          exit $retVal