From a0f45f2b4f4a623d75bccbd135b7fdb23c69a42a Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Tue, 23 Apr 2024 13:17:47 -0400 Subject: [PATCH 01/12] remove drone CI Signed-off-by: Alexandre Lamarre --- .drone.yml | 249 ----------------------------------------------------- 1 file changed, 249 deletions(-) delete mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 30e58b83..00000000 --- a/.drone.yml +++ /dev/null @@ -1,249 +0,0 @@ ---- -kind: pipeline -name: amd64 - -platform: - os: linux - arch: amd64 - -steps: -- name: build - image: rancher/dapper:v0.6.0 - commands: - - export K8S_VERSION_FROM_DRONE="v1.24" - - dapper ci - volumes: - - name: docker - path: /var/run/docker.sock - settings: - no_cache: true - -- name: build_no_psp - image: rancher/dapper:v0.6.0 - commands: - - export K8S_VERSION_FROM_DRONE="stable" - - dapper ci - volumes: - - name: docker - path: /var/run/docker.sock - -- name: fossa - image: registry.suse.com/suse/sle15:15.5 - failure: ignore - environment: - FOSSA_API_KEY: - from_secret: FOSSA_API_KEY - commands: - - zypper -n install curl unzip - - "curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/spectrometer/master/install.sh | sh" - - fossa analyze - - fossa test - when: - instance: - - drone-publish.rancher.io - ref: - include: - - "refs/heads/master" - event: - - push - - tag - -- name: github_binary_prerelease - image: plugins/github-release - settings: - api_key: - from_secret: github_token - prerelease: true - title: "Pre-release ${DRONE_TAG}" - checksum: - - sha256 - checksum_file: CHECKSUMsum-amd64.txt - checksum_flatten: true - files: - - "dist/artifacts/*" - when: - instance: - - drone-publish.rancher.io - ref: - include: - - "refs/tags/*rc*" - - "refs/tags/*alpha*" - event: - - tag - -- name: github_binary_release - image: plugins/github-release - settings: - api_key: - from_secret: github_token - title: "Release ${DRONE_TAG}" - checksum: - - sha256 - checksum_file: CHECKSUMsum-amd64.txt - checksum_flatten: true - files: - - "dist/artifacts/*" - when: - instance: - - drone-publish.rancher.io - ref: - exclude: - - "refs/tags/*rc*" - - "refs/tags/*alpha*" - event: - - tag - -- name: docker-publish - image: plugins/docker - settings: - dockerfile: package/Dockerfile - password: - from_secret: docker_password - repo: "rancher/backup-restore-operator" - tag: "${DRONE_TAG}-amd64" - username: - from_secret: docker_username - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -volumes: -- name: docker - host: - path: /var/run/docker.sock - ---- -kind: pipeline -name: arm64 - -platform: - os: linux - arch: arm64 - -steps: -- name: build - image: rancher/dapper:v0.6.0 - commands: - - export K8S_VERSION_FROM_DRONE="v1.24" - - dapper ci - volumes: - - name: docker - path: /var/run/docker.sock - -- name: build_no_psp - image: rancher/dapper:v0.6.0 - commands: - - export K8S_VERSION_FROM_DRONE="stable" - - dapper ci - volumes: - - name: docker - path: /var/run/docker.sock - -- name: github_binary_prerelease - image: plugins/github-release - settings: - api_key: - from_secret: github_token - prerelease: true - title: "Pre-release ${DRONE_TAG}" - checksum: - - sha256 - checksum_file: CHECKSUMsum-arm64.txt - checksum_flatten: true - files: - - "dist/artifacts/*" - when: - instance: - - drone-publish.rancher.io - ref: - include: - - "refs/tags/*rc*" - - "refs/tags/*alpha*" - event: - - tag - -- name: github_binary_release - image: plugins/github-release - settings: - api_key: - from_secret: github_token - title: "Release ${DRONE_TAG}" - checksum: - - sha256 - checksum_file: CHECKSUMsum-arm64.txt - checksum_flatten: true - files: - - "dist/artifacts/*" - when: - instance: - - drone-publish.rancher.io - ref: - exclude: - - "refs/tags/*rc*" - - "refs/tags/*alpha*" - event: - - tag - -- name: docker-publish - image: plugins/docker - settings: - dockerfile: package/Dockerfile - password: - from_secret: docker_password - repo: "rancher/backup-restore-operator" - tag: "${DRONE_TAG}-arm64" - username: - from_secret: docker_username - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -volumes: -- name: docker - host: - path: /var/run/docker.sock - ---- - -kind: pipeline -name: manifest - -platform: - os: linux - arch: amd64 - -steps: -- name: manifest - image: plugins/manifest:1.4.0 - settings: - username: - from_secret: docker_username - password: - from_secret: docker_password - platforms: - - linux/amd64 - - linux/arm64 - target: "rancher/backup-restore-operator:${DRONE_TAG}" - template: "rancher/backup-restore-operator:${DRONE_TAG}-ARCH" - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -depends_on: -- amd64 -- arm64 From 5b9c7b54bbbc7fc8e5d5abdb4bdad759da20a932 Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Tue, 23 Apr 2024 13:18:06 -0400 Subject: [PATCH 02/12] add goreleaser to publish BRO Signed-off-by: Alexandre Lamarre --- .goreleaser.yaml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .goreleaser.yaml diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 00000000..7e13edf4 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,46 @@ +# This is an example .goreleaser.yml file with some sensible defaults. +# Make sure to check the documentation at https://goreleaser.com +before: + hooks: + # You may remove this if you don't use go modules. + - go mod tidy +builds: + - id: backup-restore-operator + main: ./main.go + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + binary: backup-restore-operator + ldflags: + - -extldflags + - -static + - -s + - -X main.Version={{.Version}} -X main.GitCommit={{.Commit}} + flags: + - -trimpath + env: + - CGO_ENABLED=0 +archives: + - id: backup-restore-operator + builds: + - backup-restore-operator + name_template: '{{ .Binary }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ if .Mips }}_{{ .Mips }}{{ end }}' +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ incpatch .Version }}-next" +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' + +# The lines beneath this are called `modelines`. See `:help modeline` +# Feel free to remove those if you don't want/use them. +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +# vim: set ts=2 sw=2 tw=0 fo=cnqoj \ No newline at end of file From 0d566ae72a1efe87d68a83eda0d771674d2ea1e2 Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Tue, 23 Apr 2024 13:18:23 -0400 Subject: [PATCH 03/12] Add CI install/setup scripts Signed-off-by: Alexandre Lamarre --- .github/workflows/scripts/install-k3d.sh | 17 ++++++ .github/workflows/scripts/install-mc.sh | 25 +++++++++ .github/workflows/scripts/setup-cluster.sh | 55 +++++++++++++++++++ .../workflows/scripts/supported-versions.sh | 23 ++++++++ 4 files changed, 120 insertions(+) create mode 100755 .github/workflows/scripts/install-k3d.sh create mode 100755 .github/workflows/scripts/install-mc.sh create mode 100755 .github/workflows/scripts/setup-cluster.sh create mode 100755 .github/workflows/scripts/supported-versions.sh diff --git a/.github/workflows/scripts/install-k3d.sh b/.github/workflows/scripts/install-k3d.sh new file mode 100755 index 00000000..51ed39f9 --- /dev/null +++ b/.github/workflows/scripts/install-k3d.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e +set -x + +K3D_URL=https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh +DEFAULT_K3D_VERSION=v5.4.6 + +install_k3d(){ + local k3dVersion=${K3D_VERSION:-${DEFAULT_K3D_VERSION}} + echo -e "Downloading k3d@${k3dVersion} see: ${K3D_URL}" + curl --silent --fail ${K3D_URL} | TAG=${k3dVersion} bash +} + +install_k3d + +k3d version \ No newline at end of file diff --git a/.github/workflows/scripts/install-mc.sh b/.github/workflows/scripts/install-mc.sh new file mode 100755 index 00000000..40653507 --- /dev/null +++ b/.github/workflows/scripts/install-mc.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -e +set -x + + +# initArch discovers the architecture for this system. +initArch() { + ARCH=$(uname -m) + case $ARCH in + armv7*) ARCH="arm";; + aarch64) ARCH="arm64";; + x86_64) ARCH="amd64";; + esac +} + + +initArch + +curl -sL --fail https://dl.min.io/client/mc/release/linux-${ARCH}/mc > mc; +chmod +x mc; + +cp mc /usr/local/bin/mc + +mc --version \ No newline at end of file diff --git a/.github/workflows/scripts/setup-cluster.sh b/.github/workflows/scripts/setup-cluster.sh new file mode 100755 index 00000000..3e25a1bc --- /dev/null +++ b/.github/workflows/scripts/setup-cluster.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -e + +source ./scripts/version + +if [ -z "$CLUSTER_NAME" ]; then + echo "CLUSTER_NAME must be specified when setting up a cluster" + exit 1 +fi + +if [ -z "$K3S_VERSION" ]; then + echo "K3S_VERSION must be specified when setting up a cluster, use `k3d version list k3s` to find valid versions" + exit 1 +fi + +# waits until all nodes are ready +wait_for_nodes(){ + echo "wait until all agents are ready" + while : + do + readyNodes=1 + statusList=$(kubectl get nodes --no-headers | awk '{ print $2}') + # shellcheck disable=SC2162 + while read status + do + if [ "$status" == "NotReady" ] || [ "$status" == "" ] + then + readyNodes=0 + break + fi + done <<< "$(echo -e "$statusList")" + # all nodes are ready; exit + if [[ $readyNodes == 1 ]] + then + break + fi + sleep 1 + done +} + +k3d cluster delete ${CLUSTER_NAME} || true +k3d cluster create ${CLUSTER_NAME} --image "docker.io/rancher/k3s:${K3S_VERSION}" + +wait_for_nodes + +echo "${CLUSTER_NAME} ready" + +kubectl cluster-info --context k3d-${CLUSTER_NAME} +kubectl config use-context k3d-${CLUSTER_NAME} +kubectl get nodes -o wide + +IMAGE=${REPO}/backup-restore-operator:${TAG} + +k3d image import ${IMAGE} -c ${CLUSTER_NAME} diff --git a/.github/workflows/scripts/supported-versions.sh b/.github/workflows/scripts/supported-versions.sh new file mode 100755 index 00000000..650953a9 --- /dev/null +++ b/.github/workflows/scripts/supported-versions.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Prints supported versions based on the current release branch targeted +# Version output is in JSON + +set -e +set -x + +if git merge-base --is-ancestor origin/release/v5.0 HEAD +then + echo -n "[\"v1.23.9-k3s1\", \"v1.29.3-k3s1\"]" + exit 0 +elif git merge-base --is-ancestor origin/release/v4.0 HEAD +then + echo -n "[\"v1.23.9-k3s1\", \"v1.28.8-k3s1\"]" + exit 0 +elif git merge-base --is-ancestor origin/release/v3.0 HEAD +then + echo -n "[\"v1.16.9-k3s1\", \"v1.27.9-k3s1\"]" + exit 0 +fi + + +exit 1 \ No newline at end of file From 186355870af9a7daf0ff569d018205a77f11cebc Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Tue, 23 Apr 2024 13:19:28 -0400 Subject: [PATCH 04/12] Update dapper scripts to be more ergonomic Signed-off-by: Alexandre Lamarre --- Dockerfile.dapper | 19 ----------------- scripts/build | 1 + scripts/ci | 2 -- scripts/deploy | 42 ++++++------------------------------ scripts/integration | 52 ++++++++++++++------------------------------- scripts/package | 13 +++++------- 6 files changed, 29 insertions(+), 100 deletions(-) diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 8e950c0d..0bca8f2d 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -17,25 +17,6 @@ RUN if [ "${ARCH}" != "s390x" ]; then \ helm plugin install https://github.com/quintush/helm-unittest; \ fi -ENV K3S_BINARY_amd64=k3s \ - K3S_BINARY_arm64=k3s-arm64 \ - K3S_BINARY=K3S_BINARY_${ARCH} - -ARG K8S_VERSION_FROM_DRONE -ENV K8S_VERSION $K8S_VERSION_FROM_DRONE - -RUN if [ -z "${K8S_VERSION}" ]; then export K8S_VERSION="v1.24" && echo $(date +%s%N); fi -# ENV K8S_VERSION="${K8S_VERSION_FROM_DRONE:-'v1.24'}" - -RUN echo "${K8S_VERSION}" - -RUN curl -sL https://github.com/rancher/k3s/releases/download/$(curl -Ls -o /dev/null -w %{url_effective} https://update.k3s.io/v1-release/channels/${K8S_VERSION} | awk -F/ '{ print $NF }')/${!K3S_BINARY} > /usr/local/bin/k3s && \ - chmod +x /usr/local/bin/k3s - -RUN if [ "${ARCH}" != "s390x" ]; then \ - curl -sL https://dl.min.io/client/mc/release/linux-${ARCH}/mc > /usr/local/bin/mc && \ - chmod +x /usr/local/bin/mc; \ - fi ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS CROSS_ARCH USE_DOCKER_BUILDX ENV DAPPER_SOURCE /go/src/github.com/rancher/backup-restore-operator/ diff --git a/scripts/build b/scripts/build index c78fbd16..82c9f6d8 100755 --- a/scripts/build +++ b/scripts/build @@ -9,6 +9,7 @@ mkdir -p bin if [ "$(uname)" = "Linux" ]; then OTHER_LINKFLAGS="-extldflags -static -s" fi + LINKFLAGS="-X main.Version=$VERSION" LINKFLAGS="-X main.GitCommit=$COMMIT $LINKFLAGS" diff --git a/scripts/ci b/scripts/ci index d3828726..9b0e3181 100755 --- a/scripts/ci +++ b/scripts/ci @@ -3,11 +3,9 @@ set -e cd $(dirname $0) -./build ./test ./validate ./validate-ci ./package ./chart/test ./hull -./integration diff --git a/scripts/deploy b/scripts/deploy index b04912eb..60d74f72 100755 --- a/scripts/deploy +++ b/scripts/deploy @@ -41,11 +41,7 @@ EOF ) KUBECTL_CMD="kubectl" - if command -v k3s &> /dev/null ; then - KUBECTL_CMD="k3s kubectl" - else - check_kubeconfig - fi + check_kubeconfig helm repo add minio https://charts.min.io/ helm repo update @@ -66,11 +62,7 @@ EOF list_minio_files() { KUBECTL_CMD="kubectl" - if command -v k3s &> /dev/null ; then - KUBECTL_CMD="k3s kubectl" - else - check_kubeconfig - fi + check_kubeconfig local POD_NAME POD_NAME=$("${KUBECTL_CMD}" get pods --namespace minio -l "release=minio" -o jsonpath="{.items[0].metadata.name}") @@ -88,11 +80,7 @@ list_minio_files() { retrieve_minio_files() { KUBECTL_CMD="kubectl" - if command -v k3s &> /dev/null ; then - KUBECTL_CMD="k3s kubectl" - else - check_kubeconfig - fi + check_kubeconfig local POD_NAME POD_NAME=$("${KUBECTL_CMD}" get pods --namespace minio -l "release=minio" -o jsonpath="{.items[0].metadata.name}") @@ -115,11 +103,7 @@ retrieve_minio_files() { copy_minio_files() { KUBECTL_CMD="kubectl" - if command -v k3s &> /dev/null ; then - KUBECTL_CMD="k3s kubectl" - else - check_kubeconfig - fi + check_kubeconfig local POD_NAME POD_NAME=$("${KUBECTL_CMD}" get pods --namespace minio -l "release=minio" -o jsonpath="{.items[0].metadata.name}") @@ -142,11 +126,7 @@ copy_minio_files() { reset_minio_bucket() { KUBECTL_CMD="kubectl" - if command -v k3s &> /dev/null ; then - KUBECTL_CMD="k3s kubectl" - else - check_kubeconfig - fi + check_kubeconfig local POD_NAME POD_NAME=$("${KUBECTL_CMD}" get pods --namespace minio -l "release=minio" -o jsonpath="{.items[0].metadata.name}") @@ -190,11 +170,7 @@ deploy_backup_restore() { create_backup() { KUBECTL_CMD="kubectl" - if command -v k3s &> /dev/null ; then - KUBECTL_CMD="k3s kubectl" - else - check_kubeconfig - fi + check_kubeconfig if [[ "$1" = "insecure" ]]; then ${KUBECTL_CMD} create -f - < /dev/null ; then - KUBECTL_CMD="k3s kubectl" - else - check_kubeconfig - fi + check_kubeconfig if [[ "$insecure" = "true" ]]; then ${KUBECTL_CMD} create -f - < /tmp/k3s.log 2>&1 & -k3s_pid=$! - -export KUBECONFIG=/etc/rancher/k3s/k3s.yaml - -echo_with_time 'Waiting for node to be ready ...' -time timeout 300 bash -c 'while ! (k3s kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' -time timeout 300 bash -c 'while ! (k3s kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' - -echo_with_time "using kubectl and kubernetes versions:" -k3s kubectl version - -k3s kubectl get nodes -o wide --show-labels - -docker image save rancher/backup-restore-operator:$TAG -o /tmp/bro.img - -k3s ctr images import /tmp/bro.img - ls -la ./dist/artifacts # In case short commit only conists of numbers, it is regarded valid by Helm when packaging @@ -41,13 +21,13 @@ fi helm install rancher-backup-crd ./dist/artifacts/rancher-backup-crd-$HELM_CHART_VERSION.tgz -n cattle-resources-system --create-namespace --wait helm install rancher-backup ./dist/artifacts/rancher-backup-$HELM_CHART_VERSION.tgz -n cattle-resources-system --set image.tag=$TAG --set imagePullPolicy=IfNotPresent -time timeout 300 bash -c 'while ! (k3s kubectl --namespace cattle-resources-system rollout status --timeout 10s deploy/rancher-backup 2>/dev/null); do sleep 5; done' +time timeout 300 bash -c 'while ! (kubectl --namespace cattle-resources-system rollout status --timeout 10s deploy/rancher-backup 2>/dev/null); do sleep 5; done' -k3s kubectl get pods -n cattle-resources-system +kubectl get pods -n cattle-resources-system -time timeout 300 bash -c 'while ! (k3s kubectl --namespace cattle-resources-system rollout status --timeout 10s deploy/rancher-backup 2>/dev/null); do sleep 5; done' +time timeout 300 bash -c 'while ! (kubectl --namespace cattle-resources-system rollout status --timeout 10s deploy/rancher-backup 2>/dev/null); do sleep 5; done' -k3s kubectl get pods -n cattle-resources-system +kubectl get pods -n cattle-resources-system # Minio not available for s390x, only test on amd64 and arm64 if [ "$ARCH" = "s390x" ]; then @@ -58,8 +38,8 @@ fi #Deploy Minio ./scripts/deploy minio -export POD_NAME=$(k3s kubectl get pods --namespace minio -l "release=minio" -o jsonpath="{.items[0].metadata.name}") -k3s kubectl port-forward $POD_NAME 9000 --namespace minio & +export POD_NAME=$(kubectl get pods --namespace minio -l "release=minio" -o jsonpath="{.items[0].metadata.name}") +kubectl port-forward $POD_NAME 9000 --namespace minio & sleep 10 mkdir -p $HOME/.mc/certs/CAs @@ -97,7 +77,7 @@ for BACKUP in rancherbackups-insecure rancherbackups; do BACKUPRS_NAME="s3-recurring-backup" fi - time timeout 60 bash -c 'while ! (k3s kubectl wait --for condition=ready backup.resources.cattle.io/'"${BACKUPRS_NAME}"' 2>/dev/null); do k3s kubectl get backup.resources.cattle.io -A; k3s kubectl -n cattle-resources-system logs -l app.kubernetes.io/name=rancher-backup --tail=-1; sleep 2; done' + time timeout 60 bash -c 'while ! (kubectl wait --for condition=ready backup.resources.cattle.io/'"${BACKUPRS_NAME}"' 2>/dev/null); do kubectl get backup.resources.cattle.io -A; kubectl -n cattle-resources-system logs -l app.kubernetes.io/name=rancher-backup --tail=-1; sleep 2; done' mc ls --quiet --no-color "miniolocal/${BACKUP}" FIRSTBACKUP=$(mc ls --quiet --no-color miniolocal/${BACKUP} | awk '{ print $NF }') @@ -119,20 +99,20 @@ for BACKUP in rancherbackups-insecure rancherbackups; do fi done # Disable the recurring back-ups by deleting the backup CRD - k3s kubectl delete "backup.resources.cattle.io/${BACKUPRS_NAME}" + kubectl delete "backup.resources.cattle.io/${BACKUPRS_NAME}" done # Restore resource with spec.preserveUnknownFields # https://github.com/rancher/backup-restore-operator/issues/186 -cd tests/files/preserve-unknown-fields +cd ./tests/files/preserve-unknown-fields tar cvzf /tmp/preserve-unknown-fields.tar.gz -- * cd - mc cp --quiet --no-color /tmp/preserve-unknown-fields.tar.gz miniolocal/rancherbackups mc ls --quiet --no-color miniolocal/rancherbackups -k3s kubectl create -f - </dev/null); do k3s kubectl get restore.resources.cattle.io -A; k3s kubectl -n cattle-resources-system logs -l app.kubernetes.io/name=rancher-backup --tail=15; sleep 5; done' +time timeout 60 bash -c 'while ! (kubectl wait --for condition=ready restore.resources.cattle.io/restore-preserve-unknown-fields 2>/dev/null); do kubectl get restore.resources.cattle.io -A; kubectl -n cattle-resources-system logs -l app.kubernetes.io/name=rancher-backup --tail=15; sleep 5; done' # Restore resource with metadata.deletionGracePeriodSeconds # https://github.com/rancher/backup-restore-operator/issues/188 -cd tests/files/deletion-grace-period-seconds +cd ./tests/files/deletion-grace-period-seconds tar cvzf /tmp/deletion-grace-period-seconds.tar.gz -- * cd - mc cp --quiet --no-color /tmp/deletion-grace-period-seconds.tar.gz miniolocal/rancherbackups @@ -165,7 +145,7 @@ mc ls --quiet --no-color miniolocal/rancherbackups for i in $(seq 1 2); do echo "Running restore #${i} with resource having metadata.deletionGracePeriodSeconds" - k3s kubectl create -f - </dev/null); do k3s kubectl get restore.resources.cattle.io -A; k3s kubectl -n cattle-resources-system logs -l app.kubernetes.io/name=rancher-backup --tail=15; sleep 5; done' - k3s kubectl delete restore.resources.cattle.io/restore-deletion-grace-period-seconds + time timeout 60 bash -c 'while ! (kubectl wait --for condition=ready restore.resources.cattle.io/restore-deletion-grace-period-seconds 2>/dev/null); do kubectl get restore.resources.cattle.io -A; kubectl -n cattle-resources-system logs -l app.kubernetes.io/name=rancher-backup --tail=15; sleep 5; done' + kubectl delete restore.resources.cattle.io/restore-deletion-grace-period-seconds done diff --git a/scripts/package b/scripts/package index e0a4b80f..3866713b 100755 --- a/scripts/package +++ b/scripts/package @@ -6,23 +6,20 @@ source $(dirname $0)/version cd $(dirname $0)/.. mkdir -p dist/artifacts -if [ "$CROSS_ARCH" != "true" ]; then - cp bin/backup-restore-operator dist/artifacts/backup-restore-operator${SUFFIX} -else - cp bin/backup-restore-operator${SUFFIX} dist/artifacts/ -fi IMAGE=${REPO}/backup-restore-operator:${TAG} -DOCKERFILE=package/Dockerfile +DOCKERFILE=./package/Dockerfile if [ -e ${DOCKERFILE}.${ARCH} ]; then DOCKERFILE=${DOCKERFILE}.${ARCH} fi if [[ ${USE_DOCKER_BUILDX} -eq 1 ]]; then - docker buildx build --build-arg K8S_VERSION_FROM_DRONE="${K8S_VERSION}" --platform linux/amd64 -f ${DOCKERFILE} . -t ${IMAGE} + docker buildx build --platform linux/amd64 -f ${DOCKERFILE} . -t ${IMAGE} else - docker build --build-arg K8S_VERSION_FROM_DRONE="${K8S_VERSION}" -f ${DOCKERFILE} -t ${IMAGE} . + docker build -f ${DOCKERFILE} -t ${IMAGE} . fi echo Built ${IMAGE} +docker image save rancher/backup-restore-operator:$TAG -o ./dist/artifacts/backup-restore-operator.img + ./scripts/package-helm From 0aefcadc1a1243b5a0d2528dfb0f46f5fcb6a102 Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Tue, 23 Apr 2024 13:19:49 -0400 Subject: [PATCH 05/12] Update dockerfile to not copy binary Signed-off-by: Alexandre Lamarre --- package/Dockerfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/package/Dockerfile b/package/Dockerfile index 188d4a4d..2c378e77 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -1,3 +1,10 @@ +FROM registry.suse.com/bci/golang:1.22 AS builder +WORKDIR /usr/src/app +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN ./scripts/build + FROM registry.suse.com/bci/bci-micro:latest -COPY bin/backup-restore-operator /usr/bin/ -ENTRYPOINT ["backup-restore-operator"] +COPY --from=builder /usr/src/app/bin/backup-restore-operator /usr/bin/ +ENTRYPOINT ["backup-restore-operator"] \ No newline at end of file From d045a2d7004ecf1ca5e76a3b0a5ce286f206e752 Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Tue, 23 Apr 2024 13:20:19 -0400 Subject: [PATCH 06/12] Introduce GHA for build & testing in CI Signed-off-by: Alexandre Lamarre --- .github/workflows/ci.yaml | 87 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..943b4b05 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,87 @@ +name: Backup Restore CI + +permissions: + contents : read + +on: + push: + branches: + - release/v5.0 + - release/v4.0 + - release/v3.0 + paths-ignore: + - 'docs/**' + - '*.md' + - '.gitignore' + - 'CODEOWNERS' + - 'LICENSE' + pull_request: + paths-ignore: + - 'docs/**' + - '*.md' + - '.gitignore' + - 'CODEOWNERS' + - 'LICENSE' +jobs: + build: + strategy: + matrix: + os : [ + ubuntu-latest + # org-rancher-arm64-containers + ] + name : CI + runs-on : ${{ matrix.os }} + outputs: + k3sversions: ${{ steps.support.outputs.k3sversions }} + steps: + - name : Checkout repository + uses : actions/checkout@v4 + with : + fetch-depth : 0 + - name: Set K3s support output + id : support + run: ./.github/workflows/scripts/supported-versions.sh | awk '{print "k3sversions="$0}' >> $GITHUB_OUTPUT + - name : CI + run : make ci + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: bro-build-artifacts-${{ matrix.os }} + path: ./dist/artifacts + + test: + name : integration-test + needs : [ + build + ] + runs-on : ${{ matrix.os }} + strategy: + matrix: + os : [ + ubuntu-latest + # org-rancher-arm64-containers, + ] + K3S_VERSION : ${{ fromJSON(needs.build.outputs.k3sversions) }} + steps: + - name : Checkout repository + uses : actions/checkout@v4 + - name : Fetch build artifacts + uses: actions/download-artifact@v4 + with: + name: bro-build-artifacts-${{ matrix.os }} + path: ./dist/artifacts/ + - name : Debug + run : ls -R ./dist/artifacts + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name : Load backup-restore-operator image + run : docker image load -i ./dist/artifacts/backup-restore-operator.img + - name : Setup up K3d + run : ./.github/workflows/scripts/install-k3d.sh + - name : Setup up mc + run : ./.github/workflows/scripts/install-mc.sh + - name : Setup k3d cluster + run : CLUSTER_NAME=backup-restore K3S_VERSION=${{ matrix.K3S_VERSION }} ./.github/workflows/scripts/setup-cluster.sh + - name : Run integration tests + run : ./scripts/integration \ No newline at end of file From c76cb834d568048068db9eb0dab89b4d24cd555d Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Tue, 23 Apr 2024 13:21:39 -0400 Subject: [PATCH 07/12] Introduce GHA for binary release CI Signed-off-by: Alexandre Lamarre --- .github/workflows/release.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000..89dc75ef --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,33 @@ +name: goreleaser + +on: + push: + tags: + - '*' + +permissions: + contents: write + +jobs: + ci: + uses: rancher/backup-restore/.github/workflows/ci.yaml@release/v5.0 + goreleaser: + needs: [ + ci + ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: git fetch --force --tags + - uses: actions/setup-go@v5 + with: + go-version: 1.22 + - uses: goreleaser/goreleaser-action@v5 + with: + distribution: goreleaser + version: latest + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From bba8c65a45cf1413af16e13300298086894a27df Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Tue, 23 Apr 2024 13:22:03 -0400 Subject: [PATCH 08/12] Introduce GHA for publishing images to docker Signed-off-by: Alexandre Lamarre --- .github/workflows/publish.yaml | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/publish.yaml diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 00000000..c6a5ff26 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,46 @@ +name : Publish Images + +on: + push: + tags: + - "*" + +env: + REGISTRY: docker.io + REPO : rancher + + +jobs: + ci : + uses: rancher/backup-restore/.github/workflows/ci.yaml@release/v5.0 + permissions: + contents: read + push: + permissions: + contents : read + id-token: write + name : Build and push BRO images + runs-on : ubuntu-latest + steps: + - name : Checkout repository + uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + # setup tag name + - if: ${{ startsWith(github.ref, 'refs/tags/') }} + run: | + echo TAG_NAME=$(echo $GITHUB_REF | sed -e "s|refs/tags/||") >> $GITHUB_ENV + - name: Build and push BRO image + uses: docker/build-push-action@v5 + with: + context: . + file: ./package/Dockerfile + push: true + tags: ${{ env.REGISTRY }}/${{ env.REPO }}/backup-restore-operator:${{ env.TAG_NAME }} + platforms: linux/amd64,linux/arm64 \ No newline at end of file From 11cdf6fb83efc0e3cf9dedaef5f3b3010d9010de Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Tue, 23 Apr 2024 13:22:50 -0400 Subject: [PATCH 09/12] document integration testing methods Signed-off-by: Alexandre Lamarre --- docs/integration-testing.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docs/integration-testing.md diff --git a/docs/integration-testing.md b/docs/integration-testing.md new file mode 100644 index 00000000..5ea67c6a --- /dev/null +++ b/docs/integration-testing.md @@ -0,0 +1,22 @@ +## Integration testing + +### Requirements + +- [mc](https://min.io/docs/minio/linux/reference/minio-mc.html), a command line client for minio +- [k3d](https://k3d.io/v5.6.3/), a command line tool for managing k3s clusters in docker + +See CI install scripts in `./.github/workflows/scripts/` + +### Running + +Set up a test cluster: + +```bash +CLUSTER_NAME="test-cluster" ./.github/workflows/scripts/setup-cluster.sh +``` + +Run: + +```bash +./scripts/integration +``` From 53fe15e5a3af3facc4c53f0fa6a4a97565fc2385 Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Tue, 23 Apr 2024 13:34:04 -0400 Subject: [PATCH 10/12] read vault secrets from EIO Signed-off-by: Alexandre Lamarre --- .github/workflows/publish.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index c6a5ff26..3b50150b 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -22,6 +22,12 @@ jobs: name : Build and push BRO images runs-on : ubuntu-latest steps: + - name : "Read vault secrets" + uses : rancher-eio/read-vault-secrets@main + with: + secrets: | + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials username | DOCKER_USERNAME ; + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials password | DOCKER_PASSWORD - name : Checkout repository uses: actions/checkout@v4 - name: Set up Docker Buildx @@ -30,8 +36,8 @@ jobs: uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} # setup tag name - if: ${{ startsWith(github.ref, 'refs/tags/') }} run: | From 3ecff94f7ff0c362d154e2a957481ba84690691b Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Thu, 25 Apr 2024 11:42:57 -0400 Subject: [PATCH 11/12] linting, style & typos Signed-off-by: Alexandre Lamarre --- .github/workflows/scripts/setup-cluster.sh | 10 +++++----- scripts/integration | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/scripts/setup-cluster.sh b/.github/workflows/scripts/setup-cluster.sh index 3e25a1bc..fa02f717 100755 --- a/.github/workflows/scripts/setup-cluster.sh +++ b/.github/workflows/scripts/setup-cluster.sh @@ -10,7 +10,7 @@ if [ -z "$CLUSTER_NAME" ]; then fi if [ -z "$K3S_VERSION" ]; then - echo "K3S_VERSION must be specified when setting up a cluster, use `k3d version list k3s` to find valid versions" + echo "K3S_VERSION must be specified when setting up a cluster, use $(k3d version list k3s) to find valid versions" exit 1 fi @@ -39,12 +39,12 @@ wait_for_nodes(){ done } -k3d cluster delete ${CLUSTER_NAME} || true -k3d cluster create ${CLUSTER_NAME} --image "docker.io/rancher/k3s:${K3S_VERSION}" +k3d cluster delete $CLUSTER_NAME || true +k3d cluster create $CLUSTER_NAME --image "docker.io/rancher/k3s:${K3S_VERSION}" wait_for_nodes -echo "${CLUSTER_NAME} ready" +echo "$CLUSTER_NAME ready" kubectl cluster-info --context k3d-${CLUSTER_NAME} kubectl config use-context k3d-${CLUSTER_NAME} @@ -52,4 +52,4 @@ kubectl get nodes -o wide IMAGE=${REPO}/backup-restore-operator:${TAG} -k3d image import ${IMAGE} -c ${CLUSTER_NAME} +k3d image import ${IMAGE} -c $CLUSTER_NAME diff --git a/scripts/integration b/scripts/integration index 8d76a2a9..b5e470bb 100755 --- a/scripts/integration +++ b/scripts/integration @@ -1,7 +1,7 @@ #!/bin/bash set -ex function echo_with_time { - echo "$(date --utc +%Y-%m-%dT%H:%M:%SZ) "$@"" + echo "$(date --utc +%Y-%m-%dT%H:%M:%SZ) $*" } source ./scripts/version @@ -10,7 +10,7 @@ echo_with_time "INFO - Running $0" ls -la ./dist/artifacts -# In case short commit only conists of numbers, it is regarded valid by Helm when packaging +# In case short commit only consists of numbers, it is regarded valid by Helm when packaging # Or if a tag is set (if its a (pre) release) if [[ $HELM_VERSION =~ ^[0-9]+$ ]] || [[ -n $GIT_TAG ]]; then HELM_CHART_VERSION=$HELM_VERSION From 8b3a8ff0c9cc623612abfb60c79a8d8de38a8928 Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Thu, 25 Apr 2024 11:54:56 -0400 Subject: [PATCH 12/12] add timeout to setup-cluster Signed-off-by: Alexandre Lamarre --- .github/workflows/scripts/setup-cluster.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/scripts/setup-cluster.sh b/.github/workflows/scripts/setup-cluster.sh index fa02f717..31c0a0fd 100755 --- a/.github/workflows/scripts/setup-cluster.sh +++ b/.github/workflows/scripts/setup-cluster.sh @@ -16,14 +16,29 @@ fi # waits until all nodes are ready wait_for_nodes(){ + timeout=120 + start_time=$(date +%s) echo "wait until all agents are ready" while : do + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ $elapsed_time -ge $timeout ]; then + echo "Timeout reached, exiting..." + exit 1 + fi + readyNodes=1 statusList=$(kubectl get nodes --no-headers | awk '{ print $2}') # shellcheck disable=SC2162 while read status do + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ $elapsed_time -ge $timeout ]; then + echo "Timeout reached, exiting..." + exit 1 + fi if [ "$status" == "NotReady" ] || [ "$status" == "" ] then readyNodes=0