diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 390b8bf607c..46773d2adc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,34 +31,31 @@ jobs: with: submodules: true - - name: Setup Go - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5 + - name: Setup Earthly + uses: earthly/actions-setup@v1 with: - go-version: ${{ env.GO_VERSION }} + github-token: ${{ secrets.GITHUB_TOKEN }} + version: "0.8.9" - - name: Find the Go Build Cache - id: go - run: echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT - - - name: Cache the Go Build Cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 - with: - path: ${{ steps.go.outputs.cache }} - key: ${{ runner.os }}-build-check-diff-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-build-check-diff- - - - name: Cache Go Dependencies - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 + - name: Login to DockerHub + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3 + if: env.DOCKER_USR != '' with: - path: .work/pkg - key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-pkg- + username: ${{ secrets.DOCKER_USR }} + password: ${{ secrets.DOCKER_PSW }} - - name: Download Go Modules - run: make modules.download modules.check + - name: Generate Files + run: earthly +generate - - name: Check Diff - run: make check-diff + - name: Count Changed Files + id: changed_files + run: echo "count=$(git status --porcelain | wc -l)" >> $GITHUB_OUTPUT + + - name: Fail if Files Changed + if: steps.changed_files.outputs.count != 0 + uses: actions/github-script + with: + script: core.setFailed('Found changed files after running earthly +generate.'') detect-noop: runs-on: ubuntu-22.04 @@ -90,35 +87,13 @@ jobs: with: go-version: ${{ env.GO_VERSION }} - - name: Find the Go Build Cache - id: go - run: echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT - - - name: Cache the Go Build Cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 - with: - path: ${{ steps.go.outputs.cache }} - key: ${{ runner.os }}-build-lint-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-build-lint- - - - name: Cache Go Dependencies - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 - with: - path: .work/pkg - key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-pkg- - - - name: Download Go Modules - run: make modules.download modules.check - - # We could run 'make lint' to ensure our desired Go version, but we prefer - # this action because it leaves 'annotations' (i.e. it comments on PRs to - # point out linter violations). + # We could run 'earthly _lint' to ensure our desired Go version, but we + # prefer this action because it leaves 'annotations' (i.e. it comments on + # PRs to point out linter violations). - name: Lint uses: golangci/golangci-lint-action@d6238b002a20823d52840fda27e2d4891c5952dc # v4 with: version: ${{ env.GOLANGCI_VERSION }} - skip-cache: true # We do our own caching. codeql: runs-on: ubuntu-22.04 @@ -145,7 +120,7 @@ jobs: with: path: ${{ steps.go.outputs.cache }} key: ${{ runner.os }}-build-check-diff-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-build-check-diff- + restore-keys: ${{ runner.os }}-build-codeql- - name: Cache Go Dependencies uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 @@ -154,9 +129,6 @@ jobs: key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-pkg- - - name: Download Go Modules - run: make modules.download modules.check - - name: Initialize CodeQL uses: github/codeql-action/init@b7cec7526559c32f1616476ff32d17ba4c59b2d6 # v3 with: @@ -196,43 +168,32 @@ jobs: with: submodules: true - - name: Fetch History - run: git fetch --prune --unshallow - - name: Setup Go uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5 with: go-version: ${{ env.GO_VERSION }} - - name: Find the Go Build Cache - id: go - run: echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT - - - name: Cache the Go Build Cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 + - name: Setup Earthly + uses: earthly/actions-setup@v1 with: - path: ${{ steps.go.outputs.cache }} - key: ${{ runner.os }}-build-unit-tests-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-build-unit-tests- + github-token: ${{ secrets.GITHUB_TOKEN }} + version: "0.8.9" - - name: Cache Go Dependencies - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 + - name: Login to DockerHub + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3 + if: env.DOCKER_USR != '' with: - path: .work/pkg - key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-pkg- - - - name: Download Go Modules - run: make modules.download modules.check + username: ${{ secrets.DOCKER_USR }} + password: ${{ secrets.DOCKER_PSW }} - name: Run Unit Tests - run: make -j2 test + run: earthly +test - name: Publish Unit Test Coverage uses: codecov/codecov-action@6d798873df2b1b8e5846dba6fb86631229fbcb17 # v4 with: flags: unittests - file: _output/tests/linux_amd64/coverage.txt + file: _output/tests/coverage.txt token: ${{ secrets.CODECOV_TOKEN }} e2e-tests: @@ -249,71 +210,27 @@ jobs: - ssa-claims steps: - - name: Setup QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3 - with: - platforms: all - - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3 - with: - version: ${{ env.DOCKER_BUILDX_VERSION }} - install: true - - name: Checkout uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: submodules: true - - name: Fetch History - run: git fetch --prune --unshallow - - - name: Setup Go - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5 + - name: Setup Earthly + uses: earthly/actions-setup@v1 with: - go-version: ${{ env.GO_VERSION }} - - - name: Find the Go Build Cache - id: go - run: echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT + github-token: ${{ secrets.GITHUB_TOKEN }} + version: "0.8.9" - - name: Cache the Go Build Cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 - with: - path: ${{ steps.go.outputs.cache }} - key: ${{ runner.os }}-build-e2e-tests-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-build-e2e-tests- - - - name: Cache Go Dependencies - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 + - name: Login to DockerHub + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3 + if: env.DOCKER_USR != '' with: - path: .work/pkg - key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-pkg- - - - name: Download Go Modules - run: make modules.download modules.check - - - name: Build Helm Chart - run: make -j2 build - env: - # We're using docker buildx, which doesn't actually load the images it - # builds by default. Specifying --load does so. - BUILD_ARGS: "--load" + username: ${{ secrets.DOCKER_USR }} + password: ${{ secrets.DOCKER_PSW }} - name: Run E2E Tests - run: make e2e E2E_TEST_FLAGS="-test.v -test.failfast -fail-fast --kind-logs-location ./logs-kind --test-suite ${{ matrix.test-suite }}" + run: earthly +e2e --FLAGS="-test.failfast -fail-fast --test-suite ${{ matrix.test-suite }}" - - name: Upload artifacts - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 - if: failure() - with: - name: e2e-kind-logs-${{ matrix.test-suite }} - path: ./logs-kind - if-no-files-found: error - retention-days: 7 - - name: Publish E2E Test Flakes if: '!cancelled()' uses: buildpulse/buildpulse-action@d0d30f53585cf16b2e01811a5a753fd47968654a # v0.11.0 @@ -330,67 +247,26 @@ jobs: if: needs.detect-noop.outputs.noop != 'true' steps: - - name: Cleanup Disk - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - with: - android: true - dotnet: true - haskell: true - tool-cache: true - large-packages: false - swap-storage: false - - - name: Setup QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3 - with: - platforms: all - - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3 - with: - version: ${{ env.DOCKER_BUILDX_VERSION }} - install: true - - name: Checkout uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: submodules: true - - name: Fetch History - run: git fetch --prune --unshallow - - - name: Setup Go - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5 - with: - go-version: ${{ env.GO_VERSION }} - - - name: Find the Go Build Cache - id: go - run: echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT - - - name: Cache the Go Build Cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 + - name: Setup Earthly + uses: earthly/actions-setup@v1 with: - path: ${{ steps.go.outputs.cache }} - key: ${{ runner.os }}-build-publish-artifacts-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-build-publish-artifacts- + github-token: ${{ secrets.GITHUB_TOKEN }} + version: "0.8.9" - - name: Cache Go Dependencies - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 + - name: Login to DockerHub + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3 + if: env.DOCKER_USR != '' with: - path: .work/pkg - key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-pkg- - - - name: Download Go Modules - run: make modules.download modules.check + username: ${{ secrets.DOCKER_USR }} + password: ${{ secrets.DOCKER_PSW }} - name: Build Artifacts - run: make -j2 build.all - env: - # We're using docker buildx, which doesn't actually load the images it - # builds by default. Specifying --load does so. - BUILD_ARGS: "--load" + run: earthly +multiplatform-build - name: Publish Artifacts to GitHub uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 @@ -398,12 +274,8 @@ jobs: name: output path: _output/** - - name: Login to DockerHub - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3 - if: env.DOCKER_USR != '' - with: - username: ${{ secrets.DOCKER_USR }} - password: ${{ secrets.DOCKER_PSW }} + - name: Build Image + run: earthly +multiplatform-image - name: Login to Upbound uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3 @@ -413,27 +285,6 @@ jobs: username: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }} password: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW }} - - name: Publish Artifacts to S3, Marketplace, DockerHub - run: make -j2 publish BRANCH_NAME=${GITHUB_REF##*/} - if: env.AWS_USR != '' && env.DOCKER_USR != '' && env.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR != '' - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_USR }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PSW }} - AWS_DEFAULT_REGION: us-east-1 - GIT_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCS_GIT_USR: ${{ secrets.UPBOUND_BOT_GITHUB_USR }} - DOCS_GIT_PSW: ${{ secrets.UPBOUND_BOT_GITHUB_PSW }} - - - name: Promote Artifacts in S3, DockerHub - if: github.ref == 'refs/heads/master' && env.AWS_USR != '' && env.DOCKER_USR != '' && env.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR != '' - run: make -j2 promote - env: - BRANCH_NAME: master - CHANNEL: master - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_USR }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PSW }} - AWS_DEFAULT_REGION: us-east-1 - fuzz-test: runs-on: ubuntu-22.04 needs: detect-noop diff --git a/Earthfile b/Earthfile new file mode 100644 index 00000000000..0ee38346ce2 --- /dev/null +++ b/Earthfile @@ -0,0 +1,232 @@ +# The --try flag enables the experimental TRY FINALLY block. +VERSION --try 0.8 + +# TODO(negz): Figure out how to produce versions derived from git tags. Earthly +# only exposes a limited set of git metadata. We can't produce versions +# identical to those produced by the build submodule. Deriving versions from git +# metadata can also invalidate cached layers unnecessarily. +# https://docs.earthly.dev/docs/earthfile/builtin-args#git-related-args +# https://docs.earthly.dev/docs/earthfile#global + +reviewable: + BUILD +generate + BUILD +lint + BUILD +test + +test: + BUILD +go-test + +lint: + BUILD +go-lint + BUILD +helm-lint + +build: + BUILD +go-build + BUILD +helm-build + +generate: + BUILD +go-generate + BUILD +helm-generate + +go-modules: + ARG NATIVEPLATFORM + ARG TARGETARCH + ARG TARGETOS + ARG GOARCH=${TARGETARCH} + ARG GOOS=${TARGETOS} + FROM --platform=${NATIVEPLATFORM} golang:1.22 + WORKDIR /crossplane + COPY go.mod go.sum ./ + RUN \ + --mount=type=cache,sharing=private,target=/go/pkg/mod,id=go-modules-${GOOS}-${GOARCH} \ + go mod download + COPY --dir apis/ cmd/ internal/ pkg/ test/ . + RUN \ + --mount=type=cache,sharing=private,target=/go/pkg/mod,id=go-modules-${GOOS}-${GOARCH} \ + go mod tidy + RUN \ + --mount=type=cache,sharing=private,target=/go/pkg/mod,id=go-modules-${GOOS}-${GOARCH} \ + go mod verify + SAVE ARTIFACT go.mod AS LOCAL go.mod + SAVE ARTIFACT go.sum AS LOCAL go.sum + +go-generate: + ARG TARGETARCH + ARG TARGETOS + ARG GOARCH=${TARGETARCH} + ARG GOOS=${TARGETOS} + FROM +go-modules + COPY --dir hack/ . + RUN \ + --mount=type=cache,target=/go/pkg/mod,id=go-modules-${GOOS}-${GOARCH} \ + --mount=type=cache,target=/root/.cache/go-build,id=go-build-${GOOS}-${GOARCH} \ + go generate -tags 'generate' ./apis/... + SAVE ARTIFACT apis/ AS LOCAL apis + +go-build: + ARG NATIVEPLATFORM + ARG TARGETARCH + ARG TARGETOS + ARG GOARCH=${TARGETARCH} + ARG GOOS=${TARGETOS} + ARG CGO_ENABLED=0 + FROM +go-modules + COPY --platform=${NATIVEPLATFORM} (+go-generate/apis --GOOS=${TARGETOS} --GOARCH=${TARGETARCH}) ./ + RUN \ + --mount=type=cache,target=/go/pkg/mod,id=go-modules-${GOOS}-${GOARCH} \ + --mount=type=cache,target=/root/.cache/go-build,id=go-build-${GOOS}-${GOARCH} \ + go build -o crossplane ./cmd/crossplane + RUN \ + --mount=type=cache,sharing=private,target=/go/pkg/mod,id=go-modules-${GOOS}-${GOARCH} \ + --mount=type=cache,sharing=private,target=/root/.cache/go-build,id=go-build-${GOOS}-${GOARCH} \ + go build -o crank ./cmd/crank + SAVE ARTIFACT crossplane AS LOCAL _output/bin/${GOOS}_${GOARCH}/crossplane + SAVE ARTIFACT crank AS LOCAL _output/bin/${GOOS}_${GOARCH}/crank + +go-multiplatform-build: + BUILD \ + --platform=linux/amd64 \ + --platform=linux/arm64 \ + --platform=linux/arm \ + --platform=linux/ppc64le \ + --platform=darwin/arm64 \ + --platform=darwin/amd64 \ + --platform=windows/amd64 \ + +go-build + +# TODO(negz): Test ./... - this hacks around the fact that ./pkg tests are broken. +go-test: + ARG TARGETARCH + ARG TARGETOS + ARG GOARCH=${TARGETARCH} + ARG GOOS=${TARGETOS} + FROM +go-modules + RUN \ + --mount=type=cache,sharing=private,target=/go/pkg/mod,id=go-modules-${GOOS}-${GOARCH} \ + --mount=type=cache,sharing=private,target=/root/.cache/go-build,id=go-build-${GOOS}-${GOARCH} \ + go test -covermode=count -coverprofile=coverage.txt ./apis/... ./cmd/... ./internal/... + SAVE ARTIFACT coverage.txt AS LOCAL _output/tests/coverage.txt + +go-lint: + ARG TARGETARCH + ARG TARGETOS + ARG GOARCH=${TARGETARCH} + ARG GOOS=${TARGETOS} + FROM golangci/golangci-lint:v1.57.2 + WORKDIR /crossplane + COPY .golangci.yml . + COPY go.mod go.sum . + COPY --dir apis/ cmd/ internal/ pkg/ test/ . + RUN \ + --mount=type=cache,target=/go/pkg/mod,id=go-modules-${GOOS}-${GOARCH} \ + --mount=type=cache,target=/root/.cache/go-build,id=go-build-${GOOS}-${GOARCH} \ + --mount=type=cache,target=/root/.cache/golangci-lint \ + golangci-lint run + +image: + ARG NATIVEPLATFORM + ARG TARGETPLATFORM + ARG TARGETARCH + ARG TARGETOS + ARG EARTHLY_GIT_SHORT_HASH + ARG IMAGE_VERSION=v0.0.0-${EARTHLY_GIT_SHORT_HASH} + FROM --platform=${TARGETPLATFORM} gcr.io/distroless/static@sha256:41972110a1c1a5c0b6adb283e8aa092c43c31f7c5d79b8656fbffff2c3e61f05 + COPY --platform=${NATIVEPLATFORM} (+go-build/crossplane --GOOS=${TARGETOS} --GOARCH=${TARGETARCH}) /usr/local/bin/ + COPY --dir cluster/crds/ /crds + COPY --dir cluster/webhookconfigurations/ /webhookconfigurations + EXPOSE 8080 + USER 65532 + ENTRYPOINT ["crossplane"] + + SAVE IMAGE crossplane:${IMAGE_VERSION} + +multiplatform-image: + BUILD \ + --platform=linux/amd64 \ + --platform=linux/arm64 \ + --platform=linux/arm \ + --platform=linux/ppc64le \ + +image + +helm-setup: + ARG HELM_VERSION=v3.14.4 + ARG HELM_DOCS_VERSION=v1.11.0 + ARG CGO_ENABLED=0 + FROM golang:1.22 + WORKDIR /chart + RUN \ + --mount=type=cache,target=/go/pkg/mod,id=helm-go-modules \ + --mount=type=cache,target=/root/.cache/go-build,id=helm-go-build \ + go install helm.sh/helm/v3/cmd/helm@${HELM_VERSION} + RUN \ + --mount=type=cache,target=/go/pkg/mod,id=helm-go-modules \ + --mount=type=cache,target=/root/.cache/go-build,id=helm-go-build \ + go install github.com/norwoodj/helm-docs/cmd/helm-docs@${HELM_DOCS_VERSION} + COPY cluster/charts/crossplane/ . + SAVE ARTIFACT /go/bin/helm + SAVE ARTIFACT /go/bin/helm-docs + +helm-lint: + FROM +helm-setup + RUN helm lint + +helm-generate: + FROM +helm-setup + RUN helm-docs + SAVE ARTIFACT . AS LOCAL cluster/charts/crossplane + +helm-build: + ARG EARTHLY_GIT_SHORT_HASH + ARG HELM_CHART_VERSION=0.0.0+${EARTHLY_GIT_SHORT_HASH} + FROM +helm-setup + RUN helm dependency update + RUN helm package --version ${HELM_CHART_VERSION} --app-version ${HELM_CHART_VERSION} -d output . + SAVE ARTIFACT output AS LOCAL _output/charts + +kind-setup: + FROM alpine:3.19 + ARG NATIVEOS + ARG NATIVEARCH + ARG KIND_VERSION=v0.21.0 + RUN apk add --no-cache curl + RUN curl -fsSLo kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${NATIVEOS}-${NATIVEARCH}&&chmod +x kind + SAVE ARTIFACT kind + +gotestsum-setup: + ARG GOTESTSUM_VERSION=v1.11.0 + ARG CGO_ENABLED=0 + FROM golang:1.22 + RUN \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + go install gotest.tools/gotestsum@${GOTESTSUM_VERSION} + SAVE ARTIFACT /go/bin/gotestsum + +# TODO(negz): We only compile the e2e test binary because it affects the path of +# the manifests in the test code. We could let gotestsum just invoke go test if +# we stripped "test/e2e" from the beginning of the paths. +e2e: + ARG FLAGS + ARG TARGETARCH + ARG TARGETOS + ARG GOARCH=${TARGETARCH} + ARG GOOS=${TARGETOS} + ARG CGO_ENABLED=0 + FROM +go-modules + DO github.com/earthly/lib+INSTALL_DIND + COPY +go-generate/apis ./ + COPY +helm-setup/helm /usr/local/bin/helm + COPY +kind-setup/kind /usr/local/bin/kind + COPY +gotestsum-setup/gotestsum /usr/local/bin/gotestsum + COPY --dir cluster . + RUN \ + --mount=type=cache,sharing=private,target=/go/pkg/mod,id=go-modules-${GOOS}-${GOARCH} \ + --mount=type=cache,sharing=private,target=/root/.cache/go-build,id=go-build-${GOOS}-${GOARCH} \ + go test -c -o e2e ./test/e2e + WITH DOCKER --load crossplane-e2e/crossplane:latest=+image + TRY + RUN gotestsum --format testname --junitfile e2e-tests.xml --raw-command go tool test2json -t -p E2E ./e2e -test.v ${FLAGS} + FINALLY + SAVE ARTIFACT e2e-tests.xml AS LOCAL _output/tests/e2e-tests.xml + END + END