From 66e7badccbfbbdf136a86e881ad6552d958712cb Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Fri, 2 Feb 2024 16:10:58 -0800 Subject: [PATCH] ci: remote multiplatform builder --- .github/workflows/ci.yml | 149 +++--------------- depot.json | 1 + package.json | 6 +- packages/synthetic-chain/cli.ts | 25 ++- packages/synthetic-chain/docker-bake.hcl | 3 + packages/synthetic-chain/src/cli/build.ts | 5 +- .../synthetic-chain/src/cli/dockerfileGen.ts | 10 +- 7 files changed, 61 insertions(+), 138 deletions(-) create mode 100644 depot.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f42b615..f6531523 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,40 +32,20 @@ concurrency: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} - # Name these so they look less similar than AMD/ARM; omit 64 as uninformative. - X86_PLATFORM: linux/amd64 - ARM_PLATFORM: linux/arm64/v8 jobs: - platforms: - runs-on: ubuntu-latest - outputs: - platforms: '${{ steps.platforms.outputs.platforms }}' - steps: - - name: Compute Docker platforms - id: platforms - run: | - if ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' }}; then - # JSON-encoded list consisting only of the default platform. - platforms='["'"$X86_PLATFORM"'"]' - else - platforms='["$X86_PLATFORM","$ARM_PLATFORM"]' - fi - echo "platforms=$platforms" >> $GITHUB_OUTPUT - # see https://docs.docker.com/build/ci/github-actions/test-before-push/ test-proposals: - needs: [platforms] - # UNTIL https://github.com/Agoric/agoric-3-proposals/issues/2 - timeout-minutes: 120 - strategy: - matrix: - platform: ${{ fromJSON(needs.platforms.outputs.platforms) }} - # Run on our own self-hosted ARM64 machine if the platform is ARMish. - runs-on: ${{ contains(matrix.platform, '/arm') && fromJSON('["self-hosted","Linux","ARM64"]') || 'ubuntu-latest' }} + permissions: + # allow issuing OIDC tokens for this workflow run + id-token: write + # allow at least reading the repo contents, add other permissions if necessary + contents: read + # to push the resulting images + packages: write + runs-on: 'ubuntu-latest' steps: - name: free up disk space - if: ${{ !contains(matrix.platform, '/arm') }} run: | # Workaround to provide additional free space for testing. # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 @@ -82,11 +62,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - uses: depot/setup-action@v1 + with: + oidc: true # to set DEPOT_TOKEN for later steps - - name: Set up QEMU for cross-platform builds - uses: docker/setup-qemu-action@v3 + # make Docker's CLI use depot to build + - run: depot configure-docker - name: Log in to the Container registry uses: docker/login-action@v3 @@ -102,24 +83,6 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Compute Docker tags - id: docker-tags - run: | - sep= - # A list of comma-separated tags to merge in the final image. - SUFFIXED= - # Our platform, replacing slashes with underscores. - uarch=$(echo "${{ matrix.platform }}" | tr / _) - for TAG in ${{ steps.meta.outputs.tags }}; do - SUFFIXED="$SUFFIXED$sep$TAG-$uarch" - if test -z "$sep"; then - # The first tag (suffixed with our architecture) is the one we build. - sep=, - echo "tag=$TAG-$uarch" | tee -a $GITHUB_OUTPUT - fi - done - echo "tags=$SUFFIXED" | tee -a $GITHUB_OUTPUT - - name: Setup synthetic-chain run: | # The .ts scripts depend upon this @@ -128,25 +91,17 @@ jobs: corepack enable || sudo corepack enable yarn install - - run: docker system df - - run: docker buildx du --verbose - - run: df -h + - name: Build proposal "use" images + run: | + tsx packages/synthetic-chain prepare-build + depot bake use # Test before pushing the images. - name: Build and run proposal tests - if: ${{ matrix.platform == env.X86_PLATFORM }} run: yarn test - - run: docker system df - - run: docker buildx du --verbose - - run: df -h - - # Build a "use" image for each proposal. This uses Docker Bake's - # matrix feature. We could have each "use" image built in a different runner - # by including https://github.com/docker/bake-action?tab=readme-ov-file#list-targets - # in the GHA matrix, but that wouldn't be able to resolve the DAG of what to build first. - name: Push proposal "use" images - uses: docker/bake-action@v4 + uses: depot/bake-action@v1 # If we pushed from PRs, each one would overwrite main's (e.g. use-upgrade-8) # To push PR "use" images we'll need to qualify the tag (e.g. use-upgrade-8-pr-2). if: ${{ github.event_name != 'pull_request' }} @@ -156,75 +111,7 @@ jobs: ./docker-bake.hcl ${{ steps.meta.outputs.bake-file }} targets: use - - - run: docker system df - - run: docker buildx du --verbose - - run: df -h - - - name: Build and push default image - uses: docker/build-push-action@v5 - with: - context: . - platforms: ${{ matrix.platform }} - # push to registry on every repo push. A PR #2 will push with tag `pr-2` and `main` will have tag `main`. - # See https://github.com/docker/metadata-action?tab=readme-ov-file#basic. push: true - tags: ${{ steps.docker-tags.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - - run: docker system df - - run: docker buildx du --verbose - - run: df -h - - # Merge the default image from each platform into one multi-arch image, - # then publish that multiarch image. - docker-publish-multiarch: - needs: [test-proposals, platforms] - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - buildkitd-flags: --debug - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: ${{ env.REGISTRY }} - - name: Compute tags - id: docker-tags - run: | - echo "tags=${{ steps.meta.outputs.tags }}" >> $GITHUB_OUTPUT - - - name: Push multiarch image - run: | - set -ex - # Push all tags, comprised of all architectures, to the registry. - for TAG in ${{ steps.docker-tags.outputs.tags }}; do - sources= - for ARCH in ${{ join(fromJson(needs.platforms.outputs.platforms), ' ') }}; do - uarch=$(echo "$ARCH" | tr / _) - BUILD_TAG="$TAG-$uarch" - sources="$sources $BUILD_TAG" - done - docker buildx imagetools create --tag "$TAG"$sources - done - name: notify on failure if: failure() && github.event_name != 'pull_request' diff --git a/depot.json b/depot.json new file mode 100644 index 00000000..591f93f8 --- /dev/null +++ b/depot.json @@ -0,0 +1 @@ +{"id":"bqgtmlhmh8"} diff --git a/package.json b/package.json index 650a4c03..70025d6f 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,11 @@ "@agoric/synthetic-chain": "workspace:*" }, "agoricSyntheticChain": { - "fromTag": null + "fromTag": null, + "platforms": [ + "linux/amd64", + "linux/arm64" + ] }, "license": "Apache-2.0", "packageManager": "yarn@4.0.2", diff --git a/packages/synthetic-chain/cli.ts b/packages/synthetic-chain/cli.ts index 62882649..a1e3d257 100755 --- a/packages/synthetic-chain/cli.ts +++ b/packages/synthetic-chain/cli.ts @@ -41,7 +41,9 @@ const proposals = match const [cmd] = positionals; // TODO consider a lib like Commander for auto-gen help -const usage = `USAGE: +const USAGE = `USAGE: +prepare-build - generate Docker build configs + build - build the synthetic-chain "use" images test [--debug] - build the "test" images and run them @@ -50,6 +52,15 @@ test -m - target a particular proposal by substring match doctor - diagnostics and quick fixes `; +const EXPLAIN_MULTIPLATFORM = ` +ERROR: docker exporter does not currently support exporting manifest lists + +Multiple platforms are configured but Docker does not support multiplatform in one builder. +Until https://github.com/docker/roadmap/issues/371, attempting it will error as above. + +Instead use a builder that supports multiplatform such as depot.dev. +`; + /** * Put into places files that building depends upon. */ @@ -58,7 +69,7 @@ const prepareDockerBuild = () => { // copy and generate files of the build context that aren't in the build contents execSync(`cp -r ${path.resolve(cliPath, '..', 'docker-bake.hcl')} .`); writeDockerfile(allProposals, buildConfig.fromTag); - writeBakefileProposals(allProposals); + writeBakefileProposals(allProposals, buildConfig.platforms); // copy and generate files to include in the build execSync(`cp -r ${path.resolve(cliPath, '..', 'upgrade-test-scripts')} .`); buildProposalSubmissions(proposals); @@ -70,8 +81,16 @@ const prepareDockerBuild = () => { }; switch (cmd) { + case 'prepare-build': + prepareDockerBuild(); + break; case 'build': { prepareDockerBuild(); + // do not encapsulate running Depot. It's a special case which the user should understand. + if (buildConfig.platforms) { + console.error(EXPLAIN_MULTIPLATFORM); + process.exit(1); + } bakeTarget('use', values.dry); break; } @@ -103,5 +122,5 @@ switch (cmd) { runDoctor(allProposals); break; default: - console.log(usage); + console.log(USAGE); } diff --git a/packages/synthetic-chain/docker-bake.hcl b/packages/synthetic-chain/docker-bake.hcl index 75f2e728..17358f35 100644 --- a/packages/synthetic-chain/docker-bake.hcl +++ b/packages/synthetic-chain/docker-bake.hcl @@ -14,9 +14,11 @@ group "default" { ] } +// Images to use the result of all local proposals, optionally built multi-platform target "use" { inherits = ["docker-metadata-action"] name = "use-${proposal}" + platforms = PLATFORMS matrix = { proposal = PROPOSALS } @@ -29,6 +31,7 @@ target "use" { target = "use-${proposal}" } +// Image to test the result of a proposal, always current platform target "test" { name = "test-${proposal}" matrix = { diff --git a/packages/synthetic-chain/src/cli/build.ts b/packages/synthetic-chain/src/cli/build.ts index 3b0570a4..6d7aebaf 100755 --- a/packages/synthetic-chain/src/cli/build.ts +++ b/packages/synthetic-chain/src/cli/build.ts @@ -3,6 +3,8 @@ import fs from 'node:fs'; import path from 'node:path'; import { ProposalInfo } from './proposals.js'; +export type Platform = 'linux/amd64' | 'linux/arm64'; + export type AgoricSyntheticChainConfig = { /** * The agoric-3-proposals tag to build the agoric synthetic chain from. @@ -10,6 +12,7 @@ export type AgoricSyntheticChainConfig = { * Defaults to `main`, which containing all passed proposals */ fromTag: string | null; + platforms?: Platform[]; }; const defaultConfig: AgoricSyntheticChainConfig = { @@ -66,7 +69,7 @@ export const buildProposalSubmissions = (proposals: ProposalInfo[]) => { /** * Bake images using the docker buildx bake command. - * + * * Note this uses `--load` which pushes the completed images to the builder, * consuming 2-3 GB per image. * @see {@link https://docs.docker.com/engine/reference/commandline/buildx_build/#load} diff --git a/packages/synthetic-chain/src/cli/dockerfileGen.ts b/packages/synthetic-chain/src/cli/dockerfileGen.ts index 058d3837..ae69c03b 100755 --- a/packages/synthetic-chain/src/cli/dockerfileGen.ts +++ b/packages/synthetic-chain/src/cli/dockerfileGen.ts @@ -3,7 +3,6 @@ import fs from 'node:fs'; import { - lastPassedProposal, type CoreEvalProposal, type ProposalInfo, type SoftwareUpgradeProposal, @@ -11,6 +10,7 @@ import { imageNameForProposal, isPassed, } from './proposals.js'; +import { Platform } from './build.ts'; /** * Templates for Dockerfile stages @@ -182,9 +182,15 @@ ENTRYPOINT ./start_agd.sh }, }; -export function writeBakefileProposals(allProposals: ProposalInfo[]) { +export function writeBakefileProposals( + allProposals: ProposalInfo[], + platforms?: Platform[], +) { const json = { variable: { + PLATFORMS: { + default: platforms || null, + }, PROPOSALS: { default: allProposals.map(p => p.proposalName), },