From 376f31de215a677ad874adebe4c179be675c6ec3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 10 Apr 2024 23:57:38 +0200 Subject: [PATCH 001/139] ci(ct): do no longer run base push workflow in PRs --- .github/workflows/container_base_push.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index b938851f816..20083c90f33 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -3,17 +3,11 @@ name: Base Container Image on: push: + tags: + - 'v[6-9].*' branches: - 'develop' - - 'master' - paths: - - 'modules/container-base/**' - - 'modules/dataverse-parent/pom.xml' - - '.github/workflows/container_base_push.yml' - pull_request: - branches: - - 'develop' - - 'master' + - '10478-version-base-img' paths: - 'modules/container-base/**' - 'modules/dataverse-parent/pom.xml' From b8b95a995b3cebab201577429622ba086a81634e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:05:25 +0200 Subject: [PATCH 002/139] ci(ct): add a job for base images to discover branches to work on For scheduled maintenance, we will provide updates to the last three base images as well as the develop branch. --- .github/workflows/container_base_push.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 20083c90f33..41935a025a8 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -20,6 +20,25 @@ env: PLATFORMS: linux/amd64,linux/arm64 jobs: + discover: + name: Discover Release Matrix + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled + if: ${{ github.repository_owner == 'IQSS' }} + steps: + - name: Build branch matrix options + id: matrix + run: | + # Get last three releases and include develop branch as matrix elements + if [[ "${{ github.event_name }}" == "schedule" ]]; then + echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq '[ .[0:3] | .[].tag_name, "develop" ]')" | tee -a "$GITHUB_OUTPUT" + else + echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" + fi + build: name: Build image runs-on: ubuntu-latest From 6f046208086bc7f3bb4078c92b6ae8723f78b030 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:06:24 +0200 Subject: [PATCH 003/139] ci(ct): setup base image build job as matrix of discovered branches Checkout the branch/tag we receive from the discovery job --- .github/workflows/container_base_push.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 41935a025a8..b424936383c 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -45,15 +45,18 @@ jobs: permissions: contents: read packages: read + needs: discover strategy: matrix: - jdk: [ '17' ] + branch: ${{ fromJson(needs.discover.outputs.branches) }} # Only run in upstream repo - avoid unnecessary runs in forks if: ${{ github.repository_owner == 'IQSS' }} steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 + with: + ref: ${{ matrix.branch }} - name: Set up JDK ${{ matrix.jdk }} uses: actions/setup-java@v3 From 4fc0267296944a7dec92e1a70a651649cf9941db Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:07:40 +0200 Subject: [PATCH 004/139] ci(ct): upgrade java action in base image and determine java version from Maven Using Maven properties, we find out which is the projects preferred Java version. We don't need a build matrix here. --- .github/workflows/container_base_push.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index b424936383c..970e67794c2 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -57,14 +57,19 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ matrix.branch }} + - name: Determine Java version from package + run: | + echo "JAVA_VERSION=$(mvn -f modules/container-base -Pct help:evaluate -Dexpression=target.java.version -q -DforceStdout)" >> ${GITHUB_ENV} - - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v3 + - name: Set up JDK ${{ env.JAVA_VERSION }} + uses: actions/setup-java@v4 with: - java-version: ${{ matrix.jdk }} - distribution: 'adopt' - - name: Cache Maven packages - uses: actions/cache@v3 + java-version: ${{ env.JAVA_VERSION }} + distribution: 'temurin' + cache: 'maven' + cache-dependency-path: | + modules/container-base/pom.xml + with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} From 8d3097181296f70e1948adf43c2823862e8cb1fd Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:08:57 +0200 Subject: [PATCH 005/139] ci(ct): remove pull request safeguards from base image workflow We no longer work on PRs in this workflow, so we don't need to safeguard against not running steps in this case. --- .github/workflows/container_base_push.yml | 33 +++++++++-------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 970e67794c2..31a044ef02a 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -78,8 +78,18 @@ jobs: - name: Build base container image with local architecture run: mvn -f modules/container-base -Pct package - # Run anything below only if this is not a pull request. - # Accessing, pushing tags etc. to DockerHub will only succeed in upstream because secrets. + # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and + # on events in context of upstream because secrets. PRs run in context of forks by default! + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU for multi-arch builds + uses: docker/setup-qemu-action@v2 + - name: Deploy multi-arch base container image to Docker Hub + run: mvn -f modules/container-base -Pct deploy -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} name: Push description to DockerHub @@ -90,23 +100,6 @@ jobs: repository: gdcc/base short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" readme-filepath: ./modules/container-base/README.md - - - if: ${{ github.event_name != 'pull_request' }} - name: Log in to the Container registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - if: ${{ github.event_name != 'pull_request' }} - name: Set up QEMU for multi-arch builds - uses: docker/setup-qemu-action@v2 - - name: Re-set image tag based on branch - if: ${{ github.ref_name == 'master' }} - run: echo "IMAGE_TAG=alpha" >> $GITHUB_ENV - - if: ${{ github.event_name != 'pull_request' }} - name: Deploy multi-arch base container image to Docker Hub - run: mvn -f modules/container-base -Pct deploy -Dbase.image.tag=${{ env.IMAGE_TAG }} -Ddocker.platforms=${{ env.PLATFORMS }} push-app-img: name: "Rebase & Publish App Image" permissions: @@ -115,6 +108,6 @@ jobs: pull-requests: write needs: build # We do not release a new base image for pull requests, so do not trigger. - if: ${{ github.event_name != 'pull_request' }} + # if: ${{ github.event_name != 'pull_request' }} uses: ./.github/workflows/container_app_push.yml secrets: inherit From bb06a94e27e6edf6f9ccd11dbf6209d90b7d4dc2 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:09:43 +0200 Subject: [PATCH 006/139] ci(ct): remove pull request safeguards from base image workflow We no longer work on PRs in this workflow, so we don't need to safeguard against not running steps in this case. Also we no longer use the static image names, but rely on the truth Maven has in the container-base module about the version of the image. --- .github/workflows/container_base_push.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 31a044ef02a..4295280b639 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -16,7 +16,6 @@ on: - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC env: - IMAGE_TAG: unstable PLATFORMS: linux/amd64,linux/arm64 jobs: From 0ba86b2e4f68709d49c72a99580ec5c5b1ac4d5a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:10:23 +0200 Subject: [PATCH 007/139] ci(ct): stop on purpose to test the feature branch before we actually push to Docker Hub --- .github/workflows/container_base_push.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 4295280b639..bfccefaf9eb 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -69,13 +69,11 @@ jobs: cache-dependency-path: | modules/container-base/pom.xml + - name: Stop on purpose for testing + uses: actions/github-script@v3 with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - - name: Build base container image with local architecture - run: mvn -f modules/container-base -Pct package + script: | + core.setFailed('Stopped on purpose.') # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and # on events in context of upstream because secrets. PRs run in context of forks by default! From 37f4a51b2ff0341f7bb6d930614f04c2238c213b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:11:00 +0200 Subject: [PATCH 008/139] feat(ct): make the container base image follow sequential, incrementing versions --- modules/container-base/pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index fc672696df4..2754b081986 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -12,6 +12,8 @@ io.gdcc container-base + + 1 ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. @@ -40,7 +42,7 @@ docker-build gdcc/base:${base.image.tag} - unstable + R${project.version} eclipse-temurin:${target.java.version}-jre 1000 1000 From 20c6a58915711efd9e18d8454813623ead9dbb9b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:17:47 +0200 Subject: [PATCH 009/139] ci(ct): add missing output definitions to base image workflow matrix discovery --- .github/workflows/container_base_push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index bfccefaf9eb..29abab230e3 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -27,6 +27,8 @@ jobs: packages: read # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled if: ${{ github.repository_owner == 'IQSS' }} + outputs: + branches: ${{ steps.matrix.outputs.branches }} steps: - name: Build branch matrix options id: matrix From e9236638eb2921b46318c3a4fb51e663ec021a56 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:20:41 +0200 Subject: [PATCH 010/139] test(ct): for testing, determine if matrix discovery works for schedule event --- .github/workflows/container_base_push.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 29abab230e3..ab91252efc6 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -34,11 +34,11 @@ jobs: id: matrix run: | # Get last three releases and include develop branch as matrix elements - if [[ "${{ github.event_name }}" == "schedule" ]]; then + #if [[ "${{ github.event_name }}" == "schedule" ]]; then echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq '[ .[0:3] | .[].tag_name, "develop" ]')" | tee -a "$GITHUB_OUTPUT" - else - echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" - fi + #else + # echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" + #fi build: name: Build image From def5d9e2842c09c3d647619f0cea266528c71a82 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:23:56 +0200 Subject: [PATCH 011/139] fix(ct): base image build matrix output must be on a single line Fix jq output by removing newlines and superfluous whitespace. --- .github/workflows/container_base_push.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index ab91252efc6..705b152d1bc 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -35,7 +35,9 @@ jobs: run: | # Get last three releases and include develop branch as matrix elements #if [[ "${{ github.event_name }}" == "schedule" ]]; then - echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq '[ .[0:3] | .[].tag_name, "develop" ]')" | tee -a "$GITHUB_OUTPUT" + echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ + jq '[ .[0:3] | .[].tag_name, "develop" ]')" | tr -d "\n" | tr -s " " | \ + tee -a "$GITHUB_OUTPUT" #else # echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" #fi From b028c2610db5688bf443f436e1bbdb1c7c8fcb6c Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:27:06 +0200 Subject: [PATCH 012/139] fix(ct): do not stop building other base images if one build fails --- .github/workflows/container_base_push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 705b152d1bc..f36c5a8de2d 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -50,6 +50,7 @@ jobs: packages: read needs: discover strategy: + fail-fast: false matrix: branch: ${{ fromJson(needs.discover.outputs.branches) }} # Only run in upstream repo - avoid unnecessary runs in forks From b453a12b780c4149a3ca2ffb5469d6cfb85c5290 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:45:07 +0200 Subject: [PATCH 013/139] fix(ct): avoid costly lookup of Java version to be used via Maven, grep from XML instead --- .github/workflows/container_base_push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index f36c5a8de2d..d6af36702f6 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -61,9 +61,9 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ matrix.branch }} - - name: Determine Java version from package + - name: Determine Java version from Parent POM run: | - echo "JAVA_VERSION=$(mvn -f modules/container-base -Pct help:evaluate -Dexpression=target.java.version -q -DforceStdout)" >> ${GITHUB_ENV} + echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} - name: Set up JDK ${{ env.JAVA_VERSION }} uses: actions/setup-java@v4 From cb2fac02fb9370f565ef613c0f2e57f5ce7957a0 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:48:19 +0200 Subject: [PATCH 014/139] test(ct): no longer stop before Docker, but tell DMP to skip pushing --- .github/workflows/container_base_push.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index d6af36702f6..24b2640f3e3 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -74,12 +74,6 @@ jobs: cache-dependency-path: | modules/container-base/pom.xml - - name: Stop on purpose for testing - uses: actions/github-script@v3 - with: - script: | - core.setFailed('Stopped on purpose.') - # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and # on events in context of upstream because secrets. PRs run in context of forks by default! @@ -91,7 +85,7 @@ jobs: - name: Set up QEMU for multi-arch builds uses: docker/setup-qemu-action@v2 - name: Deploy multi-arch base container image to Docker Hub - run: mvn -f modules/container-base -Pct deploy -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy -Ddocker.skip.push -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} name: Push description to DockerHub From afa51c78327748d14a1e9bb6f1d6eac78e357f48 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 01:12:13 +0200 Subject: [PATCH 015/139] fix(ct): skip building and pushing the base image for v6.0 and v6.1, which is compatible with the v6.2 image --- .github/workflows/container_base_push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 24b2640f3e3..cb83a2799bb 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -85,6 +85,8 @@ jobs: - name: Set up QEMU for multi-arch builds uses: docker/setup-qemu-action@v2 - name: Deploy multi-arch base container image to Docker Hub + # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. + if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} run: mvn -f modules/container-base -Pct deploy -Ddocker.skip.push -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} From 8afac921361fcb76752c9425f0ce2e2a5708244e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 01:12:47 +0200 Subject: [PATCH 016/139] test(ct): try to limit QEMU architectures, maybe shaving off a few seconds of setup --- .github/workflows/container_base_push.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index cb83a2799bb..ed9e2a1cc1e 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -83,7 +83,10 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up QEMU for multi-arch builds - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ env.PLATFORMS }} + - name: Deploy multi-arch base container image to Docker Hub # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} From 7c4242ea384e06b70664bffde4c5d1828ac70b1a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 01:18:27 +0200 Subject: [PATCH 017/139] feat(ct): add additional tags for base images --- .github/workflows/container_base_push.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index ed9e2a1cc1e..21a15e02c1d 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -86,11 +86,20 @@ jobs: uses: docker/setup-qemu-action@v3 with: platforms: ${{ env.PLATFORMS }} + - name: Add additional tags as options + run: | + # For the development branch, update the latest tag in addition + if [[ "${{ matrix.branch }}" == "develop" ]]; then + echo "DOCKER_TAGS=-Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" + # In case of releases <=6.2, we still need to provide backward compatible names "alpha" and "unstable" + elif [[ "${{ matrix.branch }}" == "v6.2" ]]; then + echo "DOCKER_TAGS=-Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" + fi - name: Deploy multi-arch base container image to Docker Hub # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} - run: mvn -f modules/container-base -Pct deploy -Ddocker.skip.push -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy -Ddocker.skip.push ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} name: Push description to DockerHub From c08220dd4e40895973206d178f9cdeb7c7b5bf1c Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 01:35:45 +0200 Subject: [PATCH 018/139] test(ct): add feature branch to matrix and build for everything but it. --- .github/workflows/container_base_push.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 21a15e02c1d..af22b6b1a55 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -32,11 +32,12 @@ jobs: steps: - name: Build branch matrix options id: matrix + # TODO: remove the feature branch and re-enable the if/else! run: | # Get last three releases and include develop branch as matrix elements #if [[ "${{ github.event_name }}" == "schedule" ]]; then echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ - jq '[ .[0:3] | .[].tag_name, "develop" ]')" | tr -d "\n" | tr -s " " | \ + jq '[ .[0:3] | .[].tag_name, "develop", "10478-version-base-img" ]')" | tr -d "\n" | tr -s " " | \ tee -a "$GITHUB_OUTPUT" #else # echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" @@ -87,19 +88,28 @@ jobs: with: platforms: ${{ env.PLATFORMS }} - name: Add additional tags as options + # TODO: remove the feature branch and re-enable the if/else! run: | # For the development branch, update the latest tag in addition - if [[ "${{ matrix.branch }}" == "develop" ]]; then + if [[ "${{ matrix.branch }}" == "develop" || "${{ matrix.branch }}" == "10478-version-base-img" ]]; then echo "DOCKER_TAGS=-Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" # In case of releases <=6.2, we still need to provide backward compatible names "alpha" and "unstable" elif [[ "${{ matrix.branch }}" == "v6.2" ]]; then echo "DOCKER_TAGS=-Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" fi + # TODO: remove when feature branch is done + - name: Skip all but feature-branch + if: ${{ matrix.branch != '10478-version-base-img' }} + uses: actions/github-script@v3 + with: + script: | + core.setFailed('Stopping on purpose to avoid mayhem') + - name: Deploy multi-arch base container image to Docker Hub # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} - run: mvn -f modules/container-base -Pct deploy -Ddocker.skip.push ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} name: Push description to DockerHub From c4cbc576ef3dba974e8bffa338ad7a74d3962960 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 01:56:10 +0200 Subject: [PATCH 019/139] chore,test(ct): update script action to v7 --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index af22b6b1a55..f7f8e889d56 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -100,7 +100,7 @@ jobs: # TODO: remove when feature branch is done - name: Skip all but feature-branch if: ${{ matrix.branch != '10478-version-base-img' }} - uses: actions/github-script@v3 + uses: actions/github-script@v7 with: script: | core.setFailed('Stopping on purpose to avoid mayhem') From 05e48875387f769fb9cc966868699cbd4edce8c1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 01:56:44 +0200 Subject: [PATCH 020/139] fix(ct): add missing option for DMP to add additional tags to base image --- .github/workflows/container_base_push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index f7f8e889d56..cefa63786c9 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -92,10 +92,10 @@ jobs: run: | # For the development branch, update the latest tag in addition if [[ "${{ matrix.branch }}" == "develop" || "${{ matrix.branch }}" == "10478-version-base-img" ]]; then - echo "DOCKER_TAGS=-Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" + echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" # In case of releases <=6.2, we still need to provide backward compatible names "alpha" and "unstable" elif [[ "${{ matrix.branch }}" == "v6.2" ]]; then - echo "DOCKER_TAGS=-Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" + echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" fi # TODO: remove when feature branch is done - name: Skip all but feature-branch From be5b9c98cbcab64085ec8a5ffc5b8275a4fec335 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 14:09:53 +0200 Subject: [PATCH 021/139] feat(ct): trigger build of app image when all builds of base image succeed We use the branch name from the matrix as input argument. It's not being used yet. --- .github/workflows/container_app_push.yml | 5 +++++ .github/workflows/container_base_push.yml | 22 ++++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index b3e247e376c..fba693eee05 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -5,6 +5,11 @@ on: # We are deliberately *not* running on push events here to avoid double runs. # Instead, push events will trigger from the base image and maven unit tests via workflow_call. workflow_call: + inputs: + branch: + type: string + description: "A tag or branch to checkout for building the image" + required: true pull_request: branches: - develop diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index cefa63786c9..4e80caa8586 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -107,6 +107,7 @@ jobs: - name: Deploy multi-arch base container image to Docker Hub + id: build # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} run: mvn -f modules/container-base -Pct deploy ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} @@ -120,14 +121,27 @@ jobs: repository: gdcc/base short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" readme-filepath: ./modules/container-base/README.md + + # - if: always() + # name: Save status (workaround for matrix outputs) + # run: | + # # steps.build.outcome is the status BEFORE continue-on-error + # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" + push-app-img: name: "Rebase & Publish App Image" permissions: contents: read packages: write pull-requests: write - needs: build - # We do not release a new base image for pull requests, so do not trigger. - # if: ${{ github.event_name != 'pull_request' }} + needs: + - discover + - build + strategy: + fail-fast: false + matrix: + branch: ${{ fromJson(needs.discover.outputs.branches) }} uses: ./.github/workflows/container_app_push.yml - secrets: inherit + with: + branch: ${{ matrix.branch }} + From d4a196b8a6247542051bf3d0daf24789121d6b94 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 14:12:00 +0200 Subject: [PATCH 022/139] test(ct): let's skip the build for anything but our feature branch --- .github/workflows/container_app_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index fba693eee05..a92ca4aecbd 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -35,7 +35,7 @@ jobs: packages: write pull-requests: write # Only run in upstream repo - avoid unnecessary runs in forks - if: ${{ github.repository_owner == 'IQSS' }} + if: ${{ github.repository_owner == 'IQSS' && inputs.branch == '10478-version-base-img' }} steps: - name: Checkout repository From 77aec0d62dcb0aad1e3e1bf7abdf740fc7bed540 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 14:19:35 +0200 Subject: [PATCH 023/139] style(ct): fix wording to test CI pipeline --- modules/container-base/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index f093ced37c1..0905ebb62a1 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -221,7 +221,7 @@ RUN < Date: Thu, 11 Apr 2024 14:22:08 +0200 Subject: [PATCH 024/139] test(ct): try what happens when removing the on purpose failure of base image workflow --- .github/workflows/container_base_push.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 4e80caa8586..8c15c0a7ff7 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -97,13 +97,14 @@ jobs: elif [[ "${{ matrix.branch }}" == "v6.2" ]]; then echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" fi + # TODO: remove when feature branch is done - - name: Skip all but feature-branch - if: ${{ matrix.branch != '10478-version-base-img' }} - uses: actions/github-script@v7 - with: - script: | - core.setFailed('Stopping on purpose to avoid mayhem') + #- name: Skip all but feature-branch + # if: ${{ matrix.branch != '10478-version-base-img' }} + # uses: actions/github-script@v7 + # with: + # script: | + # core.setFailed('Stopping on purpose to avoid mayhem') - name: Deploy multi-arch base container image to Docker Hub From 2887d13ca93103420ad78973e4e975dd0da90b41 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 14:42:40 +0200 Subject: [PATCH 025/139] fix(ct): make secrets available to app workflow again Deleted by accident when working on alternativ matrix way to build app image after base. --- .github/workflows/container_base_push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 8c15c0a7ff7..1631c08f10e 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -135,6 +135,7 @@ jobs: contents: read packages: write pull-requests: write + secrets: inherit needs: - discover - build From bbe527d4646b09fd41fa4e57c52e2759183d2042 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 08:44:06 +0200 Subject: [PATCH 026/139] chore(ct,build): upgrade to DMP plugin v0.44.0 - Enable -Ddocker.noCache for BuildX - Hopefully straightens out some bugs on Darwin with M1/M2 --- modules/dataverse-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 612902b47a4..d03d3e242fc 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -199,7 +199,7 @@ 1.7.0 - 0.43.4 + 0.44.0 From c496ef31304f8ced339aace0b903a52d69991b71 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 08:45:35 +0200 Subject: [PATCH 027/139] feat(ct): pin down name of the builder for base image This way we have a distinct path and name to inject a BuildX builder instance configuration --- modules/container-base/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index 2754b081986..72811e34e84 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -100,6 +100,7 @@ ${docker.platforms} + dataverse ${project.build.directory}/buildx-state Dockerfile From 82c8e725365c72bbc4a2a906779bc1c61a71ff89 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 08:46:05 +0200 Subject: [PATCH 028/139] ci(ct): make number of past release configurable via env var --- .github/workflows/container_base_push.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 1631c08f10e..fe3b38d0284 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -17,6 +17,7 @@ on: env: PLATFORMS: linux/amd64,linux/arm64 + NUM_PAST_RELEASES: 3 jobs: discover: @@ -37,7 +38,7 @@ jobs: # Get last three releases and include develop branch as matrix elements #if [[ "${{ github.event_name }}" == "schedule" ]]; then echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ - jq '[ .[0:3] | .[].tag_name, "develop", "10478-version-base-img" ]')" | tr -d "\n" | tr -s " " | \ + jq '[ .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, "develop", "10478-version-base-img" ]')" | tr -d "\n" | tr -s " " | \ tee -a "$GITHUB_OUTPUT" #else # echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" From 0c91541f5c11c3daead61f657dbdee6af4addc43 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 08:54:49 +0200 Subject: [PATCH 029/139] fix(ct): correction of tags specification for base image workflow Per example at https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-only-when-a-push-of-specific-tags-occurs it should be two asterisks. --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index fe3b38d0284..838321b10f6 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -4,7 +4,7 @@ name: Base Container Image on: push: tags: - - 'v[6-9].*' + - 'v[6-9].**' branches: - 'develop' - '10478-version-base-img' From eef60f295ced438a7c3e91fbef61edc273ec51d9 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 08:56:21 +0200 Subject: [PATCH 030/139] refactor(ct): when pushes for a tag or branch occur, build for it Before, we would have run the workflow for develop only. In case develop has merges before the run starts because of a tag being created, this may result in undesired side effects. Keep in mind that pushes of tags here will also trigger releasing a new application image! --- .github/workflows/container_base_push.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 838321b10f6..ec8881b3ef5 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -41,7 +41,8 @@ jobs: jq '[ .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, "develop", "10478-version-base-img" ]')" | tr -d "\n" | tr -s " " | \ tee -a "$GITHUB_OUTPUT" #else - # echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" + # # Note: github.ref_name will be the name of the branch or the tag pushed + # echo "branches=['${{ github.ref_name }}']" | tee -a "$GITHUB_OUTPUT" #fi build: From 90cb9ce129ff1b3b9de9e0585232418b820d7dbb Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 09:59:01 +0200 Subject: [PATCH 031/139] feat(ct): replace QEMU with remote builder for ARM64 in base image --- .github/workflows/container_base_push.yml | 35 +++++++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index ec8881b3ef5..324f8134a64 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -85,10 +85,39 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up QEMU for multi-arch builds - uses: docker/setup-qemu-action@v3 + + # This is replaced by adding a remote ARM64 build host in addition to the local AMD64 + # - name: Set up QEMU for multi-arch builds + # uses: docker/setup-qemu-action@v3 + # with: + # platforms: ${{ env.PLATFORMS }} + # Setup SSH access to ARM64 builder node + - name: Setup SSH agent + uses: webfactory/ssh-agent@v0.9.0 with: - platforms: ${{ env.PLATFORMS }} + ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} + - name: Provide the known hosts key and the builder config + run: | + echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts + mkdir -p modules/container-base/target/buildx-state/buildx/instances + cat modules/container-base/target/buildx-state/buildx/instances/dataverse << EOF + { "Name": "dataverse", + "Driver": "docker-container", + "Dynamic": false, + "Nodes": [{"Name": "dataverse0", + "Endpoint": "unix:///var/run/docker.sock", + "Platforms": [{"os": "linux", "architecture": "amd64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}, + {"Name": "dataverse1", + "Endpoint": "ssh://${{ secret.BUILDER_ARM64_SSH_CONNECTION }}", + "Platforms": [{"os": "linux", "architecture": "arm64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}]} + EOF + - name: Add additional tags as options # TODO: remove the feature branch and re-enable the if/else! run: | From d7fbf871cffe8685e75533280e4ee2fa1f1add60 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 10:00:17 +0200 Subject: [PATCH 032/139] feat(ct): disable caching during base image build The ARM64 remote build host will otherwise reuse the cache with potentially stale content --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 324f8134a64..aaf20397f9d 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -142,7 +142,7 @@ jobs: id: build # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} - run: mvn -f modules/container-base -Pct deploy ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} name: Push description to DockerHub From b6fb1224b9ca6fd0f345efc6ad73ebddea962450 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 10:02:31 +0200 Subject: [PATCH 033/139] style(ct): fix typo in secrets reference --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index aaf20397f9d..9243c2725e1 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -111,7 +111,7 @@ jobs: "Flags": ["--allow-insecure-entitlement=network.host"], "Files": null}, {"Name": "dataverse1", - "Endpoint": "ssh://${{ secret.BUILDER_ARM64_SSH_CONNECTION }}", + "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", "Platforms": [{"os": "linux", "architecture": "arm64"}], "DriverOpts": null, "Flags": ["--allow-insecure-entitlement=network.host"], From b3ac714f98971ef21cd47650e04a370226334fc8 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 10:04:08 +0200 Subject: [PATCH 034/139] fix(ct): add missing pipe redirection for builder config --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 9243c2725e1..69cd31afcf4 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -100,7 +100,7 @@ jobs: run: | echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts mkdir -p modules/container-base/target/buildx-state/buildx/instances - cat modules/container-base/target/buildx-state/buildx/instances/dataverse << EOF + cat > modules/container-base/target/buildx-state/buildx/instances/dataverse << EOF { "Name": "dataverse", "Driver": "docker-container", "Dynamic": false, From 898d9053becaa64151c06ec645080f6544292776 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 11:06:32 +0200 Subject: [PATCH 035/139] refactor(ct): switch back to "maven" as builder name Using "maven" (the default name) again allows us to use the ARM64 builder for previously released images as well as develop before we merge the feature branch --- .github/workflows/container_base_push.yml | 8 ++++---- modules/container-base/pom.xml | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 69cd31afcf4..e4733cca027 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -100,17 +100,17 @@ jobs: run: | echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts mkdir -p modules/container-base/target/buildx-state/buildx/instances - cat > modules/container-base/target/buildx-state/buildx/instances/dataverse << EOF - { "Name": "dataverse", + cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF + { "Name": "maven", "Driver": "docker-container", "Dynamic": false, - "Nodes": [{"Name": "dataverse0", + "Nodes": [{"Name": "maven0", "Endpoint": "unix:///var/run/docker.sock", "Platforms": [{"os": "linux", "architecture": "amd64"}], "DriverOpts": null, "Flags": ["--allow-insecure-entitlement=network.host"], "Files": null}, - {"Name": "dataverse1", + {"Name": "maven1", "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", "Platforms": [{"os": "linux", "architecture": "arm64"}], "DriverOpts": null, diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index 72811e34e84..2754b081986 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -100,7 +100,6 @@ ${docker.platforms} - dataverse ${project.build.directory}/buildx-state Dockerfile From 577804b3776847131b6e5f05727c9f0f69232d63 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 13:02:15 +0200 Subject: [PATCH 036/139] refactor(ct): use remote ARM64 builder only for pushes, but not scheduled maintenance --- .github/workflows/container_base_push.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index e4733cca027..36741ccd211 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -86,17 +86,21 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - # This is replaced by adding a remote ARM64 build host in addition to the local AMD64 - # - name: Set up QEMU for multi-arch builds - # uses: docker/setup-qemu-action@v3 - # with: - # platforms: ${{ env.PLATFORMS }} - # Setup SSH access to ARM64 builder node + # In case of scheduled maintenance, we don't care about buildtime: use QEMU for AMD64 + ARM64 + - name: Set up QEMU for multi-arch builds + if: ${{ github.event_name == 'schedule' }} + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ env.PLATFORMS }} + # In case this is a push to develop, we care about buildtime. + # Configure a remote ARM64 build host in addition to the local AMD64 in two steps. - name: Setup SSH agent + if: ${{ github.event_name != 'schedule' }} uses: webfactory/ssh-agent@v0.9.0 with: ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} - name: Provide the known hosts key and the builder config + if: ${{ github.event_name != 'schedule' }} run: | echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts mkdir -p modules/container-base/target/buildx-state/buildx/instances From 25cbdd7b19478f9110ceb55436f9f84bcbb3aff5 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 14:19:35 +0200 Subject: [PATCH 037/139] test(ct): use DMP v0.45-SNAPSHOT to see if pushing images that have a ARG in their ref works with it --- modules/dataverse-parent/pom.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index d03d3e242fc..473c143296a 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -199,7 +199,7 @@ 1.7.0 - 0.44.0 + 0.45-SNAPSHOT @@ -215,6 +215,14 @@ never + + oss.sonatype.org + https://oss.sonatype.org/content/repositories/snapshots + + true + always + + From 2de674f6d80249ea2e446275533f4e3b6daa630f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 15:23:21 +0200 Subject: [PATCH 038/139] feat(build,ci): workflow pre-seeding Maven cache from develop #10428 Contains the feature branch for now, too. Will be deleted later before merging it to develop. --- .github/workflows/maven_cache_seed.yaml | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/maven_cache_seed.yaml diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml new file mode 100644 index 00000000000..e82ce36f829 --- /dev/null +++ b/.github/workflows/maven_cache_seed.yaml @@ -0,0 +1,37 @@ +name: Maven Cache Seeding + +on: + push: + branches: + - develop + - 10478-version-base-img + +env: + COMMON_CACHE_NAME: dataverse-maven-cache + +jobs: + preseed: + name: Drop and Re-Seed Local Repository + runs-on: ubuntu-latest + if: ${{ github.event_name == 'push' }} + steps: + - name: Drop common cache + uses: prantlf/delete-cache-action@v2 + with: + key: ${{ env.COMMON_CACHE_NAME }} + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: "17" + distribution: temurin + - name: Re-Seed common cache + run: | + mvn -B -f modules/dataverse-parent dependency:go-offline dependency:resolve-plugins + - name: Save the cache + uses: actions/cache/save@v4 + with: + path: ~/.m2/repository + key: ${{ env.COMMON_CACHE_NAME }} + From b1943c114110405a613c4093fb2d7eee5fa904d7 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 15:35:38 +0200 Subject: [PATCH 039/139] fix(build,ci): ignore not found when deleting common cache --- .github/workflows/maven_cache_seed.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index e82ce36f829..4650cfcf930 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -16,6 +16,7 @@ jobs: if: ${{ github.event_name == 'push' }} steps: - name: Drop common cache + continue-on-error: true # we don't care if the cache is not around uses: prantlf/delete-cache-action@v2 with: key: ${{ env.COMMON_CACHE_NAME }} From 941fbc904d5700cebcff9d64b0a8a1f1c2268d36 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 15:51:33 +0200 Subject: [PATCH 040/139] style(ct): remove some stale comments --- .github/workflows/container_base_push.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 36741ccd211..7f2f194cbcd 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -123,7 +123,7 @@ jobs: EOF - name: Add additional tags as options - # TODO: remove the feature branch and re-enable the if/else! + # TODO: remove the feature branch run: | # For the development branch, update the latest tag in addition if [[ "${{ matrix.branch }}" == "develop" || "${{ matrix.branch }}" == "10478-version-base-img" ]]; then @@ -133,15 +133,6 @@ jobs: echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" fi - # TODO: remove when feature branch is done - #- name: Skip all but feature-branch - # if: ${{ matrix.branch != '10478-version-base-img' }} - # uses: actions/github-script@v7 - # with: - # script: | - # core.setFailed('Stopping on purpose to avoid mayhem') - - - name: Deploy multi-arch base container image to Docker Hub id: build # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. From b64b6aff9ce38009fdf90c451493b257690472f7 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 15:52:28 +0200 Subject: [PATCH 041/139] feat(ct): make use of common cache in app container builds #10428 --- .github/workflows/container_app_push.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index a92ca4aecbd..347fcf06b86 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -42,11 +42,18 @@ jobs: uses: actions/checkout@v3 - name: Set up JDK + id: setup-java uses: actions/setup-java@v3 with: java-version: "17" distribution: temurin cache: maven + - name: Download common cache on branch cache miss + if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} + uses: actions/cache/restore@v4 + with: + key: dataverse-maven-cache + - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests) run: > @@ -111,11 +118,19 @@ jobs: if: needs.check-secrets.outputs.available == 'true' && ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name))) steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - name: Set up JDK + id: setup-java + uses: actions/setup-java@v3 with: java-version: "17" distribution: temurin + cache: maven + - name: Download common cache on branch cache miss + if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} + uses: actions/cache/restore@v4 + with: + key: dataverse-maven-cache # Depending on context, we push to different targets. Login accordingly. - if: github.event_name != 'pull_request' From 4f48123f73674eb56f3f687cc2e7df55908e13db Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 15:52:49 +0200 Subject: [PATCH 042/139] feat(ci): remove feature branch from maven seed workflow --- .github/workflows/maven_cache_seed.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index 4650cfcf930..d002a446b99 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -4,7 +4,6 @@ on: push: branches: - develop - - 10478-version-base-img env: COMMON_CACHE_NAME: dataverse-maven-cache From 721814a62672a8483d1fd0708071444bc239e806 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:01:47 +0200 Subject: [PATCH 043/139] style(ct): rename base image revision from RX to revX It's just much more readable --- modules/container-base/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index 2754b081986..e5de8841530 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -42,7 +42,7 @@ docker-build gdcc/base:${base.image.tag} - R${project.version} + rev${project.version} eclipse-temurin:${target.java.version}-jre 1000 1000 From 3c0650f6710bcda94c0e83278654a454466b5a41 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:02:12 +0200 Subject: [PATCH 044/139] feat(ct): make dataverse app image use base rev1 image --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8f9d06b8744..fa267b6fbee 100644 --- a/pom.xml +++ b/pom.xml @@ -956,7 +956,8 @@ unstable false gdcc/base:${base.image.tag} - unstable + + rev1 gdcc/configbaker:${conf.image.tag} ${app.image.tag} From 06a21612347e0a64cfe0adfa05d1823c4927f5b2 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:15:00 +0200 Subject: [PATCH 045/139] refactor(ci): delete common cache after we downloaded the old one #10428 --- .github/workflows/maven_cache_seed.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index d002a446b99..5a38abed767 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -14,11 +14,6 @@ jobs: runs-on: ubuntu-latest if: ${{ github.event_name == 'push' }} steps: - - name: Drop common cache - continue-on-error: true # we don't care if the cache is not around - uses: prantlf/delete-cache-action@v2 - with: - key: ${{ env.COMMON_CACHE_NAME }} - name: Checkout repository uses: actions/checkout@v4 - name: Set up JDK @@ -26,9 +21,16 @@ jobs: with: java-version: "17" distribution: temurin - - name: Re-Seed common cache + - name: Seed common cache run: | mvn -B -f modules/dataverse-parent dependency:go-offline dependency:resolve-plugins + # This non-obvious order is due to the fact that the download via Maven will take a very long time. + # Jobs should not be left without a cache. Deleting and saving in one go leaves only a small chance for a cache miss. + - name: Drop common cache + continue-on-error: true # we don't care if the cache is not around + uses: prantlf/delete-cache-action@v2 + with: + key: ${{ env.COMMON_CACHE_NAME }} - name: Save the cache uses: actions/cache/save@v4 with: From 818bfd4ec32fa0d67f39611f6f3235af730fbfb9 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:15:20 +0200 Subject: [PATCH 046/139] test(ci): readd feature branch to test cache handling logic --- .github/workflows/maven_cache_seed.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index 5a38abed767..f4b13725e70 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -4,6 +4,7 @@ on: push: branches: - develop + - 10478-version-base-img env: COMMON_CACHE_NAME: dataverse-maven-cache From a28997c815685b0527e3250c7526d9a03521ddae Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:16:33 +0200 Subject: [PATCH 047/139] style(ct): add common cache download to base image Without accurate measuring it seems like we use ~1 minute to download Maven artifacts for this workflow. Lets try to cut that down some by reusing the common cache. --- .github/workflows/container_base_push.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 7f2f194cbcd..667808d4d08 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -69,6 +69,7 @@ jobs: echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} - name: Set up JDK ${{ env.JAVA_VERSION }} + id: setup-java uses: actions/setup-java@v4 with: java-version: ${{ env.JAVA_VERSION }} @@ -76,6 +77,12 @@ jobs: cache: 'maven' cache-dependency-path: | modules/container-base/pom.xml + - name: Download common cache on branch cache miss + if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} + uses: actions/cache/restore@v4 + with: + key: dataverse-maven-cache + path: ~/.m2/repository # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and # on events in context of upstream because secrets. PRs run in context of forks by default! From 085e41e9c1bd7c550e76ad47b5450ac5fe197450 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:19:59 +0200 Subject: [PATCH 048/139] style(ct): remove stale comment about platforms empty in base image --- modules/container-base/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index e5de8841530..60be420bd9a 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -97,7 +97,6 @@ - ${docker.platforms} ${project.build.directory}/buildx-state From 2fe0ca39ac5c9cf044ad833b7e66de449d61efa3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:52:12 +0200 Subject: [PATCH 049/139] fix(build,ci): downgrade to cache action v2 Per https://github.com/actions/cache/issues/1361 restores in v3 and v4 were not working, maybe saving is botched, too? --- .github/workflows/maven_cache_seed.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index f4b13725e70..148808a4b04 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -33,7 +33,7 @@ jobs: with: key: ${{ env.COMMON_CACHE_NAME }} - name: Save the cache - uses: actions/cache/save@v4 + uses: actions/cache@v2 with: path: ~/.m2/repository key: ${{ env.COMMON_CACHE_NAME }} From afad97e4160d60e31b01611e8a1e28ad0192631f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 17:09:50 +0200 Subject: [PATCH 050/139] fix(build,ci,ct): fix missing path for cache restore --- .github/workflows/container_app_push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 347fcf06b86..3344a8e2d0d 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -52,6 +52,7 @@ jobs: if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} uses: actions/cache/restore@v4 with: + path: ~/.m2/repository key: dataverse-maven-cache @@ -130,6 +131,7 @@ jobs: if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} uses: actions/cache/restore@v4 with: + path: ~/.m2/repository key: dataverse-maven-cache # Depending on context, we push to different targets. Login accordingly. From 1c1c30e8b1d11f830354535f5164a92112eb3ca3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 17:11:51 +0200 Subject: [PATCH 051/139] test(ci): fix a comment to trigger base image workflow --- modules/container-base/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index 0905ebb62a1..a8bd3a32ca3 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -203,7 +203,7 @@ RUN < Date: Fri, 12 Apr 2024 17:17:49 +0200 Subject: [PATCH 052/139] style(ci): remove feature branch from maven cache seeding again --- .github/workflows/maven_cache_seed.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index 148808a4b04..85f1381c789 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -4,7 +4,6 @@ on: push: branches: - develop - - 10478-version-base-img env: COMMON_CACHE_NAME: dataverse-maven-cache From 43402435ae4d619b872ac9bd14704f54306fc49e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 17:20:08 +0200 Subject: [PATCH 053/139] doc(ci): add note about cache availability to seeding workflow --- .github/workflows/maven_cache_seed.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index 85f1381c789..e11d8c91c0e 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -31,6 +31,8 @@ jobs: uses: prantlf/delete-cache-action@v2 with: key: ${{ env.COMMON_CACHE_NAME }} + # NOTE: It is vital here to remember that only caches with the scope of the default branch are + # available to other branches. We use the v2 action here to save it anyway. - name: Save the cache uses: actions/cache@v2 with: From 899e8aa6548d8b6076c266807cc265d4c0bb049b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 17:31:16 +0200 Subject: [PATCH 054/139] feat(ci): make sure to rejuvenate the common cache every 7 days --- .github/workflows/maven_cache_seed.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index e11d8c91c0e..d31559138b8 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -4,6 +4,10 @@ on: push: branches: - develop + # According to https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy + # all caches are depleted after 7 days of no access. Make sure we rejuvenate every 7 days to keep it available. + schedule: + - cron: '23 2 * * 0' # Run for 'develop' every Sunday at 02:23 UTC env: COMMON_CACHE_NAME: dataverse-maven-cache @@ -12,7 +16,6 @@ jobs: preseed: name: Drop and Re-Seed Local Repository runs-on: ubuntu-latest - if: ${{ github.event_name == 'push' }} steps: - name: Checkout repository uses: actions/checkout@v4 From f538896971022cc4fa011a38aca41936125a30dc Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Apr 2024 14:14:56 +0200 Subject: [PATCH 055/139] chore(ct): replace wait-for with wait4x Aligning configbaker and base image with same tool. wait4x has much more features to wait for different services. --- doc/sphinx-guides/source/container/base-image.rst | 2 +- modules/container-base/src/main/docker/Dockerfile | 12 ++++++------ .../docker/scripts/init_3_wait_dataverse_db_host.sh | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index c41250d48c5..29c357b91f6 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -46,7 +46,7 @@ The base image provides: - CLI tools necessary to run Dataverse (i. e. ``curl`` or ``jq`` - see also :doc:`../installation/prerequisites` in Installation Guide) - Linux tools for analysis, monitoring and so on - `Jattach `__ (attach to running JVM) -- `wait-for `__ (tool to "wait for" a service to be available) +- `wait4x `__ (tool to "wait for" a service to be available) - `dumb-init `__ (see :ref:`below ` for details) This image is created as a "multi-arch image", see :ref:`below `. diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index a8bd3a32ca3..e3fa6e477a9 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -92,8 +92,7 @@ EOF ARG JATTACH_VERSION="v2.1" ARG JATTACH_CHECKSUM="07885fdc782e02e7302c6d190f54c3930afa10a38140365adf54076ec1086a8e" -ARG WAIT_FOR_VERSION="v2.2.3" -ARG WAIT_FOR_CHECKSUM="70271181be69cd2c7265b2746f97fccfd7e8aa1059894138a775369c23589ff4" +ARG WAIT4X_VERSION="v2.14.0" ARG PKGS="jq imagemagick curl unzip wget acl dirmngr gpg lsof procps netcat dumb-init" # Installing the packages in an extra container layer for better caching @@ -108,10 +107,11 @@ RUN < Date: Mon, 15 Apr 2024 14:17:55 +0200 Subject: [PATCH 056/139] build(ct): make target architecture available in base image build As per https://docs.docker.com/reference/dockerfile/#automatic-platform-args-in-the-global-scope BuildKit / buildx will expose the target architecture. It requires adding an ARG in the Dockerfile to inject the data. --- modules/container-base/src/main/docker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index e3fa6e477a9..256d9159d2d 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -69,6 +69,9 @@ ENV PATH="${PATH}:${PAYARA_DIR}/bin:${SCRIPT_DIR}" \ ### PART 1: SYSTEM ### ARG UID=1000 ARG GID=1000 +# Auto-populated by BuildKit / buildx +#ARG TARGETARCH="amd64" +ARG TARGETARCH USER root WORKDIR / SHELL ["/bin/bash", "-euo", "pipefail", "-c"] From e86133468a0086f0701efc69ad1549e21102f284 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Apr 2024 14:20:18 +0200 Subject: [PATCH 057/139] chore(ct): upgrade base image with jattach v2.2 jattach binary is now available for ARM64 and AMD64, but requires special handling with download URLs and checksums. --- .../container-base/src/main/docker/Dockerfile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index 256d9159d2d..fbd43604eac 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -93,8 +93,9 @@ RUN < Date: Tue, 16 Apr 2024 17:42:18 +0200 Subject: [PATCH 058/139] chore: remove obsolete Maven Cache seed workflow --- .github/workflows/maven_cache_seed.yaml | 44 ------------------------- 1 file changed, 44 deletions(-) delete mode 100644 .github/workflows/maven_cache_seed.yaml diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml deleted file mode 100644 index d31559138b8..00000000000 --- a/.github/workflows/maven_cache_seed.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: Maven Cache Seeding - -on: - push: - branches: - - develop - # According to https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy - # all caches are depleted after 7 days of no access. Make sure we rejuvenate every 7 days to keep it available. - schedule: - - cron: '23 2 * * 0' # Run for 'develop' every Sunday at 02:23 UTC - -env: - COMMON_CACHE_NAME: dataverse-maven-cache - -jobs: - preseed: - name: Drop and Re-Seed Local Repository - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Set up JDK - uses: actions/setup-java@v4 - with: - java-version: "17" - distribution: temurin - - name: Seed common cache - run: | - mvn -B -f modules/dataverse-parent dependency:go-offline dependency:resolve-plugins - # This non-obvious order is due to the fact that the download via Maven will take a very long time. - # Jobs should not be left without a cache. Deleting and saving in one go leaves only a small chance for a cache miss. - - name: Drop common cache - continue-on-error: true # we don't care if the cache is not around - uses: prantlf/delete-cache-action@v2 - with: - key: ${{ env.COMMON_CACHE_NAME }} - # NOTE: It is vital here to remember that only caches with the scope of the default branch are - # available to other branches. We use the v2 action here to save it anyway. - - name: Save the cache - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ env.COMMON_CACHE_NAME }} - From 54fe365964f8c726ec2a3e42bee5b28b41f952d0 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 16 Apr 2024 17:43:42 +0200 Subject: [PATCH 059/139] style(ci): remove superfluous empty line --- .github/workflows/container_base_push.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 667808d4d08..660f9bdd861 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -86,7 +86,6 @@ jobs: # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and # on events in context of upstream because secrets. PRs run in context of forks by default! - - name: Log in to the Container registry uses: docker/login-action@v3 with: From 50a9ede0037ae255b1f363881a8a0cb42e236746 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 21:12:40 +0200 Subject: [PATCH 060/139] ci(ct): temporarily allow other repos to run the base image workflow Needs to be reverted when done with testing --- .github/workflows/container_base_push.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 660f9bdd861..4843800a3d1 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -26,8 +26,9 @@ jobs: permissions: contents: read packages: read + # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled - if: ${{ github.repository_owner == 'IQSS' }} + #if: ${{ github.repository_owner == 'IQSS' }} outputs: branches: ${{ steps.matrix.outputs.branches }} steps: @@ -56,8 +57,9 @@ jobs: fail-fast: false matrix: branch: ${{ fromJson(needs.discover.outputs.branches) }} + # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks - if: ${{ github.repository_owner == 'IQSS' }} + #if: ${{ github.repository_owner == 'IQSS' }} steps: - name: Checkout repository From 19de76d77e72b8da9558365ee0510b714a465cb5 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 21:33:29 +0200 Subject: [PATCH 061/139] ci(ct): temporarily disable remote docker builder in fork --- .github/workflows/container_base_push.yml | 55 ++++++++++++----------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 4843800a3d1..ca05a30c95f 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -102,33 +102,34 @@ jobs: platforms: ${{ env.PLATFORMS }} # In case this is a push to develop, we care about buildtime. # Configure a remote ARM64 build host in addition to the local AMD64 in two steps. - - name: Setup SSH agent - if: ${{ github.event_name != 'schedule' }} - uses: webfactory/ssh-agent@v0.9.0 - with: - ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} - - name: Provide the known hosts key and the builder config - if: ${{ github.event_name != 'schedule' }} - run: | - echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts - mkdir -p modules/container-base/target/buildx-state/buildx/instances - cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF - { "Name": "maven", - "Driver": "docker-container", - "Dynamic": false, - "Nodes": [{"Name": "maven0", - "Endpoint": "unix:///var/run/docker.sock", - "Platforms": [{"os": "linux", "architecture": "amd64"}], - "DriverOpts": null, - "Flags": ["--allow-insecure-entitlement=network.host"], - "Files": null}, - {"Name": "maven1", - "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", - "Platforms": [{"os": "linux", "architecture": "arm64"}], - "DriverOpts": null, - "Flags": ["--allow-insecure-entitlement=network.host"], - "Files": null}]} - EOF + # TODO: re-enable for final PR + #- name: Setup SSH agent + # if: ${{ github.event_name != 'schedule' }} + # uses: webfactory/ssh-agent@v0.9.0 + # with: + # ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} + #- name: Provide the known hosts key and the builder config + # if: ${{ github.event_name != 'schedule' }} + # run: | + # echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts + # mkdir -p modules/container-base/target/buildx-state/buildx/instances + # cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF + # { "Name": "maven", + # "Driver": "docker-container", + # "Dynamic": false, + # "Nodes": [{"Name": "maven0", + # "Endpoint": "unix:///var/run/docker.sock", + # "Platforms": [{"os": "linux", "architecture": "amd64"}], + # "DriverOpts": null, + # "Flags": ["--allow-insecure-entitlement=network.host"], + # "Files": null}, + # {"Name": "maven1", + # "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", + # "Platforms": [{"os": "linux", "architecture": "arm64"}], + # "DriverOpts": null, + # "Flags": ["--allow-insecure-entitlement=network.host"], + # "Files": null}]} + # EOF - name: Add additional tags as options # TODO: remove the feature branch From a9a5709412b9640e1e595ca989a926b62090d3a9 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 21:14:11 +0200 Subject: [PATCH 062/139] feat(ct): introduce backports for older base images This way we can apply important changes if need be. --- .github/workflows/container_base_push.yml | 17 ++++++++-- .../src/backports/v6.1/001-pom.xml.patch | 33 +++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 modules/container-base/src/backports/v6.1/001-pom.xml.patch diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index ca05a30c95f..147e5e28d12 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -142,10 +142,23 @@ jobs: echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" fi + # Try to retrieve backport patches for this git ref (but don't fail if there aren't any) + # and try to apply them if present + - name: Get backport patches + continue-on-error: true + uses: actions/checkout@v4 + with: + path: backports + sparse-checkout: | + modules/container-base/src/backports/${{ matrix.branch }} + - name: Apply backport patches + id: patch + run: | + find ${GITHUB_WORKSPACE}/backports/modules/container-base/src/backports/${{ matrix.branch }} -type f -name '*.patch' -print0 | \ + xargs -0 patch -p1 -s -i + - name: Deploy multi-arch base container image to Docker Hub id: build - # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. - if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} diff --git a/modules/container-base/src/backports/v6.1/001-pom.xml.patch b/modules/container-base/src/backports/v6.1/001-pom.xml.patch new file mode 100644 index 00000000000..590eeae962a --- /dev/null +++ b/modules/container-base/src/backports/v6.1/001-pom.xml.patch @@ -0,0 +1,33 @@ +--- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 ++++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 +@@ -12,6 +12,7 @@ + + io.gdcc + container-base ++ 0.6.1 + ${packaging.type} + Container Base Image + This module provides an application server base image to be decorated with the Dataverse app. +@@ -40,8 +42,10 @@ + + docker-build + gdcc/base:${base.image.tag} +- unstable +- eclipse-temurin:${target.java.version}-jre ++ ${project.version}-pyr${payara.version} ++ 0 ++ eclipse-temurin:${target.java.version}-jre-${java.image.flavor} ++ jammy + 1000 + 1000 + linux/amd64,linux/arm64 +@@ -110,6 +113,9 @@ + + assembly.xml + ++ ++ ${base.image.tag}-r${base.image.revision} ++ + + + From 0691d4d707c60818b3c285d398973ab35ac9b102 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 23:18:38 +0200 Subject: [PATCH 063/139] feat(ct): add patches for v6.1, 6.2 and 6.3 base images - Patching versions into the POM and it's tags. - Adding security fixes for 6.2 and 6.3 base image (missing user statement) - Adding labels to keep information about origins --- .../src/backports/v6.1/001-pom.xml.patch | 4 +-- .../src/backports/v6.1/002-Dockerfile.patch | 10 ++++++ .../src/backports/v6.2/001-pom.xml.patch | 33 +++++++++++++++++++ .../v6.2/002-Dockerfile.labels.patch | 10 ++++++ .../v6.2/003-Dockerfile.security.patch | 10 ++++++ .../src/backports/v6.3/001-pom.xml.patch | 33 +++++++++++++++++++ .../v6.3/002-Dockerfile.labels.patch | 18 ++++++++++ .../v6.3/003-Dockerfile.security.patch | 7 ++++ 8 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 modules/container-base/src/backports/v6.1/002-Dockerfile.patch create mode 100644 modules/container-base/src/backports/v6.2/001-pom.xml.patch create mode 100644 modules/container-base/src/backports/v6.2/002-Dockerfile.labels.patch create mode 100644 modules/container-base/src/backports/v6.2/003-Dockerfile.security.patch create mode 100644 modules/container-base/src/backports/v6.3/001-pom.xml.patch create mode 100644 modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch create mode 100644 modules/container-base/src/backports/v6.3/003-Dockerfile.security.patch diff --git a/modules/container-base/src/backports/v6.1/001-pom.xml.patch b/modules/container-base/src/backports/v6.1/001-pom.xml.patch index 590eeae962a..4e1c672b521 100644 --- a/modules/container-base/src/backports/v6.1/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.1/001-pom.xml.patch @@ -4,7 +4,7 @@ io.gdcc container-base -+ 0.6.1 ++ 0.8.0 ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. @@ -14,7 +14,7 @@ gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-pyr${payara.version} ++ ${project.version}-${java.image.flavor}-pyr${payara.version}-tmrn${target.java.version} + 0 + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy diff --git a/modules/container-base/src/backports/v6.1/002-Dockerfile.patch b/modules/container-base/src/backports/v6.1/002-Dockerfile.patch new file mode 100644 index 00000000000..4bb7a1eac91 --- /dev/null +++ b/modules/container-base/src/backports/v6.1/002-Dockerfile.patch @@ -0,0 +1,10 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -233,4 +233,6 @@ LABEL org.opencontainers.image.created="@git.build.time@" \ + org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.title="Dataverse Base Image" \ +- org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" ++ org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" \ ++ org.opencontainers.image.base.name="@java.image@" \ ++ org.dataverse.deps.payara.version="@payara.version@" diff --git a/modules/container-base/src/backports/v6.2/001-pom.xml.patch b/modules/container-base/src/backports/v6.2/001-pom.xml.patch new file mode 100644 index 00000000000..5a0fe404148 --- /dev/null +++ b/modules/container-base/src/backports/v6.2/001-pom.xml.patch @@ -0,0 +1,33 @@ +--- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 ++++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 +@@ -12,6 +12,7 @@ + + io.gdcc + container-base ++ 0.9.0 + ${packaging.type} + Container Base Image + This module provides an application server base image to be decorated with the Dataverse app. +@@ -40,8 +42,10 @@ + + docker-build + gdcc/base:${base.image.tag} +- unstable +- eclipse-temurin:${target.java.version}-jre ++ ${project.version}-${java.image.flavor}-pyr${payara.version}-tmrn${target.java.version} ++ 0 ++ eclipse-temurin:${target.java.version}-jre-${java.image.flavor} ++ jammy + 1000 + 1000 + linux/amd64,linux/arm64 +@@ -110,6 +113,9 @@ + + assembly.xml + ++ ++ ${base.image.tag}-r${base.image.revision} ++ + + + diff --git a/modules/container-base/src/backports/v6.2/002-Dockerfile.labels.patch b/modules/container-base/src/backports/v6.2/002-Dockerfile.labels.patch new file mode 100644 index 00000000000..fbb7f80c4ce --- /dev/null +++ b/modules/container-base/src/backports/v6.2/002-Dockerfile.labels.patch @@ -0,0 +1,10 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -242,4 +242,6 @@ LABEL org.opencontainers.image.created="@git.build.time@" \ + org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.title="Dataverse Base Image" \ +- org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" ++ org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" \ ++ org.opencontainers.image.base.name="@java.image@" \ ++ org.dataverse.deps.payara.version="@payara.version@" diff --git a/modules/container-base/src/backports/v6.2/003-Dockerfile.security.patch b/modules/container-base/src/backports/v6.2/003-Dockerfile.security.patch new file mode 100644 index 00000000000..1ab4c3a980a --- /dev/null +++ b/modules/container-base/src/backports/v6.2/003-Dockerfile.security.patch @@ -0,0 +1,10 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -226,6 +226,7 @@ USER root + RUN true && \ + chgrp -R 0 "${DOMAIN_DIR}" && \ + chmod -R g=u "${DOMAIN_DIR}" ++USER payara + + # Set the entrypoint to tini (as a process supervisor) + ENTRYPOINT ["/usr/bin/dumb-init", "--"] diff --git a/modules/container-base/src/backports/v6.3/001-pom.xml.patch b/modules/container-base/src/backports/v6.3/001-pom.xml.patch new file mode 100644 index 00000000000..595a43beed4 --- /dev/null +++ b/modules/container-base/src/backports/v6.3/001-pom.xml.patch @@ -0,0 +1,33 @@ +--- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 ++++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 +@@ -12,6 +12,7 @@ + + io.gdcc + container-base ++ 0.9.1 + ${packaging.type} + Container Base Image + This module provides an application server base image to be decorated with the Dataverse app. +@@ -40,8 +42,10 @@ + + docker-build + gdcc/base:${base.image.tag} +- unstable +- eclipse-temurin:${target.java.version}-jre ++ ${project.version}-${java.image.flavor}-pyr${payara.version}-tmrn${target.java.version} ++ 0 ++ eclipse-temurin:${target.java.version}-jre-${java.image.flavor} ++ jammy + 1000 + 1000 + linux/amd64,linux/arm64 +@@ -110,6 +113,9 @@ + + assembly.xml + ++ ++ ${base.image.tag}-r${base.image.revision} ++ + + + diff --git a/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch b/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch new file mode 100644 index 00000000000..c0bbe7698d8 --- /dev/null +++ b/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch @@ -0,0 +1,18 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -224,6 +224,7 @@ USER root + RUN true && \ + chgrp -R 0 "${DOMAIN_DIR}" && \ + chmod -R g=u "${DOMAIN_DIR}" ++USER payara + + # Set the entrypoint to tini (as a process supervisor) + ENTRYPOINT ["/usr/bin/dumb-init", "--"] +@@ -240,4 +241,6 @@ LABEL org.opencontainers.image.created="@git.build.time@" \ + org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.title="Dataverse Base Image" \ +- org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" ++ org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" \ ++ org.opencontainers.image.base.name="@java.image@" \ ++ org.dataverse.deps.payara.version="@payara.version@" diff --git a/modules/container-base/src/backports/v6.3/003-Dockerfile.security.patch b/modules/container-base/src/backports/v6.3/003-Dockerfile.security.patch new file mode 100644 index 00000000000..d8487b3aacc --- /dev/null +++ b/modules/container-base/src/backports/v6.3/003-Dockerfile.security.patch @@ -0,0 +1,7 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -224,6 +224,7 @@ USER root + RUN true && \ + chgrp -R 0 "${DOMAIN_DIR}" && \ + chmod -R g=u "${DOMAIN_DIR}" ++USER payara From ae8ced2f3040b4c29648a3f9da321c25c189a1fa Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 23:25:18 +0200 Subject: [PATCH 064/139] ci,fix(ct): only apply 1 patch file at a time for base images Otherwise the patch is applied to some other patchfile, which makes no sense. --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 147e5e28d12..b0be1b72c89 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -155,7 +155,7 @@ jobs: id: patch run: | find ${GITHUB_WORKSPACE}/backports/modules/container-base/src/backports/${{ matrix.branch }} -type f -name '*.patch' -print0 | \ - xargs -0 patch -p1 -s -i + xargs -0 -n1 patch -p1 -s -i - name: Deploy multi-arch base container image to Docker Hub id: build From 37fb155739096f5e4a178ab3a851db8033eb4d34 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 23:27:31 +0200 Subject: [PATCH 065/139] ci,fix(ct): correction of v6.3 Dockerfile patches Double patching makes CI trip over. --- .../src/backports/v6.3/002-Dockerfile.labels.patch | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch b/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch index c0bbe7698d8..c7744882153 100644 --- a/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch +++ b/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch @@ -1,13 +1,5 @@ --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile -@@ -224,6 +224,7 @@ USER root - RUN true && \ - chgrp -R 0 "${DOMAIN_DIR}" && \ - chmod -R g=u "${DOMAIN_DIR}" -+USER payara - - # Set the entrypoint to tini (as a process supervisor) - ENTRYPOINT ["/usr/bin/dumb-init", "--"] @@ -240,4 +241,6 @@ LABEL org.opencontainers.image.created="@git.build.time@" \ org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ org.opencontainers.image.licenses="Apache-2.0" \ From c177d9ab7e281e5b0accbc94767c4630ac87cf92 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 23:42:01 +0200 Subject: [PATCH 066/139] ci,fix(ct): make v6.3 get alpha tag Was 6.2 - but we are at v6.3 already! --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index b0be1b72c89..ee7cb85a42e 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -138,7 +138,7 @@ jobs: if [[ "${{ matrix.branch }}" == "develop" || "${{ matrix.branch }}" == "10478-version-base-img" ]]; then echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" # In case of releases <=6.2, we still need to provide backward compatible names "alpha" and "unstable" - elif [[ "${{ matrix.branch }}" == "v6.2" ]]; then + elif [[ "${{ matrix.branch }}" == "v6.3" ]]; then echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" fi From fffcfe05374ea932fcf76112c328b77a9dcb2e95 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 27 Aug 2024 16:23:45 +0200 Subject: [PATCH 067/139] refactor(ct): align base POM tags with suitable policy across releases and develop --- modules/container-base/pom.xml | 13 ++++++++++--- .../src/backports/v6.1/001-pom.xml.patch | 10 ++++++---- .../src/backports/v6.2/001-pom.xml.patch | 10 ++++++---- .../src/backports/v6.3/001-pom.xml.patch | 10 ++++++---- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index 60be420bd9a..5de94bfc941 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -13,7 +13,7 @@ io.gdcc container-base - 1 + 1.0.0 ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. @@ -42,8 +42,12 @@ docker-build gdcc/base:${base.image.tag} - rev${project.version} - eclipse-temurin:${target.java.version}-jre + ${project.version}-${java.image.flavor} + + SNAPSHOT + ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + noble 1000 1000 linux/amd64,linux/arm64 @@ -111,6 +115,9 @@ assembly.xml + + ${base.image.tag.revision} + diff --git a/modules/container-base/src/backports/v6.1/001-pom.xml.patch b/modules/container-base/src/backports/v6.1/001-pom.xml.patch index 4e1c672b521..508b496af14 100644 --- a/modules/container-base/src/backports/v6.1/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.1/001-pom.xml.patch @@ -8,14 +8,16 @@ ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. -@@ -40,8 +42,10 @@ +@@ -40,8 +42,12 @@ docker-build gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor}-pyr${payara.version}-tmrn${target.java.version} -+ 0 ++ ${project.version}-${java.image.flavor} ++ ++ SNAPSHOT ++ ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 @@ -26,7 +28,7 @@ assembly.xml + -+ ${base.image.tag}-r${base.image.revision} ++ ${base.image.tag.revision} + diff --git a/modules/container-base/src/backports/v6.2/001-pom.xml.patch b/modules/container-base/src/backports/v6.2/001-pom.xml.patch index 5a0fe404148..256680c2595 100644 --- a/modules/container-base/src/backports/v6.2/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.2/001-pom.xml.patch @@ -8,14 +8,16 @@ ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. -@@ -40,8 +42,10 @@ +@@ -40,8 +42,12 @@ docker-build gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor}-pyr${payara.version}-tmrn${target.java.version} -+ 0 ++ ${project.version}-${java.image.flavor} ++ ++ SNAPSHOT ++ ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 @@ -26,7 +28,7 @@ assembly.xml + -+ ${base.image.tag}-r${base.image.revision} ++ ${base.image.tag.revision} + diff --git a/modules/container-base/src/backports/v6.3/001-pom.xml.patch b/modules/container-base/src/backports/v6.3/001-pom.xml.patch index 595a43beed4..b1ab9bfe589 100644 --- a/modules/container-base/src/backports/v6.3/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.3/001-pom.xml.patch @@ -8,14 +8,16 @@ ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. -@@ -40,8 +42,10 @@ +@@ -40,8 +42,12 @@ docker-build gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor}-pyr${payara.version}-tmrn${target.java.version} -+ 0 ++ ${project.version}-${java.image.flavor} ++ ++ SNAPSHOT ++ ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 @@ -26,7 +28,7 @@ assembly.xml + -+ ${base.image.tag}-r${base.image.revision} ++ ${base.image.tag.revision} + From c5b9ccc3e370fa7c7d3e83be538a5c359de4b2e9 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 28 Aug 2024 15:53:51 +0200 Subject: [PATCH 068/139] docs(ct): document new tagging and release policy for base image --- .../source/container/base-image.rst | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 0005265fb1c..e364579b568 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -26,15 +26,26 @@ Supported Image Tags This image is sourced from the main upstream code `repository of the Dataverse software `_. Development and maintenance of the `image's code `_ -happens there (again, by the community). Community-supported image tags are based on the two most important -upstream branches: - -- The ``unstable`` tag corresponds to the ``develop`` branch, where pull requests are merged. - (`Dockerfile `__) -- The ``alpha`` tag corresponds to the ``master`` branch, where releases are cut from. - (`Dockerfile `__) - - +happens there (again, by the community). + +In our tagging, we `follow the Bitname example `_: + +- ``latest``: Rolling tag on par with whatever happens on the ``develop`` branch. (`Dockerfile `__) + Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. + Usage is recommended for development purposes only. +- ``..--p-j``: + Rolling tag corresponding to functionality provided by the ``container-base`` module at a certain version. + Auto-released for changes on ``develop`` branch as well as scheduled maintenance for the *three* most recent Dataverse releases. + Semantic versioning applies to functionality of the base image only. + (`Dockerfile `__ or same location within tagged Dataverse releases) +- ``..--p-j-r``: + Immutable tag, otherwise same as above. + Revision is incremented for rebuilds of the base image. + Recommended for production builds and usage. + +**Note:** It is not possible to provide shorter rolling tags, as the source for a tag like ``x.y.z-jammy`` would not be unambiguous. +If the version number does not change between releases and/or the ``develop`` branch, one is going to win over another image. +If at the same time Payara or Java version change (which is independent from the module version!), it becomes unclear which version people end up using. Image Contents ++++++++++++++ From c8f95a0e008d8e004dd2b25f3f450083968b73ef Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 28 Aug 2024 15:59:49 +0200 Subject: [PATCH 069/139] refactor(ct): align base image tagging with documented policy --- modules/container-base/pom.xml | 4 ++-- modules/container-base/src/backports/v6.1/001-pom.xml.patch | 4 ++-- modules/container-base/src/backports/v6.2/001-pom.xml.patch | 4 ++-- modules/container-base/src/backports/v6.3/001-pom.xml.patch | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index 5de94bfc941..8e5c476c64a 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -42,10 +42,10 @@ docker-build gdcc/base:${base.image.tag} - ${project.version}-${java.image.flavor} + ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} SNAPSHOT - ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} + ${base.image.tag}-r${base.image.revision} eclipse-temurin:${target.java.version}-jre-${java.image.flavor} noble 1000 diff --git a/modules/container-base/src/backports/v6.1/001-pom.xml.patch b/modules/container-base/src/backports/v6.1/001-pom.xml.patch index 508b496af14..a61f6b6fb14 100644 --- a/modules/container-base/src/backports/v6.1/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.1/001-pom.xml.patch @@ -14,10 +14,10 @@ gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor} ++ ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} + + SNAPSHOT -+ ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} ++ ${base.image.tag}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.2/001-pom.xml.patch b/modules/container-base/src/backports/v6.2/001-pom.xml.patch index 256680c2595..90fccffe53d 100644 --- a/modules/container-base/src/backports/v6.2/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.2/001-pom.xml.patch @@ -14,10 +14,10 @@ gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor} ++ ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} + + SNAPSHOT -+ ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} ++ ${base.image.tag}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.3/001-pom.xml.patch b/modules/container-base/src/backports/v6.3/001-pom.xml.patch index b1ab9bfe589..83e151b6656 100644 --- a/modules/container-base/src/backports/v6.3/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.3/001-pom.xml.patch @@ -14,10 +14,10 @@ gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor} ++ ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} + + SNAPSHOT -+ ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} ++ ${base.image.tag}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 From 136021316726bcfe2ff3c7ba48ad008f24973854 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 28 Aug 2024 18:03:25 +0200 Subject: [PATCH 070/139] ci(ct): split up push releases and maintenance workflows --- .github/workflows/container_base_push.yml | 107 +++----------- .github/workflows/container_maintenance.yml | 152 ++++++++++++++++++++ 2 files changed, 171 insertions(+), 88 deletions(-) create mode 100644 .github/workflows/container_maintenance.yml diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index ee7cb85a42e..f0e1b9d9f14 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -1,5 +1,5 @@ --- -name: Base Container Image +name: Container Images Releasing on: push: @@ -7,56 +7,23 @@ on: - 'v[6-9].**' branches: - 'develop' - - '10478-version-base-img' + # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore paths: - 'modules/container-base/**' - 'modules/dataverse-parent/pom.xml' - '.github/workflows/container_base_push.yml' - schedule: - - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC env: PLATFORMS: linux/amd64,linux/arm64 - NUM_PAST_RELEASES: 3 + DEVELOPMENT_BRANCH: develop jobs: - discover: - name: Discover Release Matrix - runs-on: ubuntu-latest - permissions: - contents: read - packages: read - # TODO: re-enable for final PR - # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled - #if: ${{ github.repository_owner == 'IQSS' }} - outputs: - branches: ${{ steps.matrix.outputs.branches }} - steps: - - name: Build branch matrix options - id: matrix - # TODO: remove the feature branch and re-enable the if/else! - run: | - # Get last three releases and include develop branch as matrix elements - #if [[ "${{ github.event_name }}" == "schedule" ]]; then - echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ - jq '[ .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, "develop", "10478-version-base-img" ]')" | tr -d "\n" | tr -s " " | \ - tee -a "$GITHUB_OUTPUT" - #else - # # Note: github.ref_name will be the name of the branch or the tag pushed - # echo "branches=['${{ github.ref_name }}']" | tee -a "$GITHUB_OUTPUT" - #fi - build: name: Build image runs-on: ubuntu-latest permissions: contents: read packages: read - needs: discover - strategy: - fail-fast: false - matrix: - branch: ${{ fromJson(needs.discover.outputs.branches) }} # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks #if: ${{ github.repository_owner == 'IQSS' }} @@ -64,8 +31,6 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - with: - ref: ${{ matrix.branch }} - name: Determine Java version from Parent POM run: | echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} @@ -94,12 +59,6 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - # In case of scheduled maintenance, we don't care about buildtime: use QEMU for AMD64 + ARM64 - - name: Set up QEMU for multi-arch builds - if: ${{ github.event_name == 'schedule' }} - uses: docker/setup-qemu-action@v3 - with: - platforms: ${{ env.PLATFORMS }} # In case this is a push to develop, we care about buildtime. # Configure a remote ARM64 build host in addition to the local AMD64 in two steps. # TODO: re-enable for final PR @@ -131,52 +90,17 @@ jobs: # "Files": null}]} # EOF - - name: Add additional tags as options - # TODO: remove the feature branch - run: | - # For the development branch, update the latest tag in addition - if [[ "${{ matrix.branch }}" == "develop" || "${{ matrix.branch }}" == "10478-version-base-img" ]]; then - echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" - # In case of releases <=6.2, we still need to provide backward compatible names "alpha" and "unstable" - elif [[ "${{ matrix.branch }}" == "v6.3" ]]; then - echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" - fi + # TODO: logic of retrieving the current reversion number, adding the property to the mvn call below - # Try to retrieve backport patches for this git ref (but don't fail if there aren't any) - # and try to apply them if present - - name: Get backport patches - continue-on-error: true - uses: actions/checkout@v4 - with: - path: backports - sparse-checkout: | - modules/container-base/src/backports/${{ matrix.branch }} - - name: Apply backport patches - id: patch + - name: Configure update of "latest" tag for development branch + if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }} run: | - find ${GITHUB_WORKSPACE}/backports/modules/container-base/src/backports/${{ matrix.branch }} -type f -name '*.patch' -print0 | \ - xargs -0 -n1 patch -p1 -s -i + echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" - name: Deploy multi-arch base container image to Docker Hub id: build run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} - - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} - name: Push description to DockerHub - uses: peter-evans/dockerhub-description@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - repository: gdcc/base - short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" - readme-filepath: ./modules/container-base/README.md - - # - if: always() - # name: Save status (workaround for matrix outputs) - # run: | - # # steps.build.outcome is the status BEFORE continue-on-error - # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" - push-app-img: name: "Rebase & Publish App Image" permissions: @@ -187,11 +111,18 @@ jobs: needs: - discover - build - strategy: - fail-fast: false - matrix: - branch: ${{ fromJson(needs.discover.outputs.branches) }} uses: ./.github/workflows/container_app_push.yml with: - branch: ${{ matrix.branch }} + branch: ${{ github.ref_name }} + + # TODO: job to update the docker hub description with supported tags and all + #- if: ${{ github.event_name == 'push' && github.ref_name == env.DEVELOPMENT_BRANCH }} + # name: Push description to DockerHub + # uses: peter-evans/dockerhub-description@v3 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + # repository: gdcc/base + # short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" + # readme-filepath: ./modules/container-base/README.md diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml new file mode 100644 index 00000000000..8f7efa4300d --- /dev/null +++ b/.github/workflows/container_maintenance.yml @@ -0,0 +1,152 @@ +--- +name: Container Images Scheduled Maintenance + +on: + # TODO: remove "push" in final PR + push: + branches: + - '10478-version-base-img' + paths: + - 'modules/container-base/**' + - 'modules/dataverse-parent/pom.xml' + - '.github/workflows/container_maintenance.yml' + # Allow manual workflow triggers in case we need to repair images on Docker Hub (build and replace) + workflow_dispatch: + schedule: + - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC + +env: + PLATFORMS: linux/amd64,linux/arm64 + NUM_PAST_RELEASES: 3 + # TODO: change to "develop" in final PR + DEVELOP_BRANCH: 10478-version-base-img + +jobs: + discover: + name: Discover Release Matrix + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + # TODO: re-enable for final PR + # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled + #if: ${{ github.repository_owner == 'IQSS' }} + outputs: + branches: ${{ steps.matrix.outputs.branches }} + current_release: ${{ steps.matrix.outputs.current_release }} + steps: + - name: Build branch matrix options + id: matrix + run: | + echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ + jq '[ .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, "${{ env.DEVELOP_BRANCH }}" ]')" | tr -d "\n" | tr -s " " | \ + tee -a "$GITHUB_OUTPUT" + echo "current_release=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq '.[0].tag_name' )" | tee -a "$GITHUB_OUTPUT" + + build: + name: Build image + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + needs: discover + strategy: + fail-fast: false + matrix: + branch: ${{ fromJson(needs.discover.outputs.branches) }} + # TODO: re-enable for final PR + # Only run in upstream repo - avoid unnecessary runs in forks + #if: ${{ github.repository_owner == 'IQSS' }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ matrix.branch }} + - name: Determine Java version from Parent POM + run: | + echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} + + - name: Set up JDK ${{ env.JAVA_VERSION }} + id: setup-java + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: 'temurin' + cache: 'maven' + cache-dependency-path: | + modules/container-base/pom.xml + - name: Download common cache on branch cache miss + if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} + uses: actions/cache/restore@v4 + with: + key: dataverse-maven-cache + path: ~/.m2/repository + + # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and + # on events in context of upstream because secrets. PRs run in context of forks by default! + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU for multi-arch builds + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ env.PLATFORMS }} + + # Try to retrieve backport patches for this git ref (but don't fail if there aren't any) + # and try to apply them if present + - name: Get and apply backported patches + # There might be no patches - ignore errors + continue-on-error: true + run: | + mkdir -p "${GITHUB_WORKSPACE}/patches" + curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ + tar -zxf - -C "${GITHUB_WORKSPACE}/patches" --wildcards "*/modules/container-base/src/backports/${{ matrix.branch }}" --strip-components=6 + find "${GITHUB_WORKSPACE}/patches" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + - name: Configure update of "latest" tag for development branch + if: ${{ matrix.branch == env.DEVELOP_BRANCH }} + run: | + echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" + + # TODO: figure out if rebuild is necessary + # TODO: logic of retrieving the current reversion number, adding the property to the mvn call below + + - name: Deploy multi-arch base container image to Docker Hub + id: build + run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + + # - if: always() + # name: Save status (workaround for matrix outputs) + # run: | + # # steps.build.outcome is the status BEFORE continue-on-error + # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" + + push-app-img: + name: "Rebase & Publish App Image" + permissions: + contents: read + packages: write + pull-requests: write + secrets: inherit + needs: + - discover + - build + strategy: + fail-fast: false + matrix: + branch: ${{ fromJson(needs.discover.outputs.branches) }} + uses: ./.github/workflows/container_app_push.yml + with: + branch: ${{ matrix.branch }} + + # TODO: job to update the docker hub description with supported tags and all + # - name: Push description to DockerHub + # uses: peter-evans/dockerhub-description@v3 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + # repository: gdcc/base + # short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" + # readme-filepath: ./modules/container-base/README.md From 716e2899656f03b64395057b8e193f40b8cc485b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 11:14:13 +0200 Subject: [PATCH 071/139] ci(ct): exclude app image rebuilds for easier testing --- .github/workflows/container_maintenance.yml | 34 ++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 8f7efa4300d..4e329cb9fe7 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -123,23 +123,23 @@ jobs: # # steps.build.outcome is the status BEFORE continue-on-error # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" - push-app-img: - name: "Rebase & Publish App Image" - permissions: - contents: read - packages: write - pull-requests: write - secrets: inherit - needs: - - discover - - build - strategy: - fail-fast: false - matrix: - branch: ${{ fromJson(needs.discover.outputs.branches) }} - uses: ./.github/workflows/container_app_push.yml - with: - branch: ${{ matrix.branch }} + #push-app-img: + # name: "Rebase & Publish App Image" + # permissions: + # contents: read + # packages: write + # pull-requests: write + # secrets: inherit + # needs: + # - discover + # - build + # strategy: + # fail-fast: false + # matrix: + # branch: ${{ fromJson(needs.discover.outputs.branches) }} + # uses: ./.github/workflows/container_app_push.yml + # with: + # branch: ${{ matrix.branch }} # TODO: job to update the docker hub description with supported tags and all # - name: Push description to DockerHub From d6339a9ae26594b215aad1a6b6e236b354e71f34 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 11:08:03 +0200 Subject: [PATCH 072/139] ci(ct): detect necessary rebuilds and calculate revision number --- .github/workflows/container_maintenance.yml | 76 +++++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 4e329cb9fe7..0323c1a7227 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -105,17 +105,83 @@ jobs: curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ tar -zxf - -C "${GITHUB_WORKSPACE}/patches" --wildcards "*/modules/container-base/src/backports/${{ matrix.branch }}" --strip-components=6 find "${GITHUB_WORKSPACE}/patches" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + + # Figure out if a rebuild is necessary because either there is an updated Java image or our installed packages need updates + - name: Check for recent Temurin image updates + id: temurin-check + run: | + JAVA_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout )" + JAVA_IMAGE_NS="library" + JAVA_IMAGE_REPO="$( echo "$JAVA_IMAGE" | cut -f1 -d: )" + JAVA_IMAGE_TAG="$( echo "$JAVA_IMAGE" | cut -f2 -d: )" + JAVA_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${JAVA_IMAGE_NS}/repositories/${JAVA_IMAGE_REPO}/tags/${JAVA_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$JAVA_IMAGE_LAST_UPDATE" = "null" ]]; then + echo "::error title='Invalid Java Image'::Could not find ${JAVA_IMAGE} in the registry" + exit 1 + fi + + BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" + BASE_IMAGE_NS="$( echo "$BASE_IMAGE" | cut -f1 -d/ )" + BASE_IMAGE_REPO="$( echo "$BASE_IMAGE" | cut -f1 -d: | cut -f2 -d/ )" + BASE_IMAGE_TAG="$( echo "$BASE_IMAGE" | cut -f2 -d: )" + BASE_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${BASE_IMAGE_NS}/repositories/${BASE_IMAGE_REPO}/tags/${BASE_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$BASE_IMAGE_LAST_UPDATE" = "null" || "$BASE_IMAGE_LAST_UPDATE" < "$JAVA_IMAGE_LAST_UPDATE" ]]; then + echo "Java image $JAVA_IMAGE has a newer release ($JAVA_IMAGE_LAST_UPDATE), which is more recent than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" + echo "newer_java_image=true" >> "${GITHUB_OUTPUT}" + else + echo "Java image $JAVA_IMAGE ($JAVA_IMAGE_LAST_UPDATE) is older than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" + echo "newer_java_image=false" >> "${GITHUB_OUTPUT}" + fi + # TODO: if we introduce more flavors as a matrix, we need to adapt the install command to check for updates + - name: Check for package updates in base image + id: package-check + if: ${{ steps.temurin-check.outputs.newer_java_image == 'false' }} + run: | + BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" + PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" + if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update && apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then + echo "Base image $BASE_IMAGE needs package updates" + echo "newer_packages=true" >> "${GITHUB_OUTPUT}" + else + echo "Base image $BASE_IMAGE has no package updates" + echo "newer_packages=false" >> "${GITHUB_OUTPUT}" + fi + + - name: Calculate revision number for immutable tag + run: | + BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" + BASE_IMAGE_NS_REPO="$( echo "$BASE_IMAGE" | cut -d: -f1 )" + BASE_IMAGE_TAG=""$( echo "$BASE_IMAGE" | cut -d: -f2 )"" + + function get_all_tags() { + ref="$1" + case "$ref" in + */*) :;; # namespace/repository syntax, leave as is + *) ref="library/$ref";; # bare repository name (docker official image); must convert to namespace/repository syntax + esac + token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ref}:pull" ) + i=0 + while [ $? == 0 ]; do + i=$((i+1)) + curl -sS -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/${ref}/tags/?page=$i&page_size=100" | jq -r '."results"[]["name"]' 2>/dev/null + done + } + CURRENT=$( get_all_tags "${BASE_IMAGE_NS_REPO}" | grep "${BASE_IMAGE_TAG}-r" | sed -e "s#${BASE_IMAGE_TAG}-r##" | sort -h | tail -n1 ) + + # If there is a current number, increment it - otherwise this is the initial version, set to 0 + if [[ "$CURRENT" ]]; then + echo "REVISION_OPTION=-Dbase.image.revision=$((CURRENT+1))" | tee -a "${GITHUB_ENV}" + else + echo "REVISION_OPTION=-Dbase.image.revision=0" | tee -a "${GITHUB_ENV}" + fi - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" - - # TODO: figure out if rebuild is necessary - # TODO: logic of retrieving the current reversion number, adding the property to the mvn call below - - name: Deploy multi-arch base container image to Docker Hub + if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' }} id: build - run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${REVISION_OPTION} -Ddocker.platforms=${{ env.PLATFORMS }} # - if: always() # name: Save status (workaround for matrix outputs) From 289f352ed11db894ecb52f5f3f34592e5a700177 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 12:36:25 +0200 Subject: [PATCH 073/139] build(ct): update base image with wait4x 2.14.1 Hopefully will clear lots of reported security vulnerabilities --- modules/container-base/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index 00deb085ec9..e3717928285 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -106,7 +106,7 @@ EOF ARG JATTACH_VERSION="v2.2" ARG JATTACH_TGZ_CHECKSUM_AMD64="acd9e17f15749306be843df392063893e97bfecc5260eef73ee98f06e5cfe02f" ARG JATTACH_TGZ_CHECKSUM_ARM64="288ae5ed87ee7fe0e608c06db5a23a096a6217c9878ede53c4e33710bdcaab51" -ARG WAIT4X_VERSION="v2.14.0" +ARG WAIT4X_VERSION="v2.14.1" ARG PKGS="jq imagemagick curl unzip wget acl lsof procps netcat-openbsd dumb-init" # Installing the packages in an extra container layer for better caching From 3a771f96726c98b66ada0c5eeb8020b56be36364 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 12:40:24 +0200 Subject: [PATCH 074/139] ci(ct): add option to manual trigger of maintenance workflow to force build --- .github/workflows/container_maintenance.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 0323c1a7227..f4a91aea562 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -12,6 +12,12 @@ on: - '.github/workflows/container_maintenance.yml' # Allow manual workflow triggers in case we need to repair images on Docker Hub (build and replace) workflow_dispatch: + inputs: + force_build: + type: boolean + required: false + default: false + description: "Build and deploy even if no newer Java images or package updates are found." schedule: - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC @@ -179,7 +185,7 @@ jobs: run: | echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" - name: Deploy multi-arch base container image to Docker Hub - if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' }} + if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${REVISION_OPTION} -Ddocker.platforms=${{ env.PLATFORMS }} From 899bba62bb5aa753e4be876079b69b7c03d64e7b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 12:43:51 +0200 Subject: [PATCH 075/139] ci(ct): no longer react to pushes in maintenance workflow Only use either scheduled or manual events to run the workflow. --- .github/workflows/container_maintenance.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index f4a91aea562..6c7b9d3e9ca 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -2,14 +2,6 @@ name: Container Images Scheduled Maintenance on: - # TODO: remove "push" in final PR - push: - branches: - - '10478-version-base-img' - paths: - - 'modules/container-base/**' - - 'modules/dataverse-parent/pom.xml' - - '.github/workflows/container_maintenance.yml' # Allow manual workflow triggers in case we need to repair images on Docker Hub (build and replace) workflow_dispatch: inputs: From ed17240da719d8a8862be57299ec60cb34a73d04 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 13:33:16 +0200 Subject: [PATCH 076/139] ci(ct): extract revision logic into separate script Easier to test and reuse for different workflows --- .../workflows/scripts/get_next_revision.sh | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100755 .github/workflows/scripts/get_next_revision.sh diff --git a/.github/workflows/scripts/get_next_revision.sh b/.github/workflows/scripts/get_next_revision.sh new file mode 100755 index 00000000000..5ec10ee2fb9 --- /dev/null +++ b/.github/workflows/scripts/get_next_revision.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# This script is used to retrieve the next revision number for a given image reference. +# The image reference must be of the form "[/]:", where the namespace will default to "library" if omitted. + +set -eu + +IMAGE=${1} +IMAGE_NS_REPO="$( echo "$IMAGE" | cut -d: -f1 )" +IMAGE_TAG="$( echo "$IMAGE" | cut -d: -f2 )" + +if [[ "$IMAGE_TAG" == "$IMAGE_NS_REPO" ]]; then + >&2 echo "You must provide an image reference in the format [/]:" + exit 1 +fi + +case "$IMAGE_NS_REPO" in + */*) :;; # namespace/repository syntax, leave as is + *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax +esac + +# Without such a token we run into rate limits +token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) + +ALL_TAGS="$( + i=0 + while [ $? == 0 ]; do + i=$((i+1)) + RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then + # If we run into an error on the first attempt, that means we have a problem. + if [[ "$i" == "1" ]]; then + >&2 echo "Error when retrieving tag data: $( echo "$RESULT" | jq '.message' )" + exit 2 + # Otherwise it will just mean we reached the last page already + else + break + fi + else + echo "$RESULT" | jq -r '."results"[]["name"]' + # DEBUG: + #echo "$RESULT" | >&2 jq -r '."results"[]["name"]' + fi + done +)" + +# If a former tag could not be found, it just might not exist already. Setting to -1, will be incremented to 0 to start a new series. +CURRENT=$( echo "$ALL_TAGS" | grep "${IMAGE_TAG}-r" | sed -e "s#${IMAGE_TAG}-r##" | sort -h | tail -n1 || echo "-1" ) +echo "$((CURRENT+1))" From 454fbe6c63cd9ea84c20970abcc2727c7fc401dc Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 13:34:09 +0200 Subject: [PATCH 077/139] ci(ct): add revision logic to base image release workflow on push --- .github/workflows/container_base_push.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index f0e1b9d9f14..fad436f0dda 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -90,8 +90,10 @@ jobs: # "Files": null}]} # EOF - # TODO: logic of retrieving the current reversion number, adding the property to the mvn call below - + - name: Calculate revision number for immutable tag + run: | + BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" + echo "REVISION_OPTION=-Dbase.image.revision=$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" | tee -a "${GITHUB_ENV}" - name: Configure update of "latest" tag for development branch if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }} run: | @@ -99,7 +101,7 @@ jobs: - name: Deploy multi-arch base container image to Docker Hub id: build - run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${REVISION_OPTION} -Ddocker.platforms=${{ env.PLATFORMS }} push-app-img: name: "Rebase & Publish App Image" From f1e3a3f0890a44e51f0417bd25433f8cbadbf7b1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 13:34:32 +0200 Subject: [PATCH 078/139] ci(ct): temporarily disable release of app image when rebuilding base --- .github/workflows/container_base_push.yml | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index fad436f0dda..5ce05c6f8e6 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -103,19 +103,19 @@ jobs: id: build run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${REVISION_OPTION} -Ddocker.platforms=${{ env.PLATFORMS }} - push-app-img: - name: "Rebase & Publish App Image" - permissions: - contents: read - packages: write - pull-requests: write - secrets: inherit - needs: - - discover - - build - uses: ./.github/workflows/container_app_push.yml - with: - branch: ${{ github.ref_name }} + #push-app-img: + # name: "Rebase & Publish App Image" + # permissions: + # contents: read + # packages: write + # pull-requests: write + # secrets: inherit + # needs: + # - discover + # - build + # uses: ./.github/workflows/container_app_push.yml + # with: + # branch: ${{ github.ref_name }} # TODO: job to update the docker hub description with supported tags and all From 266c7bc29993086a6918be710baaa6f3975f7af2 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 13:34:48 +0200 Subject: [PATCH 079/139] ci(ct): temporarily enable triggering release workflow from feature branch --- .github/workflows/container_base_push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 5ce05c6f8e6..532c64bd781 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -7,6 +7,8 @@ on: - 'v[6-9].**' branches: - 'develop' + # TODO: delete for final PR + - '10478-version-base-img' # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore paths: - 'modules/container-base/**' From c04d9e406716b02265894f2b9b5b0f3f47242046 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 13:35:25 +0200 Subject: [PATCH 080/139] ci(ct): use script for revision logic in maintenance flow --- .github/workflows/container_maintenance.yml | 29 ++++----------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 6c7b9d3e9ca..2e668af5162 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -147,31 +147,12 @@ jobs: - name: Calculate revision number for immutable tag run: | + # Get the revision logic script (not present on older releases) + curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ + tar -zxf - -C "." --wildcards "*/.github/workflows/scripts/get_next_revision.sh" --strip-components=1 + # Now get the new revision number BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" - BASE_IMAGE_NS_REPO="$( echo "$BASE_IMAGE" | cut -d: -f1 )" - BASE_IMAGE_TAG=""$( echo "$BASE_IMAGE" | cut -d: -f2 )"" - - function get_all_tags() { - ref="$1" - case "$ref" in - */*) :;; # namespace/repository syntax, leave as is - *) ref="library/$ref";; # bare repository name (docker official image); must convert to namespace/repository syntax - esac - token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ref}:pull" ) - i=0 - while [ $? == 0 ]; do - i=$((i+1)) - curl -sS -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/${ref}/tags/?page=$i&page_size=100" | jq -r '."results"[]["name"]' 2>/dev/null - done - } - CURRENT=$( get_all_tags "${BASE_IMAGE_NS_REPO}" | grep "${BASE_IMAGE_TAG}-r" | sed -e "s#${BASE_IMAGE_TAG}-r##" | sort -h | tail -n1 ) - - # If there is a current number, increment it - otherwise this is the initial version, set to 0 - if [[ "$CURRENT" ]]; then - echo "REVISION_OPTION=-Dbase.image.revision=$((CURRENT+1))" | tee -a "${GITHUB_ENV}" - else - echo "REVISION_OPTION=-Dbase.image.revision=0" | tee -a "${GITHUB_ENV}" - fi + echo "REVISION_OPTION=-Dbase.image.revision=$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" | tee -a "${GITHUB_ENV}" - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | From 2efd1b534803bb04f7e171548ba3f5561db0886d Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 13:52:46 +0200 Subject: [PATCH 081/139] ci(ct): fix revision logic to return 0 when rolling tag not present before --- .github/workflows/scripts/get_next_revision.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts/get_next_revision.sh b/.github/workflows/scripts/get_next_revision.sh index 5ec10ee2fb9..631f8b6f513 100755 --- a/.github/workflows/scripts/get_next_revision.sh +++ b/.github/workflows/scripts/get_next_revision.sh @@ -45,5 +45,9 @@ ALL_TAGS="$( )" # If a former tag could not be found, it just might not exist already. Setting to -1, will be incremented to 0 to start a new series. -CURRENT=$( echo "$ALL_TAGS" | grep "${IMAGE_TAG}-r" | sed -e "s#${IMAGE_TAG}-r##" | sort -h | tail -n1 || echo "-1" ) -echo "$((CURRENT+1))" +CURRENT=$( echo "$ALL_TAGS" | grep "${IMAGE_TAG}-r" | sed -e "s#${IMAGE_TAG}-r##" | sort -h | tail -n1 ) +if [[ "$CURRENT" ]]; then + echo "$((CURRENT+1))" +else + echo "0" +fi From b61395ae57f02587ce6cc4de15ad46cf6ed72a42 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 30 Aug 2024 16:11:20 +0200 Subject: [PATCH 082/139] build,doc(parent): add special base image versioning As long as we are developing the next version of Dataverse, we don't want the develop branch to fight the container images updated by the maintenance workflow for the past releases. To avoid this, we simply use the next release version for the base image we use during development. It requires updating a property during the release process twice, as we need to set the version to the "main" version property ($revision) once we aim for creating the tag. This way, when using the release tag in forks or other ways as a foundation, people end up with the right base image. After the tag is done, we need to prepare using the virtual next version within the develop branch again. --- .../source/developers/making-releases.rst | 10 ++++++++ modules/dataverse-parent/pom.xml | 24 ++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index e436ba9e9d2..a403f249beb 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -46,6 +46,7 @@ Bump Version Numbers Increment the version number to the milestone (e.g. 5.10.1) in the following two files: - modules/dataverse-parent/pom.xml -> ```` -> ```` (e.g. `pom.xml commit `_) +- modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` as ``${revision}`` - doc/sphinx-guides/source/conf.py (two places, e.g. `conf.py commit `_) Add the version being released to the lists in the following file: @@ -213,6 +214,15 @@ Now that we've published the release, close the milestone and create a new one. Note that for milestones we use just the number without the "v" (e.g. "5.10.1"). +Update the Container Base Image Version Property +------------------------------------------------ + +Create a new branch ``prepare-next-iteration`` and update the following files to prepare for the next development cycle: + +- modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` as ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}`` + +Now create a pull request and merge it - there is no review or QA necessary, as it is a predefined necessity. + Add the Release to the Dataverse Roadmap ---------------------------------------- diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 281aa274132..3df06281547 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -449,10 +449,13 @@ ct - - + ${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion} + @@ -478,6 +481,21 @@ false + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.0 + + + parse-version + + parse-version + + initialize + + + From f95476a4d9baa1c3386ac068583732a037c4fd58 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 30 Aug 2024 16:17:48 +0200 Subject: [PATCH 083/139] build(ct,parent): adapt base image POM to new tagging policy We will use a long tag with Payara and Java version in it during development. This way we auto-update images on dev's PCs on these breaking changes. For any releases, we will use short tags using only the version and the OS flavor. These rolling tags will be accompanied by the revisioned tags. These extend the base name with the suffix as before, but only for releases, not the development branch. As the development branch is a rolling target, immutable tags make little sense. This commit also adds the necessary backporting patches to older releases for the parent module to define the base.image.version as $revision, as described in the extended release process. --- modules/container-base/pom.xml | 12 ++++-------- .../src/backports/v6.1/001-pom.xml.patch | 17 ++++------------- .../src/backports/v6.1/003-parent-pom.xml.patch | 11 +++++++++++ .../src/backports/v6.2/001-pom.xml.patch | 17 ++++------------- .../src/backports/v6.2/004-parent-pom.xml.patch | 11 +++++++++++ .../src/backports/v6.3/001-pom.xml.patch | 17 ++++------------- .../src/backports/v6.3/004-parent-pom.xml.patch | 11 +++++++++++ 7 files changed, 49 insertions(+), 47 deletions(-) create mode 100644 modules/container-base/src/backports/v6.1/003-parent-pom.xml.patch create mode 100644 modules/container-base/src/backports/v6.2/004-parent-pom.xml.patch create mode 100644 modules/container-base/src/backports/v6.3/004-parent-pom.xml.patch diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index 8e5c476c64a..e707c6a69c8 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -13,7 +13,6 @@ io.gdcc container-base - 1.0.0 ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. @@ -41,11 +40,11 @@ ct docker-build + gdcc/base:${base.image.tag} - ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} - - SNAPSHOT - ${base.image.tag}-r${base.image.revision} + + ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} + ${base.image.version}-${java.image.flavor} eclipse-temurin:${target.java.version}-jre-${java.image.flavor} noble 1000 @@ -115,9 +114,6 @@ assembly.xml - - ${base.image.tag.revision} - diff --git a/modules/container-base/src/backports/v6.1/001-pom.xml.patch b/modules/container-base/src/backports/v6.1/001-pom.xml.patch index a61f6b6fb14..ef0fbdda9fc 100644 --- a/modules/container-base/src/backports/v6.1/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.1/001-pom.xml.patch @@ -1,23 +1,14 @@ --- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 +++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 -@@ -12,6 +12,7 @@ - - io.gdcc - container-base -+ 0.8.0 - ${packaging.type} - Container Base Image - This module provides an application server base image to be decorated with the Dataverse app. -@@ -40,8 +42,12 @@ +@@ -40,8 +42,11 @@ docker-build gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} -+ -+ SNAPSHOT -+ ${base.image.tag}-r${base.image.revision} ++ gdcc/base:${base.image.tag} ++ ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${java.image.flavor} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.1/003-parent-pom.xml.patch b/modules/container-base/src/backports/v6.1/003-parent-pom.xml.patch new file mode 100644 index 00000000000..a69cfd43ea7 --- /dev/null +++ b/modules/container-base/src/backports/v6.1/003-parent-pom.xml.patch @@ -0,0 +1,11 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -457,7 +457,8 @@ + + ++ ${revision} + + + diff --git a/modules/container-base/src/backports/v6.2/001-pom.xml.patch b/modules/container-base/src/backports/v6.2/001-pom.xml.patch index 90fccffe53d..ef0fbdda9fc 100644 --- a/modules/container-base/src/backports/v6.2/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.2/001-pom.xml.patch @@ -1,23 +1,14 @@ --- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 +++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 -@@ -12,6 +12,7 @@ - - io.gdcc - container-base -+ 0.9.0 - ${packaging.type} - Container Base Image - This module provides an application server base image to be decorated with the Dataverse app. -@@ -40,8 +42,12 @@ +@@ -40,8 +42,11 @@ docker-build gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} -+ -+ SNAPSHOT -+ ${base.image.tag}-r${base.image.revision} ++ gdcc/base:${base.image.tag} ++ ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${java.image.flavor} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.2/004-parent-pom.xml.patch b/modules/container-base/src/backports/v6.2/004-parent-pom.xml.patch new file mode 100644 index 00000000000..a69cfd43ea7 --- /dev/null +++ b/modules/container-base/src/backports/v6.2/004-parent-pom.xml.patch @@ -0,0 +1,11 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -457,7 +457,8 @@ + + ++ ${revision} + + + diff --git a/modules/container-base/src/backports/v6.3/001-pom.xml.patch b/modules/container-base/src/backports/v6.3/001-pom.xml.patch index 83e151b6656..ef0fbdda9fc 100644 --- a/modules/container-base/src/backports/v6.3/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.3/001-pom.xml.patch @@ -1,23 +1,14 @@ --- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 +++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 -@@ -12,6 +12,7 @@ - - io.gdcc - container-base -+ 0.9.1 - ${packaging.type} - Container Base Image - This module provides an application server base image to be decorated with the Dataverse app. -@@ -40,8 +42,12 @@ +@@ -40,8 +42,11 @@ docker-build gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} -+ -+ SNAPSHOT -+ ${base.image.tag}-r${base.image.revision} ++ gdcc/base:${base.image.tag} ++ ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${java.image.flavor} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.3/004-parent-pom.xml.patch b/modules/container-base/src/backports/v6.3/004-parent-pom.xml.patch new file mode 100644 index 00000000000..a69cfd43ea7 --- /dev/null +++ b/modules/container-base/src/backports/v6.3/004-parent-pom.xml.patch @@ -0,0 +1,11 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -457,7 +457,8 @@ + + ++ ${revision} + + + From a4354c0e05c4cc09057e924519fb70f313b65cee Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 30 Aug 2024 16:20:24 +0200 Subject: [PATCH 084/139] ci(ct): determine base image name once in maintenance workflow Instead of retrieving the base image name again in every step, add it to the environment as variable. In the same go, enforce the shorter tags for release branches and the full length ones for the development branch. --- .github/workflows/container_maintenance.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 2e668af5162..976f01f920f 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -104,6 +104,16 @@ jobs: tar -zxf - -C "${GITHUB_WORKSPACE}/patches" --wildcards "*/modules/container-base/src/backports/${{ matrix.branch }}" --strip-components=6 find "${GITHUB_WORKSPACE}/patches" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + # Determine the base image name we are going to use from here on + - name: Determine base image name + run: | + if [[ "${{ matrix.branch }}" = "${{ env.DEVELOP_BRANCH }}" ]]; then + NAME=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) + else + NAME=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag='${base.image.tag.release}' -q -DforceStdout ) + fi + echo "BASE_IMAGE=${NAME}" | tee -a "${GITHUB_ENV}" + # Figure out if a rebuild is necessary because either there is an updated Java image or our installed packages need updates - name: Check for recent Temurin image updates id: temurin-check @@ -118,7 +128,6 @@ jobs: exit 1 fi - BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" BASE_IMAGE_NS="$( echo "$BASE_IMAGE" | cut -f1 -d/ )" BASE_IMAGE_REPO="$( echo "$BASE_IMAGE" | cut -f1 -d: | cut -f2 -d/ )" BASE_IMAGE_TAG="$( echo "$BASE_IMAGE" | cut -f2 -d: )" @@ -135,7 +144,6 @@ jobs: id: package-check if: ${{ steps.temurin-check.outputs.newer_java_image == 'false' }} run: | - BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update && apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then echo "Base image $BASE_IMAGE needs package updates" @@ -151,7 +159,6 @@ jobs: curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ tar -zxf - -C "." --wildcards "*/.github/workflows/scripts/get_next_revision.sh" --strip-components=1 # Now get the new revision number - BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" echo "REVISION_OPTION=-Dbase.image.revision=$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" | tee -a "${GITHUB_ENV}" - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} From f15c9d60bcaed58fbe667595beafe099cf1e910f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 30 Aug 2024 16:21:06 +0200 Subject: [PATCH 085/139] style,ci(ct): use shell parameter extension and less cut Little faster, little more portable. --- .github/workflows/container_maintenance.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 976f01f920f..fdf40f81750 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -120,17 +120,17 @@ jobs: run: | JAVA_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout )" JAVA_IMAGE_NS="library" - JAVA_IMAGE_REPO="$( echo "$JAVA_IMAGE" | cut -f1 -d: )" - JAVA_IMAGE_TAG="$( echo "$JAVA_IMAGE" | cut -f2 -d: )" + JAVA_IMAGE_REPO="${JAVA_IMAGE%:*}" + JAVA_IMAGE_TAG="${JAVA_IMAGE#*:}" JAVA_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${JAVA_IMAGE_NS}/repositories/${JAVA_IMAGE_REPO}/tags/${JAVA_IMAGE_TAG}" | jq -r .last_updated )" if [[ "$JAVA_IMAGE_LAST_UPDATE" = "null" ]]; then echo "::error title='Invalid Java Image'::Could not find ${JAVA_IMAGE} in the registry" exit 1 fi - BASE_IMAGE_NS="$( echo "$BASE_IMAGE" | cut -f1 -d/ )" - BASE_IMAGE_REPO="$( echo "$BASE_IMAGE" | cut -f1 -d: | cut -f2 -d/ )" - BASE_IMAGE_TAG="$( echo "$BASE_IMAGE" | cut -f2 -d: )" + BASE_IMAGE_NS="${BASE_IMAGE%/*}" + BASE_IMAGE_REPO="$( echo "${BASE_IMAGE%:*}" | cut -f2 -d/ )" + BASE_IMAGE_TAG="${BASE_IMAGE#*:}" BASE_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${BASE_IMAGE_NS}/repositories/${BASE_IMAGE_REPO}/tags/${BASE_IMAGE_TAG}" | jq -r .last_updated )" if [[ "$BASE_IMAGE_LAST_UPDATE" = "null" || "$BASE_IMAGE_LAST_UPDATE" < "$JAVA_IMAGE_LAST_UPDATE" ]]; then echo "Java image $JAVA_IMAGE has a newer release ($JAVA_IMAGE_LAST_UPDATE), which is more recent than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" From e3de4a178de418934d8036ac7a720523920959bf Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 30 Aug 2024 16:28:12 +0200 Subject: [PATCH 086/139] ci(ct): on release branches make the base image tags short and add additional revision tag --- .github/workflows/container_maintenance.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index fdf40f81750..da7d3028c3f 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -153,13 +153,15 @@ jobs: echo "newer_packages=false" >> "${GITHUB_OUTPUT}" fi - - name: Calculate revision number for immutable tag + - name: Calculate revision number for immutable tag (on release branches only) + if: ${{ matrix.branch != env.DEVELOP_BRANCH }} run: | # Get the revision logic script (not present on older releases) curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ tar -zxf - -C "." --wildcards "*/.github/workflows/scripts/get_next_revision.sh" --strip-components=1 # Now get the new revision number - echo "REVISION_OPTION=-Dbase.image.revision=$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" | tee -a "${GITHUB_ENV}" + REVISION_TAG="${BASE_IMAGE#*:}-r$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" + echo "DOCKER_TAGS=-Dbase.image.tag=\${base.image.tag.release} -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=${REVISION_TAG}" | tee -a "${GITHUB_ENV}" - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | @@ -167,7 +169,7 @@ jobs: - name: Deploy multi-arch base container image to Docker Hub if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build - run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${REVISION_OPTION} -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} # - if: always() # name: Save status (workaround for matrix outputs) From 59b676be7567aa68b1351df2b7b0bf5f16823512 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 2 Sep 2024 14:52:42 +0200 Subject: [PATCH 087/139] docs(ct): update tagging policy for base image --- .../source/container/base-image.rst | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index e364579b568..13cd393a7e2 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -28,24 +28,29 @@ This image is sourced from the main upstream code `repository of the Dataverse s Development and maintenance of the `image's code `_ happens there (again, by the community). -In our tagging, we `follow the Bitname example `_: - -- ``latest``: Rolling tag on par with whatever happens on the ``develop`` branch. (`Dockerfile `__) - Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. - Usage is recommended for development purposes only. -- ``..--p-j``: - Rolling tag corresponding to functionality provided by the ``container-base`` module at a certain version. - Auto-released for changes on ``develop`` branch as well as scheduled maintenance for the *three* most recent Dataverse releases. - Semantic versioning applies to functionality of the base image only. - (`Dockerfile `__ or same location within tagged Dataverse releases) -- ``..--p-j-r``: - Immutable tag, otherwise same as above. - Revision is incremented for rebuilds of the base image. - Recommended for production builds and usage. - -**Note:** It is not possible to provide shorter rolling tags, as the source for a tag like ``x.y.z-jammy`` would not be unambiguous. -If the version number does not change between releases and/or the ``develop`` branch, one is going to win over another image. -If at the same time Payara or Java version change (which is independent from the module version!), it becomes unclear which version people end up using. +In our tagging, we `follow the Bitname example `_. +For the time being, the only operating system *flavors* we use and support are ``noble`` (6.4+) and ``jammy`` (pre 6.4). + +- | ``latest``: + | **Rolling tag** on par with whatever happens on the ``develop`` branch. (`Dockerfile `__) + | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. + Usage is recommended for **development purposes only**. +- | ``.--p-j``: + | **Rolling tag** during a development cycle of the Dataverse software (`Dockerfile `__). + | *Note 1:* No updates for tags after a development cycle ends. + Usage is recommended for **development purposes only**. + | *Note 2:* The current tag will always be 1 minor version ahead of Datavere releases. + Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. + The rolling tag in use during the cycle will be ``6.(x+1)-ffffff-p6.202P.P-jJJ``. + See also: :doc:`release process <../developers/making-releases>`. +- | ``.-``: + | **Rolling tag** for released versions of Dataverse, will always point to latest revision (see below) of a release image. + | The **three latest releases** of the Dataverse project are supported by regular, scheduled maintenance. + Content will be fairly stable, as disruptive changes like Payara or Java version switches are extremly unlikely once released. + Expect disruptive changes in case of high risk security threats. +- | ``.--r``: + | **Immutable tag**; revision is incremented for rebuilds of the base image. + Recommended for **production builds and usage.** Image Contents ++++++++++++++ From 6567e029d996478c2e8c660bd561f4ba6827b97f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 09:43:00 +0200 Subject: [PATCH 088/139] ci: add setup-maven action Instead of repeating the same steps everywhere, use a common action under our version control. --- .github/actions/setup-maven/action.yml | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/actions/setup-maven/action.yml diff --git a/.github/actions/setup-maven/action.yml b/.github/actions/setup-maven/action.yml new file mode 100644 index 00000000000..6edbba1b1a9 --- /dev/null +++ b/.github/actions/setup-maven/action.yml @@ -0,0 +1,37 @@ +--- +name: "Setup Maven and Caches" +description: "Determine Java version and setup Maven, including necessary caches." +inputs: + git-reference: + description: 'The git reference (branch/tag) to check out' + required: true + default: 'develop' + pom-paths: + description: "List of paths to Maven POM(s) for cache dependency setup" + required: false + default: 'pom.xml' +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.git-reference }} + - name: Determine Java version by reading the Maven property + shell: bash + run: | + echo "JAVA_VERSION=$(grep '' ${GITHUB_WORKSPACE}/modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" | tee -a ${GITHUB_ENV} + - name: Set up JDK ${{ env.JAVA_VERSION }} + id: setup-java + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: 'temurin' + cache: 'maven' + cache-dependency-path: ${{ inputs.pom-paths }} + - name: Download common cache on branch cache miss + if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} + uses: actions/cache/restore@v4 + with: + key: dataverse-maven-cache + path: ~/.m2/repository From 0966edaad306e123f294effc2b7be45f4491ba60 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 09:43:29 +0200 Subject: [PATCH 089/139] ci(ct): run maintenance with the new setup-maven action Testing here - might not yet work. --- .github/workflows/container_maintenance.yml | 45 ++++++++++----------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index da7d3028c3f..3baae428180 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -2,6 +2,8 @@ name: Container Images Scheduled Maintenance on: + # TODO: think about adding a (filtered) push event trigger here in case we change the patches + # --- # Allow manual workflow triggers in case we need to repair images on Docker Hub (build and replace) workflow_dispatch: inputs: @@ -57,29 +59,13 @@ jobs: #if: ${{ github.repository_owner == 'IQSS' }} steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout and Setup Maven + # TODO: change to upstream location in final PR + # Necessary as the checked out release branch might not contain the action as files + uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - ref: ${{ matrix.branch }} - - name: Determine Java version from Parent POM - run: | - echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} - - - name: Set up JDK ${{ env.JAVA_VERSION }} - id: setup-java - uses: actions/setup-java@v4 - with: - java-version: ${{ env.JAVA_VERSION }} - distribution: 'temurin' - cache: 'maven' - cache-dependency-path: | - modules/container-base/pom.xml - - name: Download common cache on branch cache miss - if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} - uses: actions/cache/restore@v4 - with: - key: dataverse-maven-cache - path: ~/.m2/repository + git-reference: ${{ matrix.branch }} + pom-paths: modules/container-base/pom.xml # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and # on events in context of upstream because secrets. PRs run in context of forks by default! @@ -169,7 +155,9 @@ jobs: - name: Deploy multi-arch base container image to Docker Hub if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build - run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + run: | + mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + echo "rebuild=true" | tee -a "${GITHUB_OUTPUT}" # - if: always() # name: Save status (workaround for matrix outputs) @@ -177,6 +165,17 @@ jobs: # # steps.build.outcome is the status BEFORE continue-on-error # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" + - name: Rebuild application container + if: ${{ steps.build.outputs.rebuild }} + uses: ./.github/actions/deploy-app-container + with: + registry: "" + registry_token: "" + ref: "" + base_image: "" + base_image_tag: "" + + #push-app-img: # name: "Rebase & Publish App Image" # permissions: From 7177c32a9642ec99e1df1f802d1b567778a8c821 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 11:05:57 +0200 Subject: [PATCH 090/139] ci: add action to check for newer releases of base images Reusable action to determine if a base image has more recent updates than the creation of the derived image. --- .../actions/check-newer-base-image/action.yml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/actions/check-newer-base-image/action.yml diff --git a/.github/actions/check-newer-base-image/action.yml b/.github/actions/check-newer-base-image/action.yml new file mode 100644 index 00000000000..d0c2e92d60a --- /dev/null +++ b/.github/actions/check-newer-base-image/action.yml @@ -0,0 +1,53 @@ +--- +name: "Check Newer Base Image" +description: "Determine if a base image is more recent than the derived." +inputs: + base: + description: 'The name and (rolling) tag of the base image to check - or - a shell command to get it' + required: true + derived: + description: 'The name and (rolling) tag of the derived image - or - a shell command to get it' + required: true +outputs: + is-more-recent: + description: "True if base image has a more recent update, false if not." + value: "${{ steps.determine.outputs.is_more_recent }}" + +runs: + using: composite + steps: + - shell: bash + id: determine + run: | + BASE_IMAGE="${{ inputs.base }}" + # Get namespace, default to "library" if not found + BASE_IMAGE_NS="${BASE_IMAGE%/*}" + if [[ "$BASE_IMAGE_NS" = "${BASE_IMAGE}" ]]; then + BASE_IMAGE_NS="library" + fi + BASE_IMAGE_REPO="${BASE_IMAGE%:*}" + BASE_IMAGE_TAG="${BASE_IMAGE#*:}" + + BASE_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${BASE_IMAGE_NS}/repositories/${BASE_IMAGE_REPO}/tags/${BASE_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$BASE_IMAGE_LAST_UPDATE" = "null" ]]; then + echo "::error title='Invalid Base Image'::Could not find ${BASE_IMAGE} in the registry" + exit 1 + fi + + DERIVED_IMAGE="${{ inputs.derived }}" + # Get namespace, default to "library" if not found + DERIVED_IMAGE_NS="${DERIVED_IMAGE%/*}" + if [[ "${DERIVED_IMAGE_NS}" = "${DERIVED_IMAGE}" ]]; then + DERIVED_IMAGE_NS="library" + fi + DERIVED_IMAGE_REPO="$( echo "${DERIVED_IMAGE%:*}" | cut -f2 -d/ )" + DERIVED_IMAGE_TAG="${DERIVED_IMAGE#*:}" + + DERIVED_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${DERIVED_IMAGE_NS}/repositories/${DERIVED_IMAGE_REPO}/tags/${DERIVED_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$DERIVED_IMAGE_LAST_UPDATE" = "null" || "$DERIVED_IMAGE_LAST_UPDATE" < "$BASE_IMAGE_LAST_UPDATE" ]]; then + echo "Base image $BASE_IMAGE has a newer release ($BASE_IMAGE_LAST_UPDATE), which is more recent than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + echo "is_more_recent=true" >> $GITHUB_OUTPUT + else + echo "Base image $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE) is older than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + echo "is_more_recent=false" >> $GITHUB_OUTPUT + fi From 8bc6bceff3fc365800440bedad1471f1ff382c68 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 11:05:57 +0200 Subject: [PATCH 091/139] ci: add action to check for newer releases of base images Reusable action to determine if a base image has more recent updates than the creation of the derived image. --- .../check-newer-parent-image/action.yml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/actions/check-newer-parent-image/action.yml diff --git a/.github/actions/check-newer-parent-image/action.yml b/.github/actions/check-newer-parent-image/action.yml new file mode 100644 index 00000000000..6f8ce950cec --- /dev/null +++ b/.github/actions/check-newer-parent-image/action.yml @@ -0,0 +1,53 @@ +--- +name: "Check Newer Parent Image" +description: "Determine if a base image is more recent than the derived." +inputs: + parent: + description: 'The name and (rolling) tag of the parent image to check - or - a shell command to get it' + required: true + derived: + description: 'The name and (rolling) tag of the derived image - or - a shell command to get it' + required: true +outputs: + is-more-recent: + description: "True if base image has a more recent update, false if not." + value: "${{ steps.determine.outputs.is_more_recent }}" + +runs: + using: composite + steps: + - shell: bash + id: determine + run: | + PARENT_IMAGE="${{ inputs.parent }}" + # Get namespace, default to "library" if not found + PARENT_IMAGE_NS="${PARENT_IMAGE%/*}" + if [[ "$PARENT_IMAGE_NS" = "${PARENT_IMAGE}" ]]; then + PARENT_IMAGE_NS="library" + fi + PARENT_IMAGE_REPO="${PARENT_IMAGE%:*}" + PARENT_IMAGE_TAG="${PARENT_IMAGE#*:}" + + PARENT_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${PARENT_IMAGE_NS}/repositories/${PARENT_IMAGE_REPO}/tags/${PARENT_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$PARENT_IMAGE_LAST_UPDATE" = "null" ]]; then + echo "::error title='Invalid PARENT Image'::Could not find ${PARENT_IMAGE} in the registry" + exit 1 + fi + + DERIVED_IMAGE="${{ inputs.derived }}" + # Get namespace, default to "library" if not found + DERIVED_IMAGE_NS="${DERIVED_IMAGE%/*}" + if [[ "${DERIVED_IMAGE_NS}" = "${DERIVED_IMAGE}" ]]; then + DERIVED_IMAGE_NS="library" + fi + DERIVED_IMAGE_REPO="$( echo "${DERIVED_IMAGE%:*}" | cut -f2 -d/ )" + DERIVED_IMAGE_TAG="${DERIVED_IMAGE#*:}" + + DERIVED_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${DERIVED_IMAGE_NS}/repositories/${DERIVED_IMAGE_REPO}/tags/${DERIVED_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$DERIVED_IMAGE_LAST_UPDATE" = "null" || "$DERIVED_IMAGE_LAST_UPDATE" < "$PARENT_IMAGE_LAST_UPDATE" ]]; then + echo "Parent image $PARENT_IMAGE has a newer release ($PARENT_IMAGE_LAST_UPDATE), which is more recent than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + echo "is_more_recent=true" >> $GITHUB_OUTPUT + else + echo "Parent image $PARENT_IMAGE ($PARENT_IMAGE_LAST_UPDATE) is older than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + echo "is_more_recent=false" >> $GITHUB_OUTPUT + fi From 19fcf89540120d8dbb5d01ceb993868e41f2c6e6 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 11:06:26 +0200 Subject: [PATCH 092/139] ci(ct): use newer image action in maintenance workflow --- .github/workflows/container_maintenance.yml | 31 +++++---------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 3baae428180..aabfe093301 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -103,32 +103,15 @@ jobs: # Figure out if a rebuild is necessary because either there is an updated Java image or our installed packages need updates - name: Check for recent Temurin image updates id: temurin-check - run: | - JAVA_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout )" - JAVA_IMAGE_NS="library" - JAVA_IMAGE_REPO="${JAVA_IMAGE%:*}" - JAVA_IMAGE_TAG="${JAVA_IMAGE#*:}" - JAVA_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${JAVA_IMAGE_NS}/repositories/${JAVA_IMAGE_REPO}/tags/${JAVA_IMAGE_TAG}" | jq -r .last_updated )" - if [[ "$JAVA_IMAGE_LAST_UPDATE" = "null" ]]; then - echo "::error title='Invalid Java Image'::Could not find ${JAVA_IMAGE} in the registry" - exit 1 - fi - - BASE_IMAGE_NS="${BASE_IMAGE%/*}" - BASE_IMAGE_REPO="$( echo "${BASE_IMAGE%:*}" | cut -f2 -d/ )" - BASE_IMAGE_TAG="${BASE_IMAGE#*:}" - BASE_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${BASE_IMAGE_NS}/repositories/${BASE_IMAGE_REPO}/tags/${BASE_IMAGE_TAG}" | jq -r .last_updated )" - if [[ "$BASE_IMAGE_LAST_UPDATE" = "null" || "$BASE_IMAGE_LAST_UPDATE" < "$JAVA_IMAGE_LAST_UPDATE" ]]; then - echo "Java image $JAVA_IMAGE has a newer release ($JAVA_IMAGE_LAST_UPDATE), which is more recent than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" - echo "newer_java_image=true" >> "${GITHUB_OUTPUT}" - else - echo "Java image $JAVA_IMAGE ($JAVA_IMAGE_LAST_UPDATE) is older than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" - echo "newer_java_image=false" >> "${GITHUB_OUTPUT}" - fi + # TODO: change to upstream location in final PR + uses: gdcc/wip-dataverse-base-image/.github/actions/check-newer-parent-image@10478-version-base-img + with: + parent: "$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout )" + derived: "${{ env.BASE_IMAGE }}" # TODO: if we introduce more flavors as a matrix, we need to adapt the install command to check for updates - name: Check for package updates in base image id: package-check - if: ${{ steps.temurin-check.outputs.newer_java_image == 'false' }} + if: ${{ steps.temurin-check.outputs.is-more-recent == 'false' }} run: | PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update && apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then @@ -153,7 +136,7 @@ jobs: run: | echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" - name: Deploy multi-arch base container image to Docker Hub - if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} + if: ${{ steps.temurin-check.outputs.is-more-recent == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build run: | mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} From dbf2372da3e6a624c1e957da09551dd38563e95a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 12:59:40 +0200 Subject: [PATCH 093/139] ci: add action to determine next image revision --- .github/actions/get-image-revision/action.yml | 78 +++++++++++++++++++ .../workflows/scripts/get_next_revision.sh | 53 ------------- 2 files changed, 78 insertions(+), 53 deletions(-) create mode 100644 .github/actions/get-image-revision/action.yml delete mode 100755 .github/workflows/scripts/get_next_revision.sh diff --git a/.github/actions/get-image-revision/action.yml b/.github/actions/get-image-revision/action.yml new file mode 100644 index 00000000000..ca9eedeabe1 --- /dev/null +++ b/.github/actions/get-image-revision/action.yml @@ -0,0 +1,78 @@ +name: "Get Incremented Revision Tag" +description: "For a pre-existing rolling tag lookup the latest immutable revision tag, increment and return" +inputs: + image-ref: + description: "The full image reference including namespace, repo and tag" + required: true + revision-separator: + description: "The separator string to use between tag and revision number. Defaults to '-r'" + required: false + default: "-r" + tag-options-prefix: + description: "A string that the revision tag will be appended to and sent to output tag-options" + required: false + default: "" +outputs: + revision-tag: + description: "The updated immutable tag, ready to use" + value: ${{ steps.extract.outputs.revision_tag }} + tag-options: + description: "A string with some command line options (may be empty)" + value: ${{ steps.extract.outputs.tag_options }} +runs: + using: composite + steps: + - shell: bash + id: extract + run: | + IMAGE="${{ inputs.image-ref }}" + IMAGE_NS_REPO="${IMAGE%:*}" + IMAGE_TAG="${IMAGE#*:}" + + if [[ "$IMAGE_TAG" = "$IMAGE_NS_REPO" ]]; then + >&2 echo "You must provide an image reference in the format [/]:" + exit 1 + fi + + case "$IMAGE_NS_REPO" in + */*) :;; # namespace/repository syntax, leave as is + *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax + esac + + # Without such a token we run into rate limits + token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) + + ALL_TAGS="$( + i=0 + while [ $? == 0 ]; do + i=$((i+1)) + RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then + # If we run into an error on the first attempt, that means we have a problem. + if [[ "$i" == "1" ]]; then + >&2 echo "Error when retrieving tag data: $( echo "$RESULT" | jq '.message' )" + exit 2 + # Otherwise it will just mean we reached the last page already + else + break + fi + else + echo "$RESULT" | jq -r '."results"[]["name"]' + # DEBUG: + #echo "$RESULT" | >&2 jq -r '."results"[]["name"]' + fi + done + )" + + # Note: if a former tag could not be found, it just might not exist already. Start new series with rev 0 + CURRENT=$( echo "$ALL_TAGS" | grep "${IMAGE_TAG}${{ inputs.revision-separator }}" | sed -e "s#${IMAGE_TAG}${{ inputs.revision-separator }}##" | sort -h | tail -n1 ) + if [[ "$CURRENT" ]]; then + REVISION_TAG="${IMAGE_TAG}${{ inputs.revision-separator }}$((CURRENT+1))" + else + REVISION_TAG="${IMAGE_TAG}${{ inputs.revision-separator }}0" + fi + echo "revision_tag=${REVISION_TAG}" | tee -a "$GITHUB_OUTPUT" + + if [[ -n "${{ inputs.tag-options-prefix }}" ]]; then + echo "tag_options=${{ inputs.tag-options-prefix }}${REVISION_TAG}" | tee -a "$GITHUB_OUTPUT" + fi \ No newline at end of file diff --git a/.github/workflows/scripts/get_next_revision.sh b/.github/workflows/scripts/get_next_revision.sh deleted file mode 100755 index 631f8b6f513..00000000000 --- a/.github/workflows/scripts/get_next_revision.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -# This script is used to retrieve the next revision number for a given image reference. -# The image reference must be of the form "[/]:", where the namespace will default to "library" if omitted. - -set -eu - -IMAGE=${1} -IMAGE_NS_REPO="$( echo "$IMAGE" | cut -d: -f1 )" -IMAGE_TAG="$( echo "$IMAGE" | cut -d: -f2 )" - -if [[ "$IMAGE_TAG" == "$IMAGE_NS_REPO" ]]; then - >&2 echo "You must provide an image reference in the format [/]:" - exit 1 -fi - -case "$IMAGE_NS_REPO" in - */*) :;; # namespace/repository syntax, leave as is - *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax -esac - -# Without such a token we run into rate limits -token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) - -ALL_TAGS="$( - i=0 - while [ $? == 0 ]; do - i=$((i+1)) - RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) - if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then - # If we run into an error on the first attempt, that means we have a problem. - if [[ "$i" == "1" ]]; then - >&2 echo "Error when retrieving tag data: $( echo "$RESULT" | jq '.message' )" - exit 2 - # Otherwise it will just mean we reached the last page already - else - break - fi - else - echo "$RESULT" | jq -r '."results"[]["name"]' - # DEBUG: - #echo "$RESULT" | >&2 jq -r '."results"[]["name"]' - fi - done -)" - -# If a former tag could not be found, it just might not exist already. Setting to -1, will be incremented to 0 to start a new series. -CURRENT=$( echo "$ALL_TAGS" | grep "${IMAGE_TAG}-r" | sed -e "s#${IMAGE_TAG}-r##" | sort -h | tail -n1 ) -if [[ "$CURRENT" ]]; then - echo "$((CURRENT+1))" -else - echo "0" -fi From 54e21cc63e526826cfcf29776ba2279d9d7dc4d8 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 13:00:02 +0200 Subject: [PATCH 094/139] ci(ct): use image revision action in maintenance workflow --- .github/workflows/container_maintenance.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index aabfe093301..692e934e9d2 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -96,7 +96,7 @@ jobs: if [[ "${{ matrix.branch }}" = "${{ env.DEVELOP_BRANCH }}" ]]; then NAME=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) else - NAME=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag='${base.image.tag.release}' -q -DforceStdout ) + NAME=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag="\${base.image.tag.release}" -q -DforceStdout ) fi echo "BASE_IMAGE=${NAME}" | tee -a "${GITHUB_ENV}" @@ -114,7 +114,7 @@ jobs: if: ${{ steps.temurin-check.outputs.is-more-recent == 'false' }} run: | PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" - if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update && apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then + if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update && apt install -s ${PKGS}" | grep -q "0 upgraded" ) ]]; then echo "Base image $BASE_IMAGE needs package updates" echo "newer_packages=true" >> "${GITHUB_OUTPUT}" else @@ -124,13 +124,12 @@ jobs: - name: Calculate revision number for immutable tag (on release branches only) if: ${{ matrix.branch != env.DEVELOP_BRANCH }} - run: | - # Get the revision logic script (not present on older releases) - curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ - tar -zxf - -C "." --wildcards "*/.github/workflows/scripts/get_next_revision.sh" --strip-components=1 - # Now get the new revision number - REVISION_TAG="${BASE_IMAGE#*:}-r$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" - echo "DOCKER_TAGS=-Dbase.image.tag=\${base.image.tag.release} -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=${REVISION_TAG}" | tee -a "${GITHUB_ENV}" + id: revision-tag + # TODO: change to upstream location in final PR + uses: gdcc/wip-dataverse-base-image/.github/actions/get-image-revision@10478-version-base-img + with: + image-ref: ${{ env.BASE_IMAGE }} + tag-options-prefix: "-Dbase.image.tag=\\${base.image.tag.release} -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=" - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | @@ -139,7 +138,7 @@ jobs: if: ${{ steps.temurin-check.outputs.is-more-recent == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build run: | - mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${{ steps.revision-tag.outputs.tag-options }} -Ddocker.platforms=${{ env.PLATFORMS }} echo "rebuild=true" | tee -a "${GITHUB_OUTPUT}" # - if: always() From db64fdfe05e7cbc89966ca3a6744a16fc423b97e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 13:25:35 +0200 Subject: [PATCH 095/139] fix,ci(ct): don't grep for package updates from apt update output --- .github/workflows/container_maintenance.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 692e934e9d2..c09aad22916 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -114,11 +114,11 @@ jobs: if: ${{ steps.temurin-check.outputs.is-more-recent == 'false' }} run: | PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" - if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update && apt install -s ${PKGS}" | grep -q "0 upgraded" ) ]]; then - echo "Base image $BASE_IMAGE needs package updates" + if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update >&2 && apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then + echo "Base image $BASE_IMAGE needs updates for our custom installed packages" echo "newer_packages=true" >> "${GITHUB_OUTPUT}" else - echo "Base image $BASE_IMAGE has no package updates" + echo "Base image $BASE_IMAGE has no updates for our custom installed packages" echo "newer_packages=false" >> "${GITHUB_OUTPUT}" fi From 268535122a98ee2078eb548e884dcd7253956d61 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 6 Sep 2024 15:11:30 -0400 Subject: [PATCH 096/139] doc tweaks for versioned base images: making releases #10827 --- .../source/developers/making-releases.rst | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index a403f249beb..5bd92833a6e 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -40,19 +40,24 @@ Use the GitHub issue number and the release tag for the name of the branch. (e.g Make the following changes in the release branch. -Bump Version Numbers --------------------- +Bump Version Numbers and Prepare Container Tags +----------------------------------------------- Increment the version number to the milestone (e.g. 5.10.1) in the following two files: - modules/dataverse-parent/pom.xml -> ```` -> ```` (e.g. `pom.xml commit `_) -- modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` as ``${revision}`` - doc/sphinx-guides/source/conf.py (two places, e.g. `conf.py commit `_) Add the version being released to the lists in the following file: - doc/sphinx-guides/source/versions.rst (e.g. `versions.rst commit `_) +Return to the parent pom and make the following change, which is necessary for proper tagging of images: + +- modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` to ``${revision}`` + +(Before you make this change the value should be ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}``. Later on, after cutting a release, we'll change it back to that value.) + Check in the Changes Above into a Release Branch and Merge It ------------------------------------------------------------- @@ -217,11 +222,11 @@ Note that for milestones we use just the number without the "v" (e.g. "5.10.1"). Update the Container Base Image Version Property ------------------------------------------------ -Create a new branch ``prepare-next-iteration`` and update the following files to prepare for the next development cycle: +Create a new branch (any name is fine but ``prepare-next-iteration`` is suggested) and update the following files to prepare for the next development cycle: -- modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` as ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}`` +- modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` to ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}`` -Now create a pull request and merge it - there is no review or QA necessary, as it is a predefined necessity. +Now create a pull request and merge it. Add the Release to the Dataverse Roadmap ---------------------------------------- From b47d09a1d1aa6cf9c394801ae3773257b495b440 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 6 Sep 2024 16:45:48 -0400 Subject: [PATCH 097/139] iterate on "supported image tags" section #10827 --- .../source/container/base-image.rst | 51 +++++++++++++------ .../source/developers/making-releases.rst | 2 + 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 13cd393a7e2..83799397860 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -21,6 +21,8 @@ IQSS will not offer you support how to deploy or run it, please reach out to the You might be interested in taking a look at :doc:`../developers/containers`, linking you to some (community-based) efforts. +.. _base-supported-image-tags: + Supported Image Tags ++++++++++++++++++++ @@ -28,29 +30,46 @@ This image is sourced from the main upstream code `repository of the Dataverse s Development and maintenance of the `image's code `_ happens there (again, by the community). -In our tagging, we `follow the Bitname example `_. -For the time being, the only operating system *flavors* we use and support are ``noble`` (6.4+) and ``jammy`` (pre 6.4). +Our tagging is inspired by `Bitnami `_ and we offer two categories of tags: + +- rolling: images change over time +- immutable: images are fixed and never change + +In the tags below you'll see the term "flavor". This refers to flavor of Linux the container is built on. We use Ubuntu as the basis for our images and, for the time being, the only operating system flavors we use and support are ``noble`` (6.4+) and ``jammy`` (pre-6.4). + +We offer two tags for production use, a rolling tag and an immutable one: -- | ``latest``: - | **Rolling tag** on par with whatever happens on the ``develop`` branch. (`Dockerfile `__) +- | **Rolling Production** + | Definition: ``.-`` + | Example: ``6.4-noble`` + | Summary: A **rolling tag** for released versions of Dataverse that will always point to latest revision of an immutable production image (see below). + | The **three latest releases** of the Dataverse project will receive regular, scheduled updates to the image such as security patches for the underlying operating system. Content will be fairly stable as disruptive changes like Payara or Java upgrades will be handled in a new major or minor upgrade to Dataverse (a new ``.`` tag). Expect disruptive changes in case of high risk security threats. +- | **Immutable Production** + | Definition: ``.--r`` + | Example: ``6.4-noble-r1`` + | Summary: An **immutable tag** where the revision is incremented for rebuilds of the image. + | This image should be especially attractive if you want explict control over when your images are updated. + +We offer two rolling tags for developers: + +- | **Latest** + | Definition: ``latest`` + | Example: ``latest`` + | Summary: A **rolling tag** that tracks the ``develop`` branch (see also :ref:`develop-branch`). (`Dockerfile `__) | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. Usage is recommended for **development purposes only**. -- | ``.--p-j``: - | **Rolling tag** during a development cycle of the Dataverse software (`Dockerfile `__). +- | **Flexible Stack** + | Definition: ``.--p-j``: + | Example: ``6.4-noble-p6.2024.6-j17`` + | Summary: A **rolling tag** during a development cycle of the Dataverse software (`Dockerfile `__). | *Note 1:* No updates for tags after a development cycle ends. Usage is recommended for **development purposes only**. | *Note 2:* The current tag will always be 1 minor version ahead of Datavere releases. Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. The rolling tag in use during the cycle will be ``6.(x+1)-ffffff-p6.202P.P-jJJ``. - See also: :doc:`release process <../developers/making-releases>`. -- | ``.-``: - | **Rolling tag** for released versions of Dataverse, will always point to latest revision (see below) of a release image. - | The **three latest releases** of the Dataverse project are supported by regular, scheduled maintenance. - Content will be fairly stable, as disruptive changes like Payara or Java version switches are extremly unlikely once released. - Expect disruptive changes in case of high risk security threats. -- | ``.--r``: - | **Immutable tag**; revision is incremented for rebuilds of the base image. - Recommended for **production builds and usage.** + See also: :doc:`/developers/making-releases`. + +You can find these tags at https://hub.docker.com/r/gdcc/base/tags Image Contents ++++++++++++++ @@ -403,4 +422,4 @@ from `run-java-sh recommendations`_. .. _MicroProfile Config Sources: https://docs.payara.fish/community/docs/Technical%20Documentation/MicroProfile/Config/Overview.html .. _run-java-sh recommendations: https://github.com/fabric8io-images/run-java-sh/blob/master/TUNING.md#recommandations .. _Domain Master Password: https://docs.payara.fish/community/docs/Technical%20Documentation/Payara%20Server%20Documentation/Security%20Guide/Administering%20System%20Security.html#to-change-the-master-password -.. _Payara Admin Console: https://docs.payara.fish/community/docs/Technical%20Documentation/Payara%20Server%20Documentation/General%20Administration/Overview.html#administration-console \ No newline at end of file +.. _Payara Admin Console: https://docs.payara.fish/community/docs/Technical%20Documentation/Payara%20Server%20Documentation/General%20Administration/Overview.html#administration-console diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index 5bd92833a6e..4936e942389 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -228,6 +228,8 @@ Create a new branch (any name is fine but ``prepare-next-iteration`` is suggeste Now create a pull request and merge it. +For more background, see :ref:`base-supported-image-tags`. + Add the Release to the Dataverse Roadmap ---------------------------------------- From 2ecbd692bd5a4a6fbb2702af5c859a2f09e0f577 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 13 Sep 2024 12:26:04 +0200 Subject: [PATCH 098/139] chore(ct): update base image wait4x to 2.14.2 for stdlib update --- modules/container-base/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index e3717928285..802db62e5e4 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -106,7 +106,7 @@ EOF ARG JATTACH_VERSION="v2.2" ARG JATTACH_TGZ_CHECKSUM_AMD64="acd9e17f15749306be843df392063893e97bfecc5260eef73ee98f06e5cfe02f" ARG JATTACH_TGZ_CHECKSUM_ARM64="288ae5ed87ee7fe0e608c06db5a23a096a6217c9878ede53c4e33710bdcaab51" -ARG WAIT4X_VERSION="v2.14.1" +ARG WAIT4X_VERSION="v2.14.2" ARG PKGS="jq imagemagick curl unzip wget acl lsof procps netcat-openbsd dumb-init" # Installing the packages in an extra container layer for better caching From 66bcbbb12976e03b6f41feae39255b214580f481 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 15:03:39 +0200 Subject: [PATCH 099/139] chore(ci): delete duplicate action after renaming --- .../actions/check-newer-base-image/action.yml | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 .github/actions/check-newer-base-image/action.yml diff --git a/.github/actions/check-newer-base-image/action.yml b/.github/actions/check-newer-base-image/action.yml deleted file mode 100644 index d0c2e92d60a..00000000000 --- a/.github/actions/check-newer-base-image/action.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -name: "Check Newer Base Image" -description: "Determine if a base image is more recent than the derived." -inputs: - base: - description: 'The name and (rolling) tag of the base image to check - or - a shell command to get it' - required: true - derived: - description: 'The name and (rolling) tag of the derived image - or - a shell command to get it' - required: true -outputs: - is-more-recent: - description: "True if base image has a more recent update, false if not." - value: "${{ steps.determine.outputs.is_more_recent }}" - -runs: - using: composite - steps: - - shell: bash - id: determine - run: | - BASE_IMAGE="${{ inputs.base }}" - # Get namespace, default to "library" if not found - BASE_IMAGE_NS="${BASE_IMAGE%/*}" - if [[ "$BASE_IMAGE_NS" = "${BASE_IMAGE}" ]]; then - BASE_IMAGE_NS="library" - fi - BASE_IMAGE_REPO="${BASE_IMAGE%:*}" - BASE_IMAGE_TAG="${BASE_IMAGE#*:}" - - BASE_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${BASE_IMAGE_NS}/repositories/${BASE_IMAGE_REPO}/tags/${BASE_IMAGE_TAG}" | jq -r .last_updated )" - if [[ "$BASE_IMAGE_LAST_UPDATE" = "null" ]]; then - echo "::error title='Invalid Base Image'::Could not find ${BASE_IMAGE} in the registry" - exit 1 - fi - - DERIVED_IMAGE="${{ inputs.derived }}" - # Get namespace, default to "library" if not found - DERIVED_IMAGE_NS="${DERIVED_IMAGE%/*}" - if [[ "${DERIVED_IMAGE_NS}" = "${DERIVED_IMAGE}" ]]; then - DERIVED_IMAGE_NS="library" - fi - DERIVED_IMAGE_REPO="$( echo "${DERIVED_IMAGE%:*}" | cut -f2 -d/ )" - DERIVED_IMAGE_TAG="${DERIVED_IMAGE#*:}" - - DERIVED_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${DERIVED_IMAGE_NS}/repositories/${DERIVED_IMAGE_REPO}/tags/${DERIVED_IMAGE_TAG}" | jq -r .last_updated )" - if [[ "$DERIVED_IMAGE_LAST_UPDATE" = "null" || "$DERIVED_IMAGE_LAST_UPDATE" < "$BASE_IMAGE_LAST_UPDATE" ]]; then - echo "Base image $BASE_IMAGE has a newer release ($BASE_IMAGE_LAST_UPDATE), which is more recent than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" - echo "is_more_recent=true" >> $GITHUB_OUTPUT - else - echo "Base image $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE) is older than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" - echo "is_more_recent=false" >> $GITHUB_OUTPUT - fi From d3dc5c65b957aa8489168560b4fe398d7e8d8439 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 16:14:21 +0200 Subject: [PATCH 100/139] docs(ct): update base image tag policy from latest discussion #10827 As discussed during the 2024-09-12 containerization working group meeting (see ct.gdcc.io) and on https://dataverse.zulipchat.com/#narrow/stream/375812-containers/topic/change.20version.20scheme.20base.20image.3F/near/469884104 --- .../source/container/base-image.rst | 53 ++++++++++++------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 83799397860..8d08f042e5d 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -37,39 +37,54 @@ Our tagging is inspired by `Bitnami .`` tag). +Expect disruptive changes in case of high risk security threats. + +- | **Latest** + | Definition: ``latest`` + | Summary: Rolling tag, always pointing to the latest revision of the most current Dataverse release. - | **Rolling Production** | Definition: ``.-`` | Example: ``6.4-noble`` - | Summary: A **rolling tag** for released versions of Dataverse that will always point to latest revision of an immutable production image (see below). - | The **three latest releases** of the Dataverse project will receive regular, scheduled updates to the image such as security patches for the underlying operating system. Content will be fairly stable as disruptive changes like Payara or Java upgrades will be handled in a new major or minor upgrade to Dataverse (a new ``.`` tag). Expect disruptive changes in case of high risk security threats. + | Summary: Rolling tag, pointing to the latest revision of an immutable production image for released versions of Dataverse. - | **Immutable Production** | Definition: ``.--r`` | Example: ``6.4-noble-r1`` | Summary: An **immutable tag** where the revision is incremented for rebuilds of the image. | This image should be especially attractive if you want explict control over when your images are updated. -We offer two rolling tags for developers: +Tags for Development Use +^^^^^^^^^^^^^^^^^^^^^^^^ -- | **Latest** - | Definition: ``latest`` - | Example: ``latest`` - | Summary: A **rolling tag** that tracks the ``develop`` branch (see also :ref:`develop-branch`). (`Dockerfile `__) +All of the tags below are strongly recommended for development purposes only due to their fast changing nature. +In addition to updates due to PR merges, the most recent are undergoing scheduled maintenance to ensure timely security fixes. +When a development cycle of the Dataverse project finishes, maintenance ceases for any tags carrying version numbers. +For now, stale images will be kept on Docker Hub indefinitely. + +- | **Unstable** + | Definition: ``unstable`` + | Summary: Rolling tag, tracking the ``develop`` branch (see also :ref:`develop-branch`). (`Dockerfile `__) | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. - Usage is recommended for **development purposes only**. +- | **Upcoming** + | Definition: ``.-`` + | Example: ``6.5-noble`` + | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. + Will roll over to the rolling production tag after a Dataverse release. - | **Flexible Stack** | Definition: ``.--p-j``: - | Example: ``6.4-noble-p6.2024.6-j17`` - | Summary: A **rolling tag** during a development cycle of the Dataverse software (`Dockerfile `__). - | *Note 1:* No updates for tags after a development cycle ends. - Usage is recommended for **development purposes only**. - | *Note 2:* The current tag will always be 1 minor version ahead of Datavere releases. - Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. - The rolling tag in use during the cycle will be ``6.(x+1)-ffffff-p6.202P.P-jJJ``. - See also: :doc:`/developers/making-releases`. - -You can find these tags at https://hub.docker.com/r/gdcc/base/tags + | Example: ``6.5-noble-p6.2024.6-j17`` + | Summary: Rolling tag during a development cycle of the Dataverse software (`Dockerfile `__). + +**NOTE**: The current tag will always be 1 minor version ahead of Datavere releases. +Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. +The rolling tag in use during the cycle will be ``6.(x+1)-FFF`` and ``6.(x+1)-FFF-p6.202P.P-jJJ``. +See also: :doc:`/developers/making-releases`. Image Contents ++++++++++++++ From 08a94bb84542b7980330c4d428a03debe6a95316 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 16:40:09 +0200 Subject: [PATCH 101/139] style,docs(ct): minor tweaks to base image policy as per @pdurbin --- doc/sphinx-guides/source/container/base-image.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 8d08f042e5d..84f6c5ece02 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -77,11 +77,11 @@ For now, stale images will be kept on Docker Hub indefinitely. | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. Will roll over to the rolling production tag after a Dataverse release. - | **Flexible Stack** - | Definition: ``.--p-j``: + | Definition: ``.--p-j`` | Example: ``6.5-noble-p6.2024.6-j17`` | Summary: Rolling tag during a development cycle of the Dataverse software (`Dockerfile `__). -**NOTE**: The current tag will always be 1 minor version ahead of Datavere releases. +**NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. The rolling tag in use during the cycle will be ``6.(x+1)-FFF`` and ``6.(x+1)-FFF-p6.202P.P-jJJ``. See also: :doc:`/developers/making-releases`. From 8cc9d20326509af6236d51362ab5729087fb1f09 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 16:49:03 +0200 Subject: [PATCH 102/139] feat(ct): switch latest to unstable in base image flow As per latest discussion, we want to keep the unstable tag around. It shall still point to the latest from develop as it has been done before our revised tagging policy. Latest will be used for production images, much more aligned with the Bitnami policy --- .github/workflows/container_maintenance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index c09aad22916..80e063bc222 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -133,7 +133,7 @@ jobs: - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | - echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" + echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=unstable" | tee -a "${GITHUB_ENV}" - name: Deploy multi-arch base container image to Docker Hub if: ${{ steps.temurin-check.outputs.is-more-recent == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build From 466d2ef4ec0c5a33253acca82b861fd05564a571 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 17:03:46 +0200 Subject: [PATCH 103/139] style,docs(ct): minor tweaks to base image policy as per @pdurbin --- doc/sphinx-guides/source/container/base-image.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 84f6c5ece02..a0852a5465f 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -72,12 +72,12 @@ For now, stale images will be kept on Docker Hub indefinitely. | Summary: Rolling tag, tracking the ``develop`` branch (see also :ref:`develop-branch`). (`Dockerfile `__) | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. - | **Upcoming** - | Definition: ``.-`` + | Definition: ``.-`` | Example: ``6.5-noble`` | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. Will roll over to the rolling production tag after a Dataverse release. - | **Flexible Stack** - | Definition: ``.--p-j`` + | Definition: ``.--p-j`` | Example: ``6.5-noble-p6.2024.6-j17`` | Summary: Rolling tag during a development cycle of the Dataverse software (`Dockerfile `__). From 4c9e80fe09e0f607a039d61fd2dfbf190235660e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 17:31:57 +0200 Subject: [PATCH 104/139] fix(ct): remove auth for revision action Maybe we won't run into a rate limit for now, as the limit of 180/s * 100 = 18000 tags/s seems to leave us some headroom for now. --- .github/actions/get-image-revision/action.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/get-image-revision/action.yml b/.github/actions/get-image-revision/action.yml index ca9eedeabe1..6eaf89abba2 100644 --- a/.github/actions/get-image-revision/action.yml +++ b/.github/actions/get-image-revision/action.yml @@ -39,14 +39,17 @@ runs: *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax esac - # Without such a token we run into rate limits + # Without such a token we may run into rate limits + # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) ALL_TAGS="$( i=0 while [ $? == 0 ]; do i=$((i+1)) - RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. + # RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + RESULT=$( curl -s "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then # If we run into an error on the first attempt, that means we have a problem. if [[ "$i" == "1" ]]; then From 86f2c84d3ded51597f2e15f9bc86817e502afcec Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 17:44:39 +0200 Subject: [PATCH 105/139] fix(ct): try to avoid shell substitution in base image flow We might need more backslashes to avoid telling the shell to replace what we want to be a maven property --- .github/workflows/container_maintenance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 80e063bc222..c8fe2fa2937 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -129,7 +129,7 @@ jobs: uses: gdcc/wip-dataverse-base-image/.github/actions/get-image-revision@10478-version-base-img with: image-ref: ${{ env.BASE_IMAGE }} - tag-options-prefix: "-Dbase.image.tag=\\${base.image.tag.release} -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=" + tag-options-prefix: "-Dbase.image.tag=\\\\${base.image.tag.release} -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=" - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | From 997f7ebab9c61f578ddeb31197edfed847e3e64b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 16 Sep 2024 17:54:58 +0200 Subject: [PATCH 106/139] fix(ct): avoid shell substitutions going awry for base image release tag Instead of using a Maven reference and dealing with escaping of $ chars, override the suffix with an empty string --- .github/workflows/container_maintenance.yml | 4 ++-- modules/container-base/pom.xml | 4 ++-- modules/container-base/src/backports/v6.1/001-pom.xml.patch | 4 ++-- modules/container-base/src/backports/v6.2/001-pom.xml.patch | 4 ++-- modules/container-base/src/backports/v6.3/001-pom.xml.patch | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index c8fe2fa2937..b7f8fcf45fa 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -96,7 +96,7 @@ jobs: if [[ "${{ matrix.branch }}" = "${{ env.DEVELOP_BRANCH }}" ]]; then NAME=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) else - NAME=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag="\${base.image.tag.release}" -q -DforceStdout ) + NAME=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) fi echo "BASE_IMAGE=${NAME}" | tee -a "${GITHUB_ENV}" @@ -129,7 +129,7 @@ jobs: uses: gdcc/wip-dataverse-base-image/.github/actions/get-image-revision@10478-version-base-img with: image-ref: ${{ env.BASE_IMAGE }} - tag-options-prefix: "-Dbase.image.tag=\\\\${base.image.tag.release} -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=" + tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=" - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index e707c6a69c8..b6667077d98 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -43,8 +43,8 @@ gdcc/base:${base.image.tag} - ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} - ${base.image.version}-${java.image.flavor} + ${base.image.version}-${java.image.flavor}${base.image.tag.suffix} + -p${payara.version}-j${target.java.version} eclipse-temurin:${target.java.version}-jre-${java.image.flavor} noble 1000 diff --git a/modules/container-base/src/backports/v6.1/001-pom.xml.patch b/modules/container-base/src/backports/v6.1/001-pom.xml.patch index ef0fbdda9fc..6498f972889 100644 --- a/modules/container-base/src/backports/v6.1/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.1/001-pom.xml.patch @@ -7,8 +7,8 @@ - unstable - eclipse-temurin:${target.java.version}-jre + gdcc/base:${base.image.tag} -+ ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} -+ ${base.image.version}-${java.image.flavor} ++ ${base.image.version}-${java.image.flavor}${base.image.tag.suffix} ++ -p${payara.version}-j${target.java.version} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.2/001-pom.xml.patch b/modules/container-base/src/backports/v6.2/001-pom.xml.patch index ef0fbdda9fc..6498f972889 100644 --- a/modules/container-base/src/backports/v6.2/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.2/001-pom.xml.patch @@ -7,8 +7,8 @@ - unstable - eclipse-temurin:${target.java.version}-jre + gdcc/base:${base.image.tag} -+ ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} -+ ${base.image.version}-${java.image.flavor} ++ ${base.image.version}-${java.image.flavor}${base.image.tag.suffix} ++ -p${payara.version}-j${target.java.version} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.3/001-pom.xml.patch b/modules/container-base/src/backports/v6.3/001-pom.xml.patch index ef0fbdda9fc..6498f972889 100644 --- a/modules/container-base/src/backports/v6.3/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.3/001-pom.xml.patch @@ -7,8 +7,8 @@ - unstable - eclipse-temurin:${target.java.version}-jre + gdcc/base:${base.image.tag} -+ ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} -+ ${base.image.version}-${java.image.flavor} ++ ${base.image.version}-${java.image.flavor}${base.image.tag.suffix} ++ -p${payara.version}-j${target.java.version} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 From 48ccdf2bbcbdf8b0cd739fcd504b2465d4cc93c4 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 09:01:14 +0200 Subject: [PATCH 107/139] ci(ct): remove out of scope actions for #10478 We will deal with shipping the updated application container images separately as part of issue #10618. Adding some comments about why some stuff is still around. --- .github/workflows/container_maintenance.yml | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index b7f8fcf45fa..c10215a7ccb 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -141,23 +141,25 @@ jobs: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${{ steps.revision-tag.outputs.tag-options }} -Ddocker.platforms=${{ env.PLATFORMS }} echo "rebuild=true" | tee -a "${GITHUB_OUTPUT}" + # TODO: this is here to not drop the knowledge about matrix output workarounds (for now) # - if: always() # name: Save status (workaround for matrix outputs) # run: | # # steps.build.outcome is the status BEFORE continue-on-error # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" - - name: Rebuild application container - if: ${{ steps.build.outputs.rebuild }} - uses: ./.github/actions/deploy-app-container - with: - registry: "" - registry_token: "" - ref: "" - base_image: "" - base_image_tag: "" - + # TODO: As part of issue #10618 we will need to create this action, shipping updated app images + #- name: Rebuild application container + # if: ${{ steps.build.outputs.rebuild }} + # uses: ./.github/actions/deploy-app-container + # with: + # registry: "" + # registry_token: "" + # ref: "" + # base_image: "" + # base_image_tag: "" + # TODO: This job should become part of the matrix as an action, so we don't need to fiddle with matrix outputs hacks #push-app-img: # name: "Rebase & Publish App Image" # permissions: From cc39131d321eab777d0fe4be158e900692870a62 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 09:15:20 +0200 Subject: [PATCH 108/139] ci(ct): reorganise tags for develop branch #10478 Now adding the "upcoming" tag during the develop branch run. Also some reorganizing to ship the tag options using outputs, not env vars. Moving the common Maven option to enable tag overrides to the command instead of the options building. --- .github/workflows/container_maintenance.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index c10215a7ccb..fdef2aac6d4 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -94,11 +94,11 @@ jobs: - name: Determine base image name run: | if [[ "${{ matrix.branch }}" = "${{ env.DEVELOP_BRANCH }}" ]]; then - NAME=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) + echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" else - NAME=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) + echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" fi - echo "BASE_IMAGE=${NAME}" | tee -a "${GITHUB_ENV}" # Figure out if a rebuild is necessary because either there is an updated Java image or our installed packages need updates - name: Check for recent Temurin image updates @@ -129,16 +129,18 @@ jobs: uses: gdcc/wip-dataverse-base-image/.github/actions/get-image-revision@10478-version-base-img with: image-ref: ${{ env.BASE_IMAGE }} - tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=" + tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision=" - name: Configure update of "latest" tag for development branch + id: develop-tag if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | - echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=unstable" | tee -a "${GITHUB_ENV}" + echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}" - name: Deploy multi-arch base container image to Docker Hub if: ${{ steps.temurin-check.outputs.is-more-recent == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build run: | - mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${{ steps.revision-tag.outputs.tag-options }} -Ddocker.platforms=${{ env.PLATFORMS }} + mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \ + -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }} echo "rebuild=true" | tee -a "${GITHUB_OUTPUT}" # TODO: this is here to not drop the knowledge about matrix output workarounds (for now) From c6b992a1ac11e514b01be81e12cf03039295a6cb Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 10:30:13 +0200 Subject: [PATCH 109/139] chore(parent): remove OSS plugin snapshot repo after upgrade to DMP 0.45 --- modules/dataverse-parent/pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 3df06281547..6bea02569ec 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -214,14 +214,6 @@ never - - oss.sonatype.org - https://oss.sonatype.org/content/repositories/snapshots - - true - always - - From 8955ef087717b337a9ba7d777505dbd02cce303f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 10:31:22 +0200 Subject: [PATCH 110/139] build(ct): make application image use new base image flexi stack #10478 --- pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 9b4914a1640..d8fe55ba22a 100644 --- a/pom.xml +++ b/pom.xml @@ -999,8 +999,9 @@ unstable false gdcc/base:${base.image.tag} - - rev1 + noble + + ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} gdcc/configbaker:${conf.image.tag} ${app.image.tag} From 710cf3cab8b6734d12d5ec2be1134669de5af740 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 10:53:09 +0200 Subject: [PATCH 111/139] style(ct): remove stale comment from base module pom --- modules/container-base/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index b6667077d98..6417b5b91fa 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -12,7 +12,6 @@ io.gdcc container-base - ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. From b043edff6d1846f223ff89d0f0184abc5d726580 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 11:02:13 +0200 Subject: [PATCH 112/139] chore(ct): add comment about apt sec updates detection via list hack For now, we stick to not alter the image more than necessary. Only packages we installed will be upgraded, as these are not part of the normal Java base images. The Java base images receive regular updates and undergo testing. It might be unwise to just install all the security updates we could get. Leaving the option here for later saves the trouble to dig up the solution again. --- .github/workflows/container_maintenance.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index fdef2aac6d4..3a28f355e94 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -120,7 +120,13 @@ jobs: else echo "Base image $BASE_IMAGE has no updates for our custom installed packages" echo "newer_packages=false" >> "${GITHUB_OUTPUT}" - fi + fi + + # TODO: In a future version of this script, we might want to include checking for other security updates, + # not just updates to the packages we installed. + # grep security /etc/apt/sources.list > /tmp/security.list + # apt-get update -oDir::Etc::Sourcelist=/tmp/security.list + # apt-get dist-upgrade -y -oDir::Etc::Sourcelist=/tmp/security.list -oDir::Etc::SourceParts=/bin/false -s - name: Calculate revision number for immutable tag (on release branches only) if: ${{ matrix.branch != env.DEVELOP_BRANCH }} From 9b12bc234f4fe9e455bfce3e66f3df7120f413cc Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 14:43:25 +0200 Subject: [PATCH 113/139] ci(ct): use new setup-maven action in base image push workflow Simplify setup following DRY principle --- .github/workflows/container_base_push.yml | 25 +++++------------------ 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 532c64bd781..ff5c6117f2b 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -31,27 +31,12 @@ jobs: #if: ${{ github.repository_owner == 'IQSS' }} steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Determine Java version from Parent POM - run: | - echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} - - - name: Set up JDK ${{ env.JAVA_VERSION }} - id: setup-java - uses: actions/setup-java@v4 - with: - java-version: ${{ env.JAVA_VERSION }} - distribution: 'temurin' - cache: 'maven' - cache-dependency-path: | - modules/container-base/pom.xml - - name: Download common cache on branch cache miss - if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} - uses: actions/cache/restore@v4 + - name: Checkout and Setup Maven + # TODO: change to upstream location in final PR + uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - key: dataverse-maven-cache - path: ~/.m2/repository + git-reference: ${{ github.ref }} + pom-paths: modules/container-base/pom.xml # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and # on events in context of upstream because secrets. PRs run in context of forks by default! From 4f8499f79b09b8dfb619ae4ece0684c97b5f2999 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 14:44:38 +0200 Subject: [PATCH 114/139] ci(ct): replace logic in base image push workflow Using the same actions and steps as done in the maintenance action should work for this workflow in case of a push event, too. --- .github/workflows/container_base_push.yml | 26 +++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index ff5c6117f2b..29493fcbb5c 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -77,18 +77,32 @@ jobs: # "Files": null}]} # EOF - - name: Calculate revision number for immutable tag + # Determine the base image name we are going to use from here on + - name: Determine base image name run: | - BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" - echo "REVISION_OPTION=-Dbase.image.revision=$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" | tee -a "${GITHUB_ENV}" + if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then + echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + else + echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + fi + - name: Calculate revision number for immutable tag (on release branches only) + if: ${{ github.ref_name != env.DEVELOPMENT_BRANCH }} + id: revision-tag + uses: ./.github/actions/get-image-revision + with: + image-ref: ${{ env.BASE_IMAGE }} + tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision=" - name: Configure update of "latest" tag for development branch + id: develop-tag if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }} run: | - echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" - + echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}" - name: Deploy multi-arch base container image to Docker Hub id: build - run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${REVISION_OPTION} -Ddocker.platforms=${{ env.PLATFORMS }} + run: | + mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \ + -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }} #push-app-img: # name: "Rebase & Publish App Image" From b43af732f84f7d7d9899ec18ac5b7095b9fac907 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 14:45:08 +0200 Subject: [PATCH 115/139] test(ct): temporarily set dev branch in base image push flow to feature branch --- .github/workflows/container_base_push.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 29493fcbb5c..25dfac32ba8 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -17,7 +17,8 @@ on: env: PLATFORMS: linux/amd64,linux/arm64 - DEVELOPMENT_BRANCH: develop + # TODO: set back to develop for final PR + DEVELOPMENT_BRANCH: 10478-version-base-img jobs: build: From 415bae3a81ca91b0007ae0b769c07ad31bf13839 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 14:47:49 +0200 Subject: [PATCH 116/139] ci(ct): use setup-maven action in app image push workflow to simplify setup Following DRY principle, reuse the steps defined --- .github/workflows/container_app_push.yml | 45 +++++++++--------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 3344a8e2d0d..7f8c880a6bc 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -38,23 +38,15 @@ jobs: if: ${{ github.repository_owner == 'IQSS' && inputs.branch == '10478-version-base-img' }} steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up JDK - id: setup-java - uses: actions/setup-java@v3 - with: - java-version: "17" - distribution: temurin - cache: maven - - name: Download common cache on branch cache miss - if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} - uses: actions/cache/restore@v4 + - name: Checkout and Setup Maven + # TODO: change to upstream location in final PR + uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - path: ~/.m2/repository - key: dataverse-maven-cache - + git-reference: ${{ github.ref_name }} + pom-paths: | + pom.xml + modules/container-configbaker/pom.xml + modules/dataverse-parent/pom.xml - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests) run: > @@ -119,20 +111,15 @@ jobs: if: needs.check-secrets.outputs.available == 'true' && ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name))) steps: - - uses: actions/checkout@v4 - - name: Set up JDK - id: setup-java - uses: actions/setup-java@v3 - with: - java-version: "17" - distribution: temurin - cache: maven - - name: Download common cache on branch cache miss - if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} - uses: actions/cache/restore@v4 + - name: Checkout and Setup Maven + # TODO: change to upstream location in final PR + uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - path: ~/.m2/repository - key: dataverse-maven-cache + git-reference: ${{ github.ref_name }} + pom-paths: | + pom.xml + modules/container-configbaker/pom.xml + modules/dataverse-parent/pom.xml # Depending on context, we push to different targets. Login accordingly. - if: github.event_name != 'pull_request' From 5eb6d1a2cabaff69acbb30b90bc7fce0acc0146e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 14:48:28 +0200 Subject: [PATCH 117/139] test(ct): temporarily enable app image push flow execution in all forks --- .github/workflows/container_app_push.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 7f8c880a6bc..15ef7b892e8 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -35,7 +35,8 @@ jobs: packages: write pull-requests: write # Only run in upstream repo - avoid unnecessary runs in forks - if: ${{ github.repository_owner == 'IQSS' && inputs.branch == '10478-version-base-img' }} + # TODO: re-enable for final PR + #if: ${{ github.repository_owner == 'IQSS' }} steps: - name: Checkout and Setup Maven From 71f43992d4587660a298850d58211657bb8ed28f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 14:50:03 +0200 Subject: [PATCH 118/139] ci(ct): use an optional base image ref for app image push flow We need to transfer the determined base image name we might have just built from the calling workflow into this flow. As we provide a default value, this is picked up for pull_requests. --- .github/workflows/container_app_push.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 15ef7b892e8..68c079b9ee5 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -6,10 +6,11 @@ on: # Instead, push events will trigger from the base image and maven unit tests via workflow_call. workflow_call: inputs: - branch: + base-image-ref: type: string - description: "A tag or branch to checkout for building the image" - required: true + description: "Reference of the base image to build on in full qualified form [/]/:" + required: false + default: "gdcc/base:unstable" pull_request: branches: - develop @@ -21,7 +22,6 @@ on: env: IMAGE_TAG: unstable - BASE_IMAGE_TAG: unstable REGISTRY: "" # Empty means default to Docker Hub PLATFORMS: "linux/amd64,linux/arm64" MASTER_BRANCH_TAG: alpha @@ -53,6 +53,7 @@ jobs: run: > mvn -B -f modules/dataverse-parent -P ct -pl edu.harvard.iq:dataverse -am + -Dbase.image=${{ inputs.base-image-ref }} install # TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false") @@ -156,11 +157,12 @@ jobs: run: > mvn -B -f modules/dataverse-parent -P ct -pl edu.harvard.iq:dataverse -am + -Dbase.image=${{ inputs.base-image-ref }} install - name: Deploy multi-arch application and configbaker container image run: > mvn - -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }} + -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image=${{ inputs.base-image-ref }} ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }} -P ct deploy From 5a0cfa0ab3169d87cd878f40b59e5af33d7cd8ae Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 15:46:15 +0200 Subject: [PATCH 119/139] ci(setup-maven): try to auto-detect git ref It's not so easy to determine the right git ref for different scenarios like PR, etc. Unless explicitly given a ref, try to autodetect the right one to go with. --- .github/actions/setup-maven/action.yml | 4 ++-- .github/workflows/container_app_push.yml | 2 -- .github/workflows/container_base_push.yml | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-maven/action.yml b/.github/actions/setup-maven/action.yml index 6edbba1b1a9..4cf09f34231 100644 --- a/.github/actions/setup-maven/action.yml +++ b/.github/actions/setup-maven/action.yml @@ -4,8 +4,8 @@ description: "Determine Java version and setup Maven, including necessary caches inputs: git-reference: description: 'The git reference (branch/tag) to check out' - required: true - default: 'develop' + required: false + default: '${{ github.ref }}' pom-paths: description: "List of paths to Maven POM(s) for cache dependency setup" required: false diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 68c079b9ee5..555ea97cf52 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -43,7 +43,6 @@ jobs: # TODO: change to upstream location in final PR uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - git-reference: ${{ github.ref_name }} pom-paths: | pom.xml modules/container-configbaker/pom.xml @@ -117,7 +116,6 @@ jobs: # TODO: change to upstream location in final PR uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - git-reference: ${{ github.ref_name }} pom-paths: | pom.xml modules/container-configbaker/pom.xml diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 25dfac32ba8..1365dd69eaf 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -36,7 +36,6 @@ jobs: # TODO: change to upstream location in final PR uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - git-reference: ${{ github.ref }} pom-paths: modules/container-base/pom.xml # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and From 34db758a32044ab0281e8d627c59c0b3d4553c5f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 17:16:51 +0200 Subject: [PATCH 120/139] chore(ct): add notes in flows about adding a path filter We want to avoid duplicate runs which might trigger race conditions for image shipments. --- .github/workflows/container_app_push.yml | 4 ++++ .github/workflows/maven_unit_test.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 555ea97cf52..e3dbcd4d2c6 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -48,6 +48,10 @@ jobs: modules/container-configbaker/pom.xml modules/dataverse-parent/pom.xml + # TODO: Add a filter step here, that avoids building the image if this is a PR and there are other files touched than declared above. + # Use https://github.com/dorny/paths-filter to solve this. This will ensure we do not run this twice if this workflow + # will be triggered by the other workflows already (base image or java changes) + - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests) run: > mvn -B -f modules/dataverse-parent diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 4ad4798bc64..1c9c6c7d5e3 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -156,6 +156,10 @@ jobs: # NOTE: this may be extended with adding a report to the build output, leave a comment, send to Sonarcloud, ... + # TODO: Add a filter step here, that avoids calling the app image release workflow if there are changes to the base image. + # Use https://github.com/dorny/paths-filter to solve this. Will require and additional job or adding to integration-test job. + # This way we ensure that we're not running the app image flow with a non-matching base image. + push-app-img: name: Publish App Image permissions: From 314fa339a6ea162c2bb4dc069a2c71c01a24df89 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 17:24:17 +0200 Subject: [PATCH 121/139] chore(ct): add note about missing triggers for base push flow This is out of scope for #10827, but should be addressed at a later point to avoid duplicated runs with potential race conditions. Also it enables proper rebuilds for preview images when someone is just trying to create a base image change, but which should obviously be tested full chain. --- .github/workflows/container_base_push.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 1365dd69eaf..d52209c5c30 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -15,6 +15,9 @@ on: - 'modules/dataverse-parent/pom.xml' - '.github/workflows/container_base_push.yml' + # TODO: we are missing a workflow_call option here, so we can trigger this flow from pr comments and maven tests (keep the secrets availability in mind!) + # TODO: we are missing a pull_request option here (filter for stuff that would trigger the maven runs!) so we can trigger preview builds for them when coming from the main repo (keep the secrets availability in mind!) + env: PLATFORMS: linux/amd64,linux/arm64 # TODO: set back to develop for final PR From 09f7264f98fa49e303d3fd068306b60f2a7a54a8 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 17:25:05 +0200 Subject: [PATCH 122/139] fix,ci(ct): don't trigger the base push flow for backports Obviously these are meant for the maintenance workflow, not the push flow! --- .github/workflows/container_base_push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index d52209c5c30..c6283856ac8 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -12,6 +12,7 @@ on: # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore paths: - 'modules/container-base/**' + - '!modules/container-base/src/backports/**' - 'modules/dataverse-parent/pom.xml' - '.github/workflows/container_base_push.yml' From 44ec28d61f492804ca491fa6402bf3f5148bff6d Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 17 Sep 2024 17:31:39 +0200 Subject: [PATCH 123/139] ci(ct): trigger app flow from base push flow We detect the tag we have been using in a finalizing step to hand a proper base image ref to the app image workflow to make it work on the images we just pushed to the registry. --- .github/workflows/container_base_push.yml | 39 +++++++++++++++-------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index c6283856ac8..8a9a3212fe9 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -26,7 +26,7 @@ env: jobs: build: - name: Build image + name: Base Image runs-on: ubuntu-latest permissions: contents: read @@ -34,6 +34,8 @@ jobs: # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks #if: ${{ github.repository_owner == 'IQSS' }} + outputs: + base-image-ref: ${{ steps.finalize.outputs.base-image-ref }} steps: - name: Checkout and Setup Maven @@ -102,25 +104,34 @@ jobs: if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }} run: | echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}" + - name: Deploy multi-arch base container image to Docker Hub id: build run: | mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \ -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }} - #push-app-img: - # name: "Rebase & Publish App Image" - # permissions: - # contents: read - # packages: write - # pull-requests: write - # secrets: inherit - # needs: - # - discover - # - build - # uses: ./.github/workflows/container_app_push.yml - # with: - # branch: ${{ github.ref_name }} + - name: Determine appropriate base image ref for app image + id: finalize + run: | + if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then + echo "base-image-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT" + else + echo "base-image-ref=gdcc/base:${{ steps.revision-tag.outputs.revision-tag }}" | tee -a "$GITHUB_OUTPUT" + fi + + push-app-img: + name: "Rebase & Publish App Image" + permissions: + contents: read + packages: write + pull-requests: write + secrets: inherit + needs: + - build + uses: ./.github/workflows/container_app_push.yml + with: + base-image-ref: ${{ needs.build.outputs.base-image-ref }} # TODO: job to update the docker hub description with supported tags and all From 26be5c1813f7d9aae501c6872b1eb8b950abe447 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 18 Sep 2024 17:04:47 +0200 Subject: [PATCH 124/139] ci(ct): reshape maintenance workflow into external matrix script Unfortunately, matrix jobs logs and outputs cannot be aggregated in Github Actions. The only way to work around the limitations of GHA is by using a custom build script that create a similar matrix like experience. This commit introduces these scripts, probably also making some custom actions we added obsolete. --- .github/workflows/container_maintenance.yml | 125 ++-------------- .github/workflows/scripts/maintenance-job.sh | 141 +++++++++++++++++++ .github/workflows/scripts/utils.sh | 107 ++++++++++++++ 3 files changed, 260 insertions(+), 113 deletions(-) create mode 100755 .github/workflows/scripts/maintenance-job.sh create mode 100644 .github/workflows/scripts/utils.sh diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 3a28f355e94..65cbcc81d48 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -18,42 +18,14 @@ on: env: PLATFORMS: linux/amd64,linux/arm64 NUM_PAST_RELEASES: 3 - # TODO: change to "develop" in final PR - DEVELOP_BRANCH: 10478-version-base-img jobs: - discover: - name: Discover Release Matrix - runs-on: ubuntu-latest - permissions: - contents: read - packages: read - # TODO: re-enable for final PR - # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled - #if: ${{ github.repository_owner == 'IQSS' }} - outputs: - branches: ${{ steps.matrix.outputs.branches }} - current_release: ${{ steps.matrix.outputs.current_release }} - steps: - - name: Build branch matrix options - id: matrix - run: | - echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ - jq '[ .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, "${{ env.DEVELOP_BRANCH }}" ]')" | tr -d "\n" | tr -s " " | \ - tee -a "$GITHUB_OUTPUT" - echo "current_release=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq '.[0].tag_name' )" | tee -a "$GITHUB_OUTPUT" - build: name: Build image runs-on: ubuntu-latest permissions: contents: read packages: read - needs: discover - strategy: - fail-fast: false - matrix: - branch: ${{ fromJson(needs.discover.outputs.branches) }} # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks #if: ${{ github.repository_owner == 'IQSS' }} @@ -64,7 +36,6 @@ jobs: # Necessary as the checked out release branch might not contain the action as files uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - git-reference: ${{ matrix.branch }} pom-paths: modules/container-base/pom.xml # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and @@ -79,93 +50,21 @@ jobs: with: platforms: ${{ env.PLATFORMS }} - # Try to retrieve backport patches for this git ref (but don't fail if there aren't any) - # and try to apply them if present - - name: Get and apply backported patches - # There might be no patches - ignore errors - continue-on-error: true - run: | - mkdir -p "${GITHUB_WORKSPACE}/patches" - curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ - tar -zxf - -C "${GITHUB_WORKSPACE}/patches" --wildcards "*/modules/container-base/src/backports/${{ matrix.branch }}" --strip-components=6 - find "${GITHUB_WORKSPACE}/patches" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i - - # Determine the base image name we are going to use from here on - - name: Determine base image name - run: | - if [[ "${{ matrix.branch }}" = "${{ env.DEVELOP_BRANCH }}" ]]; then - echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}" - echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" - else - echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" - fi - - # Figure out if a rebuild is necessary because either there is an updated Java image or our installed packages need updates - - name: Check for recent Temurin image updates - id: temurin-check - # TODO: change to upstream location in final PR - uses: gdcc/wip-dataverse-base-image/.github/actions/check-newer-parent-image@10478-version-base-img - with: - parent: "$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout )" - derived: "${{ env.BASE_IMAGE }}" - # TODO: if we introduce more flavors as a matrix, we need to adapt the install command to check for updates - - name: Check for package updates in base image - id: package-check - if: ${{ steps.temurin-check.outputs.is-more-recent == 'false' }} + # Discover the releases we want to maintain + - name: Discover maintained releases + id: discover run: | - PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" - if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update >&2 && apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then - echo "Base image $BASE_IMAGE needs updates for our custom installed packages" - echo "newer_packages=true" >> "${GITHUB_OUTPUT}" - else - echo "Base image $BASE_IMAGE has no updates for our custom installed packages" - echo "newer_packages=false" >> "${GITHUB_OUTPUT}" - fi + DEVELOPMENT_BRANCH=$( curl -f -sS https://api.github.com/repos/${{ github.repository }} | jq -r '.default_branch' ) + echo "DEVELOPMENT_BRANCH=$DEVELOPMENT_BRANCH" | tee -a "$GITHUB_ENV" + echo "branches=$( curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq -r " .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, \"${DEVELOPMENT_BRANCH}\" " | tr "\n" " " )" | tee -a "${GITHUB_OUTPUT}" + - # TODO: In a future version of this script, we might want to include checking for other security updates, - # not just updates to the packages we installed. - # grep security /etc/apt/sources.list > /tmp/security.list - # apt-get update -oDir::Etc::Sourcelist=/tmp/security.list - # apt-get dist-upgrade -y -oDir::Etc::Sourcelist=/tmp/security.list -oDir::Etc::SourceParts=/bin/false -s - - - name: Calculate revision number for immutable tag (on release branches only) - if: ${{ matrix.branch != env.DEVELOP_BRANCH }} - id: revision-tag - # TODO: change to upstream location in final PR - uses: gdcc/wip-dataverse-base-image/.github/actions/get-image-revision@10478-version-base-img - with: - image-ref: ${{ env.BASE_IMAGE }} - tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision=" - - name: Configure update of "latest" tag for development branch - id: develop-tag - if: ${{ matrix.branch == env.DEVELOP_BRANCH }} + # Execute matrix build for the discovered branches + - name: Execute build matrix script + id: execute run: | - echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}" - - name: Deploy multi-arch base container image to Docker Hub - if: ${{ steps.temurin-check.outputs.is-more-recent == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} - id: build - run: | - mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \ - -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }} - echo "rebuild=true" | tee -a "${GITHUB_OUTPUT}" - - # TODO: this is here to not drop the knowledge about matrix output workarounds (for now) - # - if: always() - # name: Save status (workaround for matrix outputs) - # run: | - # # steps.build.outcome is the status BEFORE continue-on-error - # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" - - # TODO: As part of issue #10618 we will need to create this action, shipping updated app images - #- name: Rebuild application container - # if: ${{ steps.build.outputs.rebuild }} - # uses: ./.github/actions/deploy-app-container - # with: - # registry: "" - # registry_token: "" - # ref: "" - # base_image: "" - # base_image_tag: "" + echo "force_build=${{ inputs.force_build }}" + .github/workflows/scripts/maintenance-job.sh ${{ steps.discover.outputs.branches }} # TODO: This job should become part of the matrix as an action, so we don't need to fiddle with matrix outputs hacks #push-app-img: diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/maintenance-job.sh new file mode 100755 index 00000000000..bf1a9748bf7 --- /dev/null +++ b/.github/workflows/scripts/maintenance-job.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +# A matrix-like job to maintain a number of releases as well as the latest snap of Dataverse. + +# PREREQUISITES: +# - You have Java, Maven, QEMU and Docker all setup and ready to go +# - You obviously checked out the develop branch, otherwise you'd not be executing this script +# - You added all the branch names you want to run maintenance for as arguments +# Optional, but recommended: +# - You added a DEVELOPMENT_BRANCH env var to your runner/job env with the name of the development branch +# - You added a FORCE_BUILD=0|1 env var to indicate if the base image build should be forced +# - You added a PLATFORMS env var with all the target platforms you want to build for + +# NOTE: +# This script is a culmination of Github Action steps into a single script. +# The reason to put all of this in here is due to the complexity of the Github Action and the limitation of the +# matrix support in Github actions, where outputs cannot be aggregated or otherwise used further. + +set -euo pipefail + +# Get all the inputs +# If not within a runner, just print to stdout (duplicating the output in case of tee usage, but that's ok for testing) +GITHUB_OUTPUT=${GITHUB_OUTPUT:-"/proc/self/fd/1"} +GITHUB_ENV=${GITHUB_ENV:-"/proc/self/fd/1"} +GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-"$(pwd)"} +GITHUB_SERVER_URL=${GITHUB_SERVER_URL:-"https://github.com"} +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:-"IQSS/dataverse"} + +MAINTENANCE_WORKSPACE="${GITHUB_WORKSPACE}/maintenance-job" + +DEVELOPMENT_BRANCH="${DEVELOPMENT_BRANCH:-"develop"}" +FORCE_BUILD="${FORCE_BUILD:-"0"}" +PLATFORMS="${PLATFORMS:-"linux/amd64,linux/arm64"}" + +# Setup and validation +if [[ -z "$*" ]]; then + >&2 echo "You must give a list of branch names as arguments" + exit 1; +fi + +source "$( dirname "$0" )/utils.sh" + +# Delete old stuff if present +rm -rf "$MAINTENANCE_WORKSPACE" +mkdir -p "$MAINTENANCE_WORKSPACE" + +# Cache the image tags we maintain in this array (same order as branches array!) +# This list will be used to build the support matrix within the Docker Hub image description +SUPPORTED_ROLLING_TAGS=() + +for BRANCH in "$@"; do + echo "::group::Running maintenance for $BRANCH" + + # 0. Determine if this is a development branch and the most current release + IS_DEV=0 + if [[ "$BRANCH" = "$DEVELOPMENT_BRANCH" ]]; then + IS_DEV=1 + fi + IS_CURRENT_RELEASE=0 + if [[ "$BRANCH" = $( curl -f -sS "https://api.github.com/repos/$GITHUB_REPOSITORY/releases" | jq -r '.[0].tag_name' ) ]]; then + IS_CURRENT_RELEASE=1 + fi + + # 1. Let's get the maintained sources + git clone -c advice.detachedHead=false --depth 1 --branch "$BRANCH" "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" "$MAINTENANCE_WORKSPACE/$BRANCH" + # Switch context + cd "$MAINTENANCE_WORKSPACE/$BRANCH" + + # 2. Now let's apply the patches (we have them checked out in $GITHUB_WORKSPACE, not necessarily in this local checkout) + if [[ -d ${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH ]]; then + find "${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + fi + + # 3. Determine the base image ref (/:) + BASE_IMAGE_REF="" + # For the dev branch we want to full flexi stack tag, to detect stack upgrades requiring new build + if (( IS_DEV )); then + BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) + else + BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) + fi + + # 4. Check for Temurin image updates + JAVA_IMAGE_REF=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout ) + NEWER_JAVA_IMAGE=0 + if check_newer_parent "$JAVA_IMAGE_REF" "$BASE_IMAGE_REF"; then + NEWER_JAVA_IMAGE=1 + fi + + # 5. Check for package updates in base image + PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" + NEWER_PKGS=0 + # Don't bother with package checks if the java image is newer already + if ! (( NEWER_JAVA_IMAGE )); then + if check_newer_pkgs "$BASE_IMAGE_REF" "$PKGS"; then + NEWER_PKGS=1 + fi + fi + + # 6. Get current immutable revision tag if not on the dev branch + REV=$( current_revision "$BASE_IMAGE_REF" ) + CURRENT_REV_TAG="${BASE_IMAGE_REF#*:}-r$REV" + NEXT_REV_TAG="${BASE_IMAGE_REF#*:}-r$(( REV + 1 ))" + + # 7. Let's put together what tags we want added to this build run + TAG_OPTIONS="" + if ! (( IS_DEV )); then + TAG_OPTIONS="-Dbase.image=$BASE_IMAGE_REF -Ddocker.tags.revision=$NEXT_REV_TAG" + + # In case of the current release, add the "latest" tag as well. Also add to list of rolling tags. + if (( IS_CURRENT_RELEASE )); then + TAG_OPTIONS="$TAG_OPTIONS -Ddocker.tags.latest=latest" + SUPPORTED_ROLLING_TAGS+=("[\"latest\", \"${BASE_IMAGE_REF#*:}\"]") + else + SUPPORTED_ROLLING_TAGS+=("[\"${BASE_IMAGE_REF#*:}\"]") + fi + else + UPCOMING_TAG=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image.tag -Dbase.image.tag.suffix="" -q -DforceStdout ) + TAG_OPTIONS="-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=$UPCOMING_TAG" + + SUPPORTED_ROLLING_TAGS+=("[\"unstable\", \"$UPCOMING_TAG\", \"${BASE_IMAGE_REF#*:}\"]") + fi + echo "Determined these additional Maven tag options: $TAG_OPTIONS" + + # 7. Let's build the base image if necessary + NEWER_BASE_IMAGE=0 + if (( NEWER_JAVA_IMAGE + NEWER_PKGS + FORCE_BUILD > 0 )); then + mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ + -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS + NEWER_BASE_IMAGE=1 + else + echo "No rebuild necessary, we're done here." + fi + + if (( NEWER_BASE_IMAGE )); then + echo "Built a new base image, should continue with application images now..." + # TODO: rebuild the app images here + fi + + echo "::endgroup::" +done diff --git a/.github/workflows/scripts/utils.sh b/.github/workflows/scripts/utils.sh new file mode 100644 index 00000000000..b63787105ee --- /dev/null +++ b/.github/workflows/scripts/utils.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +set -euo pipefail + +function check_newer_parent() { + PARENT_IMAGE="$1" + # Get namespace, default to "library" if not found + PARENT_IMAGE_NS="${PARENT_IMAGE%/*}" + if [[ "$PARENT_IMAGE_NS" = "${PARENT_IMAGE}" ]]; then + PARENT_IMAGE_NS="library" + fi + PARENT_IMAGE_REPO="${PARENT_IMAGE%:*}" + PARENT_IMAGE_TAG="${PARENT_IMAGE#*:}" + + PARENT_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${PARENT_IMAGE_NS}/repositories/${PARENT_IMAGE_REPO}/tags/${PARENT_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$PARENT_IMAGE_LAST_UPDATE" = "null" ]]; then + echo "::error title='Invalid PARENT Image'::Could not find ${PARENT_IMAGE} in the registry" + exit 1 + fi + + DERIVED_IMAGE="$2" + # Get namespace, default to "library" if not found + DERIVED_IMAGE_NS="${DERIVED_IMAGE%/*}" + if [[ "${DERIVED_IMAGE_NS}" = "${DERIVED_IMAGE}" ]]; then + DERIVED_IMAGE_NS="library" + fi + DERIVED_IMAGE_REPO="$( echo "${DERIVED_IMAGE%:*}" | cut -f2 -d/ )" + DERIVED_IMAGE_TAG="${DERIVED_IMAGE#*:}" + + DERIVED_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${DERIVED_IMAGE_NS}/repositories/${DERIVED_IMAGE_REPO}/tags/${DERIVED_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$DERIVED_IMAGE_LAST_UPDATE" = "null" || "$DERIVED_IMAGE_LAST_UPDATE" < "$PARENT_IMAGE_LAST_UPDATE" ]]; then + echo "Parent image $PARENT_IMAGE has a newer release ($PARENT_IMAGE_LAST_UPDATE), which is more recent than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + return 0 + else + echo "Parent image $PARENT_IMAGE ($PARENT_IMAGE_LAST_UPDATE) is older than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + return 1 + fi +} + +function check_newer_pkgs() { + IMAGE="$1" + PKGS="$2" + + docker run --rm -u 0 "${IMAGE}" sh -c "apt update >/dev/null 2>&1 && apt install -s ${PKGS}" | tee /proc/self/fd/2 | grep -q "0 upgraded" + + if [[ ! $? ]]; then + echo "Base image $IMAGE needs updates for our custom installed packages" + return 0 + else + echo "Base image $IMAGE has no updates for our custom installed packages" + return 1 + fi + + # TODO: In a future version of this script, we might want to include checking for other security updates, + # not just updates to the packages we installed. + # grep security /etc/apt/sources.list > /tmp/security.list + # apt-get update -oDir::Etc::Sourcelist=/tmp/security.list + # apt-get dist-upgrade -y -oDir::Etc::Sourcelist=/tmp/security.list -oDir::Etc::SourceParts=/bin/false -s + +} + +function current_revision() { + IMAGE="$1" + IMAGE_NS_REPO="${IMAGE%:*}" + IMAGE_TAG="${IMAGE#*:}" + + if [[ "$IMAGE_TAG" = "$IMAGE_NS_REPO" ]]; then + >&2 echo "You must provide an image reference in the format [/]:" + exit 1 + fi + + case "$IMAGE_NS_REPO" in + */*) :;; # namespace/repository syntax, leave as is + *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax + esac + + # Without such a token we may run into rate limits + # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. + # token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) + + ALL_TAGS="$( + i=0 + while [ $? == 0 ]; do + i=$((i+1)) + # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. + # RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + RESULT=$( curl -s "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then + # If we run into an error on the first attempt, that means we have a problem. + if [[ "$i" == "1" ]]; then + >&2 echo "Error when retrieving tag data: $( echo "$RESULT" | jq '.message' )" + exit 2 + # Otherwise it will just mean we reached the last page already + else + break + fi + else + echo "$RESULT" | jq -r '."results"[]["name"]' + # DEBUG: + #echo "$RESULT" | >&2 jq -r '."results"[]["name"]' + fi + done + )" + + # Note: if a former tag could not be found, it just might not exist already. Start new series with rev 0 + echo "$ALL_TAGS" | grep "${IMAGE_TAG}-r" | sed -e "s#${IMAGE_TAG}-r##" | sort -h | tail -n1 || echo "-1" +} From ff0498bc2aca5fbef5b53f4786d9eea1d6f4ba35 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 18 Sep 2024 17:30:18 +0200 Subject: [PATCH 125/139] ci(ct): remove obsolete actions for revisions and parent image changes detection --- .../check-newer-parent-image/action.yml | 53 ------------ .github/actions/get-image-revision/action.yml | 81 ------------------- 2 files changed, 134 deletions(-) delete mode 100644 .github/actions/check-newer-parent-image/action.yml delete mode 100644 .github/actions/get-image-revision/action.yml diff --git a/.github/actions/check-newer-parent-image/action.yml b/.github/actions/check-newer-parent-image/action.yml deleted file mode 100644 index 6f8ce950cec..00000000000 --- a/.github/actions/check-newer-parent-image/action.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -name: "Check Newer Parent Image" -description: "Determine if a base image is more recent than the derived." -inputs: - parent: - description: 'The name and (rolling) tag of the parent image to check - or - a shell command to get it' - required: true - derived: - description: 'The name and (rolling) tag of the derived image - or - a shell command to get it' - required: true -outputs: - is-more-recent: - description: "True if base image has a more recent update, false if not." - value: "${{ steps.determine.outputs.is_more_recent }}" - -runs: - using: composite - steps: - - shell: bash - id: determine - run: | - PARENT_IMAGE="${{ inputs.parent }}" - # Get namespace, default to "library" if not found - PARENT_IMAGE_NS="${PARENT_IMAGE%/*}" - if [[ "$PARENT_IMAGE_NS" = "${PARENT_IMAGE}" ]]; then - PARENT_IMAGE_NS="library" - fi - PARENT_IMAGE_REPO="${PARENT_IMAGE%:*}" - PARENT_IMAGE_TAG="${PARENT_IMAGE#*:}" - - PARENT_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${PARENT_IMAGE_NS}/repositories/${PARENT_IMAGE_REPO}/tags/${PARENT_IMAGE_TAG}" | jq -r .last_updated )" - if [[ "$PARENT_IMAGE_LAST_UPDATE" = "null" ]]; then - echo "::error title='Invalid PARENT Image'::Could not find ${PARENT_IMAGE} in the registry" - exit 1 - fi - - DERIVED_IMAGE="${{ inputs.derived }}" - # Get namespace, default to "library" if not found - DERIVED_IMAGE_NS="${DERIVED_IMAGE%/*}" - if [[ "${DERIVED_IMAGE_NS}" = "${DERIVED_IMAGE}" ]]; then - DERIVED_IMAGE_NS="library" - fi - DERIVED_IMAGE_REPO="$( echo "${DERIVED_IMAGE%:*}" | cut -f2 -d/ )" - DERIVED_IMAGE_TAG="${DERIVED_IMAGE#*:}" - - DERIVED_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${DERIVED_IMAGE_NS}/repositories/${DERIVED_IMAGE_REPO}/tags/${DERIVED_IMAGE_TAG}" | jq -r .last_updated )" - if [[ "$DERIVED_IMAGE_LAST_UPDATE" = "null" || "$DERIVED_IMAGE_LAST_UPDATE" < "$PARENT_IMAGE_LAST_UPDATE" ]]; then - echo "Parent image $PARENT_IMAGE has a newer release ($PARENT_IMAGE_LAST_UPDATE), which is more recent than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" - echo "is_more_recent=true" >> $GITHUB_OUTPUT - else - echo "Parent image $PARENT_IMAGE ($PARENT_IMAGE_LAST_UPDATE) is older than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" - echo "is_more_recent=false" >> $GITHUB_OUTPUT - fi diff --git a/.github/actions/get-image-revision/action.yml b/.github/actions/get-image-revision/action.yml deleted file mode 100644 index 6eaf89abba2..00000000000 --- a/.github/actions/get-image-revision/action.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: "Get Incremented Revision Tag" -description: "For a pre-existing rolling tag lookup the latest immutable revision tag, increment and return" -inputs: - image-ref: - description: "The full image reference including namespace, repo and tag" - required: true - revision-separator: - description: "The separator string to use between tag and revision number. Defaults to '-r'" - required: false - default: "-r" - tag-options-prefix: - description: "A string that the revision tag will be appended to and sent to output tag-options" - required: false - default: "" -outputs: - revision-tag: - description: "The updated immutable tag, ready to use" - value: ${{ steps.extract.outputs.revision_tag }} - tag-options: - description: "A string with some command line options (may be empty)" - value: ${{ steps.extract.outputs.tag_options }} -runs: - using: composite - steps: - - shell: bash - id: extract - run: | - IMAGE="${{ inputs.image-ref }}" - IMAGE_NS_REPO="${IMAGE%:*}" - IMAGE_TAG="${IMAGE#*:}" - - if [[ "$IMAGE_TAG" = "$IMAGE_NS_REPO" ]]; then - >&2 echo "You must provide an image reference in the format [/]:" - exit 1 - fi - - case "$IMAGE_NS_REPO" in - */*) :;; # namespace/repository syntax, leave as is - *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax - esac - - # Without such a token we may run into rate limits - # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. - token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) - - ALL_TAGS="$( - i=0 - while [ $? == 0 ]; do - i=$((i+1)) - # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. - # RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) - RESULT=$( curl -s "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) - if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then - # If we run into an error on the first attempt, that means we have a problem. - if [[ "$i" == "1" ]]; then - >&2 echo "Error when retrieving tag data: $( echo "$RESULT" | jq '.message' )" - exit 2 - # Otherwise it will just mean we reached the last page already - else - break - fi - else - echo "$RESULT" | jq -r '."results"[]["name"]' - # DEBUG: - #echo "$RESULT" | >&2 jq -r '."results"[]["name"]' - fi - done - )" - - # Note: if a former tag could not be found, it just might not exist already. Start new series with rev 0 - CURRENT=$( echo "$ALL_TAGS" | grep "${IMAGE_TAG}${{ inputs.revision-separator }}" | sed -e "s#${IMAGE_TAG}${{ inputs.revision-separator }}##" | sort -h | tail -n1 ) - if [[ "$CURRENT" ]]; then - REVISION_TAG="${IMAGE_TAG}${{ inputs.revision-separator }}$((CURRENT+1))" - else - REVISION_TAG="${IMAGE_TAG}${{ inputs.revision-separator }}0" - fi - echo "revision_tag=${REVISION_TAG}" | tee -a "$GITHUB_OUTPUT" - - if [[ -n "${{ inputs.tag-options-prefix }}" ]]; then - echo "tag_options=${{ inputs.tag-options-prefix }}${REVISION_TAG}" | tee -a "$GITHUB_OUTPUT" - fi \ No newline at end of file From 2794d3a874012319769f463ca26dcca2b8f1cd49 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 18 Sep 2024 17:44:33 +0200 Subject: [PATCH 126/139] ci(ct): re-enable forced build for maintenance workflow --- .github/workflows/container_maintenance.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 65cbcc81d48..47bbd005764 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -54,16 +54,15 @@ jobs: - name: Discover maintained releases id: discover run: | + echo "FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 )" | tee -a "$GITHUB_ENV" DEVELOPMENT_BRANCH=$( curl -f -sS https://api.github.com/repos/${{ github.repository }} | jq -r '.default_branch' ) echo "DEVELOPMENT_BRANCH=$DEVELOPMENT_BRANCH" | tee -a "$GITHUB_ENV" echo "branches=$( curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq -r " .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, \"${DEVELOPMENT_BRANCH}\" " | tr "\n" " " )" | tee -a "${GITHUB_OUTPUT}" - # Execute matrix build for the discovered branches - name: Execute build matrix script id: execute run: | - echo "force_build=${{ inputs.force_build }}" .github/workflows/scripts/maintenance-job.sh ${{ steps.discover.outputs.branches }} # TODO: This job should become part of the matrix as an action, so we don't need to fiddle with matrix outputs hacks From 585bc8cc603c94ae8a45f6da46994aa2e4d9d85b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 00:42:56 +0200 Subject: [PATCH 127/139] ci(ct): add outputs to maintenance matrix job Can be picked up by other jobs, e.g. to create textblocks for docs or a job matrix. --- .github/workflows/container_maintenance.yml | 3 ++ .github/workflows/scripts/maintenance-job.sh | 30 +++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 47bbd005764..d6ec6486a59 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -29,6 +29,9 @@ jobs: # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks #if: ${{ github.repository_owner == 'IQSS' }} + outputs: + supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} + rebuilt_base_images: ${{ steps.execute.outputs.rebuilt_base_images }} steps: - name: Checkout and Setup Maven diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/maintenance-job.sh index bf1a9748bf7..df562aac573 100755 --- a/.github/workflows/scripts/maintenance-job.sh +++ b/.github/workflows/scripts/maintenance-job.sh @@ -44,9 +44,12 @@ source "$( dirname "$0" )/utils.sh" rm -rf "$MAINTENANCE_WORKSPACE" mkdir -p "$MAINTENANCE_WORKSPACE" -# Cache the image tags we maintain in this array (same order as branches array!) +# Store the image tags we maintain in this array (same order as branches array!) # This list will be used to build the support matrix within the Docker Hub image description SUPPORTED_ROLLING_TAGS=() +# Store the tags of base images we are actually rebuilding to base new app images upon +# Takes the from "branch-name=base-image-ref" +REBUILT_BASE_IMAGES=() for BRANCH in "$@"; do echo "::group::Running maintenance for $BRANCH" @@ -128,6 +131,13 @@ for BRANCH in "$@"; do mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS NEWER_BASE_IMAGE=1 + + # Save the information about the immutable or rolling tag we just built + if ! (( IS_DEV )); then + REBUILT_BASE_IMAGES+=("$BRANCH=${BASE_IMAGE_REF%:*}:$NEXT_REV_TAG") + else + REBUILT_BASE_IMAGES+=("$BRANCH=$BASE_IMAGE_REF") + fi else echo "No rebuild necessary, we're done here." fi @@ -139,3 +149,21 @@ for BRANCH in "$@"; do echo "::endgroup::" done + +# Built the output which base images have actually been rebuilt as JSON +REBUILT_IMAGES="[" +for IMAGE in "${REBUILT_BASE_IMAGES[@]}"; do + REBUILT_IMAGES+=" \"$IMAGE\" " +done +REBUILT_IMAGES+="]" +echo "rebuilt_base_images=${REBUILT_IMAGES// /, }" | tee -a "${GITHUB_OUTPUT}" + +# Built the supported rolling tags matrix as JSON +SUPPORTED_TAGS="{" +for (( i=0; i < ${#SUPPORTED_ROLLING_TAGS[@]} ; i++ )); do + j=$((i+1)) + SUPPORTED_TAGS+="\"${!j}\": ${SUPPORTED_ROLLING_TAGS[$i]}" + (( i < ${#SUPPORTED_ROLLING_TAGS[@]}-1 )) && SUPPORTED_TAGS+=", " +done +SUPPORTED_TAGS+="}" +echo "supported_tag_matrix=$SUPPORTED_TAGS" | tee -a "$GITHUB_OUTPUT" From 6adc5c46aa2210d04c28fe24d2e27cd55178509a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 00:43:41 +0200 Subject: [PATCH 128/139] ci(ct): remove draft of building app images in maintenance matrix job --- .github/workflows/scripts/maintenance-job.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/maintenance-job.sh index df562aac573..e78bd4979fa 100755 --- a/.github/workflows/scripts/maintenance-job.sh +++ b/.github/workflows/scripts/maintenance-job.sh @@ -125,12 +125,10 @@ for BRANCH in "$@"; do fi echo "Determined these additional Maven tag options: $TAG_OPTIONS" - # 7. Let's build the base image if necessary - NEWER_BASE_IMAGE=0 + # 8. Let's build the base image if necessary if (( NEWER_JAVA_IMAGE + NEWER_PKGS + FORCE_BUILD > 0 )); then mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS - NEWER_BASE_IMAGE=1 # Save the information about the immutable or rolling tag we just built if ! (( IS_DEV )); then @@ -142,11 +140,6 @@ for BRANCH in "$@"; do echo "No rebuild necessary, we're done here." fi - if (( NEWER_BASE_IMAGE )); then - echo "Built a new base image, should continue with application images now..." - # TODO: rebuild the app images here - fi - echo "::endgroup::" done From feeb1265060a9d60945b37b4f90c26c182078ec3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 16:57:51 +0200 Subject: [PATCH 129/139] style,ci(ct): reword the maintenance build workflow name --- .github/workflows/container_maintenance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index d6ec6486a59..88718b435f7 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -21,7 +21,7 @@ env: jobs: build: - name: Build image + name: Base Image Matrix Build runs-on: ubuntu-latest permissions: contents: read From 130102032ce9edd3f873f10987166b38ed1db3b4 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 16:58:58 +0200 Subject: [PATCH 130/139] ci(ct): make the maintenance workflow push the hub description for the base image #10478 --- .github/workflows/container_base_push.yml | 13 +------ .github/workflows/container_maintenance.yml | 41 ++++++++++++++++----- modules/container-base/README.md | 19 ++++++---- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 8a9a3212fe9..24b1102d3a7 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -13,6 +13,7 @@ on: paths: - 'modules/container-base/**' - '!modules/container-base/src/backports/**' + - '!modules/container-base/README.md' - 'modules/dataverse-parent/pom.xml' - '.github/workflows/container_base_push.yml' @@ -132,15 +133,3 @@ jobs: uses: ./.github/workflows/container_app_push.yml with: base-image-ref: ${{ needs.build.outputs.base-image-ref }} - - - # TODO: job to update the docker hub description with supported tags and all - #- if: ${{ github.event_name == 'push' && github.ref_name == env.DEVELOPMENT_BRANCH }} - # name: Push description to DockerHub - # uses: peter-evans/dockerhub-description@v3 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - # repository: gdcc/base - # short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" - # readme-filepath: ./modules/container-base/README.md diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 88718b435f7..2b1deaded5f 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -87,12 +87,35 @@ jobs: # with: # branch: ${{ matrix.branch }} - # TODO: job to update the docker hub description with supported tags and all - # - name: Push description to DockerHub - # uses: peter-evans/dockerhub-description@v3 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - # repository: gdcc/base - # short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" - # readme-filepath: ./modules/container-base/README.md + hub-description: + name: Push description to DockerHub + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Render README + id: render + run: | + TAGS_JSON='${{ needs.build.outputs.supported_tag_matrix }}' + echo "$TAGS_JSON" | jq -r 'keys | sort | reverse | .[]' | + while IFS= read -r branch; do + echo \ + "- \`$( echo "$TAGS_JSON" | jq --arg v "$branch" -r '.[$v] | join("`, `")' )\`" \ + "([Dockerfile](https://github.com/IQSS/dataverse/blob/${branch}/modules/container-base/src/main/docker/Dockerfile)," \ + "[Patches](https://github.com/IQSS/dataverse/blob/develop/modules/container-base/src/backports/${branch}))" \ + | tee -a "${GITHUB_WORKSPACE}/tags.md" + done + sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags.md" "./modules/container-base/README.md" + + - name: Push description to DockerHub + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: gdcc/base + short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" + readme-filepath: ./modules/container-base/README.md \ No newline at end of file diff --git a/modules/container-base/README.md b/modules/container-base/README.md index dc4d185bbb5..925a053cf0b 100644 --- a/modules/container-base/README.md +++ b/modules/container-base/README.md @@ -31,17 +31,22 @@ to ask for help and guidance. ## Supported Image Tags This image is sourced within the main upstream code [repository of the Dataverse software](https://github.com/IQSS/dataverse). -Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-base) -happens there (again, by the community). Community-supported image tags are based on the two most important branches: +Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-base) happens there (again, by the community). +Community-supported image tags are based on the two most important branches: -- The `unstable` tag corresponds to the `develop` branch, where pull requests are merged. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/develop/modules/container-base/src/main/docker/Dockerfile)) -- The `alpha` tag corresponds to the `master` branch, where releases are cut from. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/master/modules/container-base/src/main/docker/Dockerfile)) +Our tagging is inspired by [Bitnami](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html>). +For more detailed information about our tagging policy, please read about our [base image tags](https://guides.dataverse.org/en/latest/container/base-image.html#supported-image-tags) in the Dataverse Containers Guide. + +For ease of use, here is a list of images that are currently maintained. + + + +All of them are rolling tags, except those ending with `-r`, which are immutable tags. +The `unstable` tags are the current development branch snapshot. Within the main repository, you may find the base image files at `/modules/container-base`. This Maven module uses the [Maven Docker Plugin](https://dmp.fabric8.io) to build and ship the image. -You may use, extend, or alter this image to your liking and/or host in some different registry if you want to. +You may use, extend, or alter this image to your liking and/or host in some different registry if you want to under the terms of the Apache 2.0 license. **Supported architectures:** This image is created as a "multi-arch image", supporting the most common architectures Dataverse usually runs on: AMD64 (Windows/Linux/...) and ARM64 (Apple M1/M2). From fbec22df4745e57b9a18a00dd9fe93ada036fbee Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 17:51:32 +0200 Subject: [PATCH 131/139] style(ct): fix simple typo in base image README --- modules/container-base/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-base/README.md b/modules/container-base/README.md index 925a053cf0b..da2a54064a4 100644 --- a/modules/container-base/README.md +++ b/modules/container-base/README.md @@ -34,7 +34,7 @@ This image is sourced within the main upstream code [repository of the Dataverse Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-base) happens there (again, by the community). Community-supported image tags are based on the two most important branches: -Our tagging is inspired by [Bitnami](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html>). +Our tagging is inspired by [Bitnami](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html). For more detailed information about our tagging policy, please read about our [base image tags](https://guides.dataverse.org/en/latest/container/base-image.html#supported-image-tags) in the Dataverse Containers Guide. For ease of use, here is a list of images that are currently maintained. From 6b7a9d618aae2fff50742a56c3f760dec3354ddf Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 17:52:49 +0200 Subject: [PATCH 132/139] fix(ct): remove bug from package upgrade detection in maintenance workflow We did not correctly compare the status code of the grep command, breaking the update detection --- .github/workflows/scripts/utils.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scripts/utils.sh b/.github/workflows/scripts/utils.sh index b63787105ee..987b58d8bb5 100644 --- a/.github/workflows/scripts/utils.sh +++ b/.github/workflows/scripts/utils.sh @@ -42,13 +42,14 @@ function check_newer_pkgs() { PKGS="$2" docker run --rm -u 0 "${IMAGE}" sh -c "apt update >/dev/null 2>&1 && apt install -s ${PKGS}" | tee /proc/self/fd/2 | grep -q "0 upgraded" + STATUS=$? - if [[ ! $? ]]; then - echo "Base image $IMAGE needs updates for our custom installed packages" - return 0 - else + if [[ $STATUS -eq 0 ]]; then echo "Base image $IMAGE has no updates for our custom installed packages" return 1 + else + echo "Base image $IMAGE needs updates for our custom installed packages" + return 0 fi # TODO: In a future version of this script, we might want to include checking for other security updates, From 85731000382f5bda0394275b73f93b4ecd06d45f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 18:22:43 +0200 Subject: [PATCH 133/139] docs,style(ct): small rewording about immutable tags for base image --- modules/container-base/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/container-base/README.md b/modules/container-base/README.md index da2a54064a4..0598d709eac 100644 --- a/modules/container-base/README.md +++ b/modules/container-base/README.md @@ -41,8 +41,9 @@ For ease of use, here is a list of images that are currently maintained. -All of them are rolling tags, except those ending with `-r`, which are immutable tags. +All of them are rolling tags, except those ending with `-r`, which are the most recent immutable tags. The `unstable` tags are the current development branch snapshot. +We strongly recommend using only immutable tags for production use cases. Within the main repository, you may find the base image files at `/modules/container-base`. This Maven module uses the [Maven Docker Plugin](https://dmp.fabric8.io) to build and ship the image. From 9c91798f168e3d77e6347249ea9e053e3bce631a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 18:23:55 +0200 Subject: [PATCH 134/139] feat,ci(ct): add immutable tags to list of base image tags in maintenance job As discussed during community meeting on 2024-09-19. --- .github/workflows/scripts/maintenance-job.sh | 25 +++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/maintenance-job.sh index e78bd4979fa..59e880e7a05 100755 --- a/.github/workflows/scripts/maintenance-job.sh +++ b/.github/workflows/scripts/maintenance-job.sh @@ -109,27 +109,25 @@ for BRANCH in "$@"; do TAG_OPTIONS="" if ! (( IS_DEV )); then TAG_OPTIONS="-Dbase.image=$BASE_IMAGE_REF -Ddocker.tags.revision=$NEXT_REV_TAG" - - # In case of the current release, add the "latest" tag as well. Also add to list of rolling tags. + # In case of the current release, add the "latest" tag as well. if (( IS_CURRENT_RELEASE )); then TAG_OPTIONS="$TAG_OPTIONS -Ddocker.tags.latest=latest" - SUPPORTED_ROLLING_TAGS+=("[\"latest\", \"${BASE_IMAGE_REF#*:}\"]") - else - SUPPORTED_ROLLING_TAGS+=("[\"${BASE_IMAGE_REF#*:}\"]") fi else UPCOMING_TAG=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image.tag -Dbase.image.tag.suffix="" -q -DforceStdout ) TAG_OPTIONS="-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=$UPCOMING_TAG" + # For the dev branch we only have rolling tags and can add them now already SUPPORTED_ROLLING_TAGS+=("[\"unstable\", \"$UPCOMING_TAG\", \"${BASE_IMAGE_REF#*:}\"]") fi echo "Determined these additional Maven tag options: $TAG_OPTIONS" # 8. Let's build the base image if necessary + NEWER_IMAGE=0 if (( NEWER_JAVA_IMAGE + NEWER_PKGS + FORCE_BUILD > 0 )); then mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS - + NEWER_IMAGE=1 # Save the information about the immutable or rolling tag we just built if ! (( IS_DEV )); then REBUILT_BASE_IMAGES+=("$BRANCH=${BASE_IMAGE_REF%:*}:$NEXT_REV_TAG") @@ -140,6 +138,21 @@ for BRANCH in "$@"; do echo "No rebuild necessary, we're done here." fi + # 9. Add list of rolling and immutable tags for release builds + if ! (( IS_DEV )); then + RELEASE_TAGS_LIST="[" + if (( IS_CURRENT_RELEASE )); then + RELEASE_TAGS_LIST+="\"latest\", " + fi + RELEASE_TAGS_LIST+="\"${BASE_IMAGE_REF#*:}\", " + if (( NEWER_IMAGE )); then + RELEASE_TAGS_LIST+="\"$NEXT_REV_TAG\"]" + else + RELEASE_TAGS_LIST+="\"$CURRENT_REV_TAG\"]" + fi + SUPPORTED_ROLLING_TAGS+=("${RELEASE_TAGS_LIST}") + fi + echo "::endgroup::" done From 452849450e9890dfd54aecb2a4eaaa83e0404a70 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 18:24:24 +0200 Subject: [PATCH 135/139] style,ci(ct): add some more verbosity about progress in maintenance job --- .github/workflows/scripts/maintenance-job.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/maintenance-job.sh index 59e880e7a05..370988b9812 100755 --- a/.github/workflows/scripts/maintenance-job.sh +++ b/.github/workflows/scripts/maintenance-job.sh @@ -70,7 +70,9 @@ for BRANCH in "$@"; do cd "$MAINTENANCE_WORKSPACE/$BRANCH" # 2. Now let's apply the patches (we have them checked out in $GITHUB_WORKSPACE, not necessarily in this local checkout) + echo "Checking for patches..." if [[ -d ${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH ]]; then + echo "Applying patches now." find "${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i fi @@ -82,9 +84,11 @@ for BRANCH in "$@"; do else BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) fi + echo "Determined BASE_IMAGE_REF=$BASE_IMAGE_REF from Maven" # 4. Check for Temurin image updates JAVA_IMAGE_REF=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout ) + echo "Determined JAVA_IMAGE_REF=$JAVA_IMAGE_REF from Maven" NEWER_JAVA_IMAGE=0 if check_newer_parent "$JAVA_IMAGE_REF" "$BASE_IMAGE_REF"; then NEWER_JAVA_IMAGE=1 @@ -92,6 +96,7 @@ for BRANCH in "$@"; do # 5. Check for package updates in base image PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" + echo "Determined installed packages=\"$PKGS\" from Maven" NEWER_PKGS=0 # Don't bother with package checks if the java image is newer already if ! (( NEWER_JAVA_IMAGE )); then From 9b9ab1024f25d4634372d3d29ea99d9155f3c869 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 18:55:47 +0200 Subject: [PATCH 136/139] refactor,ci(ct): finishing touches for #10478 Re-enable and change everything necessary to reference the upstream IQSS context as of now. --- .github/workflows/container_app_push.yml | 10 ++- .github/workflows/container_base_push.yml | 67 ++++++++++----------- .github/workflows/container_maintenance.yml | 12 ++-- .github/workflows/maven_unit_test.yml | 4 ++ 4 files changed, 44 insertions(+), 49 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index e3dbcd4d2c6..8e38a17209c 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -35,13 +35,11 @@ jobs: packages: write pull-requests: write # Only run in upstream repo - avoid unnecessary runs in forks - # TODO: re-enable for final PR - #if: ${{ github.repository_owner == 'IQSS' }} + if: ${{ github.repository_owner == 'IQSS' }} steps: - name: Checkout and Setup Maven - # TODO: change to upstream location in final PR - uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: pom-paths: | pom.xml @@ -51,6 +49,7 @@ jobs: # TODO: Add a filter step here, that avoids building the image if this is a PR and there are other files touched than declared above. # Use https://github.com/dorny/paths-filter to solve this. This will ensure we do not run this twice if this workflow # will be triggered by the other workflows already (base image or java changes) + # To become a part of #10618. - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests) run: > @@ -117,8 +116,7 @@ jobs: ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name))) steps: - name: Checkout and Setup Maven - # TODO: change to upstream location in final PR - uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: pom-paths: | pom.xml diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 24b1102d3a7..c2340576c78 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -7,8 +7,6 @@ on: - 'v[6-9].**' branches: - 'develop' - # TODO: delete for final PR - - '10478-version-base-img' # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore paths: - 'modules/container-base/**' @@ -17,13 +15,13 @@ on: - 'modules/dataverse-parent/pom.xml' - '.github/workflows/container_base_push.yml' + # These TODOs are left for #10618 # TODO: we are missing a workflow_call option here, so we can trigger this flow from pr comments and maven tests (keep the secrets availability in mind!) # TODO: we are missing a pull_request option here (filter for stuff that would trigger the maven runs!) so we can trigger preview builds for them when coming from the main repo (keep the secrets availability in mind!) env: PLATFORMS: linux/amd64,linux/arm64 - # TODO: set back to develop for final PR - DEVELOPMENT_BRANCH: 10478-version-base-img + DEVELOPMENT_BRANCH: develop jobs: build: @@ -32,16 +30,14 @@ jobs: permissions: contents: read packages: read - # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks - #if: ${{ github.repository_owner == 'IQSS' }} + if: ${{ github.repository_owner == 'IQSS' }} outputs: base-image-ref: ${{ steps.finalize.outputs.base-image-ref }} steps: - name: Checkout and Setup Maven - # TODO: change to upstream location in final PR - uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: pom-paths: modules/container-base/pom.xml @@ -55,34 +51,33 @@ jobs: # In case this is a push to develop, we care about buildtime. # Configure a remote ARM64 build host in addition to the local AMD64 in two steps. - # TODO: re-enable for final PR - #- name: Setup SSH agent - # if: ${{ github.event_name != 'schedule' }} - # uses: webfactory/ssh-agent@v0.9.0 - # with: - # ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} - #- name: Provide the known hosts key and the builder config - # if: ${{ github.event_name != 'schedule' }} - # run: | - # echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts - # mkdir -p modules/container-base/target/buildx-state/buildx/instances - # cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF - # { "Name": "maven", - # "Driver": "docker-container", - # "Dynamic": false, - # "Nodes": [{"Name": "maven0", - # "Endpoint": "unix:///var/run/docker.sock", - # "Platforms": [{"os": "linux", "architecture": "amd64"}], - # "DriverOpts": null, - # "Flags": ["--allow-insecure-entitlement=network.host"], - # "Files": null}, - # {"Name": "maven1", - # "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", - # "Platforms": [{"os": "linux", "architecture": "arm64"}], - # "DriverOpts": null, - # "Flags": ["--allow-insecure-entitlement=network.host"], - # "Files": null}]} - # EOF + - name: Setup SSH agent + if: ${{ github.event_name != 'schedule' }} + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} + - name: Provide the known hosts key and the builder config + if: ${{ github.event_name != 'schedule' }} + run: | + echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts + mkdir -p modules/container-base/target/buildx-state/buildx/instances + cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF + { "Name": "maven", + "Driver": "docker-container", + "Dynamic": false, + "Nodes": [{"Name": "maven0", + "Endpoint": "unix:///var/run/docker.sock", + "Platforms": [{"os": "linux", "architecture": "amd64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}, + {"Name": "maven1", + "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", + "Platforms": [{"os": "linux", "architecture": "arm64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}]} + EOF # Determine the base image name we are going to use from here on - name: Determine base image name diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 2b1deaded5f..986fe25cdf5 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -26,18 +26,15 @@ jobs: permissions: contents: read packages: read - # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks - #if: ${{ github.repository_owner == 'IQSS' }} + if: ${{ github.repository_owner == 'IQSS' }} outputs: supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} rebuilt_base_images: ${{ steps.execute.outputs.rebuilt_base_images }} steps: - name: Checkout and Setup Maven - # TODO: change to upstream location in final PR - # Necessary as the checked out release branch might not contain the action as files - uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: pom-paths: modules/container-base/pom.xml @@ -68,7 +65,9 @@ jobs: run: | .github/workflows/scripts/maintenance-job.sh ${{ steps.discover.outputs.branches }} - # TODO: This job should become part of the matrix as an action, so we don't need to fiddle with matrix outputs hacks + # TODO: Use the needs.build.outputs.rebuilt_base_images with fromJSON() to create a matrix job. + # Must be a single rank matrix (vector), the branch and base image tag information ships as "branch=tag" string + # Will be part of working on #10618, app image versioned tags. #push-app-img: # name: "Rebase & Publish App Image" # permissions: @@ -77,7 +76,6 @@ jobs: # pull-requests: write # secrets: inherit # needs: - # - discover # - build # strategy: # fail-fast: false diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 1c9c6c7d5e3..a94b17a67ba 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -30,6 +30,7 @@ jobs: continue-on-error: ${{ matrix.experimental }} runs-on: ubuntu-latest steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.jdk }} @@ -95,6 +96,7 @@ jobs: # status: "Experimental" continue-on-error: ${{ matrix.experimental }} steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.jdk }} @@ -128,6 +130,7 @@ jobs: needs: integration-test name: Coverage Report Submission steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - uses: actions/checkout@v3 - uses: actions/setup-java@v3 @@ -159,6 +162,7 @@ jobs: # TODO: Add a filter step here, that avoids calling the app image release workflow if there are changes to the base image. # Use https://github.com/dorny/paths-filter to solve this. Will require and additional job or adding to integration-test job. # This way we ensure that we're not running the app image flow with a non-matching base image. + # To become a part of #10618. push-app-img: name: Publish App Image From 45b412b139761bea7404da341be6f55e8db950dd Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 22:46:47 +0200 Subject: [PATCH 137/139] fix,ci(ct): only add base image Maven option when the input is defined Without this in case of the pull_request event the input is null and the build fails because we have base image defined at all. Simply not adding the option if the input is undefined means we stick to what is defined within the POM. --- .github/workflows/container_app_push.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 8e38a17209c..3b7ce066d73 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -55,7 +55,7 @@ jobs: run: > mvn -B -f modules/dataverse-parent -P ct -pl edu.harvard.iq:dataverse -am - -Dbase.image=${{ inputs.base-image-ref }} + $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) install # TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false") @@ -157,12 +157,13 @@ jobs: run: > mvn -B -f modules/dataverse-parent -P ct -pl edu.harvard.iq:dataverse -am - -Dbase.image=${{ inputs.base-image-ref }} + $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) install - name: Deploy multi-arch application and configbaker container image run: > mvn - -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image=${{ inputs.base-image-ref }} + -Dapp.image.tag=${{ env.IMAGE_TAG }} + $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }} -P ct deploy From 52e6408a5a0d33629139c2098253e6ca8f08588a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 23:00:22 +0200 Subject: [PATCH 138/139] docs(ct): add release note for maintenance workflow #10478 --- doc/release-notes/10478-version-base-image.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/release-notes/10478-version-base-image.md diff --git a/doc/release-notes/10478-version-base-image.md b/doc/release-notes/10478-version-base-image.md new file mode 100644 index 00000000000..34f444a2122 --- /dev/null +++ b/doc/release-notes/10478-version-base-image.md @@ -0,0 +1,7 @@ +### Adding versioned tags to Container Base Images + +With this release we introduce a detailed maintenance workflow for our container images. +As output of the GDCC Containerization Working Group, the community takes another step towards production ready containers available directly from the core project. + +The maintenance workflow regularly updates the Container Base Image, which contains the operating system, Java, Payara Application Server, as well as tools and libraries required by the Dataverse application. +Shipping these rolling releases as well as immutable revisions is the foundation for secure and reliable Dataverse Application Container images. From dc6b59742550106513fb018f9872512ab7cd1039 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 23:34:36 +0200 Subject: [PATCH 139/139] style(ct): add comment explaining what a flavor is in base image Maven props Co-authored-by: Philip Durbin --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index d8fe55ba22a..edf72067976 100644 --- a/pom.xml +++ b/pom.xml @@ -999,6 +999,7 @@ unstable false gdcc/base:${base.image.tag} + noble ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version}