From 2a9fdf3feb644f80cabc6bb9540c6466b77141ba Mon Sep 17 00:00:00 2001 From: Jonathan Louie Date: Thu, 13 Jun 2024 16:53:22 -0700 Subject: [PATCH 1/7] WIP Create initial workflow for publishing to Maven Central --- .github/workflows/gradle-cd.yml | 189 ++++++++++++++++++++++++++++++++ java/client/build.gradle | 1 + 2 files changed, 190 insertions(+) create mode 100644 .github/workflows/gradle-cd.yml diff --git a/.github/workflows/gradle-cd.yml b/.github/workflows/gradle-cd.yml new file mode 100644 index 0000000000..bb3859584d --- /dev/null +++ b/.github/workflows/gradle-cd.yml @@ -0,0 +1,189 @@ +name: Continuous Deployment + +env: + MATRIX: '[{"OS": "ubuntu","NAMED_OS": "linux","RUNNER": "ubuntu-latest","ARCH": "x64","TARGET": "x86_64-unknown-linux-gnu"}, + {"OS": "ubuntu","NAMED_OS": "linux","RUNNER": ["self-hosted", "Linux", "ARM64"],"ARCH": "arm64","TARGET": "aarch64-unknown-linux-gnu"}, + {"OS": "macos","NAMED_OS": "darwin","RUNNER": "macos-12","ARCH": "x64","TARGET": "x86_64-apple-darwin"}, + {"OS": "macos","NAMED_OS": "darwin","RUNNER": "macos-13-xlarge","ARCH": "arm64","TARGET": "aarch64-apple-darwin"}]' + #, + #{"OS": "ubuntu","NAMED_OS": "linux","ARCH": "arm64","TARGET": "aarch64-unknown-linux-musl","RUNNER": ["self-hosted", "Linux", "ARM64"],"IMAGE": "node:alpine","CONTAINER_OPTIONS": "--user root --privileged --rm"}, + #{"OS": "ubuntu","NAMED_OS": "linux","ARCH": "x64","TARGET": "x86_64-unknown-linux-musl","RUNNER": "ubuntu-latest","IMAGE": "node:alpine","CONTAINER_OPTIONS": "--user root --privileged"}]' + +on: + pull_request: + paths: + - .github/workflows/gradle-cd.yml + - .github/workflows/start-self-hosted-runner/action.yml + - .github/workflows/install-rust-and-protoc/action.yml + push: + tags: + - "v*.*" + +concurrency: + group: npm-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +permissions: + id-token: write + +jobs: + output-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Set matrix + id: set-matrix + run: | + echo "::set-output name=matrix::${{toJson( env.MATRIX )}}" + + start-self-hosted-runner: + if: github.repository_owner == 'aws' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Start self hosted EC2 runner + uses: ./.github/workflows/start-self-hosted-runner + with: + role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} + + publish-binaries: + needs: [start-self-hosted-runner, output-matrix] + if: github.repository_owner == 'aws' + name: Publish packages to Maven Central + runs-on: ${{ matrix.build.RUNNER }} + container: + image: ${{ matrix.build.IMAGE || '' }} + options: ${{ matrix.build.CONTAINER_OPTIONS || 'none'}} + strategy: + fail-fast: false + matrix: + build: + ${{fromJson(needs.output-matrix.outputs.matrix)}} + + steps: + - name: Setup self-hosted runner access + if: ${{ contains(matrix.build.RUNNER, 'self-hosted') && matrix.build.TARGET != 'aarch64-unknown-linux-musl' }} + run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/glide-for-redis + + # For MUSL on X64 we need to install git since we use the checkout action + - name: Install git for musl + if: ${{ contains(matrix.build.TARGET, 'x86_64-unknown-linux-musl')}} + run: | + apk update + apk add git + + - name: Checkout + if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl' }} + uses: actions/checkout@v4 + with: + submodules: "true" + fetch-depth: 0 + + #- name: Setup for musl + # if: ${{ contains(matrix.build.TARGET, 'musl')}} + # uses: ./.github/workflows/setup-musl-on-linux + # with: + # workspace: $GITHUB_WORKSPACE + # npm-scope: ${{ vars.NPM_SCOPE }} + # npm-auth-token: ${{ secrets.NPM_AUTH_TOKEN }} + # arch: ${{ matrix.build.ARCH }} + + - name: Set the release version + shell: bash + run: | + export version=`if ${{ github.event_name == 'pull_request' }}; then echo '255.255.255'; else echo ${GITHUB_REF:11}; fi` + echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: ${{ matrix.java }} + + - name: Install shared software dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install protoc (protobuf) + uses: arduino/setup-protoc@v3 + with: + version: "26.1" + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Update package version in config.toml + uses: ./.github/workflows/update-glide-version + with: + folder_path: "${{ github.workspace }}/java/.cargo" + named_os: ${{ matrix.build.NAMED_OS }} + + - name: Create build.gradle file + shell: bash + working-directory: ./java/client + run: | + SED_FOR_MACOS=`if [[ "${{ inputs.os }}" =~ .*"macos".* ]]; then echo "''"; fi` + sed -i $SED_FOR_MACOS 's/placeholder/${{ matrix. }}/g' ./build.gradle + + #- name: Build Java wrapper + # working-directory: java + # run: ./gradlew --continue build + + - name: Publish to Maven Central + if: github.event_name != 'pull_request' + shell: bash + working-directory: java + run: | + ./gradlew :client:publishToMavenLocal + # set +e + # Redirect only stderr + # { npm_publish_err=$(npm publish --access public 2>&1 >&3 3>&-); } 3>&1 + # if [[ "$npm_publish_err" == *"You cannot publish over the previously published versions"* ]] + # then + # echo "Skipping publishing, package already published" + # elif [[ ! -z "$npm_publish_err" ]] + # then + # echo "Failed to publish with error: ${npm_publish_err}" + # exit 1 + # fi + # env: + # NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} + + #- name: Pack the Java JAR + # shell: bash + # working-directory: ./node + # run: | + # # Remove the "cpu" and "os" fileds so the base package would be able to install it on ubuntu + # SED_FOR_MACOS=`if [[ "${{ matrix.build.OS }}" =~ .*"macos".* ]]; then echo "''"; fi` + # sed -i $SED_FOR_MACOS '/"\/\/\/cpu": \[/,/]/d' ./package.json && sed -i $SED_FOR_MACOS '/"\/\/\/os": \[/,/]/d' ./package.json + # mkdir -p bin + # npm pack --pack-destination ./bin + # ls ./bin + # env: + # NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} + + - name: Upload the JAR + if: github.event_name != 'pull_request' + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.build.TARGET }} + path: ./java/client/build/libs/glide-placeholder.jar + if-no-files-found: error + + # Reset the repository to make sure we get the clean checkout of the action later in other actions. + # It is not required since in other actions we are cleaning before the action, but it is a good practice to do it here as well. + - name: Reset repository + if: ${{ matrix.build.ARCH == 'arm64' }} + shell: bash + run: | + git reset --hard + git clean -xdf + diff --git a/java/client/build.gradle b/java/client/build.gradle index 13fccbfea6..b27824df77 100644 --- a/java/client/build.gradle +++ b/java/client/build.gradle @@ -223,4 +223,5 @@ tasks.withType(Test) { jar { archiveBaseName = "glide" + archiveClassifier = "placeholder" } From 432d1a65481fae6f3f2e3729be7fa052806da16d Mon Sep 17 00:00:00 2001 From: Jonathan Louie Date: Fri, 14 Jun 2024 14:28:17 -0700 Subject: [PATCH 2/7] Add classifier to workflow --- .github/workflows/gradle-cd.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gradle-cd.yml b/.github/workflows/gradle-cd.yml index bb3859584d..416ccd31f1 100644 --- a/.github/workflows/gradle-cd.yml +++ b/.github/workflows/gradle-cd.yml @@ -1,10 +1,10 @@ name: Continuous Deployment env: - MATRIX: '[{"OS": "ubuntu","NAMED_OS": "linux","RUNNER": "ubuntu-latest","ARCH": "x64","TARGET": "x86_64-unknown-linux-gnu"}, - {"OS": "ubuntu","NAMED_OS": "linux","RUNNER": ["self-hosted", "Linux", "ARM64"],"ARCH": "arm64","TARGET": "aarch64-unknown-linux-gnu"}, - {"OS": "macos","NAMED_OS": "darwin","RUNNER": "macos-12","ARCH": "x64","TARGET": "x86_64-apple-darwin"}, - {"OS": "macos","NAMED_OS": "darwin","RUNNER": "macos-13-xlarge","ARCH": "arm64","TARGET": "aarch64-apple-darwin"}]' + MATRIX: '[{"OS": "ubuntu","NAMED_OS": "linux","RUNNER": "ubuntu-latest","ARCH": "x64","TARGET": "x86_64-unknown-linux-gnu","CLASSIFIER": "linux-x86_64"}, + {"OS": "ubuntu","NAMED_OS": "linux","RUNNER": ["self-hosted", "Linux", "ARM64"],"ARCH": "arm64","TARGET": "aarch64-unknown-linux-gnu","CLASSIFIER": "linux-aarch_64"}, + {"OS": "macos","NAMED_OS": "darwin","RUNNER": "macos-12","ARCH": "x64","TARGET": "x86_64-apple-darwin","CLASSIFIER": "osx-x86_64"}, + {"OS": "macos","NAMED_OS": "darwin","RUNNER": "macos-13-xlarge","ARCH": "arm64","TARGET": "aarch64-apple-darwin","CLASSIFIER": "osx-aarch_64"}]' #, #{"OS": "ubuntu","NAMED_OS": "linux","ARCH": "arm64","TARGET": "aarch64-unknown-linux-musl","RUNNER": ["self-hosted", "Linux", "ARM64"],"IMAGE": "node:alpine","CONTAINER_OPTIONS": "--user root --privileged --rm"}, #{"OS": "ubuntu","NAMED_OS": "linux","ARCH": "x64","TARGET": "x86_64-unknown-linux-musl","RUNNER": "ubuntu-latest","IMAGE": "node:alpine","CONTAINER_OPTIONS": "--user root --privileged"}]' @@ -131,7 +131,7 @@ jobs: working-directory: ./java/client run: | SED_FOR_MACOS=`if [[ "${{ inputs.os }}" =~ .*"macos".* ]]; then echo "''"; fi` - sed -i $SED_FOR_MACOS 's/placeholder/${{ matrix. }}/g' ./build.gradle + sed -i $SED_FOR_MACOS 's/placeholder/${{ matrix.build.CLASSIFIER }}/g' ./build.gradle #- name: Build Java wrapper # working-directory: java From a17a5a4c480c208f598aaafabf8aba6cad962747 Mon Sep 17 00:00:00 2001 From: Jonathan Louie Date: Fri, 14 Jun 2024 14:52:49 -0700 Subject: [PATCH 3/7] Remove condition to allow all jobs to run --- .github/workflows/gradle-cd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle-cd.yml b/.github/workflows/gradle-cd.yml index 416ccd31f1..59aef3c0e5 100644 --- a/.github/workflows/gradle-cd.yml +++ b/.github/workflows/gradle-cd.yml @@ -38,7 +38,7 @@ jobs: echo "::set-output name=matrix::${{toJson( env.MATRIX )}}" start-self-hosted-runner: - if: github.repository_owner == 'aws' + #if: github.repository_owner == 'aws' runs-on: ubuntu-latest steps: - name: Checkout @@ -55,7 +55,7 @@ jobs: publish-binaries: needs: [start-self-hosted-runner, output-matrix] - if: github.repository_owner == 'aws' + #if: github.repository_owner == 'aws' name: Publish packages to Maven Central runs-on: ${{ matrix.build.RUNNER }} container: From 4754cefa04ddc28a0aff94fdb34eaa27adc35f22 Mon Sep 17 00:00:00 2001 From: Jonathan Louie Date: Sun, 16 Jun 2024 14:33:51 -0700 Subject: [PATCH 4/7] Try to fix Gradle workflow errors --- .github/workflows/gradle-cd.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/gradle-cd.yml b/.github/workflows/gradle-cd.yml index 59aef3c0e5..66938eb737 100644 --- a/.github/workflows/gradle-cd.yml +++ b/.github/workflows/gradle-cd.yml @@ -37,24 +37,24 @@ jobs: run: | echo "::set-output name=matrix::${{toJson( env.MATRIX )}}" - start-self-hosted-runner: + #start-self-hosted-runner: #if: github.repository_owner == 'aws' - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Start self hosted EC2 runner - uses: ./.github/workflows/start-self-hosted-runner - with: - role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + + # - name: Start self hosted EC2 runner + # uses: ./.github/workflows/start-self-hosted-runner + # with: + # role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} + # aws-region: ${{ secrets.AWS_REGION }} + # ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} publish-binaries: - needs: [start-self-hosted-runner, output-matrix] + needs: [output-matrix] #[start-self-hosted-runner, output-matrix] #if: github.repository_owner == 'aws' name: Publish packages to Maven Central runs-on: ${{ matrix.build.RUNNER }} From 0462c9f727ea8451af6abb2eef196be06fcf07d5 Mon Sep 17 00:00:00 2001 From: Jonathan Louie Date: Mon, 17 Jun 2024 10:27:01 -0700 Subject: [PATCH 5/7] Re-enable aws related options --- .github/workflows/gradle-cd.yml | 36 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/gradle-cd.yml b/.github/workflows/gradle-cd.yml index 66938eb737..c2c9a8eb92 100644 --- a/.github/workflows/gradle-cd.yml +++ b/.github/workflows/gradle-cd.yml @@ -37,25 +37,25 @@ jobs: run: | echo "::set-output name=matrix::${{toJson( env.MATRIX )}}" - #start-self-hosted-runner: - #if: github.repository_owner == 'aws' - # runs-on: ubuntu-latest - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - - # - name: Start self hosted EC2 runner - # uses: ./.github/workflows/start-self-hosted-runner - # with: - # role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} - # aws-region: ${{ secrets.AWS_REGION }} - # ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} + start-self-hosted-runner: + if: github.repository_owner == 'aws' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Start self hosted EC2 runner + uses: ./.github/workflows/start-self-hosted-runner + with: + role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} publish-binaries: - needs: [output-matrix] #[start-self-hosted-runner, output-matrix] - #if: github.repository_owner == 'aws' + needs: [start-self-hosted-runner, output-matrix] + if: github.repository_owner == 'aws' name: Publish packages to Maven Central runs-on: ${{ matrix.build.RUNNER }} container: @@ -175,7 +175,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: ${{ matrix.build.TARGET }} - path: ./java/client/build/libs/glide-placeholder.jar + path: ./java/client/build/libs/glide-${{ matrix.build.CLASSIFIER }}.jar if-no-files-found: error # Reset the repository to make sure we get the clean checkout of the action later in other actions. From 6cc5fba787385063bb6faac0885abeb78a7a49b6 Mon Sep 17 00:00:00 2001 From: Jonathan Louie Date: Mon, 17 Jun 2024 15:38:40 -0700 Subject: [PATCH 6/7] Add missing property --- .github/workflows/gradle-cd.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gradle-cd.yml b/.github/workflows/gradle-cd.yml index c2c9a8eb92..7bdb8d4bfd 100644 --- a/.github/workflows/gradle-cd.yml +++ b/.github/workflows/gradle-cd.yml @@ -49,7 +49,8 @@ jobs: - name: Start self hosted EC2 runner uses: ./.github/workflows/start-self-hosted-runner with: - role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} + aws-access-key-id: ${{ secrets.AWS_EC2_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_EC2_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_REGION }} ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} From bcc0053dbf6f18611dca58935b1c030cdf78b410 Mon Sep 17 00:00:00 2001 From: Jonathan Louie Date: Mon, 17 Jun 2024 16:13:25 -0700 Subject: [PATCH 7/7] Revert "Add missing property" This reverts commit 6cc5fba787385063bb6faac0885abeb78a7a49b6. --- .github/workflows/gradle-cd.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/gradle-cd.yml b/.github/workflows/gradle-cd.yml index 7bdb8d4bfd..c2c9a8eb92 100644 --- a/.github/workflows/gradle-cd.yml +++ b/.github/workflows/gradle-cd.yml @@ -49,8 +49,7 @@ jobs: - name: Start self hosted EC2 runner uses: ./.github/workflows/start-self-hosted-runner with: - aws-access-key-id: ${{ secrets.AWS_EC2_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_EC2_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} aws-region: ${{ secrets.AWS_REGION }} ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }}