fix(ci): switch osx build to codesign and notarytool (#1078)

See details in #1078 Closes #1066
ipfs · Apr 10, 2024 · 8c700ab · 8c700ab
1 parent 22116ba
commit 8c700ab
Show file tree

Hide file tree

Showing 2 changed files with 113 additions and 28 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -16,9 +16,19 @@ on:
 
 env:
  DIST_ROOT: ${{ github.event.inputs.custom_dist_root || '/ipns/dist.ipfs.tech' }} # content root used for calculating diff to build
- KUBO_VER: 'v0.26.0'       # kubo daemon used for chunking and applying diff
+ KUBO_VER: 'v0.27.0'       # kubo daemon used for chunking and applying diff
  CLUSTER_CTL_VER: 'v1.0.8' # ipfs-cluster-ctl used for pinning
 
+concurrency:
+  # we want only one job running at the time because it is expensive
+  # expecially when building artifact for multiple platforms
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  # IMPORTANT: we want to save resources and cancell old builds on PRs,
+  # but we can't cancel jobs in master branch because they update DNSLink
+  # which is used as DIST_ROOT of the next job, so if we cancel a master job
+  # we will "forget" about releases added in skipped build.
+  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+
 jobs:
   build:
     runs-on: ${{ fromJSON(vars.CI_BUILD_RUNS_ON || '"ubuntu-latest"') }}
@@ -56,6 +66,14 @@ jobs:
   sign-macos:
     runs-on: "macos-latest"
     needs: build
+    concurrency:
+      # notarization depends on remote HTTP service provided by Apple
+      # and we want to have only one instance at a time, across all branches
+      # and PRs to avoid triggering throttling / blacklisting when multiple
+      # jobs try to notarize at the same time
+      group: sign-macos
+      # never cancel ongoing notarization, it could me one for master branch
+      cancel-in-progress: false
     steps:
       - uses: actions/checkout@v4
       - name: Retrieve unsigned artifacts
@@ -66,24 +84,51 @@ jobs:
         continue-on-error: true # skip if no releases
       - name: List ./releases before
         run: ls -Rhl ./releases || echo "No ./releases"
-      - name: Install gon via HomeBrew for code signing and app notarization
+      - name: Install dependencies of sign-new-macos-releases.sh
+        run: |
+          brew install ipfs coreutils gawk gnu-sed jq
+      - name: Set up rcodesign rust tool (TODO)
+        if: false
         run: |
-          brew tap mitchellh/gon
-          brew install ipfs coreutils gawk gnu-sed jq mitchellh/gon/gon
-          ipfs init --profile test # needed for calculating NEW_CID later
+          cargo install apple-codesign
       - name: Import Keychain Certs
+        # if this ever breaks, we should replace this magic with epxlicit security commands executed inside of it via.. nodejs
+        # prior art: https://github.com/lando/code-sign-action/blob/f35d0b777ee592c758351252fa3f0d58f21e5129/action.yml#L106-L123
         uses: apple-actions/import-codesign-certs@8f3fb608891dd2244cdab3d69cd68c0d37a7fe93 # v2
         with:
           p12-file-base64: ${{ secrets.APPLE_CERTS_P12 }}
           p12-password: ${{ secrets.APPLE_CERTS_PASS }}
       - name: Verify identity used for signing
         run: security find-identity -v
+      - name: Secrets for signing (TODO rcodesign)
+        # TODO: revisit switch to rcodesign once we have to switch mode due to move to new org
+        # we dont use this yet, we use codesign from Apple and run on macOS
+        # because rcodesign errored on 'invalid password'
+        if: false
+        run: |
+          echo -n "${{ secrets.APPLE_CERTS_P12 }}" | base64 --decode > ~/.apple-certs.p12
+          echo -n "{{ secrets.APPLE_CERTS_PASS }}" > ~/.apple-certs.pass
+      - name: Secrets for notarization (TODO rcodesign)
+        # TODO: revisit switch to rcodesign once we have to switch mode due to move to new org
+        # we dont use this yet, we use notarytool from Apple and run on macOS
+        # because (afaik) rcodesign does not support App-specific password mode
+        # we use for legacy reasons
+        if: false
+        run: |
+          rcodesign encode-app-store-connect-api-key \
+            "${{ secrets.APPLE_APIKEY_ISSUER_ID }}" \
+            "${{ secrets.APPLE_APIKEY_ID }}" \
+            "${{ secrets.APPLE_APIKEY_FILE }}" \
+            > ~/.apple-api-key
+      - name: Kubo init
+        run: ipfs init --profile test # needed for calculating NEW_CID in sign-new-macos-releases.sh
       - name: Sign any new releases
         run: ./scripts/ci/sign-new-macos-releases.sh
         env:
           WORK_DIR: ${{ github.workspace }}
-          AC_USERNAME: ${{ secrets.APPLE_AC_USERNAME }} # implicitly read from env by gon
-          AC_PASSWORD: ${{ secrets.APPLE_AC_PASSWORD }}
+          APPLE_AC_USERNAME: ${{ secrets.APPLE_AC_USERNAME }}
+          APPLE_AC_PASSWORD: ${{ secrets.APPLE_AC_PASSWORD }}
+          APPLE_AC_TEAM_ID: ${{ secrets.APPLE_AC_TEAM_ID }}
       - name: List ./releases after
         run: ls -Rhl ./releases || echo "No ./releases"
       - name: Temporarily save notarized artifacts

diff --git a/scripts/ci/sign-new-macos-releases.sh b/scripts/ci/sign-new-macos-releases.sh
@@ -1,6 +1,11 @@
 #!/usr/bin/env bash
 set -e
 
+echo "::group::Store credentials to avoid GUI prompt in CI"
+    xcrun notarytool store-credentials "notarytool-profile" \
+        --apple-id "${APPLE_AC_USERNAME}" --team-id "${APPLE_AC_TEAM_ID}" --password "${APPLE_AC_PASSWORD}"
+echo "::endgroup::"
+
 echo "::group::Unpack any new darwin arm64 and amd64 binaries to ./tmp"
     # ./releases/{DIST_NAME}/{DIST_VERSION}/*_darwin-${arch}.tar.gz
     # -> ./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/
@@ -18,7 +23,7 @@ echo "::group::Unpack any new darwin arm64 and amd64 binaries to ./tmp"
     ls -Rhl ./tmp || echo "Nothing new in ./tmp"
 echo "::endgroup::"
 
-echo "::group::Sign and notarize the mac binaries"
+echo "::group::Unpack .zip and sign the binaries"
     # Find and sign executables in
     # ./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/
     for NEW_DIR in ./releases/*/*; do
@@ -27,22 +32,60 @@ echo "::group::Sign and notarize the mac binaries"
         DIST_NAME=$(basename $(dirname "$NEW_DIR"))
         DIST_MAC_ARCHS=$(gawk '{ print $2; }' <(grep darwin "./dists/${DIST_NAME}/build_matrix"))
         for arch in $DIST_MAC_ARCHS; do
-            EXECUTABLES=$(jq -nc '$ARGS.positional' --args $(find "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/" -perm +111 -type f -print))
-            echo "-> Signing ${EXECUTABLES}"
-            echo "{
-                \"source\" : $EXECUTABLES,
-                \"bundle_id\" : \"io.ipfs.dist.${DIST_NAME}\",
-                \"apple_id\": {
-                \"password\":  \"@env:AC_PASSWORD\"
-                },
-                \"sign\" :{
-                \"application_identity\" : \"Developer ID Application: Protocol Labs, Inc. (7Y229E2YRL)\"
-                },
-                \"zip\" :{
-                    \"output_path\" : \"./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed.zip\"
-                }
-            }" | tee | jq > "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-gon.json"
-            gon -log-level=info -log-json "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-gon.json"
+            # create destination dir matching .tar.gz structure
+            mkdir -p "${WORK_DIR}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/${DIST_NAME}"
+            # find executable files, and process each one
+            find "${WORK_DIR}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned" -perm +111 -type f -print | while read -r file; do
+                # -perm +111 will return all executables, including .sh scripts
+                # so we need to skip them
+                if [[ "$file" == *.sh ]]; then
+                    echo "-- Skipping shell script ${file}"
+                    continue
+                fi
+
+                echo "-> Processing ${file}"
+                ls -hl "${file}"
+
+                echo "-> Signing ${file}"
+
+                # Sign with Apple's tool
+                # All credentials are imported to macOS keychain
+                # and will be found via TEAM_ID match
+                xcrun codesign --force --verbose --display --timestamp --options=runtime --sign "$APPLE_AC_TEAM_ID" "${file}"
+
+                # TODO: revisit switch to rcodesign once we have to generate new credentials anyway
+                # if we use rcodesign if we ever swithc away from macos runner
+                #rcodesign sign \
+                #    --p12-file ~/.apple-certs.p12 --p12-password-file ~/.apple-certs.pass \
+                #    --code-signature-flags runtime --for-notarization \
+                #    "${file}"
+
+                echo "-> Notarizing ${file}"
+                # The tool (or Apple API) seems to only accept.zip, even if it is a single binary
+                TMP_ZIP=$(mktemp -u -t "${DIST_NAME}_${DIST_VERSION}_${arch}-signed-for-notarization.zip")
+                zip "${TMP_ZIP}" "${file}"
+
+                # Notarize with Apple's notarytool for now (only reason we use macOS runner)
+                xcrun notarytool submit --progress --keychain-profile "notarytool-profile" --wait "${TMP_ZIP}"
+
+                # NOTE: no stappling, because it would break signatures of Mach-O Binaries (which we publish without any .app or .dmg envelope)
+                # This means out binaries will rely on online notarization the first time macOS Gatewkeeper sees a new binary.
+
+                # Verify produced blob is a-ok
+                if ! xcrun spctl --assess --type install --context context:primary-signature --ignore-cache --verbose=2 "${file}"; then
+                    echo "error: Signature of ${file} will not be accepted by Apple Gatekeeper!" 1>&2
+                    exit 1
+                fi
+                #
+                # TODO: revisit switching notarization to rcodesign once we have to generate new credentials anyway
+                # (rcodesign uses "api key" thing which is 3 things, and codesigns appleid + app-specific password
+                # and it was easier to use notarytool on macOS worker than to make rcodesign work)
+                # rcodesign notary-submit --api-key-path ~/.apple-api-key --wait "${file}"
+
+
+                # move signed binaries to a directory matching .tar.gz structure
+                mv "${file}" "${WORK_DIR}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/${DIST_NAME}/"
+            done
         done
     done
 echo "::endgroup::"
@@ -59,13 +102,10 @@ echo "::group::Update changed binaries in ./releases"
         DIST_MAC_ARCHS=$(gawk '{ print $2; }' <(grep darwin "./dists/${DIST_NAME}/build_matrix"))
         for arch in $DIST_MAC_ARCHS; do
             echo "-> Starting the update of darwin_${arch}.tar.gz for name='${DIST_NAME}' and version='${DIST_VERSION}'"
-            # unzip signed binaries to a directory matching .tar.gz structure
             cd "${WORK_DIR}"
             mkdir -p "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/${DIST_NAME}"
             cd "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/${DIST_NAME}/"
-            echo "-> Unpacking gon .zip for ${arch}"
-            unzip "${WORK_DIR}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed.zip"
-            echo "-> Unpacked contents"
+            echo "-> Signed contents"
             ls -Rhl "${WORK_DIR}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/"
             # replace .tar.gz with one that has the same structure, but signed binaries
             PKG_NAME="${DIST_NAME}_${DIST_VERSION}_darwin-${arch}.tar.gz"