deploy with script

.github/workflows/_extension_deploy.yml

@@ -0,0 +1,122 @@
+#
+# Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml
+#
+# note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the
+# deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as
+# this workflow can be configured to use a custom deploy script.
+
+
+name: Extension Deployment
+on:
+  workflow_call:
+    inputs:
+      # The name of the extension
+      extension_name:
+        required: true
+        type: string
+      # DuckDB version to build against
+      duckdb_version:
+        required: true
+        type: string
+      # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64'
+      exclude_archs:
+        required: false
+        type: string
+        default: ""
+      # Whether to upload this deployment as the latest. This may overwrite a previous deployment.
+      deploy_latest:
+        required: false
+        type: boolean
+        default: false
+      # Whether to upload this deployment under a versioned path. These will not be deleted automatically
+      deploy_versioned:
+        required: false
+        type: boolean
+        default: false
+      # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times
+      artifact_postfix:
+        required: false
+        type: string
+        default: ""
+      # Override the default deploy script with a custom script
+      deploy_script:
+        required: false
+        type: string
+        default: "./scripts/extension-upload.sh"
+      # Override the default matrix parse script with a custom script
+      matrix_parse_script:
+        required: false
+        type: string
+        default: "./duckdb/scripts/modify_distribution_matrix.py"
+
+jobs:
+  generate_matrix:
+    name: Generate matrix
+    runs-on: ubuntu-latest
+    outputs:
+      deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: Checkout DuckDB to version
+        run: |
+          cd duckdb
+          git checkout ${{ inputs.duckdb_version }}
+
+      - id: parse-matrices
+        run: |
+          python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
+          deploy_matrix="`cat deploy_matrix.json`"
+          echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT
+          echo `cat $GITHUB_OUTPUT`
+
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+    needs: generate_matrix
+    if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }}
+    strategy:
+      matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: Checkout DuckDB to version
+        run: |
+          cd duckdb
+          git checkout ${{ inputs.duckdb_version }}
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}}
+          path: |
+            /tmp/extension
+
+      - name: Deploy
+        shell: bash
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_KEY }}
+          AWS_DEFAULT_REGION: ${{ secrets.S3_DUCKDB_ORG_REGION }}
+          AWS_ENDPOINT_URL: ${{ secrets.S3_DUCKDB_ORG_ENDPOINT }}
+          BUCKET_NAME: ${{ secrets.S3_DUCKDB_ORG_BUCKET }}
+          DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.S3_DUCKDB_ORG_EXTENSION_SIGNING_PK }}
+        run: |
+          pwd
+          python3 -m pip install pip awscli
+          git config --global --add safe.directory '*'
+          cd duckdb
+          git fetch --tags
+          export DUCKDB_VERSION=`git tag --points-at HEAD`
+          export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`}
+          cd ..
+          git fetch --tags
+          export EXT_VERSION=`git tag --points-at HEAD`
+          export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`}
+          ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}}

.github/workflows/test.yml

@@ -21,13 +21,14 @@ jobs:
       extension_name: h3ext
       exclude_archs: "windows_amd64_rtools"
 
-  # duckdb-stable-deploy:
-  #   name: Deploy extension binaries
-  #   needs: duckdb-stable-build
-  #   uses: ./.github/workflows/_extension_deploy.yml
-  #   secrets: inherit
-  #   with:
-  #     duckdb_version: v0.10.1
-  #     extension_name: quack
-  #     deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
-  #     deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
+  duckdb-stable-deploy:
+    name: Deploy extension binaries
+    needs: duckdb-stable-build
+    uses: ./.github/workflows/_extension_deploy.yml
+    secrets: inherit
+    with:
+      duckdb_version: v0.10.1
+      extension_name: h3ext
+      exclude_archs: "windows_amd64_rtools"
+      deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
+      deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}

scripts/extension-upload.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# Extension upload script
+
+# Usage: ./extension-upload.sh <name> <extension_version> <duckdb_version> <architecture> <s3_bucket> <copy_to_latest> <copy_to_versioned>
+# <name>                : Name of the extension
+# <extension_version>   : Version (commit / version tag) of the extension
+# <duckdb_version>      : Version (commit / version tag) of DuckDB
+# <architecture>        : Architecture target of the extension binary
+# <s3_bucket>           : S3 bucket to upload to
+# <copy_to_latest>      : Set this as the latest version ("true" / "false", default: "false")
+# <copy_to_versioned>   : Set this as a versioned version that will prevent its deletion
+
+set -e
+
+if [[ $4 == wasm* ]]; then
+  ext="/tmp/extension/$1.duckdb_extension.wasm"
+else
+  ext="/tmp/extension/$1.duckdb_extension"
+fi
+
+echo $ext
+
+script_dir="$(dirname "$(readlink -f "$0")")"
+
+# calculate SHA256 hash of extension binary
+cat $ext > $ext.append
+
+if [[ $4 == wasm* ]]; then
+  # 0 for custom section
+  # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256)
+  # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02]
+  echo -n -e '\x00' >> $ext.append
+  echo -n -e '\x93\x02' >> $ext.append
+  # 10 in hex = 16 in decimal, lenght of name, 1 byte
+  echo -n -e '\x10' >> $ext.append
+  echo -n -e 'duckdb_signature' >> $ext.append
+  # the name of the WebAssembly custom section, 16 bytes
+  # 100 in hex, 256 in decimal
+  # [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)],
+  # for a grand total of 2 bytes
+  echo -n -e '\x80\x02' >> $ext.append
+fi
+
+# (Optionally) Sign binary
+if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then
+  echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem
+  $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash
+  openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign
+  rm -f private.pem
+fi
+
+# Signature is always there, potentially defaulting to 256 zeros
+truncate -s 256 $ext.sign
+
+# append signature to extension binary
+cat $ext.sign >> $ext.append
+
+# compress extension binary
+if [[ $4 == wasm_* ]]; then
+  brotli < $ext.append > "$ext.compressed"
+else
+  gzip < $ext.append > "$ext.compressed"
+fi
+
+set -e
+
+# Abort if AWS key is not set
+if [ -z "$AWS_ACCESS_KEY_ID" ]; then
+    echo "No AWS key found, skipping.."
+    exit 0
+fi
+
+# upload versioned version
+if [[ $7 = 'true' ]]; then
+  if [[ $4 == wasm* ]]; then
+    aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm"
+  else
+    aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read
+  fi
+fi
+
+# upload to latest version
+if [[ $6 = 'true' ]]; then
+  if [[ $4 == wasm* ]]; then
+    aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm"
+  else
+    aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read
+  fi
+fi