-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* deploy with script * deploy latest from branch (testing) * revert test
- Loading branch information
1 parent
7030144
commit 8cf4118
Showing
3 changed files
with
223 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,122 @@ | ||
# | ||
# Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml | ||
# | ||
# note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the | ||
# deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as | ||
# this workflow can be configured to use a custom deploy script. | ||
|
||
|
||
name: Extension Deployment | ||
on: | ||
workflow_call: | ||
inputs: | ||
# The name of the extension | ||
extension_name: | ||
required: true | ||
type: string | ||
# DuckDB version to build against | ||
duckdb_version: | ||
required: true | ||
type: string | ||
# ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' | ||
exclude_archs: | ||
required: false | ||
type: string | ||
default: "" | ||
# Whether to upload this deployment as the latest. This may overwrite a previous deployment. | ||
deploy_latest: | ||
required: false | ||
type: boolean | ||
default: false | ||
# Whether to upload this deployment under a versioned path. These will not be deleted automatically | ||
deploy_versioned: | ||
required: false | ||
type: boolean | ||
default: false | ||
# Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times | ||
artifact_postfix: | ||
required: false | ||
type: string | ||
default: "" | ||
# Override the default deploy script with a custom script | ||
deploy_script: | ||
required: false | ||
type: string | ||
default: "./scripts/extension-upload.sh" | ||
# Override the default matrix parse script with a custom script | ||
matrix_parse_script: | ||
required: false | ||
type: string | ||
default: "./duckdb/scripts/modify_distribution_matrix.py" | ||
|
||
jobs: | ||
generate_matrix: | ||
name: Generate matrix | ||
runs-on: ubuntu-latest | ||
outputs: | ||
deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} | ||
steps: | ||
- uses: actions/checkout@v3 | ||
with: | ||
fetch-depth: 0 | ||
submodules: 'true' | ||
|
||
- name: Checkout DuckDB to version | ||
run: | | ||
cd duckdb | ||
git checkout ${{ inputs.duckdb_version }} | ||
- id: parse-matrices | ||
run: | | ||
python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty | ||
deploy_matrix="`cat deploy_matrix.json`" | ||
echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT | ||
echo `cat $GITHUB_OUTPUT` | ||
deploy: | ||
name: Deploy | ||
runs-on: ubuntu-latest | ||
needs: generate_matrix | ||
if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }} | ||
strategy: | ||
matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}} | ||
|
||
steps: | ||
- uses: actions/checkout@v3 | ||
with: | ||
fetch-depth: 0 | ||
submodules: 'true' | ||
|
||
- name: Checkout DuckDB to version | ||
run: | | ||
cd duckdb | ||
git checkout ${{ inputs.duckdb_version }} | ||
- uses: actions/download-artifact@v2 | ||
with: | ||
name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}} | ||
path: | | ||
/tmp/extension | ||
- name: Deploy | ||
shell: bash | ||
env: | ||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_ID }} | ||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_KEY }} | ||
AWS_DEFAULT_REGION: ${{ secrets.S3_DUCKDB_ORG_REGION }} | ||
AWS_ENDPOINT_URL: ${{ secrets.S3_DUCKDB_ORG_ENDPOINT }} | ||
BUCKET_NAME: ${{ secrets.S3_DUCKDB_ORG_BUCKET }} | ||
DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.S3_DUCKDB_ORG_EXTENSION_SIGNING_PK }} | ||
run: | | ||
pwd | ||
python3 -m pip install pip awscli | ||
git config --global --add safe.directory '*' | ||
cd duckdb | ||
git fetch --tags | ||
export DUCKDB_VERSION=`git tag --points-at HEAD` | ||
export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} | ||
cd .. | ||
git fetch --tags | ||
export EXT_VERSION=`git tag --points-at HEAD` | ||
export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} | ||
${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,90 @@ | ||
#!/bin/bash | ||
|
||
# Extension upload script | ||
|
||
# Usage: ./extension-upload.sh <name> <extension_version> <duckdb_version> <architecture> <s3_bucket> <copy_to_latest> <copy_to_versioned> | ||
# <name> : Name of the extension | ||
# <extension_version> : Version (commit / version tag) of the extension | ||
# <duckdb_version> : Version (commit / version tag) of DuckDB | ||
# <architecture> : Architecture target of the extension binary | ||
# <s3_bucket> : S3 bucket to upload to | ||
# <copy_to_latest> : Set this as the latest version ("true" / "false", default: "false") | ||
# <copy_to_versioned> : Set this as a versioned version that will prevent its deletion | ||
|
||
set -e | ||
|
||
if [[ $4 == wasm* ]]; then | ||
ext="/tmp/extension/$1.duckdb_extension.wasm" | ||
else | ||
ext="/tmp/extension/$1.duckdb_extension" | ||
fi | ||
|
||
echo $ext | ||
|
||
script_dir="$(dirname "$(readlink -f "$0")")" | ||
|
||
# calculate SHA256 hash of extension binary | ||
cat $ext > $ext.append | ||
|
||
if [[ $4 == wasm* ]]; then | ||
# 0 for custom section | ||
# 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) | ||
# [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] | ||
echo -n -e '\x00' >> $ext.append | ||
echo -n -e '\x93\x02' >> $ext.append | ||
# 10 in hex = 16 in decimal, lenght of name, 1 byte | ||
echo -n -e '\x10' >> $ext.append | ||
echo -n -e 'duckdb_signature' >> $ext.append | ||
# the name of the WebAssembly custom section, 16 bytes | ||
# 100 in hex, 256 in decimal | ||
# [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], | ||
# for a grand total of 2 bytes | ||
echo -n -e '\x80\x02' >> $ext.append | ||
fi | ||
|
||
# (Optionally) Sign binary | ||
if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then | ||
echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem | ||
$script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash | ||
openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign | ||
rm -f private.pem | ||
fi | ||
|
||
# Signature is always there, potentially defaulting to 256 zeros | ||
truncate -s 256 $ext.sign | ||
|
||
# append signature to extension binary | ||
cat $ext.sign >> $ext.append | ||
|
||
# compress extension binary | ||
if [[ $4 == wasm_* ]]; then | ||
brotli < $ext.append > "$ext.compressed" | ||
else | ||
gzip < $ext.append > "$ext.compressed" | ||
fi | ||
|
||
set -e | ||
|
||
# Abort if AWS key is not set | ||
if [ -z "$AWS_ACCESS_KEY_ID" ]; then | ||
echo "No AWS key found, skipping.." | ||
exit 0 | ||
fi | ||
|
||
# upload versioned version | ||
if [[ $7 = 'true' ]]; then | ||
if [[ $4 == wasm* ]]; then | ||
aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" | ||
else | ||
aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read | ||
fi | ||
fi | ||
|
||
# upload to latest version | ||
if [[ $6 = 'true' ]]; then | ||
if [[ $4 == wasm* ]]; then | ||
aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" | ||
else | ||
aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read | ||
fi | ||
fi |