Skip to content

refactor: consolidate datastore and burn datastore data #3734

refactor: consolidate datastore and burn datastore data

refactor: consolidate datastore and burn datastore data #3734

Workflow file for this run

name: CI
on:
pull_request:
branches:
- main
- develop
- rc/next
paths-ignore:
- "**/CHANGELOG.md"
push:
branches:
- main
- develop
- rc/next
paths-ignore:
- "**/CHANGELOG.md"
workflow_dispatch:
jobs:
pre_run:
name: Cancel previous runs
runs-on: ubuntu-latest
steps:
- name: Cancel Previous Runs
uses: styfle/cancel-workflow-action@ad6cb1b847ffb509a69b745b6ee2f1d14dfe14b8
with:
access_token: ${{ github.token }}
persist-credentials: false
get_release_info:
name: Get Release Info
runs-on: ubuntu-latest
needs: pre_run
outputs:
tag: ${{ steps.new_release_tag.outputs.TAG }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Get latest release
if: startsWith(github.ref, 'refs/heads/main')
id: release
uses: pozetroninc/github-action-get-latest-release@master
with:
repository: ${{ github.repository }}
excludes: prerelease, draft
- name: Determine if release build
if: startsWith(github.ref, 'refs/heads/main')
id: new_release_tag
env:
LATEST_RELEASE: ${{ steps.release.outputs.release }}
run: |
CARGO_VERSION=v$(grep "version" Cargo.toml | head -n 1 | cut -d\" -f2)
if [[ "${CARGO_VERSION}" != "${LATEST_RELEASE}" ]]; then
echo "::set-output name=TAG::${CARGO_VERSION}"
echo "::warning::Will create release for version: ${CARGO_VERSION}"
else
echo "::warning::Will not create a release"
fi
audit:
name: Audit and format
runs-on: ubuntu-latest
needs: pre_run
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install Rust toolchain
run: |
rustup toolchain install stable --profile minimal --component rustfmt
echo "RUST_VERSION_HASH=$(rustc --version | sha256sum | awk '{print $1}')" >> $GITHUB_ENV
- name: Cache cargo
uses: actions/cache@v3
with:
path: |
~/.cargo/
./target/debug/build/
key: ${{ runner.os }}-rust-${{ env.RUST_VERSION_HASH }}-${{ hashFiles('./Cargo.lock') }}
- name: Install dependencies
run: cargo install cargo-audit
- name: Run audit
run: cargo audit --ignore RUSTSEC-2022-0093 --ignore RUSTSEC-2021-0076 --ignore RUSTSEC-2022-0090 --ignore RUSTSEC-2022-002 --ignore RUSTSEC-2022-0028 --ignore RUSTSEC-2024-0344
- name: Run rustfmt
run: cargo fmt --all -- --check
test_coverage_cargo:
name: Generate test coverage
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install Rust toolchain
run: |
rustup toolchain install stable --profile minimal
echo "RUST_VERSION_HASH=$(rustc --version | sha256sum | awk '{print $1}')" >> $GITHUB_ENV
- name: Cache cargo
uses: actions/cache@v3
with:
path: |
~/.cargo/
./target/debug/build/
key: ${{ runner.os }}-rust-${{ env.RUST_VERSION_HASH }}-${{ hashFiles('./Cargo.lock') }}
- name: install dependencies
uses: taiki-e/install-action@v2
with:
tool: cargo-llvm-cov,nextest
- name: Run unit test with coverage
run: cargo cov
- name: Upload coverage data to codecov
uses: codecov/codecov-action@v4
with:
files: lcov.info
token: ${{ secrets.CODECOV_TOKEN }}
dist_clarinet:
name: Build Clarinet Distributions
runs-on: ${{ matrix.os }}
needs: pre_run
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
platform: linux
target: x86_64-unknown-linux-gnu
architecture: x64
libc: glibc
- os: windows-latest
platform: windows
target: x86_64-pc-windows-msvc
architecture: x64
- os: macos-latest
platform: darwin
target: x86_64-apple-darwin
architecture: x64
- os: macos-latest
platform: darwin
target: aarch64-apple-darwin
architecture: arm64
steps:
- name: Configure git to use LF (Windows)
if: matrix.os == 'windows-latest'
run: |
git config --global core.autocrlf false
git config --global core.eol lf
- name: Checkout repository
uses: actions/checkout@v4
- name: Install Rust toolchain
run: rustup toolchain install stable --profile minimal --target ${{ matrix.target }}
- name: "Get Rust version (unix)"
if: matrix.os != 'windows-latest'
run: echo "RUST_VERSION_HASH=$(rustc --version | shasum -a 256 | awk '{print $1}')" >> $GITHUB_ENV
- name: "Get Rust version (windows)"
if: matrix.os == 'windows-latest'
shell: bash
run: echo "RUST_VERSION_HASH=$(rustc --version | sha256sum | awk '{print $1}')" >> $GITHUB_ENV
- name: Cache cargo
uses: actions/cache@v3
with:
path: |
~/.cargo/
./target/${{ matrix.target }}/release/
key: ${{ runner.os }}-rust-${{ env.RUST_VERSION_HASH }}-cargo-${{ hashFiles('./Cargo.lock') }}
- name: Install wix (Windows)
if: matrix.os == 'windows-latest' && steps.cache-cargo.outputs.cache-hit != 'true'
run: cargo install cargo-wix
# Set environment variables required from cross compiling from macos-x86_64 to macos-arm64
- name: Configure macos-arm64 cross compile config
if: matrix.target == 'aarch64-apple-darwin'
run: |
echo "SDKROOT=$(xcrun -sdk macosx --show-sdk-path)" >> $GITHUB_ENV
echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)" >> $GITHUB_ENV
- name: Configure artifact names (libc)
if: ${{ matrix.libc }}
shell: bash
run: |
echo "SHORT_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}-${{ matrix.libc }}" >> $GITHUB_ENV
echo "PRE_GYP_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}-${{ matrix.libc }}" >> $GITHUB_ENV
- name: Configure artifact names (not libc)
if: ${{ ! matrix.libc }}
shell: bash
run: |
echo "SHORT_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}" >> $GITHUB_ENV
echo "PRE_GYP_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}-unknown" >> $GITHUB_ENV
- name: Build - Cargo
if: matrix.target != 'x86_64-unknown-linux-musl'
run: cargo build --release --features=telemetry --locked --target ${{ matrix.target }}
# Steps for Windows Code Signing with DigiCert
- name: Windows - Setup Certificate
if: startsWith(github.ref, 'refs/heads/main') && matrix.os == 'windows-latest'
run: |
echo "${{ secrets.SM_CLIENT_CERT_FILE_B64 }}" | base64 --decode > /d/Certificate_pkcs12.p12
cat /d/Certificate_pkcs12.p12
shell: bash
- name: Windows - Set variables
if: startsWith(github.ref, 'refs/heads/main') && matrix.os == 'windows-latest'
id: variables
run: |
dir
echo "::set-output name=version::${GITHUB_REF#refs/tags/v}"
echo "::set-output name=KEYPAIR_NAME::gt-standard-keypair"
echo "::set-output name=CERTIFICATE_NAME::gt-certificate"
echo "SM_HOST=${{ secrets.SM_HOST }}" >> "$GITHUB_ENV"
echo "SM_API_KEY=${{ secrets.SM_API_KEY }}" >> "$GITHUB_ENV"
echo "SM_CLIENT_CERT_FILE=D:\\Certificate_pkcs12.p12" >> "$GITHUB_ENV"
echo "SM_CLIENT_CERT_PASSWORD=${{ secrets.SM_CLIENT_CERT_PASSWORD }}" >> "$GITHUB_ENV"
echo "C:\Program Files (x86)\Windows Kits\10\App Certification Kit" >> $GITHUB_PATH
echo "C:\Program Files (x86)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools" >> $GITHUB_PATH
echo "C:\Program Files\DigiCert\DigiCert Keylocker Tools" >> $GITHUB_PATH
shell: bash
- name: Windows - Setup Keylocker KSP
if: startsWith(github.ref, 'refs/heads/main') && matrix.os == 'windows-latest'
run: |
curl -X GET https://one.digicert.com/signingmanager/api-ui/v1/releases/Keylockertools-windows-x64.msi/download -H "x-api-key:%SM_API_KEY%" -o Keylockertools-windows-x64.msi
msiexec /i Keylockertools-windows-x64.msi /quiet /qn
smksp_registrar.exe list
smctl.exe keypair ls
C:\Windows\System32\certutil.exe -csp "DigiCert Signing Manager KSP" -key -user
shell: cmd
- name: Windows - Certificates Sync
if: startsWith(github.ref, 'refs/heads/main') && matrix.os == 'windows-latest'
run: |
smctl windows certsync
shell: cmd
- name: Code sign bin (Windows)
if: startsWith(github.ref, 'refs/heads/main') && matrix.os == 'windows-latest'
run: |
$signtool_path = ((Resolve-Path -Path "${env:ProgramFiles(x86)}/Windows Kits/10/bin/10*/x86").Path[-1]) + "/signtool.exe"
$bin_path = (Resolve-Path -Path "target/${{ matrix.target }}/release/clarinet.exe").Path
& ${signtool_path} sign `
/d "Clarinet is a clarity runtime packaged as a command line tool, designed to facilitate smart contract understanding, development, testing and deployment." `
/du "https://github.com/hirosystems/clarinet" `
/tr http://timestamp.digicert.com `
/sha1 "${{ secrets.SM_CODE_SIGNING_CERT_SHA1_HASH }}" `
/td sha256 `
/fd sha256 `
"${bin_path}"
& ${signtool_path} verify /v /pa "${bin_path}"
- name: Build Installer (Windows)
if: matrix.os == 'windows-latest'
run: cargo wix -v --no-build --nocapture -p clarinet-cli
- name: Code sign installed (Windows)
if: startsWith(github.ref, 'refs/heads/main') && matrix.os == 'windows-latest'
run: |
$signtool_path = ((Resolve-Path -Path "${env:ProgramFiles(x86)}/Windows Kits/10/bin/10*/x86").Path[-1]) + "/signtool.exe"
$msi_path = (Resolve-Path -Path "target/wix/*.msi").Path
& ${signtool_path} sign `
/d "Clarinet is a clarity runtime packaged as a command line tool, designed to facilitate smart contract understanding, development, testing and deployment." `
/du "https://github.com/hirosystems/clarinet" `
/tr http://timestamp.digicert.com `
/sha1 "${{ secrets.SM_CODE_SIGNING_CERT_SHA1_HASH }}" `
/td sha256 `
/fd sha256 `
"${msi_path}"
& ${signtool_path} verify /v /pa "${msi_path}"
# Don't compress for Windows because winget can't yet unzip files
- name: Compress cargo artifact (Linux)
if: matrix.os != 'windows-latest'
run: tar -C target/${{ matrix.target }}/release -zcvf clarinet-${{ env.SHORT_TARGET_NAME }}.tar.gz clarinet
- name: Rename cargo artifact (Windows)
if: matrix.os == 'windows-latest'
shell: bash
run: mv target/wix/*.msi clarinet-${{ env.SHORT_TARGET_NAME }}.msi
# Separate uploads to prevent paths from being preserved
- name: Upload cargo artifacts (Linux)
if: matrix.os != 'windows-latest'
uses: actions/upload-artifact@v4
with:
name: clarinet-${{ env.SHORT_TARGET_NAME }}
path: clarinet-${{ env.SHORT_TARGET_NAME }}.tar.gz
- name: Upload cargo artifact (Windows)
if: matrix.os == 'windows-latest'
uses: actions/upload-artifact@v4
with:
name: clarinet-${{ env.SHORT_TARGET_NAME }}
path: clarinet-${{ env.SHORT_TARGET_NAME }}.msi
- name: Unit Tests - Cargo
# can't easily run mac-arm64 tests in GH without native runners for that arch
if: matrix.target != 'aarch64-apple-darwin'
run: cargo test --workspace --release --locked --exclude clarinet-sdk-wasm --exclude clarity-jupyter-kernel --target ${{ matrix.target }}
dist_stacks_devnet_js:
name: Build Stacks Devnet JS Distributions
runs-on: ${{ matrix.os }}
needs:
- pre_run
- get_release_info
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
platform: linux
target: x86_64-unknown-linux-gnu
architecture: x64
libc: glibc
- os: windows-latest
platform: windows
target: x86_64-pc-windows-msvc
architecture: x64
- os: macos-latest
platform: darwin
target: x86_64-apple-darwin
architecture: x64
- os: macos-latest
platform: darwin
target: aarch64-apple-darwin
architecture: arm64
- os: ubuntu-latest
platform: linux
target: x86_64-unknown-linux-musl
architecture: x64
libc: musl
steps:
- name: Configure git to use LF (Windows)
if: matrix.os == 'windows-latest'
run: |
git config --global core.autocrlf false
git config --global core.eol lf
- name: Checkout repository
uses: actions/checkout@v4
- name: Install Rust toolchain
run: rustup toolchain install stable --profile minimal --target ${{ matrix.target }}
- name: "Get Rust version (unix)"
if: matrix.os != 'windows-latest'
run: echo "RUST_VERSION_HASH=$(rustc --version | shasum -a 256 | awk '{print $1}')" >> $GITHUB_ENV
- name: "Get Rust version (windows)"
if: matrix.os == 'windows-latest'
shell: bash
run: echo "RUST_VERSION_HASH=$(rustc --version | sha256sum | awk '{print $1}')" >> $GITHUB_ENV
- name: Install and cache Node
if: matrix.target != 'x86_64-unknown-linux-musl'
uses: actions/setup-node@v4
with:
node-version: "18"
# Host nodejs arch does not matter in our usecase for compiling cross-platform npm packages,
# if enabled, this will fail after installing an arm64 nodejs runtime for an x86_64 arch macos system
# architecture: ${{ matrix.architecture }}
cache: npm
cache-dependency-path: components/stacks-devnet-js/package-lock.json
- name: Cache cargo
uses: actions/cache@v3
with:
path: |
~/.cargo/
./target/${{ matrix.target }}/release/
key: ${{ runner.os }}-rust-${{ env.RUST_VERSION_HASH }}-cargo-${{ hashFiles('./Cargo.lock') }}
# Set environment variables required from cross compiling from macos-x86_64 to macos-arm64
- name: Configure macos-arm64 cross compile config
if: matrix.target == 'aarch64-apple-darwin'
run: |
echo "SDKROOT=$(xcrun -sdk macosx --show-sdk-path)" >> $GITHUB_ENV
echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)" >> $GITHUB_ENV
- name: Configure artifact names (libc)
if: ${{ matrix.libc }}
shell: bash
run: |
echo "SHORT_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}-${{ matrix.libc }}" >> $GITHUB_ENV
echo "PRE_GYP_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}-${{ matrix.libc }}" >> $GITHUB_ENV
- name: Configure artifact names (not libc)
if: ${{ ! matrix.libc }}
shell: bash
run: |
echo "SHORT_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}" >> $GITHUB_ENV
echo "PRE_GYP_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}-unknown" >> $GITHUB_ENV
- name: Build - Node
if: matrix.target != 'x86_64-unknown-linux-musl'
working-directory: components/stacks-devnet-js
run: |
npm install --ignore-scripts
npm run build-${{ env.SHORT_TARGET_NAME }}
- name: Build - Node (linux-musl)
if: matrix.target == 'x86_64-unknown-linux-musl'
uses: docker://rust:alpine3.19
env:
RUSTFLAGS: -C target-feature=-crt-static
with:
entrypoint: /bin/sh
args: -c "
cd components/stacks-devnet-js &&
apk add --update alpine-sdk musl-dev clang-dev llvm openssl-dev clang-static nodejs npm git &&
npm install --ignore-scripts &&
npm run build-${{ env.SHORT_TARGET_NAME }}"
- name: Compress node artifact
if: github.event_name != 'pull_request' || matrix.target == 'x86_64-unknown-linux-gnu'
shell: bash
run: tar -C components/stacks-devnet-js -zcvf stacks-devnet-js-${{ env.PRE_GYP_TARGET_NAME }}.tar.gz native/index.node
- name: Upload node artifact
if: github.event_name != 'pull_request' || matrix.target == 'x86_64-unknown-linux-gnu'
uses: actions/upload-artifact@v4
with:
name: stacks-devnet-js-${{ env.PRE_GYP_TARGET_NAME }}
path: stacks-devnet-js-${{ env.PRE_GYP_TARGET_NAME }}.tar.gz
# - name: Unit Tests - Node
# working-directory: node-bindings
# run: npm run spec
- name: NPM Publish
uses: JS-DevTools/npm-publish@v1
if: matrix.target == 'x86_64-unknown-linux-gnu' && startsWith(github.ref, 'refs/heads/main') && needs.get_release_info.outputs.tag != ''
with:
package: components/stacks-devnet-js/package.json
access: public
greater-version-only: true
token: ${{ secrets.NPM_TOKEN }}
dist_clarity_repl:
name: Build Clarity REPL Distribution
runs-on: ubuntu-latest
needs: pre_run
defaults:
run:
working-directory: components/clarity-repl
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install Rust toolchain
run: |
rustup toolchain install stable --profile minimal
echo "RUST_VERSION_HASH=$(rustc --version | sha256sum | awk '{print $1}')" >> $GITHUB_ENV
- name: Cache cargo
uses: actions/cache@v3
with:
path: |
~/.cargo/
./target/x86_64-unknown-linux-gnu/release/
key: ${{ runner.os }}-rust-${{ env.RUST_VERSION_HASH }}-cargo-${{ hashFiles('./Cargo.lock') }}
- name: Configure artifact names
run: |
echo "SHORT_TARGET_NAME=linux-x64-glibc" >> $GITHUB_ENV
echo "PRE_GYP_TARGET_NAME=linux-x64-glibc" >> $GITHUB_ENV
- name: Build - Cargo
run: cargo build --release --locked --target x86_64-unknown-linux-gnu
- name: Compress cargo artifact
working-directory: "."
run: tar -C target/x86_64-unknown-linux-gnu/release -zcvf clarity-repl-${{ env.SHORT_TARGET_NAME }}.tar.gz clarity-repl
- name: Upload cargo artifact
uses: actions/upload-artifact@v4
with:
name: clarity-repl-${{ env.SHORT_TARGET_NAME }}
path: clarity-repl-${{ env.SHORT_TARGET_NAME }}.tar.gz
- name: Publish clarity-repl to crates.io
if: startsWith(github.ref, 'refs/heads/main') && needs.get_release_info.outputs.tag != ''
run: 4
cargo login ${{ secrets.CARGO_CRATES_IO_API_KEY }}
cargo publish
docker_images:
name: Create ${{ matrix.name }} Docker Image
runs-on: ubuntu-latest
needs:
- get_release_info
- dist_clarinet
outputs:
version: ${{ steps.docker_meta.outputs.version }}
strategy:
fail-fast: false
matrix:
include:
- name: Clarinet
description: Clarinet is a simple, modern and opinionated runtime for testing, integrating and deploying Clarity smart contracts.
image: ${{ github.repository }}
artifact: clarinet-linux-x64-glibc
dockerfile: dockerfiles/components/clarinet-dev.dockerfile
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Generate Docker tags/labels
id: docker_meta
uses: docker/metadata-action@v4
with:
images: ${{ matrix.image }}
tags: |
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}},value=${{ needs.get_release_info.outputs.tag }},enable=${{ needs.get_release_info.outputs.tag != '' }}
type=semver,pattern={{major}}.{{minor}},value=${{ needs.get_release_info.outputs.tag }},enable=${{ needs.get_release_info.outputs.tag != '' }}
labels: |
org.opencontainers.image.title=${{ matrix.name }}
org.opencontainers.image.description=${{ matrix.description }}
- name: Login to Dockerhub
uses: docker/login-action@v2
if: github.event_name != 'pull_request'
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Download pre-built dist
uses: actions/download-artifact@v4
with:
name: ${{ matrix.artifact }}
- name: Untar pre-built dist
run: tar zxvf *.tar.gz
- name: Create Image
uses: docker/build-push-action@v3
with:
context: .
file: ${{ matrix.dockerfile }}
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
release:
name: Release
runs-on: ubuntu-latest
if: startsWith(github.ref, 'refs/heads/main') && needs.get_release_info.outputs.tag != ''
needs:
- get_release_info
- audit
- test_coverage_cargo
- dist_stacks_devnet_js
- dist_clarity_repl
- docker_images
permissions:
actions: write
contents: write
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Download pre-built dists
uses: actions/download-artifact@v4
- name: Tag and Release
uses: ncipollo/release-action@v1
with:
artifacts: "**/*.tar.gz,**/*.msi"
tag: ${{ needs.get_release_info.outputs.tag }}
commit: ${{ env.GITHUB_SHA }}
- name: Trigger pkg-version-bump workflow
uses: peter-evans/repository-dispatch@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
event-type: released
client-payload: '{"tag": "${{ needs.get_release_info.outputs.tag }}"}'