Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CI] Dry x86-64-v3 builds/default images to use glibc #4460

Merged
merged 11 commits into from
Mar 11, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions .github/actions/dockerfiles/Dockerfile.alpine-binary
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,12 @@ ARG TARGETVARIANT
ARG REPO=stacks-network/stacks-core

RUN case ${TARGETARCH} in \
"amd64") BIN_ARCH=linux-musl-x64 ;; \
"amd64") BIN_ARCH=linux-musl-x64-v3 ;; \
wileyj marked this conversation as resolved.
Show resolved Hide resolved
"arm64") BIN_ARCH=linux-musl-arm64 ;; \
"arm") BIN_ARCH=linux-musl-armv7 ;; \
"*") exit 1 ;; \
esac \
&& echo "wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip" \
&& wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip \
&& wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip \
&& unzip ${BIN_ARCH}.zip -d /out

FROM --platform=${TARGETPLATFORM} alpine
Expand Down
9 changes: 4 additions & 5 deletions .github/actions/dockerfiles/Dockerfile.debian-binary
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,12 @@ ARG TARGETVARIANT
ARG REPO=stacks-network/stacks-core

RUN case ${TARGETARCH} in \
"amd64") BIN_ARCH=linux-musl-x64 ;; \
"arm64") BIN_ARCH=linux-musl-arm64 ;; \
"arm") BIN_ARCH=linux-musl-armv7 ;; \
"amd64") BIN_ARCH=linux-glibc-x64-v3 ;; \
wileyj marked this conversation as resolved.
Show resolved Hide resolved
"arm64") BIN_ARCH=linux-glibc-arm64 ;; \
"arm") BIN_ARCH=linux-glibc-armv7 ;; \
"*") exit 1 ;; \
esac \
&& echo "wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip" \
&& wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip \
&& wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip \
&& unzip ${BIN_ARCH}.zip -d /out

FROM --platform=${TARGETPLATFORM} debian:bookworm
Expand Down
3 changes: 3 additions & 0 deletions .github/actions/dockerfiles/Dockerfile.debian-source
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ ARG GIT_BRANCH='No Branch Info'
ARG GIT_COMMIT='No Commit Info'
ARG BUILD_DIR=/build
ARG TARGET=x86_64-unknown-linux-gnu
# Allow us to override the default `--target-cpu` for the given target triplet
ARG TARGET_CPU
ENV RUSTFLAGS="${TARGET_CPU:+${RUSTFLAGS} -Ctarget-cpu=${TARGET_CPU}}"
WORKDIR /src

COPY . .
Expand Down
78 changes: 0 additions & 78 deletions .github/workflows/create-source-binary-x64.yml

This file was deleted.

68 changes: 54 additions & 14 deletions .github/workflows/create-source-binary.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,6 @@ on:
description: "Tag name of this release (x.y.z)"
required: true
type: string
arch:
description: "Stringified JSON object listing of platform matrix"
required: false
type: string
default: >-
["linux-glibc-arm64", "linux-glibc-armv7", "linux-musl-arm64", "linux-musl-armv7"]

## change the display name to the tag being built
run-name: ${{ inputs.tag }}
Expand All @@ -30,41 +24,87 @@ jobs:
## - workflow is building default branch (master)
artifact:
if: |
inputs.tag != '' &&
inputs.tag != '' &&
github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
name: Build Binaries
runs-on: ubuntu-latest
strategy:
## Run a maximum of 10 builds concurrently, using the matrix defined in inputs.arch
max-parallel: 10
matrix:
platform: ${{ fromJson(inputs.arch) }}
arch:
- linux-musl
- linux-glibc
- macos
- windows
cpu:
- arm64
- armv7
- x86-64
- x86-64-v3
exclude:
- arch: windows # excludes windows-arm64
cpu: arm64
- arch: windows # excludes windows-armv7
cpu: armv7
- arch: macos # excludes macos-armv7
cpu: armv7

steps:
## Setup Docker for the builds
- name: Docker setup
id: docker_setup
uses: stacks-network/actions/docker@main

- name: Set Local env vars
id: set_envars
run: |
case ${{ matrix.cpu }} in
x86-64)
TARGET_CPU="${{ matrix.cpu }}"
DOCKERFILE_CPU="x64"
ARCHIVE_NAME="x64"
;;
x86-64-v3)
TARGET_CPU="${{ matrix.cpu }}"
DOCKERFILE_CPU="x64"
ARCHIVE_NAME="x64-v3"
;;
*)
TARGET_CPU=""
DOCKERFILE_CPU="${{ matrix.cpu }}"
ARCHIVE_NAME="${{ matrix.cpu }}"
;;
esac
echo "DOCKERFILE=Dockerfile.${{ matrix.arch }}-${DOCKERFILE_CPU}" >> "$GITHUB_ENV"
echo "ZIPFILE=${{ matrix.arch }}-${ARCHIVE_NAME}" >> "$GITHUB_ENV"
echo "TARGET_CPU=${TARGET_CPU}" >> "$GITHUB_ENV"
echo "DOCKERFILE: ${DOCKERFILE}"
echo "ZIPFILE: ${ZIPFILE}"
echo "TARGET_CPU: ${TARGET_CPU}"

## Build the binaries using defined dockerfiles
- name: Build Binary (${{ matrix.platform }})
- name: Build Binary (${{ matrix.arch }}_${{ matrix.cpu }})
id: build_binaries
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # 5.0.0
with:
file: build-scripts/Dockerfile.${{ matrix.platform }}
outputs: type=local,dest=./release/${{ matrix.platform }}
file: build-scripts/${{ env.DOCKERFILE }}
outputs: type=local,dest=./release/${{ matrix.arch }}
build-args: |
STACKS_NODE_VERSION=${{ inputs.tag || env.GITHUB_SHA_SHORT }}
OS_ARCH=${{ matrix.platform }}
OS_ARCH=${{ matrix.arch }}
TARGET_CPU=${{ env.TARGET_CPU }}
GIT_BRANCH=${{ env.GITHUB_REF_SHORT }}
GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }}

## Compress the binary artifact
- name: Compress artifact
id: compress_artifact
run: zip --junk-paths ${{ matrix.platform }} ./release/${{ matrix.platform }}/*
run: zip --junk-paths ${{ env.ZIPFILE }} ./release/${{ matrix.arch }}/*

## Upload the binary artifact to the github action (used in `github-release.yml` to create a release)
- name: Upload artifact
id: upload_artifact
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
path: ${{ matrix.platform }}.zip
path: ${{ env.ZIPFILE }}.zip
17 changes: 0 additions & 17 deletions .github/workflows/github-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,21 +36,6 @@ jobs:
tag: ${{ inputs.tag }}
secrets: inherit

## Build x86_64 binaries from source
##
## Runs when the following is true:
## - tag is provided
## - workflow is building default branch (master)
build-binaries-x64:
if: |
inputs.tag != '' &&
github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
name: Build Binaries (x64_64)
uses: ./.github/workflows/create-source-binary-x64.yml
with:
tag: ${{ inputs.tag }}
secrets: inherit

## Runs when the following is true:
## - tag is provided
## - workflow is building default branch (master)
Expand All @@ -62,7 +47,6 @@ jobs:
runs-on: ubuntu-latest
needs:
- build-binaries
- build-binaries-x64
steps:
## Downloads the artifacts built in `create-source-binary.yml`
- name: Download Artifacts
Expand Down Expand Up @@ -111,7 +95,6 @@ jobs:
uses: ./.github/workflows/image-build-binary.yml
needs:
- build-binaries
- build-binaries-x64
- create-release
with:
tag: ${{ inputs.tag }}
Expand Down
31 changes: 18 additions & 13 deletions .github/workflows/image-build-binary.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,7 @@ on:
tag:
required: true
type: string
description: "Version tag for alpine images"
docker-org:
required: false
type: string
description: "Docker repo org for uploading images (defaults to github org)"
default: "${GITHUB_REPOSITORY_OWNER}"
description: "Version tag for docker images"

## Define which docker arch to build for
env:
Expand Down Expand Up @@ -48,14 +43,24 @@ jobs:
steps:
## Setup Docker for the builds
- name: Docker setup
id: docker_setup
uses: stacks-network/actions/docker@main
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}

## if the repo owner is not `stacks-network`, default to a docker-org of the repo owner (i.e. github user id)
## this allows forks to run the docker push workflows without having to hardcode a dockerhub org (but it does require docker hub user to match github username)
- name: Set Local env vars
id: set_envars
if: |
github.repository_owner != 'stacks-network'
run: |
echo "docker-org=${{ github.repository_owner }}" >> "$GITHUB_ENV"

## Set docker metatdata
## - depending on the matrix.dist, different tags will be enabled
## ex. alpine will have this tag: `type=ref,event=tag,enable=${{ matrix.dist == 'alpine' }}`
## ex. debian will have this tag: `type=ref,event=tag,enable=${{ matrix.dist == 'debian' }}`
- name: Docker Metadata ( ${{matrix.dist}} )
id: docker_metadata
uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 #v5.0.0
Expand All @@ -64,12 +69,12 @@ jobs:
${{env.docker-org}}/${{ github.event.repository.name }}
${{env.docker-org}}/stacks-blockchain
tags: |
type=raw,value=latest,enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'alpine' }}
type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'alpine'}}
type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' && matrix.dist == 'alpine' }}
type=ref,event=tag,enable=${{ matrix.dist == 'alpine' }}
type=raw,value=latest-${{ matrix.dist }},enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'debian' }}
type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'debian' }}
type=raw,value=latest,enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'debian' }}
type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'debian'}}
type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' && matrix.dist == 'debian' }}
type=ref,event=tag,enable=${{ matrix.dist == 'debian' }}
type=raw,value=latest-${{ matrix.dist }},enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'alpine' }}
type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'alpine' }}

## Build docker image for release
- name: Build and Push ( ${{matrix.dist}} )
Expand Down
11 changes: 11 additions & 0 deletions .github/workflows/image-build-source.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,21 @@ jobs:
steps:
## Setup Docker for the builds
- name: Docker setup
id: docker_setup
uses: stacks-network/actions/docker@main
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}

## if the repo owner is not `stacks-network`, default to a docker-org of the repo owner (i.e. github user id)
## this allows forks to run the docker push workflows without having to hardcode a dockerhub org (but it does require docker hub user to match github username)
- name: Set Local env vars
id: set_envars
if: |
github.repository_owner != 'stacks-network'
run: |
echo "docker-org=${{ github.repository_owner }}" >> "$GITHUB_ENV"

## Set docker metatdata
- name: Docker Metadata ( ${{matrix.dist}} )
id: docker_metadata
Expand All @@ -62,4 +72,5 @@ jobs:
STACKS_NODE_VERSION=${{ env.GITHUB_SHA_SHORT }}
GIT_BRANCH=${{ env.GITHUB_REF_SHORT }}
GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }}
TARGET_CPU=x86-64-v3
push: ${{ env.DOCKER_PUSH }}
1 change: 0 additions & 1 deletion .github/workflows/stacks-core-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,6 @@ jobs:

# Core contract tests on Clarinet v1
# Check for false positives/negatives
# https://github.com/stacks-network/stacks-blockchain/pull/4031#pullrequestreview-1713341208
core-contracts-clarinet-test-clarinet-v1:
name: Core Contracts Test Clarinet V1
runs-on: ubuntu-latest
Expand Down
3 changes: 2 additions & 1 deletion build-scripts/Dockerfile.linux-glibc-arm64
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \
cargo build --features monitoring_prom,slog_json --release --workspace --target ${TARGET} --bin stacks-node --bin stacks-inspect --bin clarity-cli --bin blockstack-cli \
&& mkdir -p /out \
&& cp -R ${BUILD_DIR}/target/${TARGET}/release/. /out
# --bin stacks-node --bin stacks-inspect --bin clarity-cli --bin blockstack-cli
wileyj marked this conversation as resolved.
Show resolved Hide resolved

FROM scratch AS export-stage
COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node /
COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node /
3 changes: 2 additions & 1 deletion build-scripts/Dockerfile.linux-glibc-armv7
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \
cargo build --features monitoring_prom,slog_json --release --workspace --target ${TARGET} --bin stacks-node --bin stacks-inspect --bin clarity-cli --bin blockstack-cli \
&& mkdir -p /out \
&& cp -R ${BUILD_DIR}/target/${TARGET}/release/. /out
# --bin stacks-node --bin stacks-inspect --bin clarity-cli --bin blockstack-cli
wileyj marked this conversation as resolved.
Show resolved Hide resolved

FROM scratch AS export-stage
COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node /
COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node /
Loading