Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert "[gha][docker] only push to GAR on PR; push to ECR on postcommit" #8514

Merged
merged 1 commit into from
Jun 4, 2023
Merged

Revert "[gha][docker] only push to GAR on PR; push to ECR on postcommit" #8514

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 4 additions & 19 deletions .github/workflows/docker-build-test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,6 @@ env:
# We use `pr-<pr_number>` as cache-id for PRs and simply <branch_name> otherwise.
TARGET_CACHE_ID: ${{ github.event.number && format('pr-{0}', github.event.number) || github.ref_name }}

# On PRs, only build and push to GCP
# On push, build and push to all remote registries
TARGET_REGISTRY: ${{ github.event_name == 'pull_request_target' && 'gcp' || 'remote' }}

permissions:
contents: read
id-token: write #required for GCP Workload Identity federation which we use to login into Google Artifact Registry
Expand Down Expand Up @@ -103,11 +99,9 @@ jobs:
run: |
echo "GIT_SHA: ${GIT_SHA}"
echo "TARGET_CACHE_ID: ${TARGET_CACHE_ID}"
echo "TARGET_REGISTRY: ${TARGET_REGISTRY}"
outputs:
gitSha: ${{ env.GIT_SHA }}
targetCacheId: ${{ env.TARGET_CACHE_ID }}
targetRegistry: ${{ env.TARGET_REGISTRY }}

rust-images:
needs: [permission-check, determine-docker-build-metadata]
Expand All @@ -118,7 +112,6 @@ jobs:
TARGET_CACHE_ID: ${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
PROFILE: release
BUILD_ADDL_TESTING_IMAGES: true
TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }}

rust-images-indexer:
needs: [permission-check, determine-docker-build-metadata]
Expand All @@ -134,7 +127,6 @@ jobs:
PROFILE: release
FEATURES: indexer
BUILD_ADDL_TESTING_IMAGES: true
TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }}

rust-images-failpoints:
needs: [permission-check, determine-docker-build-metadata]
Expand All @@ -150,7 +142,6 @@ jobs:
PROFILE: release
FEATURES: failpoints
BUILD_ADDL_TESTING_IMAGES: true
TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }}

rust-images-performance:
needs: [permission-check, determine-docker-build-metadata]
Expand All @@ -165,7 +156,6 @@ jobs:
TARGET_CACHE_ID: ${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
PROFILE: performance
BUILD_ADDL_TESTING_IMAGES: true
TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }}

rust-images-consensus-only-perf-test:
needs: [permission-check, determine-docker-build-metadata]
Expand All @@ -180,7 +170,6 @@ jobs:
PROFILE: release
FEATURES: consensus-only-perf-test
BUILD_ADDL_TESTING_IMAGES: true
TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }}

rust-images-all:
needs:
Expand Down Expand Up @@ -260,8 +249,7 @@ jobs:

forge-e2e-test:
needs: [rust-images-all, determine-docker-build-metadata]
if:
| # always() ensures that the job will run even if some of the previous docker variant build jobs were skipped https://docs.github.com/en/actions/learn-github-actions/expressions#status-check-functions
if: | # always() ensures that the job will run even if some of the previous docker variant build jobs were skipped https://docs.github.com/en/actions/learn-github-actions/expressions#status-check-functions
always() && needs.rust-images-all.result == 'success' && (
(github.event_name == 'push' && github.ref_name != 'main') ||
github.event_name == 'workflow_dispatch' ||
Expand All @@ -285,8 +273,7 @@ jobs:
# Run e2e compat test against testnet branch
forge-compat-test:
needs: [rust-images-all, determine-docker-build-metadata]
if:
| # always() ensures that the job will run even if some of the previous docker variant build jobs were skipped https://docs.github.com/en/actions/learn-github-actions/expressions#status-check-functions
if: | # always() ensures that the job will run even if some of the previous docker variant build jobs were skipped https://docs.github.com/en/actions/learn-github-actions/expressions#status-check-functions
always() && needs.rust-images-all.result == 'success' && (
(github.event_name == 'push' && github.ref_name != 'main') ||
github.event_name == 'workflow_dispatch' ||
Expand All @@ -307,8 +294,7 @@ jobs:
# Run forge framework upgradability test
forge-framework-upgrade-test:
needs: [rust-images-all, determine-docker-build-metadata]
if:
| # always() ensures that the job will run even if some of the previous docker variant build jobs were skipped https://docs.github.com/en/actions/learn-github-actions/expressions#status-check-functions
if: | # always() ensures that the job will run even if some of the previous docker variant build jobs were skipped https://docs.github.com/en/actions/learn-github-actions/expressions#status-check-functions
always() && needs.rust-images-all.result == 'success' && (
(github.event_name == 'push' && github.ref_name != 'main') ||
github.event_name == 'workflow_dispatch' ||
Expand All @@ -328,8 +314,7 @@ jobs:

forge-consensus-only-perf-test:
needs: [rust-images-all, determine-docker-build-metadata]
if:
| # always() ensures that the job will run even if some of the previous docker variant build jobs were skipped https://docs.github.com/en/actions/learn-github-actions/expressions#status-check-functions
if: | # always() ensures that the job will run even if some of the previous docker variant build jobs were skipped https://docs.github.com/en/actions/learn-github-actions/expressions#status-check-functions
always() && needs.rust-images-all.result == 'success' &&
contains(github.event.pull_request.labels.*.name, 'CICD:run-consensus-only-perf-test')
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
Expand Down
17 changes: 0 additions & 17 deletions .github/workflows/workflow-run-docker-rust-build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,6 @@ on:
required: false
type: boolean
description: Whether to build additional testing images. If not specified, only the base release images will be built
TARGET_REGISTRY:
default: remote
required: false
type: string
description: The target docker registry to push to

workflow_dispatch:
inputs:
GIT_SHA:
Expand All @@ -51,11 +45,6 @@ on:
required: false
type: boolean
description: Whether to build additional testing images. If not specified, only the base release images will be built
TARGET_REGISTRY:
default: remote
required: false
type: string
description: The target docker registry to push to

env:
GIT_SHA: ${{ inputs.GIT_SHA }}
Expand All @@ -66,11 +55,6 @@ env:
GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}
GCP_DOCKER_ARTIFACT_REPO_US: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO_US }}
AWS_ECR_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}
TARGET_REGISTRY: ${{ inputs.TARGET_REGISTRY }}

permissions:
contents: read
id-token: write #required for GCP Workload Identity federation which we use to login into Google Artifact Registry

jobs:
rust-all:
Expand All @@ -96,4 +80,3 @@ jobs:
FEATURES: ${{ env.FEATURES }}
BUILD_ADDL_TESTING_IMAGES: ${{ env.BUILD_ADDL_TESTING_IMAGES }}
GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}
TARGET_REGISTRY: ${{ env.TARGET_REGISTRY }}
77 changes: 35 additions & 42 deletions docker/builder/docker-bake-rust-all.hcl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ variable "GCP_DOCKER_ARTIFACT_REPO_US" {}
variable "AWS_ECR_ACCOUNT_NUM" {}

variable "TARGET_REGISTRY" {
// must be "gcp" | "local" | "remote", informs which docker tags are being generated
// must be "aws" | "remote" | "local", informs which docker tags are being generated
default = CI == "true" ? "remote" : "local"
}

Expand Down Expand Up @@ -74,8 +74,8 @@ target "debian-base" {

target "builder-base" {
dockerfile = "docker/builder/builder.Dockerfile"
target = "builder-base"
context = "."
target = "builder-base"
context = "."
contexts = {
rust = "docker-image://rust:1.66.1-bullseye@sha256:f72949bcf1daf8954c0e0ed8b7e10ac4c641608f6aa5f0ef7c172c49f35bd9b5"
}
Expand All @@ -92,7 +92,7 @@ target "builder-base" {

target "aptos-node-builder" {
dockerfile = "docker/builder/builder.Dockerfile"
target = "aptos-node-builder"
target = "aptos-node-builder"
contexts = {
builder-base = "target:builder-base"
}
Expand All @@ -103,9 +103,9 @@ target "aptos-node-builder" {

target "tools-builder" {
dockerfile = "docker/builder/builder.Dockerfile"
target = "tools-builder"
target = "tools-builder"
contexts = {
builder-base = "target:builder-base"
builder-base = "target:builder-base"
}
secret = [
"id=GIT_CREDENTIALS"
Expand All @@ -114,8 +114,8 @@ target "tools-builder" {

target "_common" {
contexts = {
debian-base = "target:debian-base"
node-builder = "target:aptos-node-builder"
debian-base = "target:debian-base"
node-builder = "target:aptos-node-builder"
tools-builder = "target:tools-builder"
}
labels = {
Expand All @@ -124,20 +124,20 @@ target "_common" {
"org.label-schema.git-sha" = "${GIT_SHA}"
}
args = {
PROFILE = "${PROFILE}"
FEATURES = "${FEATURES}"
GIT_SHA = "${GIT_SHA}"
GIT_BRANCH = "${GIT_BRANCH}"
GIT_TAG = "${GIT_TAG}"
BUILD_DATE = "${BUILD_DATE}"
PROFILE = "${PROFILE}"
FEATURES = "${FEATURES}"
GIT_SHA = "${GIT_SHA}"
GIT_BRANCH = "${GIT_BRANCH}"
GIT_TAG = "${GIT_TAG}"
BUILD_DATE = "${BUILD_DATE}"
}
}

target "validator-testing" {
inherits = ["_common"]
dockerfile = "docker/builder/validator-testing.Dockerfile"
target = "validator-testing"
cache-from = generate_cache_from("validator-testing")
cache-from = generate_cache_from("validator-testing")
cache-to = generate_cache_to("validator-testing")
tags = generate_tags("validator-testing")
}
Expand All @@ -146,7 +146,7 @@ target "tools" {
inherits = ["_common"]
dockerfile = "docker/builder/tools.Dockerfile"
target = "tools"
cache-from = generate_cache_from("tools")
cache-from = generate_cache_from("tools")
cache-to = generate_cache_to("tools")
tags = generate_tags("tools")
}
Expand All @@ -155,7 +155,7 @@ target "forge" {
inherits = ["_common"]
dockerfile = "docker/builder/forge.Dockerfile"
target = "forge"
cache-from = generate_cache_from("forge")
cache-from = generate_cache_from("forge")
cache-to = generate_cache_to("forge")
tags = generate_tags("forge")
}
Expand All @@ -164,7 +164,7 @@ target "validator" {
inherits = ["_common"]
dockerfile = "docker/builder/validator.Dockerfile"
target = "validator"
cache-from = generate_cache_from("validator")
cache-from = generate_cache_from("validator")
cache-to = generate_cache_to("validator")
tags = generate_tags("validator")
}
Expand All @@ -173,7 +173,7 @@ target "tools" {
inherits = ["_common"]
dockerfile = "docker/builder/tools.Dockerfile"
target = "tools"
cache-from = generate_cache_from("tools")
cache-from = generate_cache_from("tools")
cache-to = generate_cache_to("tools")
tags = generate_tags("tools")
}
Expand All @@ -182,7 +182,7 @@ target "node-checker" {
inherits = ["_common"]
dockerfile = "docker/builder/node-checker.Dockerfile"
target = "node-checker"
cache-from = generate_cache_from("node-checker")
cache-from = generate_cache_from("node-checker")
cache-to = generate_cache_to("node-checker")
tags = generate_tags("node-checker")
}
Expand All @@ -191,26 +191,26 @@ target "faucet" {
inherits = ["_common"]
dockerfile = "docker/builder/faucet.Dockerfile"
target = "faucet"
cache-from = generate_cache_from("faucet")
cache-to = generate_cache_to("faucet")
cache-from = generate_cache_from("faucet")
cache-to = generate_cache_to("faucet")
tags = generate_tags("faucet")
}

target "telemetry-service" {
inherits = ["_common"]
dockerfile = "docker/builder/telemetry-service.Dockerfile"
target = "telemetry-service"
cache-from = generate_cache_from("telemetry-service")
cache-to = generate_cache_to("telemetry-service")
tags = generate_tags("telemetry-service")
cache-from = generate_cache_from("telemetry-service")
cache-to = generate_cache_to("telemetry-service")
tags = generate_tags("telemetry-service")
}

target "indexer-grpc" {
inherits = ["_common"]
inherits = ["_common"]
dockerfile = "docker/builder/indexer-grpc.Dockerfile"
target = "indexer-grpc"
cache-to = generate_cache_to("indexer-grpc")
tags = generate_tags("indexer-grpc")
target = "indexer-grpc"
cache-to = generate_cache_to("indexer-grpc")
tags = generate_tags("indexer-grpc")
}

function "generate_cache_from" {
Expand All @@ -233,21 +233,14 @@ function "generate_cache_to" {
function "generate_tags" {
params = [target]
result = TARGET_REGISTRY == "remote" ? [
"${GCP_DOCKER_ARTIFACT_REPO}/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}",
"${GCP_DOCKER_ARTIFACT_REPO}/${target}:${IMAGE_TAG_PREFIX}${NORMALIZED_GIT_BRANCH_OR_PR}",
"${GCP_DOCKER_ARTIFACT_REPO_US}/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}",
"${GCP_DOCKER_ARTIFACT_REPO_US}/${target}:${IMAGE_TAG_PREFIX}${NORMALIZED_GIT_BRANCH_OR_PR}",
"${ecr_base}/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}",
"${ecr_base}/${target}:${IMAGE_TAG_PREFIX}${NORMALIZED_GIT_BRANCH_OR_PR}",
] : (
TARGET_REGISTRY == "gcp" ? [
"${GCP_DOCKER_ARTIFACT_REPO}/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}",
"${GCP_DOCKER_ARTIFACT_REPO}/${target}:${IMAGE_TAG_PREFIX}${NORMALIZED_GIT_BRANCH_OR_PR}",
"${GCP_DOCKER_ARTIFACT_REPO_US}/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}",
"${GCP_DOCKER_ARTIFACT_REPO_US}/${target}:${IMAGE_TAG_PREFIX}${NORMALIZED_GIT_BRANCH_OR_PR}",
] : [
"aptos-core/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}-from-local",
"aptos-core/${target}:${IMAGE_TAG_PREFIX}from-local",
]
)
"${ecr_base}/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}",
"${ecr_base}/${target}:${IMAGE_TAG_PREFIX}${NORMALIZED_GIT_BRANCH_OR_PR}",
] : [
"aptos-core/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}-from-local",
"aptos-core/${target}:${IMAGE_TAG_PREFIX}from-local",
]
}
4 changes: 2 additions & 2 deletions docker/builder/docker-bake-rust-all.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ echo "To build only a specific target, run: docker/builder/docker-bake-rust-all.
echo "E.g. docker/builder/docker-bake-rust-all.sh forge-images"

if [ "$CI" == "true" ]; then
docker buildx bake --progress=plain --file docker/builder/docker-bake-rust-all.hcl --push $BUILD_TARGET
TARGET_REGISTRY=remote docker buildx bake --progress=plain --file docker/builder/docker-bake-rust-all.hcl --push $BUILD_TARGET
else
docker buildx bake --file docker/builder/docker-bake-rust-all.hcl $BUILD_TARGET
TARGET_REGISTRY=local docker buildx bake --file docker/builder/docker-bake-rust-all.hcl $BUILD_TARGET
fi

echo "Build complete. Docker buildx cache usage:"
Expand Down