From 8c01f358fd567529888022cd23f826aa52a062c1 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 12:39:10 -0800 Subject: [PATCH 01/18] chore: prevent CI double runs on push + pull_request --- .github/workflows/check_db_migration_confict.yml | 6 ++++++ .github/workflows/prefer-typescript.yml | 5 +++-- .github/workflows/superset-cli.yml | 4 ++-- .github/workflows/superset-docs.yml | 3 +++ .github/workflows/superset-e2e.yml | 6 ++---- .github/workflows/superset-frontend.yml | 9 ++++++--- .github/workflows/superset-helm-lint.yml | 2 ++ .github/workflows/superset-python-integrationtest.yml | 4 ++-- .github/workflows/superset-python-misc.yml | 8 ++++++-- .github/workflows/superset-python-presto-hive.yml | 8 ++++++-- .github/workflows/superset-python-unittest.yml | 8 ++++++-- .github/workflows/superset-translations.yml | 4 ++-- .github/workflows/superset-websocket.yml | 3 +++ 13 files changed, 49 insertions(+), 21 deletions(-) diff --git a/.github/workflows/check_db_migration_confict.yml b/.github/workflows/check_db_migration_confict.yml index 8dc7ab08824e9..85058c3cd3290 100644 --- a/.github/workflows/check_db_migration_confict.yml +++ b/.github/workflows/check_db_migration_confict.yml @@ -3,6 +3,12 @@ on: push: paths: - "superset/migrations/**" + branches: + - 'master' + pull_request: + paths: + - "superset/migrations/**" + types: [synchronize, opened, reopened, ready_for_review] jobs: check_db_migration_conflict: diff --git a/.github/workflows/prefer-typescript.yml b/.github/workflows/prefer-typescript.yml index a1e75c73d422a..08480966b6c5a 100644 --- a/.github/workflows/prefer-typescript.yml +++ b/.github/workflows/prefer-typescript.yml @@ -2,9 +2,10 @@ name: Prefer TypeScript on: push: - branches-ignore: - - "dependabot/**" + branches: + - 'master' pull_request: + types: [synchronize, opened, reopened, ready_for_review] jobs: prefer_typescript: diff --git a/.github/workflows/superset-cli.yml b/.github/workflows/superset-cli.yml index faf6d7d0b21a0..91be375b31b8e 100644 --- a/.github/workflows/superset-cli.yml +++ b/.github/workflows/superset-cli.yml @@ -2,8 +2,8 @@ name: Superset CLI tests on: push: - branches-ignore: - - "dependabot/npm_and_yarn/**" + branches: + - 'master' pull_request: types: [synchronize, opened, reopened, ready_for_review] diff --git a/.github/workflows/superset-docs.yml b/.github/workflows/superset-docs.yml index 6a47df4400dd8..e1c2df7d1261b 100644 --- a/.github/workflows/superset-docs.yml +++ b/.github/workflows/superset-docs.yml @@ -4,9 +4,12 @@ on: push: paths: - "docs/**" + branches: + - 'master' pull_request: paths: - "docs/**" + types: [synchronize, opened, reopened, ready_for_review] jobs: config: diff --git a/.github/workflows/superset-e2e.yml b/.github/workflows/superset-e2e.yml index 5eb794a4859f2..0f01a1be157f0 100644 --- a/.github/workflows/superset-e2e.yml +++ b/.github/workflows/superset-e2e.yml @@ -2,10 +2,8 @@ name: E2E on: push: - branches-ignore: - - "dependabot/**/docs/**" - paths-ignore: - - "docs/**" + branches: + - 'master' pull_request: types: [synchronize, opened, reopened, ready_for_review] diff --git a/.github/workflows/superset-frontend.yml b/.github/workflows/superset-frontend.yml index 35e1a989a4179..f48a65bd2959d 100644 --- a/.github/workflows/superset-frontend.yml +++ b/.github/workflows/superset-frontend.yml @@ -2,11 +2,14 @@ name: Frontend on: push: - branches-ignore: - - "dependabot/**/docs/**" - - "dependabot/**/cypress-base/**" + branches: + - "master" + paths: + - "superset-frontend/**" pull_request: types: [synchronize, opened, reopened, ready_for_review] + paths: + - "superset-frontend/**" jobs: frontend-build: diff --git a/.github/workflows/superset-helm-lint.yml b/.github/workflows/superset-helm-lint.yml index 4bb888e15d74e..7ca8f11887459 100644 --- a/.github/workflows/superset-helm-lint.yml +++ b/.github/workflows/superset-helm-lint.yml @@ -3,6 +3,8 @@ name: Lint and Test Charts on: pull_request: types: [opened, edited, reopened, synchronize] + paths: + - "helm/**" jobs: lint-test: diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml index 655b5c85d5944..76187250f4c7d 100644 --- a/.github/workflows/superset-python-integrationtest.yml +++ b/.github/workflows/superset-python-integrationtest.yml @@ -3,8 +3,8 @@ name: Python-Integration on: push: - branches-ignore: - - "dependabot/npm_and_yarn/**" + branches: + - 'master' pull_request: types: [synchronize, opened, reopened, ready_for_review] diff --git a/.github/workflows/superset-python-misc.yml b/.github/workflows/superset-python-misc.yml index e42c743c9ec16..58d49be5264bb 100644 --- a/.github/workflows/superset-python-misc.yml +++ b/.github/workflows/superset-python-misc.yml @@ -3,10 +3,14 @@ name: Python Misc on: push: - branches-ignore: - - "dependabot/npm_and_yarn/**" + branches: + - 'master' + paths: + - "superset/**" pull_request: types: [synchronize, opened, reopened, ready_for_review] + paths: + - "superset/**" jobs: python-lint: diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml index 7e326759ea40e..8f71289b4de23 100644 --- a/.github/workflows/superset-python-presto-hive.yml +++ b/.github/workflows/superset-python-presto-hive.yml @@ -3,10 +3,14 @@ name: Python Presto/Hive on: push: - branches-ignore: - - "dependabot/npm_and_yarn/**" + branches: + - 'master' + paths: + - "superset/**" pull_request: types: [synchronize, opened, reopened, ready_for_review] + paths: + - "superset/**" jobs: test-postgres-presto: diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml index 5f7639e67ccf7..401c6ad89c3d5 100644 --- a/.github/workflows/superset-python-unittest.yml +++ b/.github/workflows/superset-python-unittest.yml @@ -3,10 +3,14 @@ name: Python-Unit on: push: - branches-ignore: - - "dependabot/npm_and_yarn/**" + branches: + - 'master' + paths: + - "superset/**" pull_request: types: [synchronize, opened, reopened, ready_for_review] + paths: + - "superset/**" jobs: unit-tests: diff --git a/.github/workflows/superset-translations.yml b/.github/workflows/superset-translations.yml index e42a9ac9e72e4..d3d957fde2f27 100644 --- a/.github/workflows/superset-translations.yml +++ b/.github/workflows/superset-translations.yml @@ -2,8 +2,8 @@ name: Translations on: push: - branches-ignore: - - "dependabot/npm_and_yarn/**" + branches: + - 'master' pull_request: types: [synchronize, opened, reopened, ready_for_review] diff --git a/.github/workflows/superset-websocket.yml b/.github/workflows/superset-websocket.yml index 73a532cb98108..345aa4ddf92bb 100644 --- a/.github/workflows/superset-websocket.yml +++ b/.github/workflows/superset-websocket.yml @@ -1,11 +1,14 @@ name: WebSocket server on: push: + branches: + - 'master' paths: - "superset-websocket/**" pull_request: paths: - "superset-websocket/**" + types: [synchronize, opened, reopened, ready_for_review] jobs: app-checks: From dfd95675024f9d7592beee754c264b0de4d14fca Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 14:01:10 -0800 Subject: [PATCH 02/18] adressing comments --- .github/workflows/prefer-typescript.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/prefer-typescript.yml b/.github/workflows/prefer-typescript.yml index 08480966b6c5a..04d92449485e1 100644 --- a/.github/workflows/prefer-typescript.yml +++ b/.github/workflows/prefer-typescript.yml @@ -4,8 +4,12 @@ on: push: branches: - 'master' + paths: + - "superset-frontend/src/**" pull_request: types: [synchronize, opened, reopened, ready_for_review] + paths: + - "superset-frontend/src/**" jobs: prefer_typescript: From 61ade6afd753d34674500e0eec49649456c36704 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 14:47:54 -0800 Subject: [PATCH 03/18] dummy --- .github/workflows/prefer-typescript.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/prefer-typescript.yml b/.github/workflows/prefer-typescript.yml index 04d92449485e1..65fa8de11dc14 100644 --- a/.github/workflows/prefer-typescript.yml +++ b/.github/workflows/prefer-typescript.yml @@ -10,7 +10,6 @@ on: types: [synchronize, opened, reopened, ready_for_review] paths: - "superset-frontend/src/**" - jobs: prefer_typescript: if: github.ref == 'ref/heads/master' && github.event_name == 'pull_request' From 1930ae5b850f20e1be8682087aa06218250dd53f Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 15:20:50 -0800 Subject: [PATCH 04/18] testing no-op --- .github/workflows/no-op.yml | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/no-op.yml diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml new file mode 100644 index 0000000000000..7e3e6b089e415 --- /dev/null +++ b/.github/workflows/no-op.yml @@ -0,0 +1,48 @@ +# no-op.yml +# +# Purpose: +# This workflow provides a workaround for the "required status checks" feature in GitHub Actions +# when using path-specific conditions in other workflows. Required checks might remain in a "Pending" +# state if the conditions are not met, thus blocking pull requests from being merged. +# This no-op (no operation) workflow provides dummy success statuses for these required jobs when +# the real jobs do not run due to path-specific conditions. +# +# How it works: +# - It defines jobs with the same names as the required jobs in the main workflows. +# - These jobs simply execute a command (`exit 0`) to succeed immediately. +# - When a pull request is created or updated, both this no-op workflow and the main workflows are triggered. +# - If the main workflows' jobs don't run (due to path conditions), these no-op jobs provide successful statuses. +# - If the main workflows' jobs do run and fail, their failure statuses take precedence, +# ensuring that pull requests are not merged with failing checks. +# +# Usage: +# - Ensure that the job names in this workflow match exactly the names of the corresponding jobs in the main workflows. +# - This workflow should be kept as-is, without path-specific conditions. + +name: No Operation Checks +on: pull_request + +jobs: + frontend-build: + runs-on: ubuntu-latest + steps: + - name: No-op for frontend-build + run: | + echo "This is a no-op step for frontend-build to ensure a successful status." + exit 0 + + pre-commit: + runs-on: ubuntu-latest + steps: + - name: No-op for pre-commit + run: | + echo "This is a no-op step for pre-commit to ensure a successful status." + exit 0 + + python-lint: + runs-on: ubuntu-latest + steps: + - name: No-op for python-lint + run: | + echo "This is a no-op step for python-lint to ensure a successful status." + exit 0 From 2028a7933a54b3ac28e3beb00298b7cbf072377a Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 15:34:10 -0800 Subject: [PATCH 05/18] catching matrix-specific no-op --- .github/workflows/license-check.yml | 5 +++-- .github/workflows/no-op.yml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml index 8e2b9bbed3096..e9b75a100983a 100644 --- a/.github/workflows/license-check.yml +++ b/.github/workflows/license-check.yml @@ -2,9 +2,10 @@ name: License Check on: push: - branches-ignore: - - "dependabot/**" + branches: + - 'master' pull_request: + types: [synchronize, opened, reopened, ready_for_review] jobs: config: diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml index 7e3e6b089e415..933192916474f 100644 --- a/.github/workflows/no-op.yml +++ b/.github/workflows/no-op.yml @@ -31,7 +31,7 @@ jobs: echo "This is a no-op step for frontend-build to ensure a successful status." exit 0 - pre-commit: + pre-commit (3.9): runs-on: ubuntu-latest steps: - name: No-op for pre-commit @@ -39,7 +39,7 @@ jobs: echo "This is a no-op step for pre-commit to ensure a successful status." exit 0 - python-lint: + python-lint (3.9): runs-on: ubuntu-latest steps: - name: No-op for python-lint From 2626bb60160643fe6abce6ef5d3415d335dff11b Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 15:40:00 -0800 Subject: [PATCH 06/18] fix yaml --- .github/workflows/no-op.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml index 933192916474f..a570b207494bc 100644 --- a/.github/workflows/no-op.yml +++ b/.github/workflows/no-op.yml @@ -31,7 +31,7 @@ jobs: echo "This is a no-op step for frontend-build to ensure a successful status." exit 0 - pre-commit (3.9): + "pre-commit (3.9)": runs-on: ubuntu-latest steps: - name: No-op for pre-commit @@ -39,7 +39,7 @@ jobs: echo "This is a no-op step for pre-commit to ensure a successful status." exit 0 - python-lint (3.9): + "python-lint (3.9)": runs-on: ubuntu-latest steps: - name: No-op for python-lint From ab0b564fca8e15c9773f3d60e2d508c4f0fea53d Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 15:47:01 -0800 Subject: [PATCH 07/18] testing --- .asf.yaml | 4 ++-- .github/workflows/no-op.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index ab980e0c3fb2b..61681b13a37a9 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -66,8 +66,8 @@ github: - cypress-matrix (3, chrome) - docker-build - frontend-build - - pre-commit (3.9) - - python-lint (3.9) + - pre-commit + - python-lint - test-mysql (3.9) - test-postgres (3.9) - test-postgres (3.10) diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml index a570b207494bc..7e3e6b089e415 100644 --- a/.github/workflows/no-op.yml +++ b/.github/workflows/no-op.yml @@ -31,7 +31,7 @@ jobs: echo "This is a no-op step for frontend-build to ensure a successful status." exit 0 - "pre-commit (3.9)": + pre-commit: runs-on: ubuntu-latest steps: - name: No-op for pre-commit @@ -39,7 +39,7 @@ jobs: echo "This is a no-op step for pre-commit to ensure a successful status." exit 0 - "python-lint (3.9)": + python-lint: runs-on: ubuntu-latest steps: - name: No-op for python-lint From 12f1c4e707d9d61eab96b65c7a3a8736c4bf6099 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 15:48:53 -0800 Subject: [PATCH 08/18] testing --- .asf.yaml | 4 ++-- .github/workflows/no-op.yml | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 61681b13a37a9..ab980e0c3fb2b 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -66,8 +66,8 @@ github: - cypress-matrix (3, chrome) - docker-build - frontend-build - - pre-commit - - python-lint + - pre-commit (3.9) + - python-lint (3.9) - test-mysql (3.9) - test-postgres (3.9) - test-postgres (3.10) diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml index 7e3e6b089e415..835063640aa96 100644 --- a/.github/workflows/no-op.yml +++ b/.github/workflows/no-op.yml @@ -32,6 +32,9 @@ jobs: exit 0 pre-commit: + strategy: + matrix: + python-version: ["3.9"] runs-on: ubuntu-latest steps: - name: No-op for pre-commit @@ -40,6 +43,9 @@ jobs: exit 0 python-lint: + strategy: + matrix: + python-version: ["3.9"] runs-on: ubuntu-latest steps: - name: No-op for python-lint From 554eb4e2d0506f1efe25853e52d1d1d3ba0eef76 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 16:27:26 -0800 Subject: [PATCH 09/18] fix docker cache hit --- scripts/docker_build_push.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh index 10737447eaa6b..cc8978a61106b 100755 --- a/scripts/docker_build_push.sh +++ b/scripts/docker_build_push.sh @@ -69,28 +69,29 @@ else fi BUILD_ARG="3.9-slim-bookworm" +MAIN_UNIQUE_TAG="${REPO_NAME}:${SHA}:${TARGET}:${BUILD_PLATFORM}:${BUILD_ARG}" case "${TARGET}" in "dev") - DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-dev -t ${REPO_NAME}:${REFSPEC}-dev -t ${DEV_TAG}" + DOCKER_TAGS="-t ${MAIN_UNIQUE_TAG} -t ${REPO_NAME}:${SHA}-dev -t ${REPO_NAME}:${REFSPEC}-dev -t ${DEV_TAG}" BUILD_TARGET="dev" ;; "lean") - DOCKER_TAGS="-t ${REPO_NAME}:${SHA} -t ${REPO_NAME}:${REFSPEC} -t ${REPO_NAME}:${LATEST_TAG}" + DOCKER_TAGS="-t ${MAIN_UNIQUE_TAG} -t ${REPO_NAME}:${SHA} -t ${REPO_NAME}:${REFSPEC} -t ${REPO_NAME}:${LATEST_TAG}" BUILD_TARGET="lean" ;; "lean310") - DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-py310 -t ${REPO_NAME}:${REFSPEC}-py310 -t ${REPO_NAME}:${LATEST_TAG}-py310" + DOCKER_TAGS="-t ${MAIN_UNIQUE_TAG} -t ${REPO_NAME}:${SHA}-py310 -t ${REPO_NAME}:${REFSPEC}-py310 -t ${REPO_NAME}:${LATEST_TAG}-py310" BUILD_TARGET="lean" BUILD_ARG="3.10-slim-bookworm" ;; "websocket") - DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-websocket -t ${REPO_NAME}:${REFSPEC}-websocket -t ${REPO_NAME}:${LATEST_TAG}-websocket" + DOCKER_TAGS="-t ${MAIN_UNIQUE_TAG} -t ${REPO_NAME}:${SHA}-websocket -t ${REPO_NAME}:${REFSPEC}-websocket -t ${REPO_NAME}:${LATEST_TAG}-websocket" BUILD_TARGET="" DOCKER_CONTEXT="superset-websocket" ;; "dockerize") - DOCKER_TAGS="-t ${REPO_NAME}:dockerize" + DOCKER_TAGS="-t ${MAIN_UNIQUE_TAG} -t ${REPO_NAME}:dockerize" BUILD_TARGET="" DOCKER_CONTEXT="-f dockerize.Dockerfile ." ;; @@ -128,8 +129,8 @@ fi docker buildx build \ ${TARGET_ARGUMENT} \ ${DOCKER_ARGS} \ - --cache-from=type=registry,ref=apache/superset:${TARGET} \ - --cache-to=type=registry,mode=max,ref=apache/superset:${TARGET} \ + --cache-from=type=registry,ref=${MAIN_UNIQUE_TAG} \ + --cache-to=type=registry,mode=max,ref=${MAIN_UNIQUE_TAG} \ ${DOCKER_TAGS} \ --platform ${BUILD_PLATFORM} \ --label "sha=${SHA}" \ From d856e1731c992ebaff8840ea7abc4ab4cb78c1cc Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 16:33:46 -0800 Subject: [PATCH 10/18] fix docker cache hit --- scripts/docker_build_push.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh index cc8978a61106b..9271fe04ea6ef 100755 --- a/scripts/docker_build_push.sh +++ b/scripts/docker_build_push.sh @@ -69,7 +69,8 @@ else fi BUILD_ARG="3.9-slim-bookworm" -MAIN_UNIQUE_TAG="${REPO_NAME}:${SHA}:${TARGET}:${BUILD_PLATFORM}:${BUILD_ARG}" +MAIN_UNIQUE_TAG="${REPO_NAME}:${SHA}-${TARGET}-${BUILD_PLATFORM}-${BUILD_ARG}" + case "${TARGET}" in "dev") From 52b1fa276d35260891e5944649a16bec748b25c3 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 16:36:37 -0800 Subject: [PATCH 11/18] fix docker cache hit --- scripts/docker_build_push.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh index 9271fe04ea6ef..db5d8de2af883 100755 --- a/scripts/docker_build_push.sh +++ b/scripts/docker_build_push.sh @@ -69,7 +69,10 @@ else fi BUILD_ARG="3.9-slim-bookworm" -MAIN_UNIQUE_TAG="${REPO_NAME}:${SHA}-${TARGET}-${BUILD_PLATFORM}-${BUILD_ARG}" + +# Replace '/' with '-' in BUILD_PLATFORM +SAFE_BUILD_PLATFORM=$(echo "${BUILD_PLATFORM}" | sed 's/\//-/g') +MAIN_UNIQUE_TAG="${REPO_NAME}:${SHA}-${TARGET}-${SAFE_BUILD_PLATFORM}-${BUILD_ARG}" case "${TARGET}" in From f364847355d60c78ebbc025731f351fcb9af1d71 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 17:31:57 -0800 Subject: [PATCH 12/18] dummy commit to test docker cache --- .github/workflows/superset-e2e.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/superset-e2e.yml b/.github/workflows/superset-e2e.yml index 0f01a1be157f0..5ce63ef8d6848 100644 --- a/.github/workflows/superset-e2e.yml +++ b/.github/workflows/superset-e2e.yml @@ -6,7 +6,6 @@ on: - 'master' pull_request: types: [synchronize, opened, reopened, ready_for_review] - jobs: cypress-matrix: runs-on: ubuntu-20.04 From ddd86f678d1e72430098754db00eb562ba2eed3a Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 17:53:57 -0800 Subject: [PATCH 13/18] trying a different ref --- scripts/docker_build_push.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh index db5d8de2af883..66fa654ce16ec 100755 --- a/scripts/docker_build_push.sh +++ b/scripts/docker_build_push.sh @@ -74,7 +74,6 @@ BUILD_ARG="3.9-slim-bookworm" SAFE_BUILD_PLATFORM=$(echo "${BUILD_PLATFORM}" | sed 's/\//-/g') MAIN_UNIQUE_TAG="${REPO_NAME}:${SHA}-${TARGET}-${SAFE_BUILD_PLATFORM}-${BUILD_ARG}" - case "${TARGET}" in "dev") DOCKER_TAGS="-t ${MAIN_UNIQUE_TAG} -t ${REPO_NAME}:${SHA}-dev -t ${REPO_NAME}:${REFSPEC}-dev -t ${DEV_TAG}" @@ -107,6 +106,7 @@ esac cat< Date: Tue, 23 Jan 2024 19:20:00 -0800 Subject: [PATCH 14/18] try supreset-cache hub --- scripts/docker_build_push.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh index 66fa654ce16ec..c0252d858fd7d 100755 --- a/scripts/docker_build_push.sh +++ b/scripts/docker_build_push.sh @@ -133,8 +133,8 @@ fi docker buildx build \ ${TARGET_ARGUMENT} \ ${DOCKER_ARGS} \ - --cache-from=type=registry,ref=${REPO_NAME} \ - --cache-to=type=registry,mode=max,ref=${REPO_NAME} \ + --cache-from=type=registry,ref=${REPO_NAME}-cache \ + --cache-to=type=registry,mode=max,ref=${REPO_NAME}-cache \ ${DOCKER_TAGS} \ --platform ${BUILD_PLATFORM} \ --label "sha=${SHA}" \ From 60348972a2ee777a20158c71ec43261af5470155 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 19:38:17 -0800 Subject: [PATCH 15/18] fancy CACHE_REF --- scripts/docker_build_push.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh index c0252d858fd7d..17147fb63953e 100755 --- a/scripts/docker_build_push.sh +++ b/scripts/docker_build_push.sh @@ -130,11 +130,13 @@ if [[ -n "${BUILD_TARGET}" ]]; then TARGET_ARGUMENT="--target ${BUILD_TARGET}" fi +CACHE_REF="${REPO_NAME}-cache:${BUILD_TARGET}-${BUILD_ARG}" + docker buildx build \ ${TARGET_ARGUMENT} \ ${DOCKER_ARGS} \ - --cache-from=type=registry,ref=${REPO_NAME}-cache \ - --cache-to=type=registry,mode=max,ref=${REPO_NAME}-cache \ + --cache-from=type=registry,ref=${CACHE_REF} \ + --cache-to=type=registry,mode=max,ref=${CACHE_REF} \ ${DOCKER_TAGS} \ --platform ${BUILD_PLATFORM} \ --label "sha=${SHA}" \ From 2519ba2cb1fa7227bdaaeb9cd9a0dfd61e0354c0 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 19:45:59 -0800 Subject: [PATCH 16/18] fancy CACHE_REF --- scripts/docker_build_push.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh index 17147fb63953e..5ad7d2d999734 100755 --- a/scripts/docker_build_push.sh +++ b/scripts/docker_build_push.sh @@ -130,7 +130,8 @@ if [[ -n "${BUILD_TARGET}" ]]; then TARGET_ARGUMENT="--target ${BUILD_TARGET}" fi -CACHE_REF="${REPO_NAME}-cache:${BUILD_TARGET}-${BUILD_ARG}" +CACHE_REF="${REPO_NAME}-cache:${TARGET}-${BUILD_ARG}" +CACHE_REF=$(echo "${CACHE_REF}" | tr -d '.') docker buildx build \ ${TARGET_ARGUMENT} \ From 4fbcf6a2bf8d1c0cc8a763841564eec2ea344ba3 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 19:54:46 -0800 Subject: [PATCH 17/18] dummy commit to test docker cache --- .github/workflows/superset-e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/superset-e2e.yml b/.github/workflows/superset-e2e.yml index 5ce63ef8d6848..0f01a1be157f0 100644 --- a/.github/workflows/superset-e2e.yml +++ b/.github/workflows/superset-e2e.yml @@ -6,6 +6,7 @@ on: - 'master' pull_request: types: [synchronize, opened, reopened, ready_for_review] + jobs: cypress-matrix: runs-on: ubuntu-20.04 From 26ca7a898e0682776c08796ba9e566390a41ac05 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 23 Jan 2024 20:13:53 -0800 Subject: [PATCH 18/18] disabling some builds --- .github/workflows/docker.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b0d12be103c11..5aa2040c2aa28 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,6 +33,13 @@ jobs: matrix: target: ["dev", "lean", "lean310", "websocket", "dockerize"] platform: ["linux/amd64", "linux/arm64"] + exclude: + # disabling because slow! no python wheels for arm/py39 and + # QEMU is slow! + - target: "dev" + platform: "linux/arm64" + - target: "lean" + platform: "linux/arm64" fail-fast: false steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"