Ci shared workflows for sub project (#6025)

* Drop old CI workflow * Add CI file for shared workflow * Add job for Spellbook * Add job for tokens * Simplify manifest commit * Remove temp comment
duneanalytics · May 29, 2024 · 7510f06 · 7510f06
1 parent 50deff1
commit 7510f06
Show file tree

Hide file tree

Showing 4 changed files with 66 additions and 36 deletions.
diff --git a/.github/workflows/commit_manifest.yml b/.github/workflows/commit_manifest.yml
@@ -9,7 +9,7 @@ on:
 concurrency:
   group: ${{ github.workflow }}
   cancel-in-progress: true
-      
+
 jobs:
   commit_manifest:
     runs-on: [ self-hosted, linux, spellbook-trino-ci ]
@@ -27,26 +27,25 @@ jobs:
 
     - name: Add git_sha to schema
       run: "/runner/change_schema.sh wizard"
-    
+
     - name: Setup variables
       run: |
         echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV
         echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
-        if [[ "${{ matrix.project }}" == "tokens" ]]; then
-          echo "PROJECT_DIR=tokens" >> $GITHUB_ENV
-          echo "S3_LOCATION=s3://manifest-spellbook-dunesql/tokens" >> $GITHUB_ENV
-        elif [[ "${{ matrix.project }}" == "spellbook" ]]; then
-          echo "PROJECT_DIR=." >> $GITHUB_ENV
-          echo "S3_LOCATION=s3://manifest-spellbook-dunesql/spellbook" >> $GITHUB_ENV
-        fi
+        echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{matrix.project}}" >> $GITHUB_ENV
+          PROJECT_DIR=${{ matrix.project }}
+          if [[ "$PROJECT_DIR" == "spellbook" ]]; then
+            PROJECT_DIR="."
+          fi
+          echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
 
     - name: dbt dependencies
       working-directory: ${{env.PROJECT_DIR}}
       run: "dbt deps"
-    
+
     - name: Run dbt debug on Trino if dunesql engine is used, and retry until cluster is available
       run: "./scripts/ensure_cluster.sh"
-          
+
     - name: dbt compile to create prod manifest from main
       run: "dbt compile --target-path . $PROFILE --project-dir ${PROJECT_DIR}"
 

diff --git a/.github/workflows/dbt_slim_ci.yml → .github/workflows/dbt_run.yml b/.github/workflows/dbt_slim_ci.yml → .github/workflows/dbt_run.yml
@@ -1,43 +1,32 @@
-# .github/workflows/app.yaml
-name: dbt slim ci
+name: DBT Run Reusable Workflow
 
 on:
-  workflow_dispatch:
-  pull_request:
-    paths-ignore:
-      - 'scripts/**'
-      - 'Pipfile'
-      - '.gitignore'
-
-concurrency:
-  group:  ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
+  workflow_call:
+    inputs:
+      project:
+        description: 'DBT run for project'
+        required: true
+        type: string
 
 jobs:
   dbt-test:
     runs-on: [ self-hosted, linux, spellbook-trino-ci ]
-    strategy:
-      matrix:
-        project: [ 'tokens', 'spellbook' ]
-      max-parallel: 1
     timeout-minutes: 90
 
     steps:
       - name: Check out repository code
         uses: actions/checkout@v3
 
       - name: Setup variables
-        run: |
+        run: | # Spellbook is a special case because it's not a subdirectory
           echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV
           echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
-          if [[ "${{ matrix.project }}" == "tokens" ]]; then
-            echo "PROJECT_DIR=tokens" >> $GITHUB_ENV
-            echo "S3_LOCATION=s3://manifest-spellbook-dunesql/tokens" >> $GITHUB_ENV
-          elif [[ "${{ matrix.project }}" == "spellbook" ]]; then
-            echo "PROJECT_DIR=." >> $GITHUB_ENV
-            echo "S3_LOCATION=s3://manifest-spellbook-dunesql/spellbook" >> $GITHUB_ENV
+          echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{inputs.project}}" >> $GITHUB_ENV
+          PROJECT_DIR=${{ inputs.project }}
+          if [[ "$PROJECT_DIR" == "spellbook" ]]; then
+            PROJECT_DIR="."
           fi
+          echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
       - name: Add git_sha to schema
         run: "/runner/change_schema.sh git_dunesql_$GIT_SHA"
 
@@ -78,4 +67,4 @@ jobs:
 
       - name: dbt test incremental model(s) if applicable
         if: env.INC_MODEL_COUNT > 0
-        run: "dbt test $PROFILE --select state:modified,config.materialized:incremental --exclude tag:prod_exclude tag:remove --defer --state . --project-dir ${PROJECT_DIR}"
+        run: "dbt test $PROFILE --select state:modified,config.materialized:incremental --exclude tag:prod_exclude tag:remove --defer --state . --project-dir ${PROJECT_DIR}"
diff --git a/.github/workflows/spellbook.yml b/.github/workflows/spellbook.yml
@@ -0,0 +1,22 @@
+name: DBT Spellbook run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths-ignore:
+      - 'scripts/**'
+      - 'Pipfile'
+      - '.gitignore'
+      - 'tokens/**'
+      - 'daily_spellbook/**'
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'spellbook'
+
diff --git a/.github/workflows/tokens.yml b/.github/workflows/tokens.yml
@@ -0,0 +1,20 @@
+name: DBT Tokens run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - tokens/**
+      - .github/workflows/tokens.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'tokens'
+