CI stuff

Run unit tests + postgres on circle for outside contributors Skip other integration tests on circle for outside contributors Set an actual condition in AZP to skip running secure tests on azp build wheels after we pass tests
dbt-labs · Feb 5, 2020 · ce73abd · ce73abd
1 parent b691a73
commit ce73abd
Show file tree

Hide file tree

Showing 3 changed files with 90 additions and 4 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -3,6 +3,8 @@ jobs:
   unit:
     docker: &test_and_postgres
       - image: fishtownjacob/test-container:2
+        environment:
+          DBT_INVOCATION_ENV: circle
       - image: postgres
         name: database
         environment: &pgenv
@@ -33,6 +35,8 @@ jobs:
   integration-snowflake-py36:
     docker: &test_only
       - image: fishtownjacob/test-container:2
+        environment:
+          DBT_INVOCATION_ENV: circle
     steps:
       - checkout
       - run:
@@ -97,6 +101,16 @@ jobs:
           command: tox -e integration-bigquery-py38
       - store_artifacts:
           path: ./logs
+  build-wheels:
+    docker: *test_only
+    steps:
+      - checkout
+      - run:
+          name: Setup env
+          command: /bin/bash ./scripts/build-wheels.sh
+      - store_artifacts:
+          path: ./dist
+          destination: dist
 
 workflows:
   version: 2
@@ -109,21 +123,41 @@ workflows:
       - integration-redshift-py36:
           requires:
             - integration-postgres-py36
+          filters: &no_forks
+            branches:
+              # Forked pull requests have CIRCLE_BRANCH set to pull/XXX
+              ignore: /pull\/[0-9]+/
       - integration-bigquery-py36:
           requires:
             - integration-postgres-py36
+          filters: *no_forks
       - integration-snowflake-py36:
           requires:
             - integration-postgres-py36
+          filters: *no_forks
       - integration-postgres-py38:
           requires:
             - unit
       - integration-redshift-py38:
           requires:
             - integration-postgres-py38
+          filters: *no_forks
       - integration-bigquery-py38:
           requires:
             - integration-postgres-py38
+          filters: *no_forks
       - integration-snowflake-py38:
           requires:
             - integration-postgres-py38
+          filters: *no_forks
+      - build-wheels:
+          requires:
+            - integration-postgres-py36
+            - integration-redshift-py36
+            - integration-bigquery-py36
+            - integration-snowflake-py36
+            - integration-postgres-py38
+            - integration-redshift-py38
+            - integration-bigquery-py38
+            - integration-snowflake-py38
+          filters: *no_forks
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -65,7 +65,7 @@ jobs:
   pool:
     vmImage: 'vs2017-win2016'
   dependsOn: PostgresIntegrationTest
-
+  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/users/'))
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -87,13 +87,12 @@ jobs:
   pool:
     vmImage: 'vs2017-win2016'
   dependsOn: PostgresIntegrationTest
-
+  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/users/'))
   steps:
   - task: UsePythonVersion@0
     inputs:
       versionSpec: '3.7'
       architecture: 'x64'
-
   - script: python -m pip install --upgrade pip && pip install tox
     displayName: 'Install dependencies'
   - script: python -m tox -e pywin-bigquery
@@ -105,7 +104,7 @@ jobs:
   pool:
     vmImage: 'vs2017-win2016'
   dependsOn: PostgresIntegrationTest
-
+  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/users/'))
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -123,3 +122,33 @@ jobs:
       REDSHIFT_TEST_PORT: $(REDSHIFT_TEST_PORT)
       REDSHIFT_TEST_HOST: $(REDSHIFT_TEST_HOST)
     displayName: Run integration tests
+
+- job: BuildWheel
+  pool:
+    vmImage: 'vs2017-win2016'
+  dependsOn:
+    - UnitTest
+    - PostgresIntegrationTest
+    - SnowflakeIntegrationTest
+    - BigQueryIntegrationTest
+    - RedshiftIntegrationTest
+  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/users/'))
+  steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.7'
+        architecture: 'x64'
+    - script: python -m pip install --upgrade pip && python -m pip install -r requirements.txt
+      displayName: Install dependencies
+    - task: ShellScript@2
+      inputs:
+        scriptPath: scripts/build-wheels.sh
+    - task: CopyFiles@2
+      inputs:
+        sourceFolder: '$(Build.SourcesDirectory)/dist'
+        contents: '(*.whl|*.tar.gz)'
+        TargetFolder: '$(Build.ArtifactStagingDirectory)'
+    - task: PublishBuildArtifacts@1
+      inputs:
+        pathtoPublish: '$(Build.ArtifactStagingDirectory)'
+        artifactName: dists
diff --git a/scripts/build-wheels.sh b/scripts/build-wheels.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -eo pipefail
+
+DBT_PATH="$( cd "$(dirname "$0")/.." ; pwd -P )"
+
+echo $SCRIPTPATH
+
+set -x
+
+rm -rf "$DBT_PATH"/dist
+mkdir -p "$DBT_PATH"/dist
+
+for SUBPATH in core plugins/postgres plugins/redshift plugins/bigquery plugins/snowflake
+do
+    rm -rf "$DBT_PATH"/"$SUBPATH"/dist
+    cd "$DBT_PATH"/"$SUBPATH"
+    python setup.py sdist bdist_wheel
+    cp -r "$DBT_PATH"/"$SUBPATH"/dist/* "$DBT_PATH"/dist/
+done
+
+cd "$DBT_PATH"
+python setup.py sdist
+
+set +x