CI stuff

.circleci/config.yml

@@ -1,8 +1,34 @@
 version: 2
 jobs:
   unit:
+    docker: &test_only
+      - image: fishtownjacob/test-container:3
+        environment:
+          DBT_INVOCATION_ENV: circle
+    steps:
+      - checkout
+      - run: tox -e flake8,mypy,unit-py36,unit-py38
+  build-wheels:
+    docker: *test_only
+    steps:
+      - checkout
+      - run:
+          name: Build wheels
+          command: |
+            $PYTHON_BIN -m pip install -U pip setuptools
+            $PYTHON_BIN -m pip install -r requirements.txt
+            $PYTHON_BIN -m pip install -r dev_requirements.txt
+            /bin/bash ./scripts/build-wheels.sh
+          environment:
+            PYTHON_BIN: /home/tox/venv3.8/bin/python
+      - store_artifacts:
+          path: ./dist
+          destination: dist
+  integration-postgres-py36:
     docker: &test_and_postgres
-      - image: fishtownjacob/test-container:2
+      - image: fishtownjacob/test-container:3
+        environment:
+          DBT_INVOCATION_ENV: circle
       - image: postgres
         name: database
         environment: &pgenv
@@ -19,20 +45,13 @@ jobs:
             PGUSER: root
             PGPASSWORD: password
             PGDATABASE: postgres
-      - run: tox -e flake8,mypy,unit-py36,unit-py38
-  integration-postgres-py36:
-    docker: *test_and_postgres
-    steps:
-      - checkout
-      - run: *setupdb
       - run:
           name: Run tests
           command: tox -e integration-postgres-py36
       - store_artifacts:
           path: ./logs
   integration-snowflake-py36:
-    docker: &test_only
-      - image: fishtownjacob/test-container:2
+    docker: *test_only
     steps:
       - checkout
       - run:
@@ -109,21 +128,42 @@ workflows:
       - integration-redshift-py36:
           requires:
             - integration-postgres-py36
+          filters: &no_forks
+            branches:
+              # Forked pull requests have CIRCLE_BRANCH set to pull/XXX
+              ignore: /pull\/[0-9]+/
       - integration-bigquery-py36:
           requires:
             - integration-postgres-py36
+          filters: *no_forks
       - integration-snowflake-py36:
           requires:
             - integration-postgres-py36
+          filters: *no_forks
       - integration-postgres-py38:
           requires:
             - unit
       - integration-redshift-py38:
           requires:
             - integration-postgres-py38
+          filters: *no_forks
       - integration-bigquery-py38:
           requires:
             - integration-postgres-py38
+          filters: *no_forks
       - integration-snowflake-py38:
           requires:
             - integration-postgres-py38
+          filters: *no_forks
+      - build-wheels:
+          requires:
+            - unit
+            - integration-postgres-py36
+            - integration-redshift-py36
+            - integration-bigquery-py36
+            - integration-snowflake-py36
+            - integration-postgres-py38
+            - integration-redshift-py38
+            - integration-bigquery-py38
+            - integration-snowflake-py38
+          filters: *no_forks

Dockerfile

@@ -26,19 +26,23 @@ RUN pip3 install tox wheel
 
 RUN python2.7 -m pip install virtualenv wheel && \
     python2.7 -m virtualenv /home/tox/venv2.7 && \
-    /home/tox/venv2.7/bin/python -m pip install -U pip tox
+    /home/tox/venv2.7/bin/python -m pip install -U pip tox && \
+    chown -R dbt_test_user /home/tox/venv2.7
 
 RUN python3.6 -m pip install -U pip wheel && \
     python3.6 -m venv /home/tox/venv3.6 && \
-    /home/tox/venv3.6/bin/python -m pip install -U pip tox
+    /home/tox/venv3.6/bin/python -m pip install -U pip tox && \
+    chown -R dbt_test_user /home/tox/venv3.6
 
 RUN python3.7 -m pip install -U pip wheel && \
     python3.7 -m venv /home/tox/venv3.7 && \
-    /home/tox/venv3.7/bin/python -m pip install -U pip tox
+    /home/tox/venv3.7/bin/python -m pip install -U pip tox && \
+    chown -R dbt_test_user /home/tox/venv3.7
 
 RUN python3.8 -m pip install -U pip wheel && \
     python3.8 -m venv /home/tox/venv3.8 && \
-    /home/tox/venv3.8/bin/python -m pip install -U pip tox
+    /home/tox/venv3.8/bin/python -m pip install -U pip tox && \
+    chown -R dbt_test_user /home/tox/venv3.8
 
 USER dbt_test_user
 

azure-pipelines.yml

@@ -32,6 +32,8 @@ jobs:
   dependsOn: UnitTest
 
   steps:
+  - pwsh: echo "True"
+    displayName: Skip everything
   - pwsh: |
       choco install postgresql --params '/Password:password' --params-global --version 10.6
       Set-Content "c:\program files\postgresql\10\data\pg_hba.conf" "host   all   all   ::1/128        trust"
@@ -65,7 +67,7 @@ jobs:
   pool:
     vmImage: 'vs2017-win2016'
   dependsOn: PostgresIntegrationTest
-
+  condition: and(succeeded(), ne(variables['SYSTEM.PULLREQUEST.ISFORK'], 'true'))
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -87,13 +89,12 @@ jobs:
   pool:
     vmImage: 'vs2017-win2016'
   dependsOn: PostgresIntegrationTest
-
+  condition: and(succeeded(), ne(variables['SYSTEM.PULLREQUEST.ISFORK'], 'true'))
   steps:
   - task: UsePythonVersion@0
     inputs:
       versionSpec: '3.7'
       architecture: 'x64'
-
   - script: python -m pip install --upgrade pip && pip install tox
     displayName: 'Install dependencies'
   - script: python -m tox -e pywin-bigquery
@@ -105,7 +106,7 @@ jobs:
   pool:
     vmImage: 'vs2017-win2016'
   dependsOn: PostgresIntegrationTest
-
+  condition: and(succeeded(), ne(variables['SYSTEM.PULLREQUEST.ISFORK'], 'true'))
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -123,3 +124,32 @@ jobs:
       REDSHIFT_TEST_PORT: $(REDSHIFT_TEST_PORT)
       REDSHIFT_TEST_HOST: $(REDSHIFT_TEST_HOST)
     displayName: Run integration tests
+
+- job: BuildWheel
+  pool:
+    vmImage: 'vs2017-win2016'
+  dependsOn:
+    - UnitTest
+    - PostgresIntegrationTest
+    - RedshiftIntegrationTest
+    - SnowflakeIntegrationTest
+    - BigQueryIntegrationTest
+  condition: and(succeeded(), ne(variables['SYSTEM.PULLREQUEST.ISFORK'], 'true'))
+  steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.7'
+        architecture: 'x64'
+    - script: python -m pip install --upgrade pip setuptools && python -m pip install -r requirements.txt && python -m pip install -r dev_requirements.txt
+      displayName: Install dependencies
+    - task: ShellScript@2
+      inputs:
+        scriptPath: scripts/build-wheels.sh
+    - task: CopyFiles@2
+      inputs:
+        contents: 'dist\?(*.whl|*.tar.gz)'
+        TargetFolder: '$(Build.ArtifactStagingDirectory)'
+    - task: PublishBuildArtifacts@1
+      inputs:
+        pathtoPublish: '$(Build.ArtifactStagingDirectory)'
+        artifactName: dists

dev_requirements.txt

@@ -8,3 +8,4 @@ ipdb
 pytest-xdist>=1.28.0,<2
 flaky>=3.5.3,<4
 mypy==0.761
+wheel

scripts/build-wheels.sh

@@ -0,0 +1,26 @@
+#!/bin/bash -eo pipefail
+
+DBT_PATH="$( cd "$(dirname "$0")/.." ; pwd -P )"
+
+PYTHON_BIN=${PYTHON_BIN:-python}
+
+echo $SCRIPTPATH
+echo $PYTHON_BIN
+
+set -x
+
+rm -rf "$DBT_PATH"/dist
+mkdir -p "$DBT_PATH"/dist
+
+for SUBPATH in core plugins/postgres plugins/redshift plugins/bigquery plugins/snowflake
+do
+    rm -rf "$DBT_PATH"/"$SUBPATH"/dist
+    cd "$DBT_PATH"/"$SUBPATH"
+    $PYTHON_BIN setup.py sdist bdist_wheel
+    cp -r "$DBT_PATH"/"$SUBPATH"/dist/* "$DBT_PATH"/dist/
+done
+
+cd "$DBT_PATH"
+$PYTHON_BIN setup.py sdist bdist_wheel
+
+set +x