From b5fc1ef4a3fe018a5993c96cb584a4676f21f8e6 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 4 Feb 2020 13:58:14 -0700 Subject: [PATCH] CI stuff circle: Run unit tests + postgres on circle for outside contributors automatically circle+azure: skip non-postgres integration tests on for outside contributors circle+azure: build wheels after tests pass Update the docker image so venvs can pip install -U pip --- .circleci/config.yml | 58 ++++++++++++++++++++++++++++++++++------- Dockerfile | 12 ++++++--- azure-pipelines.yml | 38 ++++++++++++++++++++++++--- dev_requirements.txt | 1 + scripts/build-wheels.sh | 26 ++++++++++++++++++ 5 files changed, 118 insertions(+), 17 deletions(-) create mode 100755 scripts/build-wheels.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index a4a57277b27..fbdbe95b4f8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,8 +1,34 @@ version: 2 jobs: unit: + docker: &test_only + - image: fishtownjacob/test-container:3 + environment: + DBT_INVOCATION_ENV: circle + steps: + - checkout + - run: tox -e flake8,mypy,unit-py36,unit-py38 + build-wheels: + docker: *test_only + steps: + - checkout + - run: + name: Build wheels + command: | + $PYTHON_BIN -m pip install -U pip setuptools + $PYTHON_BIN -m pip install -r requirements.txt + $PYTHON_BIN -m pip install -r dev_requirements.txt + /bin/bash ./scripts/build-wheels.sh + environment: + PYTHON_BIN: /home/tox/venv3.8/bin/python + - store_artifacts: + path: ./dist + destination: dist + integration-postgres-py36: docker: &test_and_postgres - - image: fishtownjacob/test-container:2 + - image: fishtownjacob/test-container:3 + environment: + DBT_INVOCATION_ENV: circle - image: postgres name: database environment: &pgenv @@ -19,20 +45,13 @@ jobs: PGUSER: root PGPASSWORD: password PGDATABASE: postgres - - run: tox -e flake8,mypy,unit-py36,unit-py38 - integration-postgres-py36: - docker: *test_and_postgres - steps: - - checkout - - run: *setupdb - run: name: Run tests command: tox -e integration-postgres-py36 - store_artifacts: path: ./logs integration-snowflake-py36: - docker: &test_only - - image: fishtownjacob/test-container:2 + docker: *test_only steps: - checkout - run: @@ -109,21 +128,42 @@ workflows: - integration-redshift-py36: requires: - integration-postgres-py36 + filters: &no_forks + branches: + # Forked pull requests have CIRCLE_BRANCH set to pull/XXX + ignore: /pull\/[0-9]+/ - integration-bigquery-py36: requires: - integration-postgres-py36 + filters: *no_forks - integration-snowflake-py36: requires: - integration-postgres-py36 + filters: *no_forks - integration-postgres-py38: requires: - unit - integration-redshift-py38: requires: - integration-postgres-py38 + filters: *no_forks - integration-bigquery-py38: requires: - integration-postgres-py38 + filters: *no_forks - integration-snowflake-py38: requires: - integration-postgres-py38 + filters: *no_forks + - build-wheels: + requires: + - unit + - integration-postgres-py36 + - integration-redshift-py36 + - integration-bigquery-py36 + - integration-snowflake-py36 + - integration-postgres-py38 + - integration-redshift-py38 + - integration-bigquery-py38 + - integration-snowflake-py38 + filters: *no_forks diff --git a/Dockerfile b/Dockerfile index 45e5dd9900c..17cc3515913 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,19 +26,23 @@ RUN pip3 install tox wheel RUN python2.7 -m pip install virtualenv wheel && \ python2.7 -m virtualenv /home/tox/venv2.7 && \ - /home/tox/venv2.7/bin/python -m pip install -U pip tox + /home/tox/venv2.7/bin/python -m pip install -U pip tox && \ + chown -R dbt_test_user /home/tox/venv2.7 RUN python3.6 -m pip install -U pip wheel && \ python3.6 -m venv /home/tox/venv3.6 && \ - /home/tox/venv3.6/bin/python -m pip install -U pip tox + /home/tox/venv3.6/bin/python -m pip install -U pip tox && \ + chown -R dbt_test_user /home/tox/venv3.6 RUN python3.7 -m pip install -U pip wheel && \ python3.7 -m venv /home/tox/venv3.7 && \ - /home/tox/venv3.7/bin/python -m pip install -U pip tox + /home/tox/venv3.7/bin/python -m pip install -U pip tox && \ + chown -R dbt_test_user /home/tox/venv3.7 RUN python3.8 -m pip install -U pip wheel && \ python3.8 -m venv /home/tox/venv3.8 && \ - /home/tox/venv3.8/bin/python -m pip install -U pip tox + /home/tox/venv3.8/bin/python -m pip install -U pip tox && \ + chown -R dbt_test_user /home/tox/venv3.8 USER dbt_test_user diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5343a3aa9f4..54175ec5232 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -32,6 +32,8 @@ jobs: dependsOn: UnitTest steps: + - pwsh: echo "True" + displayName: Skip everything - pwsh: | choco install postgresql --params '/Password:password' --params-global --version 10.6 Set-Content "c:\program files\postgresql\10\data\pg_hba.conf" "host all all ::1/128 trust" @@ -65,7 +67,7 @@ jobs: pool: vmImage: 'vs2017-win2016' dependsOn: PostgresIntegrationTest - + condition: and(succeeded(), ne(variables['SYSTEM.PULLREQUEST.ISFORK'], 'true')) steps: - task: UsePythonVersion@0 inputs: @@ -87,13 +89,12 @@ jobs: pool: vmImage: 'vs2017-win2016' dependsOn: PostgresIntegrationTest - + condition: and(succeeded(), ne(variables['SYSTEM.PULLREQUEST.ISFORK'], 'true')) steps: - task: UsePythonVersion@0 inputs: versionSpec: '3.7' architecture: 'x64' - - script: python -m pip install --upgrade pip && pip install tox displayName: 'Install dependencies' - script: python -m tox -e pywin-bigquery @@ -105,7 +106,7 @@ jobs: pool: vmImage: 'vs2017-win2016' dependsOn: PostgresIntegrationTest - + condition: and(succeeded(), ne(variables['SYSTEM.PULLREQUEST.ISFORK'], 'true')) steps: - task: UsePythonVersion@0 inputs: @@ -123,3 +124,32 @@ jobs: REDSHIFT_TEST_PORT: $(REDSHIFT_TEST_PORT) REDSHIFT_TEST_HOST: $(REDSHIFT_TEST_HOST) displayName: Run integration tests + +- job: BuildWheel + pool: + vmImage: 'vs2017-win2016' + dependsOn: + - UnitTest + - PostgresIntegrationTest + - RedshiftIntegrationTest + - SnowflakeIntegrationTest + - BigQueryIntegrationTest + condition: and(succeeded(), ne(variables['SYSTEM.PULLREQUEST.ISFORK'], 'true')) + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + architecture: 'x64' + - script: python -m pip install --upgrade pip setuptools && python -m pip install -r requirements.txt && python -m pip install -r dev_requirements.txt + displayName: Install dependencies + - task: ShellScript@2 + inputs: + scriptPath: scripts/build-wheels.sh + - task: CopyFiles@2 + inputs: + contents: 'dist\?(*.whl|*.tar.gz)' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: dists diff --git a/dev_requirements.txt b/dev_requirements.txt index 89f4b147a3a..626ed353b72 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -8,3 +8,4 @@ ipdb pytest-xdist>=1.28.0,<2 flaky>=3.5.3,<4 mypy==0.761 +wheel diff --git a/scripts/build-wheels.sh b/scripts/build-wheels.sh new file mode 100755 index 00000000000..1f0e9f6f656 --- /dev/null +++ b/scripts/build-wheels.sh @@ -0,0 +1,26 @@ +#!/bin/bash -eo pipefail + +DBT_PATH="$( cd "$(dirname "$0")/.." ; pwd -P )" + +PYTHON_BIN=${PYTHON_BIN:-python} + +echo $SCRIPTPATH +echo $PYTHON_BIN + +set -x + +rm -rf "$DBT_PATH"/dist +mkdir -p "$DBT_PATH"/dist + +for SUBPATH in core plugins/postgres plugins/redshift plugins/bigquery plugins/snowflake +do + rm -rf "$DBT_PATH"/"$SUBPATH"/dist + cd "$DBT_PATH"/"$SUBPATH" + $PYTHON_BIN setup.py sdist bdist_wheel + cp -r "$DBT_PATH"/"$SUBPATH"/dist/* "$DBT_PATH"/dist/ +done + +cd "$DBT_PATH" +$PYTHON_BIN setup.py sdist bdist_wheel + +set +x