From 2f752842a1756739ee8cbacab090238ca6c9d237 Mon Sep 17 00:00:00 2001 From: Kyle Wigley Date: Mon, 14 Dec 2020 13:50:52 -0500 Subject: [PATCH 1/4] update hologram and add new envs to tox --- .circleci/config.yml | 61 ++++++++++++++++++++++++++++++++-------- core/setup.py | 3 +- tox.ini | 66 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bb0ac29ff83..d22f846494a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -121,6 +121,45 @@ jobs: - store_artifacts: path: ./logs + integration-postgres-py39: + docker: *test_and_postgres + steps: + - checkout + - run: *setupdb + - run: + name: Run tests + command: tox -e integration-postgres-py39 + - store_artifacts: + path: ./logs + integration-snowflake-py39: + docker: *test_only + steps: + - checkout + - run: + name: Run tests + command: tox -e integration-snowflake-py39 + no_output_timeout: 1h + - store_artifacts: + path: ./logs + integration-redshift-py39: + docker: *test_only + steps: + - checkout + - run: + name: Run tests + command: tox -e integration-redshift-py39 + - store_artifacts: + path: ./logs + integration-bigquery-py39: + docker: *test_only + steps: + - checkout + - run: + name: Run tests + command: tox -e integration-bigquery-py39 + - store_artifacts: + path: ./logs + workflows: version: 2 test-everything: @@ -138,18 +177,18 @@ workflows: - integration-snowflake-py36: requires: - integration-postgres-py36 - - integration-postgres-py38: + - integration-postgres-py39: requires: - unit - - integration-redshift-py38: + - integration-redshift-py39: requires: - - integration-postgres-py38 - - integration-bigquery-py38: + - integration-postgres-py39 + - integration-bigquery-py39: requires: - - integration-postgres-py38 - - integration-snowflake-py38: + - integration-postgres-py39 + - integration-snowflake-py39: requires: - - integration-postgres-py38 + - integration-postgres-py39 - build-wheels: requires: - unit @@ -157,7 +196,7 @@ workflows: - integration-redshift-py36 - integration-bigquery-py36 - integration-snowflake-py36 - - integration-postgres-py38 - - integration-redshift-py38 - - integration-bigquery-py38 - - integration-snowflake-py38 + - integration-postgres-py39 + - integration-redshift-py39 + - integration-bigquery-py39 + - integration-snowflake-py39 diff --git a/core/setup.py b/core/setup.py index 2d5a6eb49b4..845545ef446 100644 --- a/core/setup.py +++ b/core/setup.py @@ -70,7 +70,7 @@ def read(fname): 'json-rpc>=1.12,<2', 'werkzeug>=0.15,<0.17', 'dataclasses==0.6;python_version<"3.7"', - 'hologram==0.0.11', + 'hologram==0.0.12', 'logbook>=1.5,<1.6', 'typing-extensions>=3.7.4,<3.8', # the following are all to match snowflake-connector-python @@ -91,6 +91,7 @@ def read(fname): 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], python_requires=">=3.6.3", ) diff --git a/tox.ini b/tox.ini index 52dd11e6498..c25f9a1e739 100644 --- a/tox.ini +++ b/tox.ini @@ -246,6 +246,72 @@ deps = -r ./editable_requirements.txt -r ./dev_requirements.txt +[testenv:unit-py39] +basepython = python3.9 +commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v {posargs} -n4 test/unit' +deps = + -r ./requirements.txt + -r ./dev_requirements.txt + + +[testenv:integration-postgres-py39] +basepython = python3.9 +passenv = * +setenv = + HOME=/home/tox +commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_postgres {posargs} -n4 test/integration/*' + /bin/bash -c '{envpython} -m pytest --durations 0 -v {posargs} -n4 --profile=postgres test/rpc' +deps = + ./core + ./plugins/postgres + -r ./dev_requirements.txt + +[testenv:integration-snowflake-py39] +basepython = python3.9 +passenv = * +setenv = + HOME=/home/tox +commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_snowflake {posargs} -n4 test/integration/*' + /bin/bash -c '{envpython} -m pytest --durations 0 -v {posargs} -n4 --profile=snowflake test/rpc' +deps = + ./core + ./plugins/snowflake + -r ./dev_requirements.txt + +[testenv:integration-bigquery-py39] +basepython = python3.9 +passenv = * +setenv = + HOME=/home/tox +commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_bigquery {posargs} -n4 test/integration/*' +deps = + ./core + ./plugins/bigquery + -r ./dev_requirements.txt + +[testenv:integration-redshift-py39] +basepython = python3.9 +passenv = * +setenv = + HOME=/home/tox +commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_redshift {posargs} -n4 test/integration/*' +deps = + ./core + ./plugins/postgres + ./plugins/redshift + -r ./dev_requirements.txt + + +[testenv:explicit-py39] +basepython = python3.9 +passenv = * +setenv = + HOME=/home/tox +commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v {posargs}' +deps = + -r ./editable_requirements.txt + -r ./dev_requirements.txt + [testenv:pywin] basepython = {env:PYTHON:}\python.exe passenv = * From 148f55335f2ffb4accbab68c07cec34b3765c5c1 Mon Sep 17 00:00:00 2001 From: Kyle Wigley Date: Thu, 10 Dec 2020 13:22:06 -0500 Subject: [PATCH 2/4] address issue with py39 --- .circleci/config.yml | 16 ++++++++++++++++ .gitignore | 2 +- core/dbt/flags.py | 3 +++ core/dbt/task/rpc/server.py | 3 ++- plugins/bigquery/setup.py | 1 + plugins/postgres/setup.py | 1 + plugins/redshift/setup.py | 1 + 7 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d22f846494a..2616f6eef9a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -177,6 +177,18 @@ workflows: - integration-snowflake-py36: requires: - integration-postgres-py36 + - integration-postgres-py38: + requires: + - unit + - integration-redshift-py38: + requires: + - integration-postgres-py38 + - integration-bigquery-py38: + requires: + - integration-postgres-py38 + - integration-snowflake-py38: + requires: + - integration-postgres-py38 - integration-postgres-py39: requires: - unit @@ -196,6 +208,10 @@ workflows: - integration-redshift-py36 - integration-bigquery-py36 - integration-snowflake-py36 + - integration-postgres-py38 + - integration-redshift-py38 + - integration-bigquery-py38 + - integration-snowflake-py38 - integration-postgres-py39 - integration-redshift-py39 - integration-bigquery-py39 diff --git a/.gitignore b/.gitignore index acb15d1f0c7..065a16487a4 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ __pycache__/ # Distribution / packaging .Python -env/ +env*/ dbt_env/ build/ develop-eggs/ diff --git a/core/dbt/flags.py b/core/dbt/flags.py index 053868399a7..498d174d72c 100644 --- a/core/dbt/flags.py +++ b/core/dbt/flags.py @@ -1,5 +1,8 @@ import os import multiprocessing +if os.name != 'nt': + # https://bugs.python.org/issue41567 + import multiprocessing.popen_spawn_posix # type: ignore from pathlib import Path from typing import Optional diff --git a/core/dbt/task/rpc/server.py b/core/dbt/task/rpc/server.py index 179a8e1ea50..e1babbb1d37 100644 --- a/core/dbt/task/rpc/server.py +++ b/core/dbt/task/rpc/server.py @@ -1,7 +1,8 @@ # import these so we can find them from . import sql_commands # noqa from . import project_commands # noqa -from . import deps # noqa +from . import deps # noqa +import multiprocessing.queues # noqa - https://bugs.python.org/issue41567 import json import os import signal diff --git a/plugins/bigquery/setup.py b/plugins/bigquery/setup.py index ed91f729d0c..377a30d1cb7 100644 --- a/plugins/bigquery/setup.py +++ b/plugins/bigquery/setup.py @@ -69,6 +69,7 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], python_requires=">=3.6.2", ) diff --git a/plugins/postgres/setup.py b/plugins/postgres/setup.py index ef74bc6c687..a1d0bc88c54 100644 --- a/plugins/postgres/setup.py +++ b/plugins/postgres/setup.py @@ -85,6 +85,7 @@ def _dbt_psycopg2_name(): 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], python_requires=">=3.6.2", ) diff --git a/plugins/redshift/setup.py b/plugins/redshift/setup.py index 3c0cd8f70e7..0a505e30852 100644 --- a/plugins/redshift/setup.py +++ b/plugins/redshift/setup.py @@ -65,6 +65,7 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], python_requires=">=3.6.2", ) From 692a423072fda9aa88df2da75abe59704ac86f3c Mon Sep 17 00:00:00 2001 From: Kyle Wigley Date: Mon, 14 Dec 2020 15:05:42 -0500 Subject: [PATCH 3/4] comment out snowflake py39 tests --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2616f6eef9a..1d8087901a1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -198,9 +198,9 @@ workflows: - integration-bigquery-py39: requires: - integration-postgres-py39 - - integration-snowflake-py39: - requires: - - integration-postgres-py39 + # - integration-snowflake-py39: + # requires: + # - integration-postgres-py39 - build-wheels: requires: - unit @@ -215,4 +215,4 @@ workflows: - integration-postgres-py39 - integration-redshift-py39 - integration-bigquery-py39 - - integration-snowflake-py39 + # - integration-snowflake-py39 From 43ed29c14c60994ad5f97039b628047e72338f8c Mon Sep 17 00:00:00 2001 From: Kyle Wigley Date: Wed, 16 Dec 2020 16:29:48 -0500 Subject: [PATCH 4/4] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e29e2a78670..12792eb4bbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - Add event tracking for project parser/load times ([#2823](https://github.com/fishtown-analytics/dbt/issues/2823),[#2893](https://github.com/fishtown-analytics/dbt/pull/2893)) - Bump cryptography version to be >= 3.2 and bump snowflake connector to 2.3.6 ([#2896](https://github.com/fishtown-analytics/dbt/issues/2896)) - Widen supported Google Cloud libraries dependencies ([#2794](https://github.com/fishtown-analytics/dbt/pull/2794), [#2877](https://github.com/fishtown-analytics/dbt/pull/2877)). +- Bump hologram to 0.0.12 and adding testing support for python3.9 ([#2822](https://github.com/fishtown-analytics/dbt/issues/2822),[#2960](https://github.com/fishtown-analytics/dbt/pull/2960)) Contributors: - [@feluelle](https://github.com/feluelle) ([#2841](https://github.com/fishtown-analytics/dbt/pull/2841))