diff --git a/Cargo.lock b/Cargo.lock index 8c35702f14..1965c66064 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -384,11 +384,13 @@ dependencies = [ "iai", "itertools 0.10.1", "log", + "native-tls", "ndarray", "num-traits", "owning_ref", "polars", "postgres", + "postgres-native-tls", "pprof", "r2d2", "r2d2_mysql", @@ -401,6 +403,7 @@ dependencies = [ "serde_json", "sqlparser", "thiserror", + "url", "uuid", ] @@ -427,6 +430,7 @@ dependencies = [ "ndarray", "numpy", "postgres", + "postgres-native-tls", "pprof", "pyo3", "pyo3-built", @@ -2021,6 +2025,19 @@ dependencies = [ "tokio-postgres", ] +[[package]] +name = "postgres-native-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d442770e2b1e244bb5eb03b31c79b65bb2568f413b899eaba850fa945a65954" +dependencies = [ + "futures", + "native-tls", + "tokio", + "tokio-native-tls", + "tokio-postgres", +] + [[package]] name = "postgres-protocol" version = "0.6.1" @@ -2974,6 +2991,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-postgres" version = "0.7.2" diff --git a/Justfile b/Justfile index ab9f9f7dd8..e9f955a752 100644 --- a/Justfile +++ b/Justfile @@ -5,7 +5,7 @@ build-debug: cargo build test: - cargo test --features src_postgres --features src_sqlite --features src_mysql --features src_csv --features dst_arrow --features dst_memory --features dst_polars -- --nocapture + cargo test --features all -- --nocapture bootstrap-python: cp README.md connectorx-python/README.md diff --git a/connectorx-python/Cargo.toml b/connectorx-python/Cargo.toml index b4a5be2249..5ba8428d77 100644 --- a/connectorx-python/Cargo.toml +++ b/connectorx-python/Cargo.toml @@ -32,6 +32,7 @@ sqlparser = "0.9" thiserror = "1" url = "2" uuid = "0.8" +postgres-native-tls = {version = "0.5"} [build-dependencies] built = {version = "0.5", features = ["chrono"]} diff --git a/connectorx-python/connectorx/tests/test_postgres.py b/connectorx-python/connectorx/tests/test_postgres.py index 3046719b93..8409ed8ede 100644 --- a/connectorx-python/connectorx/tests/test_postgres.py +++ b/connectorx-python/connectorx/tests/test_postgres.py @@ -13,6 +13,18 @@ def postgres_url() -> str: return conn +@pytest.fixture(scope="module") # type: ignore +def postgres_url_tls() -> str: + conn = os.environ["POSTGRES_URL_TLS"] + return conn + + +@pytest.fixture(scope="module") # type: ignore +def postgres_rootcert() -> str: + cert = os.environ["POSTGRES_ROOTCERT"] + return cert + + @pytest.mark.xfail def test_on_non_select(postgres_url: str) -> None: query = "CREATE TABLE non_select(id INTEGER NOT NULL)" @@ -36,7 +48,8 @@ def test_partition_on_aggregation(postgres_url: str) -> None: query = ( "SELECT test_bool, SUM(test_int) AS test_int FROM test_table GROUP BY test_bool" ) - df = read_sql(postgres_url, query, partition_on="test_int", partition_num=2) + df = read_sql(postgres_url, query, + partition_on="test_int", partition_num=2) expected = pd.DataFrame( index=range(3), data={ @@ -74,7 +87,8 @@ def test_partition_on_aggregation2(postgres_url: str) -> None: def test_udf(postgres_url: str) -> None: query = "select increment(test_int) as test_int from test_table ORDER BY test_int" - df = read_sql(postgres_url, query, partition_on="test_int", partition_num=2) + df = read_sql(postgres_url, query, + partition_on="test_int", partition_num=2) expected = pd.DataFrame( index=range(6), data={ @@ -484,7 +498,8 @@ def test_empty_result(postgres_url: str) -> None: def test_empty_result_on_some_partition(postgres_url: str) -> None: query = "SELECT * FROM test_table where test_int < 1" - df = read_sql(postgres_url, query, partition_on="test_int", partition_num=3) + df = read_sql(postgres_url, query, + partition_on="test_int", partition_num=3) expected = pd.DataFrame( data={ "test_int": pd.Series([0], dtype="Int64"), @@ -495,3 +510,97 @@ def test_empty_result_on_some_partition(postgres_url: str) -> None: } ) assert_frame_equal(df, expected, check_names=True) + + +@pytest.mark.skipif(os.environ.get("TEST_COVER", "main") not in ("all", "pgtls"), reason="Do not test TLS unless TEST_COVER=all/pgtls") +def test_read_sql_tls(postgres_url_tls: str) -> None: + query = "SELECT * FROM test_table" + df = read_sql( + f"{postgres_url_tls}?sslmode=require", + query, + partition_on="test_int", + partition_range=(0, 2000), + partition_num=3, + ) + expected = pd.DataFrame( + index=range(6), + data={ + "test_int": pd.Series([1, 2, 0, 3, 4, 1314], dtype="Int64"), + "test_nullint": pd.Series([3, None, 5, 7, 9, 2], dtype="Int64"), + "test_str": pd.Series( + ["str1", "str2", "a", "b", "c", None], dtype="object" + ), + "test_float": pd.Series([None, 2.2, 3.1, 3, 7.8, -10], dtype="float64"), + "test_bool": pd.Series( + [True, False, None, False, None, True], dtype="boolean" + ), + }, + ) + assert_frame_equal(df, expected, check_names=True) + + +@pytest.mark.skipif(os.environ.get("TEST_COVER", "main") not in ("all", "pgtls"), reason="Do not test TLS unless TEST_COVER=all/pgtls") +def test_read_sql_tls_with_cert(postgres_url_tls: str, postgres_rootcert: str) -> None: + query = "SELECT * FROM test_table" + df = read_sql( + f"{postgres_url_tls}?sslmode=require&sslrootcert={postgres_rootcert}", + query, + partition_on="test_int", + partition_range=(0, 2000), + partition_num=3, + ) + expected = pd.DataFrame( + index=range(6), + data={ + "test_int": pd.Series([1, 2, 0, 3, 4, 1314], dtype="Int64"), + "test_nullint": pd.Series([3, None, 5, 7, 9, 2], dtype="Int64"), + "test_str": pd.Series( + ["str1", "str2", "a", "b", "c", None], dtype="object" + ), + "test_float": pd.Series([None, 2.2, 3.1, 3, 7.8, -10], dtype="float64"), + "test_bool": pd.Series( + [True, False, None, False, None, True], dtype="boolean" + ), + }, + ) + assert_frame_equal(df, expected, check_names=True) + + +@pytest.mark.skipif(os.environ.get("TEST_COVER", "main") not in ("all", "pgtls"), reason="Do not test TLS unless TEST_COVER=all/pgtls") +def test_read_sql_tls_disable(postgres_url_tls: str) -> None: + query = "SELECT * FROM test_table" + df = read_sql( + f"{postgres_url_tls}?sslmode=disable", + query, + partition_on="test_int", + partition_range=(0, 2000), + partition_num=3, + ) + expected = pd.DataFrame( + index=range(6), + data={ + "test_int": pd.Series([1, 2, 0, 3, 4, 1314], dtype="Int64"), + "test_nullint": pd.Series([3, None, 5, 7, 9, 2], dtype="Int64"), + "test_str": pd.Series( + ["str1", "str2", "a", "b", "c", None], dtype="object" + ), + "test_float": pd.Series([None, 2.2, 3.1, 3, 7.8, -10], dtype="float64"), + "test_bool": pd.Series( + [True, False, None, False, None, True], dtype="boolean" + ), + }, + ) + assert_frame_equal(df, expected, check_names=True) + + +@pytest.mark.skipif(os.environ.get("TEST_COVER", "main") not in ("all", "pgtls"), reason="Do not test TLS unless TEST_COVER=all/pgtls") +@pytest.mark.xfail +def test_read_sql_tls_fail(postgres_url_tls: str) -> None: + query = "SELECT * FROM test_table" + df = read_sql( + f"{postgres_url_tls}?sslmode=require&sslrootcert=fake.cert", + query, + partition_on="test_int", + partition_range=(0, 2000), + partition_num=3, + ) diff --git a/connectorx-python/poetry.lock b/connectorx-python/poetry.lock index 887f22c7cf..26cf819f19 100644 --- a/connectorx-python/poetry.lock +++ b/connectorx-python/poetry.lock @@ -6,6 +6,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "appnope" +version = "0.1.2" +description = "Disable App Nap on macOS >= 10.9" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "atomicwrites" version = "1.4.0" @@ -28,6 +36,14 @@ docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] +[[package]] +name = "backcall" +version = "0.2.0" +description = "Specifications for callback functions passed in to an API" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "black" version = "20.8b1" @@ -167,7 +183,7 @@ test = ["pytest (>=6.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pret [[package]] name = "dask" -version = "2021.7.0" +version = "2021.7.1" description = "Parallel PyData with Task Scheduling" category = "main" optional = false @@ -177,6 +193,7 @@ python-versions = ">=3.7" cloudpickle = ">=1.1.1" fsspec = ">=0.6.0" numpy = {version = ">=1.16", optional = true, markers = "extra == \"dataframe\""} +packaging = ">=20.0" pandas = {version = ">=0.25.0", optional = true, markers = "extra == \"dataframe\""} partd = ">=0.3.10" pyyaml = "*" @@ -184,15 +201,23 @@ toolz = ">=0.8.2" [package.extras] array = ["numpy (>=1.16)"] -complete = ["bokeh (>=1.0.0,!=2.0.0)", "distributed (==2021.07.0)", "numpy (>=1.16)", "pandas (>=0.25.0)"] +complete = ["bokeh (>=1.0.0,!=2.0.0)", "distributed (==2021.07.1)", "numpy (>=1.16)", "pandas (>=0.25.0)"] dataframe = ["numpy (>=1.16)", "pandas (>=0.25.0)"] diagnostics = ["bokeh (>=1.0.0,!=2.0.0)"] -distributed = ["distributed (==2021.07.0)"] +distributed = ["distributed (==2021.07.1)"] test = ["pytest", "pytest-rerunfailures", "pytest-xdist"] +[[package]] +name = "decorator" +version = "5.0.9" +description = "Decorators for Humans" +category = "dev" +optional = false +python-versions = ">=3.5" + [[package]] name = "distributed" -version = "2021.7.0" +version = "2021.7.1" description = "Distributed scheduler for Dask" category = "dev" optional = false @@ -201,7 +226,7 @@ python-versions = ">=3.7" [package.dependencies] click = ">=6.6" cloudpickle = ">=1.5.0" -dask = "2021.07.0" +dask = "2021.07.1" msgpack = ">=0.6.0" psutil = ">=5.0" pyyaml = "*" @@ -307,6 +332,61 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "ipython" +version = "7.25.0" +description = "IPython: Productive Interactive Computing" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +appnope = {version = "*", markers = "sys_platform == \"darwin\""} +backcall = "*" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +decorator = "*" +jedi = ">=0.16" +matplotlib-inline = "*" +pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} +pickleshare = "*" +prompt-toolkit = ">=2.0.0,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.1.0" +pygments = "*" +traitlets = ">=4.2" + +[package.extras] +all = ["Sphinx (>=1.3)", "ipykernel", "ipyparallel", "ipywidgets", "nbconvert", "nbformat", "nose (>=0.10.1)", "notebook", "numpy (>=1.17)", "pygments", "qtconsole", "requests", "testpath"] +doc = ["Sphinx (>=1.3)"] +kernel = ["ipykernel"] +nbconvert = ["nbconvert"] +nbformat = ["nbformat"] +notebook = ["notebook", "ipywidgets"] +parallel = ["ipyparallel"] +qtconsole = ["qtconsole"] +test = ["nose (>=0.10.1)", "requests", "testpath", "pygments", "nbformat", "ipykernel", "numpy (>=1.17)"] + +[[package]] +name = "ipython-genutils" +version = "0.2.0" +description = "Vestigial utilities from IPython" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "jedi" +version = "0.18.0" +description = "An autocompletion tool for Python that can be used for text editors." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +parso = ">=0.8.0,<0.9.0" + +[package.extras] +qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +testing = ["Django (<3.1)", "colorama", "docopt", "pytest (<6.0.0)"] + [[package]] name = "jeepney" version = "0.7.0" @@ -345,6 +425,17 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "matplotlib-inline" +version = "0.1.2" +description = "Inline Matplotlib backend for Jupyter" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +traitlets = "*" + [[package]] name = "maturin" version = "0.9.4" @@ -451,6 +542,18 @@ pytz = ">=2017.3" [package.extras] test = ["hypothesis (>=3.58)", "pytest (>=6.0)", "pytest-xdist"] +[[package]] +name = "parso" +version = "0.8.2" +description = "A Python Parser" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +testing = ["docopt", "pytest (<6.0.0)"] + [[package]] name = "partd" version = "1.2.0" @@ -474,6 +577,25 @@ category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +[[package]] +name = "pexpect" +version = "4.8.0" +description = "Pexpect allows easy control of interactive console applications." +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +ptyprocess = ">=0.5" + +[[package]] +name = "pickleshare" +version = "0.7.5" +description = "Tiny 'shelve'-like database with concurrency support" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "pkginfo" version = "1.7.1" @@ -501,7 +623,7 @@ dev = ["pre-commit", "tox"] [[package]] name = "polars" -version = "0.8.12" +version = "0.8.13" description = "" category = "main" optional = false @@ -511,6 +633,17 @@ python-versions = "*" numpy = "*" pyarrow = ">=4.0.0,<4.1.0" +[[package]] +name = "prompt-toolkit" +version = "3.0.19" +description = "Library for building powerful interactive command lines in Python" +category = "dev" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +wcwidth = "*" + [[package]] name = "psutil" version = "5.8.0" @@ -530,6 +663,14 @@ category = "dev" optional = false python-versions = ">=3.6" +[[package]] +name = "ptyprocess" +version = "0.7.0" +description = "Run a subprocess in a pseudo terminal" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "py" version = "1.10.0" @@ -762,7 +903,7 @@ python-versions = "*" [[package]] name = "sqlalchemy" -version = "1.4.21" +version = "1.4.22" description = "Database Abstraction Library" category = "dev" optional = false @@ -852,6 +993,20 @@ dev = ["py-make (>=0.1.0)", "twine", "wheel"] notebook = ["ipywidgets (>=6)"] telegram = ["requests"] +[[package]] +name = "traitlets" +version = "5.0.5" +description = "Traitlets Python configuration system" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +ipython-genutils = "*" + +[package.extras] +test = ["pytest"] + [[package]] name = "twine" version = "3.4.2" @@ -911,6 +1066,14 @@ brotli = ["brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "wcwidth" +version = "0.2.5" +description = "Measures the displayed width of unicode strings in a terminal" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "webencodings" version = "0.5.1" @@ -953,13 +1116,17 @@ pyarrow = ["pyarrow"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "c85fff6fbe2886cc0ac4258cf00c1e346da0ef0c15242790596d1e8a14598799" +content-hash = "e1a6f2c336db8c104b3d561ce9995f9741c82eab4f47d89e11b63f423f589b67" [metadata.files] appdirs = [ {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, ] +appnope = [ + {file = "appnope-0.1.2-py2.py3-none-any.whl", hash = "sha256:93aa393e9d6c54c5cd570ccadd8edad61ea0c4b9ea7a01409020c9aa019eb442"}, + {file = "appnope-0.1.2.tar.gz", hash = "sha256:dd83cd4b5b460958838f6eb3000c660b1f9caf2a5b1de4264e941512f603258a"}, +] atomicwrites = [ {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, @@ -968,6 +1135,10 @@ attrs = [ {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, ] +backcall = [ + {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, + {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, +] black = [ {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, ] @@ -1106,12 +1277,16 @@ cryptography = [ {file = "cryptography-3.4.7.tar.gz", hash = "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713"}, ] dask = [ - {file = "dask-2021.7.0-py3-none-any.whl", hash = "sha256:7ef39f608a6186a4d910462449b2b75c6da7e8a10eca31686b06638837346e18"}, - {file = "dask-2021.7.0.tar.gz", hash = "sha256:a672df4831ed4a37dd9e0ae1903115ab208808b551aca3ed412eede5f7b7b1f5"}, + {file = "dask-2021.7.1-py3-none-any.whl", hash = "sha256:b56572363cfda0e57771ef39c84cbd8103e0dea5c6fcbbd7a2224240bb3a0bd9"}, + {file = "dask-2021.7.1.tar.gz", hash = "sha256:e72f8773a14e2909a2105e628d981083859915373281fec8e5a78d14ee0a2c8c"}, +] +decorator = [ + {file = "decorator-5.0.9-py3-none-any.whl", hash = "sha256:6e5c199c16f7a9f0e3a61a4a54b3d27e7dad0dbdde92b944426cb20914376323"}, + {file = "decorator-5.0.9.tar.gz", hash = "sha256:72ecfba4320a893c53f9706bebb2d55c270c1e51a28789361aa93e4a21319ed5"}, ] distributed = [ - {file = "distributed-2021.7.0-py3-none-any.whl", hash = "sha256:5baea0941c4f7966a99c6f200f83cde490aecc0c282e96a0e46977747a5bf0e4"}, - {file = "distributed-2021.7.0.tar.gz", hash = "sha256:6257f399ea564bfdcd80dcc9df6cdaf703dbadb94b7c068d103f8366dc7f8d1f"}, + {file = "distributed-2021.7.1-py3-none-any.whl", hash = "sha256:0b979c350315bdc85d75a17787b29b869ff88992f2573d503f3e28b256faae50"}, + {file = "distributed-2021.7.1.tar.gz", hash = "sha256:3a41ff1a1674d4ff0944dfdd8024e168ff451bcd72474d3f4d61d8a4ba019a78"}, ] docopt = [ {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, @@ -1191,6 +1366,18 @@ iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] +ipython = [ + {file = "ipython-7.25.0-py3-none-any.whl", hash = "sha256:aa21412f2b04ad1a652e30564fff6b4de04726ce875eab222c8430edc6db383a"}, + {file = "ipython-7.25.0.tar.gz", hash = "sha256:54bbd1fe3882457aaf28ae060a5ccdef97f212a741754e420028d4ec5c2291dc"}, +] +ipython-genutils = [ + {file = "ipython_genutils-0.2.0-py2.py3-none-any.whl", hash = "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8"}, + {file = "ipython_genutils-0.2.0.tar.gz", hash = "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"}, +] +jedi = [ + {file = "jedi-0.18.0-py2.py3-none-any.whl", hash = "sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93"}, + {file = "jedi-0.18.0.tar.gz", hash = "sha256:92550a404bad8afed881a137ec9a461fed49eca661414be45059329614ed0707"}, +] jeepney = [ {file = "jeepney-0.7.0-py3-none-any.whl", hash = "sha256:71335e7a4e93817982f473f3507bffc2eff7a544119ab9b73e089c8ba1409ba3"}, {file = "jeepney-0.7.0.tar.gz", hash = "sha256:1237cd64c8f7ac3aa4b3f332c4d0fb4a8216f39eaa662ec904302d4d77de5a54"}, @@ -1203,6 +1390,10 @@ locket = [ {file = "locket-0.2.1-py2.py3-none-any.whl", hash = "sha256:12b6ada59d1f50710bca9704dbadd3f447dbf8dac6664575c1281cadab8e6449"}, {file = "locket-0.2.1.tar.gz", hash = "sha256:3e1faba403619fe201552f083f1ecbf23f550941bc51985ac6ed4d02d25056dd"}, ] +matplotlib-inline = [ + {file = "matplotlib-inline-0.1.2.tar.gz", hash = "sha256:f41d5ff73c9f5385775d5c0bc13b424535c8402fe70ea8210f93e11f3683993e"}, + {file = "matplotlib_inline-0.1.2-py3-none-any.whl", hash = "sha256:5cf1176f554abb4fa98cb362aa2b55c500147e4bdbb07e3fda359143e1da0811"}, +] maturin = [ {file = "maturin-0.9.4-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:79ea854e014e8a601fc1b71eb3bb82bf4d81b62890dee69e4a083cba1f738ce9"}, {file = "maturin-0.9.4-py3-none-manylinux2010_x86_64.whl", hash = "sha256:e52dfb6bc6f1c1062883fd785466e246d733f9c7eb90a3cd00ff2d4c659d23ca"}, @@ -1319,6 +1510,10 @@ pandas = [ {file = "pandas-1.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:b10d7910ae9d7920a5ff7816d794d99acbc361f7b16a0f017d4fa83ced8cb55e"}, {file = "pandas-1.3.0.tar.gz", hash = "sha256:c554e6c9cf2d5ea1aba5979cc837b3649539ced0e18ece186f055450c86622e2"}, ] +parso = [ + {file = "parso-0.8.2-py2.py3-none-any.whl", hash = "sha256:a8c4922db71e4fdb90e0d0bc6e50f9b273d3397925e5e60a717e719201778d22"}, + {file = "parso-0.8.2.tar.gz", hash = "sha256:12b83492c6239ce32ff5eed6d3639d6a536170723c6f3f1506869f1ace413398"}, +] partd = [ {file = "partd-1.2.0-py3-none-any.whl", hash = "sha256:5c3a5d70da89485c27916328dc1e26232d0e270771bd4caef4a5124b6a457288"}, {file = "partd-1.2.0.tar.gz", hash = "sha256:aa67897b84d522dcbc86a98b942afab8c6aa2f7f677d904a616b74ef5ddbc3eb"}, @@ -1327,6 +1522,14 @@ pathspec = [ {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, ] +pexpect = [ + {file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"}, + {file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"}, +] +pickleshare = [ + {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, + {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, +] pkginfo = [ {file = "pkginfo-1.7.1-py2.py3-none-any.whl", hash = "sha256:37ecd857b47e5f55949c41ed061eb51a0bee97a87c969219d144c0e023982779"}, {file = "pkginfo-1.7.1.tar.gz", hash = "sha256:e7432f81d08adec7297633191bbf0bd47faf13cd8724c3a13250e51d542635bd"}, @@ -1336,10 +1539,14 @@ pluggy = [ {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] polars = [ - {file = "polars-0.8.12-cp36-abi3-macosx_10_7_x86_64.whl", hash = "sha256:f96791646ba62e3c088205f0b3afc5027cd765a17fbf73578512dd662fd4bd20"}, - {file = "polars-0.8.12-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f7aeb3f5939ad8104d8cf05e28833b3b78dd42b8738034a32eb16e5ed9b9fc5b"}, - {file = "polars-0.8.12-cp36-abi3-win_amd64.whl", hash = "sha256:f1888779f99d5acd12ec9a04b43b26edb3f6cc56c54f052cc734386f359297db"}, - {file = "polars-0.8.12.tar.gz", hash = "sha256:7d56627d815c9a75f34a3b97d4c5f9465b753940ca13d6e5e9afe2cbe00e3129"}, + {file = "polars-0.8.13-cp36-abi3-macosx_10_7_x86_64.whl", hash = "sha256:a5d55931fdd35a80fb8e450a0eb9b35b6fbf3543e2beae48bc33de5f0caf2786"}, + {file = "polars-0.8.13-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0defa18127226f7ec8fd40bdb326f38f014a8e6db76b3168128155491767fde4"}, + {file = "polars-0.8.13-cp36-abi3-win_amd64.whl", hash = "sha256:8e03fe5cf4e75a97bdec4930d31ad176ea22262543cb78fd849972433d137e74"}, + {file = "polars-0.8.13.tar.gz", hash = "sha256:dedbf4f08eed114ff6472339a6847e8995aef746bf51164f3a59db0379a77ff7"}, +] +prompt-toolkit = [ + {file = "prompt_toolkit-3.0.19-py3-none-any.whl", hash = "sha256:7089d8d2938043508aa9420ec18ce0922885304cddae87fb96eebca942299f88"}, + {file = "prompt_toolkit-3.0.19.tar.gz", hash = "sha256:08360ee3a3148bdb5163621709ee322ec34fc4375099afa4bbf751e9b7b7fa4f"}, ] psutil = [ {file = "psutil-5.8.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64"}, @@ -1402,6 +1609,10 @@ psycopg2-binary = [ {file = "psycopg2_binary-2.9.1-cp39-cp39-win32.whl", hash = "sha256:0b7dae87f0b729922e06f85f667de7bf16455d411971b2043bbd9577af9d1975"}, {file = "psycopg2_binary-2.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:b4d7679a08fea64573c969f6994a2631908bb2c0e69a7235648642f3d2e39a68"}, ] +ptyprocess = [ + {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, + {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, +] py = [ {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, @@ -1575,36 +1786,36 @@ sortedcontainers = [ {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, ] sqlalchemy = [ - {file = "SQLAlchemy-1.4.21-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:e10be2b717979260db0f0fa6a531e6ddccf0d85cca11983b41d04049214fa0fc"}, - {file = "SQLAlchemy-1.4.21-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6774f2001e6359b041b8af3b9bc7669afc6adce39438fae99bfacf4b03490d54"}, - {file = "SQLAlchemy-1.4.21-cp27-cp27m-win32.whl", hash = "sha256:ba84fb12826e4db193d5fbfdcf475f85c07fdfb76b84b3fb1504905f540db7ab"}, - {file = "SQLAlchemy-1.4.21-cp27-cp27m-win_amd64.whl", hash = "sha256:4c8dc1ca3330b716c48317b4d91911e00a54c0f2de486c9c25ec0c54ebf12b5f"}, - {file = "SQLAlchemy-1.4.21-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:20a5ecd03134c7ed2c05dfdf5bd96d84480afeebe3484e416f7d7ec8c92596ae"}, - {file = "SQLAlchemy-1.4.21-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:8a98e38cb07b63459070c3a63abd5059f254d2ddec7afe77824e160f6b9e26c3"}, - {file = "SQLAlchemy-1.4.21-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da11e254ab264f515b59d16f5d1ff24f5f02fbf0b9de2d2981e704176a75c03a"}, - {file = "SQLAlchemy-1.4.21-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8f77ad5628e82f76ace2ff9a5b10ee87688bda0867f3e269cab5ed8be7e4ccc5"}, - {file = "SQLAlchemy-1.4.21-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba8fd99b546aacac74c97bb0676dd5270a1cd84c44fb67adc71d00ccabcb34a8"}, - {file = "SQLAlchemy-1.4.21-cp36-cp36m-win32.whl", hash = "sha256:bee8b2a399c6be1642d5cfcfb9d0d438fcacdd5188e0b16366fa15dbd49ec667"}, - {file = "SQLAlchemy-1.4.21-cp36-cp36m-win_amd64.whl", hash = "sha256:ef998f03ee92e6c98acdfac464c145e0a9949301b6e83688d7194e746314fcba"}, - {file = "SQLAlchemy-1.4.21-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:decb9caf3a5695a8a4ebe7153b8ef7dcc57f85dc16896e3a33d5cf3e629ac396"}, - {file = "SQLAlchemy-1.4.21-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89dbe4a792f28fd21d3319d26ceea32a3132f1c5ae578ec513f77e4c2adb9b91"}, - {file = "SQLAlchemy-1.4.21-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:340fb8eda79e5b116f761c953879c98c423eca82481d5cdad762beb108ee763e"}, - {file = "SQLAlchemy-1.4.21-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:538544799d537684e83e697298fd5078252ee68f23b44d8271f77647f225bca3"}, - {file = "SQLAlchemy-1.4.21-cp37-cp37m-win32.whl", hash = "sha256:53b17656bacdb3b194bc6cff1bd2e044879cf015ab5352c932173c2172a4b99d"}, - {file = "SQLAlchemy-1.4.21-cp37-cp37m-win_amd64.whl", hash = "sha256:cfa0c25e4c87517a679d97d0617ddaccb46337f558beac72e7d85c2f34365a35"}, - {file = "SQLAlchemy-1.4.21-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:dae7ab0c4d34d40895e92b71149bcd72a2f7c5971dc013d1c29393b6067448e3"}, - {file = "SQLAlchemy-1.4.21-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92c9f6dbe3b3d7059beea12e5601b0b37dd7a51f9bb29fbc98ab314e2a8ffdb7"}, - {file = "SQLAlchemy-1.4.21-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:eb418ec022538b24d73260b694ddb5f3878d554614a4611decb433d8eee69acd"}, - {file = "SQLAlchemy-1.4.21-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:628120ce7ef7f31824929c244894ee22a98d706d8879fb5441e1c572e02ca2ae"}, - {file = "SQLAlchemy-1.4.21-cp38-cp38-win32.whl", hash = "sha256:70b978fb1bbb629e9ce41235511d89ef9d694e3933b5a52dd6d0a4040b6c7830"}, - {file = "SQLAlchemy-1.4.21-cp38-cp38-win_amd64.whl", hash = "sha256:5dbcb3fd1d64d0835e383ea091037ca6aa70a43bd1cabb0c71c27796f2c5173f"}, - {file = "SQLAlchemy-1.4.21-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:2ad74f0a7ae8c4fa374d3be26cdf8c0897669ba3fd8bad4607710bc2fb7f132d"}, - {file = "SQLAlchemy-1.4.21-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b7af10ecd1c3829ddf824e39129e026476af6a261388db4d26bf11525fd8d05"}, - {file = "SQLAlchemy-1.4.21-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:87cf4054632c20160592ca2917aec93bb83b12b3a39c865feab1ba44e0ed120d"}, - {file = "SQLAlchemy-1.4.21-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bc28702213988c96e394685ad4103a4e347305cf90569693bef8e3d12f233ae"}, - {file = "SQLAlchemy-1.4.21-cp39-cp39-win32.whl", hash = "sha256:640fc3556a1022a781f3f07fd5dc9da842ef87f873139402d5d98d64d776360f"}, - {file = "SQLAlchemy-1.4.21-cp39-cp39-win_amd64.whl", hash = "sha256:5042a7d43a8e0a8ffc8d2acacbd5fad1edf8336c376714632a5c61eff56ac06e"}, - {file = "SQLAlchemy-1.4.21.tar.gz", hash = "sha256:07e9054f4df612beadd12ca8a5342246bffcad74a1fa8df1368d1f2bb07d8fc7"}, + {file = "SQLAlchemy-1.4.22-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:488608953385d6c127d2dcbc4b11f8d7f2f30b89f6bd27c01b042253d985cc2f"}, + {file = "SQLAlchemy-1.4.22-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:5d856cc50fd26fc8dd04892ed5a5a3d7eeb914fea2c2e484183e2d84c14926e0"}, + {file = "SQLAlchemy-1.4.22-cp27-cp27m-win32.whl", hash = "sha256:a00d9c6d3a8afe1d1681cd8a5266d2f0ed684b0b44bada2ca82403b9e8b25d39"}, + {file = "SQLAlchemy-1.4.22-cp27-cp27m-win_amd64.whl", hash = "sha256:5908ea6c652a050d768580d01219c98c071e71910ab8e7b42c02af4010608397"}, + {file = "SQLAlchemy-1.4.22-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b7fb937c720847879c7402fe300cfdb2aeff22349fa4ea3651bca4e2d6555939"}, + {file = "SQLAlchemy-1.4.22-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:9bfe882d5a1bbde0245dca0bd48da0976bd6634cf2041d2fdf0417c5463e40e5"}, + {file = "SQLAlchemy-1.4.22-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eedd76f135461cf237534a6dc0d1e0f6bb88a1dc193678fab48a11d223462da5"}, + {file = "SQLAlchemy-1.4.22-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6a16c7c4452293da5143afa3056680db2d187b380b3ef4d470d4e29885720de3"}, + {file = "SQLAlchemy-1.4.22-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44d23ea797a5e0be71bc5454b9ae99158ea0edc79e2393c6e9a2354de88329c0"}, + {file = "SQLAlchemy-1.4.22-cp36-cp36m-win32.whl", hash = "sha256:a5e14cb0c0a4ac095395f24575a0e7ab5d1be27f5f9347f1762f21505e3ba9f1"}, + {file = "SQLAlchemy-1.4.22-cp36-cp36m-win_amd64.whl", hash = "sha256:bc34a007e604091ca3a4a057525efc4cefd2b7fe970f44d20b9cfa109ab1bddb"}, + {file = "SQLAlchemy-1.4.22-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:756f5d2f5b92d27450167247fb574b09c4cd192a3f8c2e493b3e518a204ee543"}, + {file = "SQLAlchemy-1.4.22-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fcbb4b4756b250ed19adc5e28c005b8ed56fdb5c21efa24c6822c0575b4964d"}, + {file = "SQLAlchemy-1.4.22-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:09dbb4bc01a734ccddbf188deb2a69aede4b3c153a72b6d5c6900be7fb2945b1"}, + {file = "SQLAlchemy-1.4.22-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f028ef6a1d828bc754852a022b2160e036202ac8658a6c7d34875aafd14a9a15"}, + {file = "SQLAlchemy-1.4.22-cp37-cp37m-win32.whl", hash = "sha256:68393d3fd31469845b6ba11f5b4209edbea0b58506be0e077aafbf9aa2e21e11"}, + {file = "SQLAlchemy-1.4.22-cp37-cp37m-win_amd64.whl", hash = "sha256:891927a49b2363a4199763a9d436d97b0b42c65922a4ea09025600b81a00d17e"}, + {file = "SQLAlchemy-1.4.22-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:fd2102a8f8a659522719ed73865dff3d3cc76eb0833039dc473e0ad3041d04be"}, + {file = "SQLAlchemy-1.4.22-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4014978de28163cd8027434916a92d0f5bb1a3a38dff5e8bf8bff4d9372a9117"}, + {file = "SQLAlchemy-1.4.22-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f814d80844969b0d22ea63663da4de5ca1c434cfbae226188901e5d368792c17"}, + {file = "SQLAlchemy-1.4.22-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d09a760b0a045b4d799102ae7965b5491ccf102123f14b2a8cc6c01d1021a2d9"}, + {file = "SQLAlchemy-1.4.22-cp38-cp38-win32.whl", hash = "sha256:26daa429f039e29b1e523bf763bfab17490556b974c77b5ca7acb545b9230e9a"}, + {file = "SQLAlchemy-1.4.22-cp38-cp38-win_amd64.whl", hash = "sha256:12bac5fa1a6ea870bdccb96fe01610641dd44ebe001ed91ef7fcd980e9702db5"}, + {file = "SQLAlchemy-1.4.22-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:39b5d36ab71f73c068cdcf70c38075511de73616e6c7fdd112d6268c2704d9f5"}, + {file = "SQLAlchemy-1.4.22-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5102b9face693e8b2db3b2539c7e1a5d9a5b4dc0d79967670626ffd2f710d6e6"}, + {file = "SQLAlchemy-1.4.22-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c9373ef67a127799027091fa53449125351a8c943ddaa97bec4e99271dbb21f4"}, + {file = "SQLAlchemy-1.4.22-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36a089dc604032d41343d86290ce85d4e6886012eea73faa88001260abf5ff81"}, + {file = "SQLAlchemy-1.4.22-cp39-cp39-win32.whl", hash = "sha256:b48148ceedfb55f764562e04c00539bb9ea72bf07820ca15a594a9a049ff6b0e"}, + {file = "SQLAlchemy-1.4.22-cp39-cp39-win_amd64.whl", hash = "sha256:1fdae7d980a2fa617d119d0dc13ecb5c23cc63a8b04ffcb5298f2c59d86851e9"}, + {file = "SQLAlchemy-1.4.22.tar.gz", hash = "sha256:ec1be26cdccd60d180359a527d5980d959a26269a2c7b1b327a1eea0cab37ed8"}, ] sqlalchemy-redshift = [ {file = "sqlalchemy-redshift-0.8.4.tar.gz", hash = "sha256:88f4bc73178e2c9e0023e6ac77591a1c68f9edfd222ae4d6962d7c3fe71a257e"}, @@ -1669,6 +1880,10 @@ tqdm = [ {file = "tqdm-4.61.2-py2.py3-none-any.whl", hash = "sha256:5aa445ea0ad8b16d82b15ab342de6b195a722d75fc1ef9934a46bba6feafbc64"}, {file = "tqdm-4.61.2.tar.gz", hash = "sha256:8bb94db0d4468fea27d004a0f1d1c02da3cdedc00fe491c0de986b76a04d6b0a"}, ] +traitlets = [ + {file = "traitlets-5.0.5-py3-none-any.whl", hash = "sha256:69ff3f9d5351f31a7ad80443c2674b7099df13cc41fc5fa6e2f6d3b0330b0426"}, + {file = "traitlets-5.0.5.tar.gz", hash = "sha256:178f4ce988f69189f7e523337a3e11d91c786ded9360174a3d9ca83e79bc5396"}, +] twine = [ {file = "twine-3.4.2-py3-none-any.whl", hash = "sha256:087328e9bb405e7ce18527a2dca4042a84c7918658f951110b38bc135acab218"}, {file = "twine-3.4.2.tar.gz", hash = "sha256:4caec0f1ed78dc4c9b83ad537e453d03ce485725f2aea57f1bb3fdde78dae936"}, @@ -1718,6 +1933,10 @@ urllib3 = [ {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, ] +wcwidth = [ + {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, + {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, +] webencodings = [ {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, diff --git a/connectorx-python/pyproject.toml b/connectorx-python/pyproject.toml index fb112657c6..7d8a704027 100644 --- a/connectorx-python/pyproject.toml +++ b/connectorx-python/pyproject.toml @@ -42,21 +42,22 @@ pyarrow = ["pyarrow"] PyMySQL = "^1.0.2" SQLAlchemy = "^1.3.23" black = "^20.8b1" +clickhouse-driver = "^0.2.1" contexttimer = "^0.3.3" dask = {extras = ["dataframe"], version = "^2021.7.0"} docopt = "^0.6.2" +ipython = "^7.25.0" maturin = "^0.9.4" modin = {extras = ["dask"], version = "^0.10.1"} +mysqlclient = "^2.0.3" +pandahouse = "^0.2.7" polars = "^0.8" psycopg2-binary = "^2.8.6" pyarrow = "^4" pytest = "^6.2" pytest-benchmark = "^3.4.1" -twine = "^3.4.1" -mysqlclient = "^2.0.3" -pandahouse = "^0.2.7" -clickhouse-driver = "^0.2.1" sqlalchemy-redshift = "^0.8.4" +twine = "^3.4.1" [tool.pytest.ini_options] minversion = "6.0" diff --git a/connectorx-python/src/arrow.rs b/connectorx-python/src/arrow.rs index 486a5562d5..341d1b8f46 100644 --- a/connectorx-python/src/arrow.rs +++ b/connectorx-python/src/arrow.rs @@ -7,7 +7,8 @@ use connectorx::{ sources::{ mysql::{BinaryProtocol as MySQLBinaryProtocol, MySQLSource, TextProtocol}, postgres::{ - BinaryProtocol as PgBinaryProtocol, CSVProtocol, CursorProtocol, PostgresSource, + rewrite_tls_args, BinaryProtocol as PgBinaryProtocol, CSVProtocol, CursorProtocol, + PostgresSource, }, sqlite::SQLiteSource, }, @@ -17,6 +18,8 @@ use connectorx::{ use fehler::throws; use libc::uintptr_t; use log::debug; +use postgres::NoTls; +use postgres_native_tls::MakeTlsConnector; use pyo3::prelude::*; use pyo3::{PyAny, Python}; @@ -32,47 +35,85 @@ pub fn write_arrow<'a>( // TODO: unlock gil if possible match source_conn.ty { SourceType::Postgres => { + let (config, tls) = rewrite_tls_args(&source_conn.conn[..])?; debug!("Protocol: {}", protocol); - match protocol { - "csv" => { - let sb = - PostgresSource::::new(&source_conn.conn[..], queries.len())?; - let dispatcher = Dispatcher::<_, _, PostgresArrowTransport>::new( - sb, - &mut destination, - queries, - ); - + match (protocol, tls) { + ("csv", Some(tls_conn)) => { + let sb = PostgresSource::::new( + config, + tls_conn, + queries.len(), + )?; + let dispatcher = Dispatcher::< + _, + _, + PostgresArrowTransport, + >::new(sb, &mut destination, queries); debug!("Running dispatcher"); dispatcher.run()?; } - "binary" => { - let sb = PostgresSource::::new( - &source_conn.conn[..], - queries.len(), - )?; + ("csv", None) => { + let sb = + PostgresSource::::new(config, NoTls, queries.len())?; let dispatcher = - Dispatcher::<_, _, PostgresArrowTransport>::new( + Dispatcher::<_, _, PostgresArrowTransport>::new( sb, &mut destination, queries, ); - debug!("Running dispatcher"); dispatcher.run()?; } - "cursor" => { - let sb = PostgresSource::::new( - &source_conn.conn[..], + ("binary", Some(tls_conn)) => { + let sb = PostgresSource::::new( + config, + tls_conn, queries.len(), )?; - let dispatcher = - Dispatcher::<_, _, PostgresArrowTransport>::new( - sb, - &mut destination, - queries, - ); - + let dispatcher = Dispatcher::< + _, + _, + PostgresArrowTransport, + >::new(sb, &mut destination, queries); + debug!("Running dispatcher"); + dispatcher.run()?; + } + ("binary", None) => { + let sb = PostgresSource::::new( + config, + NoTls, + queries.len(), + )?; + let dispatcher = Dispatcher::< + _, + _, + PostgresArrowTransport, + >::new(sb, &mut destination, queries); + debug!("Running dispatcher"); + dispatcher.run()?; + } + ("cursor", Some(tls_conn)) => { + let sb = PostgresSource::::new( + config, + tls_conn, + queries.len(), + )?; + let dispatcher = Dispatcher::< + _, + _, + PostgresArrowTransport, + >::new(sb, &mut destination, queries); + debug!("Running dispatcher"); + dispatcher.run()?; + } + ("cursor", None) => { + let sb = + PostgresSource::::new(config, NoTls, queries.len())?; + let dispatcher = Dispatcher::< + _, + _, + PostgresArrowTransport, + >::new(sb, &mut destination, queries); debug!("Running dispatcher"); dispatcher.run()?; } diff --git a/connectorx-python/src/pandas/mod.rs b/connectorx-python/src/pandas/mod.rs index 998b712f46..af51f74fea 100644 --- a/connectorx-python/src/pandas/mod.rs +++ b/connectorx-python/src/pandas/mod.rs @@ -14,7 +14,8 @@ use connectorx::{ sources::{ mysql::{BinaryProtocol as MySQLBinaryProtocol, MySQLSource, TextProtocol}, postgres::{ - BinaryProtocol as PgBinaryProtocol, CSVProtocol, CursorProtocol, PostgresSource, + rewrite_tls_args, BinaryProtocol as PgBinaryProtocol, CSVProtocol, CursorProtocol, + PostgresSource, }, sqlite::SQLiteSource, }, @@ -22,6 +23,8 @@ use connectorx::{ }; use fehler::throws; use log::debug; +use postgres::NoTls; +use postgres_native_tls::MakeTlsConnector; use pyo3::{PyAny, Python}; #[throws(ConnectorXPythonError)] @@ -37,46 +40,84 @@ pub fn write_pandas<'a>( match source_conn.ty { SourceType::Postgres => { debug!("Protocol: {}", protocol); - match protocol { - "csv" => { - let sb = - PostgresSource::::new(&source_conn.conn[..], queries.len())?; - let dispatcher = Dispatcher::<_, _, PostgresPandasTransport>::new( - sb, - &mut destination, - queries, - ); - + let (config, tls) = rewrite_tls_args(&source_conn.conn[..])?; + match (protocol, tls) { + ("csv", Some(tls_conn)) => { + let sb = PostgresSource::::new( + config, + tls_conn, + queries.len(), + )?; + let dispatcher = Dispatcher::< + _, + _, + PostgresPandasTransport, + >::new(sb, &mut destination, queries); debug!("Running dispatcher"); dispatcher.run()?; } - "binary" => { - let sb = PostgresSource::::new( - &source_conn.conn[..], - queries.len(), - )?; + ("csv", None) => { + let sb = + PostgresSource::::new(config, NoTls, queries.len())?; let dispatcher = - Dispatcher::<_, _, PostgresPandasTransport>::new( + Dispatcher::<_, _, PostgresPandasTransport>::new( sb, &mut destination, queries, ); - debug!("Running dispatcher"); dispatcher.run()?; } - "cursor" => { - let sb = PostgresSource::::new( - &source_conn.conn[..], + ("binary", Some(tls_conn)) => { + let sb = PostgresSource::::new( + config, + tls_conn, queries.len(), )?; - let dispatcher = - Dispatcher::<_, _, PostgresPandasTransport>::new( - sb, - &mut destination, - queries, - ); - + let dispatcher = Dispatcher::< + _, + _, + PostgresPandasTransport, + >::new(sb, &mut destination, queries); + debug!("Running dispatcher"); + dispatcher.run()?; + } + ("binary", None) => { + let sb = PostgresSource::::new( + config, + NoTls, + queries.len(), + )?; + let dispatcher = Dispatcher::< + _, + _, + PostgresPandasTransport, + >::new(sb, &mut destination, queries); + debug!("Running dispatcher"); + dispatcher.run()?; + } + ("cursor", Some(tls_conn)) => { + let sb = PostgresSource::::new( + config, + tls_conn, + queries.len(), + )?; + let dispatcher = Dispatcher::< + _, + _, + PostgresPandasTransport, + >::new(sb, &mut destination, queries); + debug!("Running dispatcher"); + dispatcher.run()?; + } + ("cursor", None) => { + let sb = + PostgresSource::::new(config, NoTls, queries.len())?; + let dispatcher = Dispatcher::< + _, + _, + PostgresPandasTransport, + >::new(sb, &mut destination, queries); debug!("Running dispatcher"); dispatcher.run()?; } diff --git a/connectorx-python/src/pandas/transports/postgres.rs b/connectorx-python/src/pandas/transports/postgres.rs index 6192c3feda..d2ba1e8f83 100644 --- a/connectorx-python/src/pandas/transports/postgres.rs +++ b/connectorx-python/src/pandas/transports/postgres.rs @@ -9,138 +9,95 @@ use connectorx::{ }, typesystem::TypeConversion, }; +use postgres::NoTls; +use postgres_native_tls::MakeTlsConnector; use rust_decimal::prelude::*; use serde_json::{to_string, Value}; use std::marker::PhantomData; use uuid::Uuid; -pub struct PostgresPandasTransport<'py, P>(&'py (), PhantomData

); +pub struct PostgresPandasTransport<'py, P, C>(&'py (), PhantomData

, PhantomData); -impl_transport!( - name = PostgresPandasTransport<'tp, BinaryProtocol>, - error = ConnectorXPythonError, - systems = PostgresTypeSystem => PandasTypeSystem, - route = PostgresSource => PandasDestination<'tp>, - mappings = { - { Float4[f32] => F64[f64] | conversion all } - { Float8[f64] => F64[f64] | conversion all } - { Numeric[Decimal] => F64[f64] | conversion half } - { Int2[i16] => I64[i64] | conversion all } - { Int4[i32] => I64[i64] | conversion all } - { Int8[i64] => I64[i64] | conversion all } - { Bool[bool] => Bool[bool] | conversion all } - { Char[i8] => Char[char] | conversion half } - { Text[&'r str] => Str[&'r str] | conversion all } - { BpChar[&'r str] => Str[&'r str] | conversion none } - { VarChar[&'r str] => Str[&'r str] | conversion none } - { Timestamp[NaiveDateTime] => DateTime[DateTime] | conversion half } - { TimestampTz[DateTime] => DateTime[DateTime] | conversion all } - { Date[NaiveDate] => DateTime[DateTime] | conversion half } - { UUID[Uuid] => String[String] | conversion half } - { JSON[Value] => String[String] | conversion half } - { JSONB[Value] => String[String] | conversion none } - { Time[NaiveTime] => String[String] | conversion half } - { ByteA[Vec] => Bytes[Vec] | conversion all } - { Enum[&'r str] => Str[&'r str] | conversion none } +macro_rules! impl_postgres_transport { + ($proto:ty, $tls:ty) => { + impl_transport!( + name = PostgresPandasTransport<'tp, $proto, $tls>, + error = ConnectorXPythonError, + systems = PostgresTypeSystem => PandasTypeSystem, + route = PostgresSource<$proto, $tls> => PandasDestination<'tp>, + mappings = { + { Float4[f32] => F64[f64] | conversion all } + { Float8[f64] => F64[f64] | conversion all } + { Numeric[Decimal] => F64[f64] | conversion half } + { Int2[i16] => I64[i64] | conversion all } + { Int4[i32] => I64[i64] | conversion all } + { Int8[i64] => I64[i64] | conversion all } + { Bool[bool] => Bool[bool] | conversion all } + { Char[i8] => Char[char] | conversion half } + { Text[&'r str] => Str[&'r str] | conversion all } + { BpChar[&'r str] => Str[&'r str] | conversion none } + { VarChar[&'r str] => Str[&'r str] | conversion none } + { Timestamp[NaiveDateTime] => DateTime[DateTime] | conversion half } + { TimestampTz[DateTime] => DateTime[DateTime] | conversion all } + { Date[NaiveDate] => DateTime[DateTime] | conversion half } + { UUID[Uuid] => String[String] | conversion half } + { JSON[Value] => String[String] | conversion half } + { JSONB[Value] => String[String] | conversion none } + { Time[NaiveTime] => String[String] | conversion half } + { ByteA[Vec] => Bytes[Vec] | conversion all } + { Enum[&'r str] => Str[&'r str] | conversion none } + } + ); } -); - -impl_transport!( - name = PostgresPandasTransport<'tp, CSVProtocol>, - error = ConnectorXPythonError, - systems = PostgresTypeSystem => PandasTypeSystem, - route = PostgresSource => PandasDestination<'tp>, - mappings = { - { Float4[f32] => F64[f64] | conversion all } - { Float8[f64] => F64[f64] | conversion all } - { Numeric[Decimal] => F64[f64] | conversion half } - { Int2[i16] => I64[i64] | conversion all } - { Int4[i32] => I64[i64] | conversion all } - { Int8[i64] => I64[i64] | conversion all } - { Bool[bool] => Bool[bool] | conversion all } - { Char[i8] => Char[char] | conversion half } - { Text[&'r str] => Str[&'r str] | conversion all } - { BpChar[&'r str] => Str[&'r str] | conversion none } - { VarChar[&'r str] => Str[&'r str] | conversion none } - { Timestamp[NaiveDateTime] => DateTime[DateTime] | conversion half } - { TimestampTz[DateTime] => DateTime[DateTime] | conversion all } - { Date[NaiveDate] => DateTime[DateTime] | conversion half } - { UUID[Uuid] => String[String] | conversion half } - { JSON[Value] => String[String] | conversion half } - { JSONB[Value] => String[String] | conversion none } - { Time[NaiveTime] => String[String] | conversion half } - { ByteA[Vec] => Bytes[Vec] | conversion all } - { Enum[&'r str] => Str[&'r str] | conversion none } - } -); +} -impl_transport!( - name = PostgresPandasTransport<'tp, CursorProtocol>, - error = ConnectorXPythonError, - systems = PostgresTypeSystem => PandasTypeSystem, - route = PostgresSource => PandasDestination<'tp>, - mappings = { - { Float4[f32] => F64[f64] | conversion all } - { Float8[f64] => F64[f64] | conversion all } - { Numeric[Decimal] => F64[f64] | conversion half } - { Int2[i16] => I64[i64] | conversion all } - { Int4[i32] => I64[i64] | conversion all } - { Int8[i64] => I64[i64] | conversion all } - { Bool[bool] => Bool[bool] | conversion all } - { Char[i8] => Char[char] | conversion half } - { Text[&'r str] => Str[&'r str] | conversion all } - { BpChar[&'r str] => Str[&'r str] | conversion none } - { VarChar[&'r str] => Str[&'r str] | conversion none } - { Timestamp[NaiveDateTime] => DateTime[DateTime] | conversion half } - { TimestampTz[DateTime] => DateTime[DateTime] | conversion all } - { Date[NaiveDate] => DateTime[DateTime] | conversion half } - { UUID[Uuid] => String[String] | conversion half } - { JSON[Value] => String[String] | conversion half } - { JSONB[Value] => String[String] | conversion none } - { Time[NaiveTime] => String[String] | conversion half } - { ByteA[Vec] => Bytes[Vec] | conversion all } - { Enum[&'r str] => Str[&'r str] | conversion none } - } -); +impl_postgres_transport!(BinaryProtocol, NoTls); +impl_postgres_transport!(BinaryProtocol, MakeTlsConnector); +impl_postgres_transport!(CSVProtocol, NoTls); +impl_postgres_transport!(CSVProtocol, MakeTlsConnector); +impl_postgres_transport!(CursorProtocol, NoTls); +impl_postgres_transport!(CursorProtocol, MakeTlsConnector); -impl<'py, P> TypeConversion for PostgresPandasTransport<'py, P> { +impl<'py, P, C> TypeConversion for PostgresPandasTransport<'py, P, C> { fn convert(val: Decimal) -> f64 { val.to_f64() .unwrap_or_else(|| panic!("cannot convert decimal {:?} to float64", val)) } } -impl<'py, P> TypeConversion for PostgresPandasTransport<'py, P> { +impl<'py, P, C> TypeConversion for PostgresPandasTransport<'py, P, C> { fn convert(val: NaiveTime) -> String { val.to_string() } } -impl<'py, P> TypeConversion for PostgresPandasTransport<'py, P> { +impl<'py, P, C> TypeConversion for PostgresPandasTransport<'py, P, C> { fn convert(val: i8) -> char { val as u8 as char } } -impl<'py, P> TypeConversion> for PostgresPandasTransport<'py, P> { +impl<'py, P, C> TypeConversion> + for PostgresPandasTransport<'py, P, C> +{ fn convert(val: NaiveDateTime) -> DateTime { DateTime::from_utc(val, Utc) } } -impl<'py, P> TypeConversion> for PostgresPandasTransport<'py, P> { +impl<'py, P, C> TypeConversion> for PostgresPandasTransport<'py, P, C> { fn convert(val: NaiveDate) -> DateTime { DateTime::from_utc(val.and_hms(0, 0, 0), Utc) } } -impl<'py, P> TypeConversion for PostgresPandasTransport<'py, P> { +impl<'py, P, C> TypeConversion for PostgresPandasTransport<'py, P, C> { fn convert(val: Uuid) -> String { val.to_string() } } -impl<'py, P> TypeConversion for PostgresPandasTransport<'py, P> { +impl<'py, P, C> TypeConversion for PostgresPandasTransport<'py, P, C> { fn convert(val: Value) -> String { to_string(&val).unwrap() } diff --git a/connectorx-python/src/source_router.rs b/connectorx-python/src/source_router.rs index 95cbd86da2..e9bee19ac2 100644 --- a/connectorx-python/src/source_router.rs +++ b/connectorx-python/src/source_router.rs @@ -1,14 +1,16 @@ use crate::errors::{ConnectorXPythonError, Result}; use anyhow::anyhow; use connectorx::{ - sources::{mysql::MySQLTypeSystem, postgres::PostgresTypeSystem}, + sources::{ + mysql::MySQLTypeSystem, + postgres::{rewrite_tls_args, PostgresTypeSystem}, + }, sql::{ get_partition_range_query, get_partition_range_query_sep, single_col_partition_query, CXQuery, }, }; use fehler::{throw, throws}; -use postgres::{Client, NoTls}; use r2d2_mysql::mysql::{prelude::Queryable, Pool, Row}; use rusqlite::{types::Type, Connection}; use sqlparser::dialect::MySqlDialect; @@ -86,7 +88,8 @@ impl SourceType { #[throws(ConnectorXPythonError)] fn pg_get_partition_range(conn: &str, query: &str, col: &str) -> (i64, i64) { - let mut client = Client::connect(conn, NoTls)?; + let (config, tls) = rewrite_tls_args(conn)?; + let mut client = config.connect(tls.unwrap())?; let range_query = get_partition_range_query(query, col, &PostgreSqlDialect {})?; let row = client.query_one(range_query.as_str(), &[])?; diff --git a/connectorx/Cargo.toml b/connectorx/Cargo.toml index fb1a8c1633..1fbfaef0a7 100644 --- a/connectorx/Cargo.toml +++ b/connectorx/Cargo.toml @@ -24,11 +24,13 @@ csv = {version = "1", optional = true} derive_more = {version = "0.99", optional = true} fallible-streaming-iterator = {version = "0.1", optional = true} hex = {version = "0.4", optional = true} +native-tls = {version = "0.2", optional = true} ndarray = {version = "0.15", optional = true} num-traits = {version = "0.2", optional = true} owning_ref = {version = "0.4", optional = true} polars = {version = "0.14", optional = true} postgres = {version = "0.19", features = ["with-chrono-0_4", "with-uuid-0_8", "with-serde_json-1"], optional = true} +postgres-native-tls = {version = "0.5", optional = true} r2d2 = {version = "0.8", optional = true} r2d2_mysql = {version = "18.0", optional = true} r2d2_postgres = {version = "0.18", optional = true} @@ -37,6 +39,7 @@ regex = {version = "1", optional = true} rusqlite = {version = "0.25", features = ["column_decltype", "chrono", "bundled"], optional = true} rust_decimal = {version = "1", features = ["db-postgres"], optional = true} serde_json = {version = "1", optional = true} +url = {version = "2", optional = true} uuid = {version = "0.8", optional = true} [lib] @@ -61,7 +64,21 @@ fptr = [] src_csv = ["csv", "regex", "chrono"] src_dummy = ["num-traits", "chrono"] src_mysql = ["r2d2_mysql", "rust_decimal", "num-traits", "chrono", "r2d2"] -src_postgres = ["postgres", "r2d2_postgres", "csv", "hex", "serde_json", "uuid", "rust_decimal", "num-traits", "chrono", "r2d2"] +src_postgres = [ + "postgres", + "r2d2_postgres", + "postgres-native-tls", + "csv", + "hex", + "serde_json", + "uuid", + "rust_decimal", + "num-traits", + "chrono", + "r2d2", + "url", + "native-tls", +] src_sqlite = ["rusqlite", "r2d2_sqlite", "fallible-streaming-iterator", "derive_more", "owning_ref", "chrono", "r2d2"] [package.metadata.docs.rs] diff --git a/connectorx/src/sources/postgres/connection.rs b/connectorx/src/sources/postgres/connection.rs new file mode 100644 index 0000000000..ab2ab254e4 --- /dev/null +++ b/connectorx/src/sources/postgres/connection.rs @@ -0,0 +1,109 @@ +use crate::sources::postgres::errors::PostgresSourceError; +use anyhow::anyhow; +use native_tls::{Certificate, TlsConnector}; +use postgres::{config::SslMode, Config}; +use postgres_native_tls::MakeTlsConnector; +use std::collections::HashMap; +use std::convert::TryFrom; +use std::path::PathBuf; +use url::Url; + +#[derive(Clone, Debug)] +pub struct TlsConfig { + /// Postgres config, pg_config.sslmode (`sslmode`). + pub pg_config: Config, + /// Location of the root certificate (`sslrootcert`). + pub root_cert: Option, +} + +impl TryFrom for MakeTlsConnector { + type Error = PostgresSourceError; + // The logic of this function adapted primarily from: + // https://github.com/sfackler/rust-postgres/pull/774 + // We only support server side authentication (`sslrootcert`) for now + fn try_from(tls_config: TlsConfig) -> Result { + let mut builder = TlsConnector::builder(); + let ssl_mode = tls_config.pg_config.get_ssl_mode(); + let (verify_ca, verify_hostname) = match ssl_mode { + SslMode::Disable | SslMode::Prefer => (false, false), + SslMode::Require => match tls_config.root_cert { + // If a root CA file exists, the behavior of sslmode=require will be the same as + // that of verify-ca, meaning the server certificate is validated against the CA. + // + // For more details, check out the note about backwards compatibility in + // https://postgresql.org/docs/current/libpq-ssl.html#LIBQ-SSL-CERTIFICATES. + Some(_) => (true, false), + None => (false, false), + }, + // These two modes will not work until upstream rust-postgres supports parsing + // them as part of the TLS config. + // + // SslMode::VerifyCa => (true, false), + // SslMode::VerifyFull => (true, true), + _ => panic!("unexpected sslmode {:?}", ssl_mode), + }; + + builder.danger_accept_invalid_certs(!verify_ca); + builder.danger_accept_invalid_hostnames(!verify_hostname); + + if let Some(root_cert) = tls_config.root_cert { + builder + .add_root_certificate(Certificate::from_pem(std::fs::read(root_cert)?.as_ref())?); + } + + let tls_connector = MakeTlsConnector::new(builder.build()?); + + Ok(tls_connector) + } +} + +// Strip URL params not accepted by upstream rust-postgres +fn strip_bad_opts(url: Url) -> Url { + let stripped_query: Vec<(_, _)> = url + .query_pairs() + .filter(|p| match &*p.0 { + "sslrootcert" => false, + _ => true, + }) + .collect(); + + let mut url2 = url.clone(); + url2.set_query(None); + + for pair in stripped_query { + url2.query_pairs_mut() + .append_pair(&pair.0.to_string()[..], &pair.1.to_string()[..]); + } + + url2 +} + +pub fn rewrite_tls_args( + conn: &str, +) -> Result<(Config, Option), PostgresSourceError> { + // We parse the config, then strip unsupported SSL opts and rewrite the URI + // before calling conn.parse(). + // + // For more details on this approach, see the conversation here: + // https://github.com/sfackler/rust-postgres/pull/774#discussion_r641784774 + let parsed_conn_str = Url::parse(conn).map_err(|e| anyhow!(e))?; + + let params: HashMap = + parsed_conn_str.clone().query_pairs().into_owned().collect(); + let root_cert = params.get("sslrootcert").map(|x| PathBuf::from(x)); + + let stripped_url = strip_bad_opts(parsed_conn_str.clone()); + let pg_config: Config = stripped_url.as_str().parse().unwrap(); + + let tls_config = TlsConfig { + pg_config: pg_config.clone(), + root_cert, + }; + + let tls_connector = match pg_config.get_ssl_mode() { + SslMode::Disable => None, + _ => Some(MakeTlsConnector::try_from(tls_config)?), + }; + + Ok((pg_config, tls_connector)) +} diff --git a/connectorx/src/sources/postgres/errors.rs b/connectorx/src/sources/postgres/errors.rs index 2e8b421767..7788a0c918 100644 --- a/connectorx/src/sources/postgres/errors.rs +++ b/connectorx/src/sources/postgres/errors.rs @@ -20,6 +20,9 @@ pub enum PostgresSourceError { #[error(transparent)] IOError(#[from] std::io::Error), + #[error(transparent)] + TlsError(#[from] native_tls::Error), + /// Any other errors that are too trivial to be put here explicitly. #[error(transparent)] Other(#[from] anyhow::Error), diff --git a/connectorx/src/sources/postgres/mod.rs b/connectorx/src/sources/postgres/mod.rs index 9ba1ac67d9..f5196f3655 100644 --- a/connectorx/src/sources/postgres/mod.rs +++ b/connectorx/src/sources/postgres/mod.rs @@ -1,7 +1,11 @@ +mod connection; mod errors; mod typesystem; pub use self::errors::PostgresSourceError; +pub use connection::rewrite_tls_args; +pub use typesystem::PostgresTypeSystem; + use crate::{ data_order::DataOrder, errors::ConnectorXError, @@ -17,16 +21,16 @@ use log::debug; use postgres::{ binary_copy::{BinaryCopyOutIter, BinaryCopyOutRow}, fallible_iterator::FallibleIterator, - CopyOutReader, Row, RowIter, + tls::{MakeTlsConnect, TlsConnect}, + Config, CopyOutReader, Row, RowIter, Socket, }; use r2d2::{Pool, PooledConnection}; -use r2d2_postgres::{postgres::NoTls, PostgresConnectionManager}; +use r2d2_postgres::PostgresConnectionManager; use rust_decimal::Decimal; use serde_json::{from_str, Value}; use sqlparser::dialect::PostgreSqlDialect; use std::io::BufRead; use std::marker::PhantomData; -pub use typesystem::PostgresTypeSystem; use uuid::Uuid; /// Protocol - Binary based bulk load @@ -38,11 +42,17 @@ pub enum CSVProtocol {} /// Protocol - use Cursor pub enum CursorProtocol {} -type PgManager = PostgresConnectionManager; -type PgConn = PooledConnection; +type PgManager = PostgresConnectionManager; +type PgConn = PooledConnection>; -pub struct PostgresSource

{ - pool: Pool, +pub struct PostgresSource +where + C: MakeTlsConnect + Clone + 'static + Sync + Send, + C::TlsConnect: Send, + C::Stream: Send, + >::Future: Send, +{ + pool: Pool>, queries: Vec>, names: Vec, schema: Vec, @@ -50,10 +60,16 @@ pub struct PostgresSource

{ _protocol: PhantomData

, } -impl

PostgresSource

{ +impl PostgresSource +where + C: MakeTlsConnect + Clone + 'static + Sync + Send, + C::TlsConnect: Send, + C::Stream: Send, + >::Future: Send, +{ #[throws(PostgresSourceError)] - pub fn new(conn: &str, nconn: usize) -> Self { - let manager = PostgresConnectionManager::new(conn.parse()?, NoTls); + pub fn new(config: Config, tls: C, nconn: usize) -> Self { + let manager = PostgresConnectionManager::new(config, tls); let pool = Pool::builder().max_size(nconn as u32).build(manager)?; Self { @@ -71,14 +87,18 @@ impl

PostgresSource

{ } } -impl

Source for PostgresSource

+impl Source for PostgresSource where - PostgresSourcePartition

: + PostgresSourcePartition: SourcePartition, P: Send, + C: MakeTlsConnect + Clone + 'static + Sync + Send, + C::TlsConnect: Send, + C::Stream: Send, + >::Future: Send, { const DATA_ORDERS: &'static [DataOrder] = &[DataOrder::RowMajor]; - type Partition = PostgresSourcePartition

; + type Partition = PostgresSourcePartition; type TypeSystem = PostgresTypeSystem; type Error = PostgresSourceError; @@ -168,7 +188,7 @@ where for query in self.queries { let conn = self.pool.get()?; - ret.push(PostgresSourcePartition::

::new( + ret.push(PostgresSourcePartition::::new( conn, &query, &self.schema, @@ -179,8 +199,14 @@ where } } -pub struct PostgresSourcePartition

{ - conn: PgConn, +pub struct PostgresSourcePartition +where + C: MakeTlsConnect + Clone + 'static + Sync + Send, + C::TlsConnect: Send, + C::Stream: Send, + >::Future: Send, +{ + conn: PgConn, query: CXQuery, schema: Vec, nrows: usize, @@ -189,9 +215,15 @@ pub struct PostgresSourcePartition

{ _protocol: PhantomData

, } -impl

PostgresSourcePartition

{ +impl PostgresSourcePartition +where + C: MakeTlsConnect + Clone + 'static + Sync + Send, + C::TlsConnect: Send, + C::Stream: Send, + >::Future: Send, +{ pub fn new( - conn: PgConn, + conn: PgConn, query: &CXQuery, schema: &[PostgresTypeSystem], buf_size: usize, @@ -208,7 +240,13 @@ impl

PostgresSourcePartition

{ } } -impl SourcePartition for PostgresSourcePartition { +impl SourcePartition for PostgresSourcePartition +where + C: MakeTlsConnect + Clone + 'static + Sync + Send, + C::TlsConnect: Send, + C::Stream: Send, + >::Future: Send, +{ type TypeSystem = PostgresTypeSystem; type Parser<'a> = PostgresBinarySourcePartitionParser<'a>; type Error = PostgresSourceError; @@ -246,7 +284,13 @@ impl SourcePartition for PostgresSourcePartition { } } -impl SourcePartition for PostgresSourcePartition { +impl SourcePartition for PostgresSourcePartition +where + C: MakeTlsConnect + Clone + 'static + Sync + Send, + C::TlsConnect: Send, + C::Stream: Send, + >::Future: Send, +{ type TypeSystem = PostgresTypeSystem; type Parser<'a> = PostgresCSVSourceParser<'a>; type Error = PostgresSourceError; @@ -281,7 +325,13 @@ impl SourcePartition for PostgresSourcePartition { } } -impl SourcePartition for PostgresSourcePartition { +impl SourcePartition for PostgresSourcePartition +where + C: MakeTlsConnect + Clone + 'static + Sync + Send, + C::TlsConnect: Send, + C::Stream: Send, + >::Future: Send, +{ type TypeSystem = PostgresTypeSystem; type Parser<'a> = PostgresRawSourceParser<'a>; type Error = PostgresSourceError; diff --git a/connectorx/src/transports/postgres_arrow.rs b/connectorx/src/transports/postgres_arrow.rs index 76a2fb90c3..c72afea214 100644 --- a/connectorx/src/transports/postgres_arrow.rs +++ b/connectorx/src/transports/postgres_arrow.rs @@ -6,6 +6,8 @@ use crate::sources::postgres::{ use crate::typesystem::TypeConversion; use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; use num_traits::ToPrimitive; +use postgres::NoTls; +use postgres_native_tls::MakeTlsConnector; use rust_decimal::Decimal; use std::marker::PhantomData; use thiserror::Error; @@ -23,92 +25,56 @@ pub enum PostgresArrowTransportError { ConnectorXError(#[from] crate::errors::ConnectorXError), } -pub struct PostgresArrowTransport

(PhantomData

); +pub struct PostgresArrowTransport(PhantomData

, PhantomData); -impl_transport!( - name = PostgresArrowTransport, - error = PostgresArrowTransportError, - systems = PostgresTypeSystem => ArrowTypeSystem, - route = PostgresSource => ArrowDestination, - mappings = { - { Float4[f32] => Float32[f32] | conversion all } - { Float8[f64] => Float64[f64] | conversion all } - { Numeric[Decimal] => Float64[f64] | conversion half } - { Int2[i16] => Int32[i32] | conversion all } - { Int4[i32] => Int32[i32] | conversion all } - { Int8[i64] => Int64[i64] | conversion all } - { Bool[bool] => Boolean[bool] | conversion all } - { Text[&'r str] => LargeUtf8[String] | conversion half } - { BpChar[&'r str] => LargeUtf8[String] | conversion none } - { VarChar[&'r str] => LargeUtf8[String] | conversion none } - { Timestamp[NaiveDateTime] => Date64[NaiveDateTime] | conversion all } - { Date[NaiveDate] => Date32[NaiveDate] | conversion all } - { Time[NaiveTime] => Time64[NaiveTime] | conversion all } - { UUID[Uuid] => LargeUtf8[String] | conversion half } - { Char[&'r str] => LargeUtf8[String] | conversion none } +macro_rules! impl_postgres_transport { + ($proto:ty, $tls:ty) => { + impl_transport!( + name = PostgresArrowTransport<$proto, $tls>, + error = PostgresArrowTransportError, + systems = PostgresTypeSystem => ArrowTypeSystem, + route = PostgresSource<$proto, $tls> => ArrowDestination, + mappings = { + { Float4[f32] => Float32[f32] | conversion all } + { Float8[f64] => Float64[f64] | conversion all } + { Numeric[Decimal] => Float64[f64] | conversion half } + { Int2[i16] => Int32[i32] | conversion all } + { Int4[i32] => Int32[i32] | conversion all } + { Int8[i64] => Int64[i64] | conversion all } + { Bool[bool] => Boolean[bool] | conversion all } + { Text[&'r str] => LargeUtf8[String] | conversion half } + { BpChar[&'r str] => LargeUtf8[String] | conversion none } + { VarChar[&'r str] => LargeUtf8[String] | conversion none } + { Timestamp[NaiveDateTime] => Date64[NaiveDateTime] | conversion all } + { Date[NaiveDate] => Date32[NaiveDate] | conversion all } + { Time[NaiveTime] => Time64[NaiveTime] | conversion all } + { UUID[Uuid] => LargeUtf8[String] | conversion half } + { Char[&'r str] => LargeUtf8[String] | conversion none } + } + ); } -); - -impl_transport!( - name = PostgresArrowTransport, - error = PostgresArrowTransportError, - systems = PostgresTypeSystem => ArrowTypeSystem, - route = PostgresSource => ArrowDestination, - mappings = { - { Float4[f32] => Float32[f32] | conversion all } - { Float8[f64] => Float64[f64] | conversion all } - { Numeric[Decimal] => Float64[f64] | conversion half } - { Int2[i16] => Int32[i32] | conversion all } - { Int4[i32] => Int32[i32] | conversion all } - { Int8[i64] => Int64[i64] | conversion all } - { Bool[bool] => Boolean[bool] | conversion all } - { Text[&'r str] => LargeUtf8[String] | conversion half } - { BpChar[&'r str] => LargeUtf8[String] | conversion none } - { VarChar[&'r str] => LargeUtf8[String] | conversion none } - { Timestamp[NaiveDateTime] => Date64[NaiveDateTime] | conversion all } - { Date[NaiveDate] => Date32[NaiveDate] | conversion all } - { Time[NaiveTime] => Time64[NaiveTime] | conversion all } - { UUID[Uuid] => LargeUtf8[String] | conversion half } - { Char[&'r str] => LargeUtf8[String] | conversion none } - } -); +} -impl_transport!( - name = PostgresArrowTransport, - error = PostgresArrowTransportError, - systems = PostgresTypeSystem => ArrowTypeSystem, - route = PostgresSource => ArrowDestination, - mappings = { - { Float4[f32] => Float32[f32] | conversion all } - { Float8[f64] => Float64[f64] | conversion all } - { Int2[i16] => Int32[i32] | conversion all } - { Int4[i32] => Int32[i32] | conversion all } - { Int8[i64] => Int64[i64] | conversion all } - { Bool[bool] => Boolean[bool] | conversion all } - { Text[&'r str] => LargeUtf8[String] | conversion half } - { BpChar[&'r str] => LargeUtf8[String] | conversion none } - { VarChar[&'r str] => LargeUtf8[String] | conversion none } - { Timestamp[NaiveDateTime] => Date64[NaiveDateTime] | conversion all } - { Date[NaiveDate] => Date32[NaiveDate] | conversion all } - { Time[NaiveTime] => Time64[NaiveTime] | conversion all } - { UUID[Uuid] => LargeUtf8[String] | conversion half } - { Char[&'r str] => LargeUtf8[String] | conversion none } - } -); +impl_postgres_transport!(BinaryProtocol, NoTls); +impl_postgres_transport!(BinaryProtocol, MakeTlsConnector); +impl_postgres_transport!(CSVProtocol, NoTls); +impl_postgres_transport!(CSVProtocol, MakeTlsConnector); +impl_postgres_transport!(CursorProtocol, NoTls); +impl_postgres_transport!(CursorProtocol, MakeTlsConnector); -impl

TypeConversion for PostgresArrowTransport

{ +impl TypeConversion for PostgresArrowTransport { fn convert(val: Uuid) -> String { val.to_string() } } -impl<'r, P> TypeConversion<&'r str, String> for PostgresArrowTransport

{ +impl<'r, P, C> TypeConversion<&'r str, String> for PostgresArrowTransport { fn convert(val: &'r str) -> String { val.to_string() } } -impl

TypeConversion for PostgresArrowTransport

{ +impl TypeConversion for PostgresArrowTransport { fn convert(val: Decimal) -> f64 { val.to_f64() .unwrap_or_else(|| panic!("cannot convert decimal {:?} to float64", val)) diff --git a/connectorx/src/transports/postgres_memory.rs b/connectorx/src/transports/postgres_memory.rs index 3e920eb5d3..12c614d599 100644 --- a/connectorx/src/transports/postgres_memory.rs +++ b/connectorx/src/transports/postgres_memory.rs @@ -1,15 +1,18 @@ use crate::destinations::memory::{MemoryDestination, MemoryDestinationError}; use crate::dummy_typesystem::DummyTypeSystem; use crate::sources::postgres::{ - BinaryProtocol, CSVProtocol, PostgresSource, PostgresSourceError, PostgresTypeSystem, + BinaryProtocol, CSVProtocol, CursorProtocol, PostgresSource, PostgresSourceError, + PostgresTypeSystem, }; use crate::typesystem::TypeConversion; use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc}; +use postgres::NoTls; +use postgres_native_tls::MakeTlsConnector; use std::marker::PhantomData; use thiserror::Error; use uuid::Uuid; -pub struct PostgresMemoryTransport

(PhantomData

); +pub struct PostgresMemoryTransport(PhantomData

, PhantomData); #[derive(Error, Debug)] pub enum PostgresMemoryTransportError { @@ -22,80 +25,67 @@ pub enum PostgresMemoryTransportError { #[error(transparent)] ConnectorXError(#[from] crate::errors::ConnectorXError), } +macro_rules! impl_postgres_transport { + ($proto:ty, $tls:ty) => { + impl_transport!( + name = PostgresMemoryTransport<$proto, $tls>, + error = PostgresMemoryTransportError, + systems = PostgresTypeSystem => DummyTypeSystem, + route = PostgresSource<$proto, $tls> => MemoryDestination, + mappings = { + { Float4[f32] => F64[f64] | conversion all } + { Float8[f64] => F64[f64] | conversion all } + { Int2[i16] => I64[i64] | conversion all } + { Int4[i32] => I64[i64] | conversion all } + { Int8[i64] => I64[i64] | conversion all } + { Bool[bool] => Bool[bool] | conversion all } + { Text[&'r str] => String[String] | conversion half } + { BpChar[&'r str] => String[String] | conversion none } + { VarChar[&'r str] => String[String] | conversion none } + { Timestamp[NaiveDateTime] => DateTime[DateTime] | conversion half } + { TimestampTz[DateTime] => DateTime[DateTime] | conversion all } + { Date[NaiveDate] => DateTime[DateTime] | conversion half } + { UUID[Uuid] => String[String] | conversion half } + { Char[&'r str] => String[String] | conversion none } + // { Time[NaiveTime] => String[String] | conversion half } + } + ); -impl_transport!( - name = PostgresMemoryTransport, - error = PostgresMemoryTransportError, - systems = PostgresTypeSystem => DummyTypeSystem, - route = PostgresSource => MemoryDestination, - mappings = { - { Float4[f32] => F64[f64] | conversion all } - { Float8[f64] => F64[f64] | conversion all } - { Int2[i16] => I64[i64] | conversion all } - { Int4[i32] => I64[i64] | conversion all } - { Int8[i64] => I64[i64] | conversion all } - { Bool[bool] => Bool[bool] | conversion all } - { Text[&'r str] => String[String] | conversion half } - { BpChar[&'r str] => String[String] | conversion none } - { VarChar[&'r str] => String[String] | conversion none } - { Timestamp[NaiveDateTime] => DateTime[DateTime] | conversion half } - { TimestampTz[DateTime] => DateTime[DateTime] | conversion all } - { Date[NaiveDate] => DateTime[DateTime] | conversion half } - { UUID[Uuid] => String[String] | conversion half } - { Char[&'r str] => String[String] | conversion none } - // { Time[NaiveTime] => String[String] | conversion half } } -); +} -impl_transport!( - name = PostgresMemoryTransport, - error = PostgresMemoryTransportError, - systems = PostgresTypeSystem => DummyTypeSystem, - route = PostgresSource => MemoryDestination, - mappings = { - { Float4[f32] => F64[f64] | conversion all } - { Float8[f64] => F64[f64] | conversion all } - { Int2[i16] => I64[i64] | conversion all } - { Int4[i32] => I64[i64] | conversion all } - { Int8[i64] => I64[i64] | conversion all } - { Bool[bool] => Bool[bool] | conversion all } - { Text[&'r str] => String[String] | conversion half } - { BpChar[&'r str] => String[String] | conversion none } - { VarChar[&'r str] => String[String] | conversion none } - { Timestamp[NaiveDateTime] => DateTime[DateTime] | conversion half } - { TimestampTz[DateTime] => DateTime[DateTime] | conversion all } - { Date[NaiveDate] => DateTime[DateTime] | conversion half } - { UUID[Uuid] => String[String] | conversion half } - { Char[&'r str] => String[String] | conversion none } - // { Time[NaiveTime] => String[String] | conversion half } - } -); +impl_postgres_transport!(BinaryProtocol, NoTls); +impl_postgres_transport!(BinaryProtocol, MakeTlsConnector); +impl_postgres_transport!(CSVProtocol, NoTls); +impl_postgres_transport!(CSVProtocol, MakeTlsConnector); +impl_postgres_transport!(CursorProtocol, NoTls); +impl_postgres_transport!(CursorProtocol, MakeTlsConnector); -impl

TypeConversion for PostgresMemoryTransport

{ +impl TypeConversion for PostgresMemoryTransport { fn convert(val: Uuid) -> String { val.to_string() } } -impl

TypeConversion for PostgresMemoryTransport

{ +impl TypeConversion for PostgresMemoryTransport { fn convert(val: NaiveTime) -> String { val.to_string() } } -impl<'r, P> TypeConversion<&'r str, String> for PostgresMemoryTransport

{ +impl<'r, P, C> TypeConversion<&'r str, String> for PostgresMemoryTransport { fn convert(val: &'r str) -> String { val.to_string() } } -impl

TypeConversion> for PostgresMemoryTransport

{ +impl TypeConversion> for PostgresMemoryTransport { fn convert(val: NaiveDateTime) -> DateTime { DateTime::from_utc(val, Utc) } } -impl

TypeConversion> for PostgresMemoryTransport

{ +impl TypeConversion> for PostgresMemoryTransport { fn convert(val: NaiveDate) -> DateTime { DateTime::from_utc(val.and_hms(0, 0, 0), Utc) } diff --git a/connectorx/tests/test_arrow.rs b/connectorx/tests/test_arrow.rs index ba6ecc4bb0..6578e47f0f 100644 --- a/connectorx/tests/test_arrow.rs +++ b/connectorx/tests/test_arrow.rs @@ -5,11 +5,12 @@ use connectorx::{ prelude::*, sources::{ dummy::DummySource, - postgres::{BinaryProtocol, PostgresSource}, + postgres::{rewrite_tls_args, BinaryProtocol, PostgresSource}, }, sql::CXQuery, transports::{DummyArrowTransport, PostgresArrowTransport}, }; +use postgres::NoTls; use std::env; #[test] @@ -130,9 +131,10 @@ fn test_postgres_arrow() { CXQuery::naked("select * from test_table where test_int < 2"), CXQuery::naked("select * from test_table where test_int >= 2"), ]; - let builder = PostgresSource::new(&dburl, 2).unwrap(); + let (config, _tls) = rewrite_tls_args(&dburl).unwrap(); + let builder = PostgresSource::::new(config, NoTls, 2).unwrap(); let mut destination = ArrowDestination::new(); - let dispatcher = Dispatcher::<_, _, PostgresArrowTransport>::new( + let dispatcher = Dispatcher::<_, _, PostgresArrowTransport>::new( builder, &mut destination, &queries, diff --git a/connectorx/tests/test_polars.rs b/connectorx/tests/test_polars.rs index eadc48195d..60becfe341 100644 --- a/connectorx/tests/test_polars.rs +++ b/connectorx/tests/test_polars.rs @@ -3,12 +3,13 @@ use connectorx::{ prelude::*, sources::{ dummy::DummySource, - postgres::{BinaryProtocol, PostgresSource}, + postgres::{rewrite_tls_args, BinaryProtocol, PostgresSource}, }, sql::CXQuery, transports::{DummyArrowTransport, PostgresArrowTransport}, }; use polars::{df, prelude::*}; +use postgres::NoTls; use std::env; #[test] @@ -58,9 +59,10 @@ fn test_postgres_arrow() { CXQuery::naked("select * from test_table where test_int < 2"), CXQuery::naked("select * from test_table where test_int >= 2"), ]; - let builder = PostgresSource::new(&dburl, 2).unwrap(); + let (config, _tls) = rewrite_tls_args(&dburl).unwrap(); + let builder = PostgresSource::::new(config, NoTls, 2).unwrap(); let mut destination = ArrowDestination::new(); - let dispatcher = Dispatcher::<_, _, PostgresArrowTransport>::new( + let dispatcher = Dispatcher::<_, _, PostgresArrowTransport>::new( builder, &mut destination, &queries, diff --git a/connectorx/tests/test_postgres.rs b/connectorx/tests/test_postgres.rs index 43abfaae1d..da2c2b2810 100644 --- a/connectorx/tests/test_postgres.rs +++ b/connectorx/tests/test_postgres.rs @@ -1,11 +1,12 @@ use connectorx::{ destinations::memory::MemoryDestination, prelude::*, - sources::postgres::{BinaryProtocol, CSVProtocol, PostgresSource}, + sources::postgres::{rewrite_tls_args, BinaryProtocol, CSVProtocol, PostgresSource}, sql::CXQuery, transports::PostgresMemoryTransport, }; use ndarray::array; +use postgres::NoTls; use std::env; #[test] @@ -16,7 +17,8 @@ fn load_and_parse() { #[derive(Debug, PartialEq)] struct Row(i32, Option, Option, Option, Option); - let mut source = PostgresSource::::new(&dburl, 1).unwrap(); + let (config, _tls) = rewrite_tls_args(&dburl).unwrap(); + let mut source = PostgresSource::::new(config, NoTls, 1).unwrap(); source.set_queries(&[CXQuery::naked("select * from test_table")]); source.fetch_metadata().unwrap(); @@ -66,9 +68,10 @@ fn test_postgres() { CXQuery::naked("select * from test_table where test_int < 2"), CXQuery::naked("select * from test_table where test_int >= 2"), ]; - let builder = PostgresSource::new(&dburl, 2).unwrap(); + let (config, _tls) = rewrite_tls_args(&dburl).unwrap(); + let builder = PostgresSource::::new(config, NoTls, 2).unwrap(); let mut destination = MemoryDestination::new(); - let dispatcher = Dispatcher::<_, _, PostgresMemoryTransport>::new( + let dispatcher = Dispatcher::<_, _, PostgresMemoryTransport>::new( builder, &mut destination, &queries, @@ -122,9 +125,10 @@ fn test_postgres_agg() { let queries = [CXQuery::naked( "SELECT test_bool, SUM(test_float) FROM test_table GROUP BY test_bool", )]; - let builder = PostgresSource::new(&dburl, 1).unwrap(); + let (config, _tls) = rewrite_tls_args(&dburl).unwrap(); + let builder = PostgresSource::::new(config, NoTls, 1).unwrap(); let mut destination = MemoryDestination::new(); - let dispatcher = Dispatcher::<_, _, PostgresMemoryTransport>::new( + let dispatcher = Dispatcher::<_, _, PostgresMemoryTransport>::new( builder, &mut destination, &queries, @@ -149,7 +153,8 @@ fn load_and_parse_csv() { #[derive(Debug, PartialEq)] struct Row(i32, Option, Option, Option, Option); - let mut source = PostgresSource::::new(&dburl, 1).unwrap(); + let (config, _tls) = rewrite_tls_args(&dburl).unwrap(); + let mut source = PostgresSource::::new(config, NoTls, 1).unwrap(); source.set_queries(&[CXQuery::naked("select * from test_table")]); source.fetch_metadata().unwrap(); @@ -199,10 +204,12 @@ fn test_postgres_csv() { CXQuery::naked("select * from test_table where test_int < 2"), CXQuery::naked("select * from test_table where test_int >= 2"), ]; - let builder = PostgresSource::::new(&dburl, 2).unwrap(); + let (config, _tls) = rewrite_tls_args(&dburl).unwrap(); + let builder = PostgresSource::::new(config, NoTls, 2).unwrap(); let mut dst = MemoryDestination::new(); - let dispatcher = - Dispatcher::<_, _, PostgresMemoryTransport>::new(builder, &mut dst, &queries); + let dispatcher = Dispatcher::<_, _, PostgresMemoryTransport>::new( + builder, &mut dst, &queries, + ); dispatcher.run().expect("run dispatcher"); assert_eq!(