From 77eb0394cfcc45641ad272ec0afed98e5018ad26 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Thu, 25 May 2023 15:28:56 +0800 Subject: [PATCH] feat: Support python binding (#112) --- .github/workflows/ci.yaml | 7 ++ .gitignore | 1 + Cargo.toml | 1 + bindings/python/.gitignore | 75 +++++++++++++++++++ bindings/python/Cargo.toml | 36 +++++++++ bindings/python/README.md | 53 +++++++++++++ bindings/python/pyproject.toml | 54 +++++++++++++ .../python/python/databend_driver/__init__.py | 3 + .../python/databend_driver/__init__.pyi | 4 + bindings/python/src/asyncio.rs | 51 +++++++++++++ bindings/python/src/lib.rs | 59 +++++++++++++++ bindings/python/test.sh | 12 +++ bindings/python/tests/binding.feature | 22 ++++++ bindings/python/tests/steps/binding.py | 37 +++++++++ driver/src/flight_sql.rs | 5 +- driver/src/lib.rs | 1 - tests/Makefile | 5 ++ 17 files changed, 423 insertions(+), 3 deletions(-) create mode 100644 bindings/python/.gitignore create mode 100644 bindings/python/Cargo.toml create mode 100644 bindings/python/README.md create mode 100644 bindings/python/pyproject.toml create mode 100644 bindings/python/python/databend_driver/__init__.py create mode 100644 bindings/python/python/databend_driver/__init__.pyi create mode 100644 bindings/python/src/asyncio.rs create mode 100644 bindings/python/src/lib.rs create mode 100755 bindings/python/test.sh create mode 100644 bindings/python/tests/binding.feature create mode 100644 bindings/python/tests/steps/binding.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 910d29a4c..7e5a7f838 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,6 +36,12 @@ jobs: target: x86_64-pc-windows-msvc steps: - uses: actions/checkout@v3 + + - name: Setup Python-3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - uses: ./.github/actions/setup with: cache-key: build @@ -61,4 +67,5 @@ jobs: - run: make -C tests test-core - run: make -C tests test-driver - run: make -C tests test-bendsql + - run: make -C tests test-bindings-python - run: sudo chown -R runner ~/.cargo/registry diff --git a/.gitignore b/.gitignore index 8d4453113..cf5585c2a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .vscode +.idea target/ Cargo.lock diff --git a/Cargo.toml b/Cargo.toml index 9ff0f9e9b..fa66c7fb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,4 +3,5 @@ members = [ "core", "driver", "cli", + "bindings/python" ] diff --git a/bindings/python/.gitignore b/bindings/python/.gitignore new file mode 100644 index 000000000..650369827 --- /dev/null +++ b/bindings/python/.gitignore @@ -0,0 +1,75 @@ +/target + +# Byte-compiled / optimized / DLL files +__pycache__/ +.pytest_cache/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +.venv/ +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +include/ +man/ +venv/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt +pip-selfcheck.json + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +.DS_Store + +# Sphinx documentation +docs/_build/ + +# PyCharm +.idea/ + +# VSCode +.vscode/ + +# Pyenv +.python-version + +# Generated docs +docs diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml new file mode 100644 index 000000000..06f36b00c --- /dev/null +++ b/bindings/python/Cargo.toml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "databend-python" +version = "0.0.1" +edition = "2021" +license = "Apache-2.0" +publish = false + +[lib] +crate-type = ["cdylib"] +doc = false + +[dependencies] +chrono = { version = "0.4.24", default-features = false, features = ["std"] } +futures = "0.3.28" +databend-driver = { path = "../../driver", version = "0.2.20", features = ["rustls", "flight-sql"] } +databend-client = { version = "0.1.15", path = "../../core" } +pyo3 = { version = "0.18", features = ["abi3-py37"] } +pyo3-asyncio = { version = "0.18", features = ["tokio-runtime"] } +tokio = "1" diff --git a/bindings/python/README.md b/bindings/python/README.md new file mode 100644 index 000000000..d792020ac --- /dev/null +++ b/bindings/python/README.md @@ -0,0 +1,53 @@ +## databend-driver + +### Build + +```shell +cd bindings/python +maturin develop +``` + +## Usage + +```python +import databend_driver +import asyncio +async def main(): + s = databend_driver.AsyncDatabendDriver('databend+http://root:root@localhost:8000/?sslmode=disable') + await s.exec("CREATE TABLE if not exists test_upload (x Int32,y VARCHAR)") + +asyncio.run(main()) +``` + +## Development + +Setup virtualenv: + +```shell +python -m venv venv +``` + +Activate venv: + +```shell +source venv/bin/activate +```` + +Install `maturin`: + +```shell +pip install maturin[patchelf] +``` + +Build bindings: + +```shell +maturin develop +``` + +Run some tests: + +```shell +maturin develop -E test +behave tests +``` \ No newline at end of file diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml new file mode 100644 index 000000000..d7acdaab4 --- /dev/null +++ b/bindings/python/pyproject.toml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +build-backend = "maturin" +requires = ["maturin>=0.14.16,<0.15"] + +[project] +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +description = "Databend Driver Python Binding" +license = { text = "Apache-2.0" } +name = "databend-driver" +readme = "README.md" +requires-python = ">=3.7" + +[project.optional-dependencies] +benchmark = [ + "gevent", + "greenify", + "greenlet", + "boto3", + "pydantic", + "boto3-stubs[essential]", +] +docs = ["pdoc"] +test = ["behave"] + +[project.urls] +Documentation = "" +Homepage = "" +Repository = "https://github.com/datafuselabs/bendsql" + +[tool.maturin] +features = ["pyo3/extension-module"] +module-name = "databend_driver._databend_driver" +python-source = "python" diff --git a/bindings/python/python/databend_driver/__init__.py b/bindings/python/python/databend_driver/__init__.py new file mode 100644 index 000000000..66ffa5d7a --- /dev/null +++ b/bindings/python/python/databend_driver/__init__.py @@ -0,0 +1,3 @@ +from ._databend_driver import * + +__all__ = _databend_driver.__all__ \ No newline at end of file diff --git a/bindings/python/python/databend_driver/__init__.pyi b/bindings/python/python/databend_driver/__init__.pyi new file mode 100644 index 000000000..ca4776896 --- /dev/null +++ b/bindings/python/python/databend_driver/__init__.pyi @@ -0,0 +1,4 @@ +class AsyncDatabendDriver: + def __init__(self, dsn: str): ... + + async def exec(self, sql: str) -> int: ... diff --git a/bindings/python/src/asyncio.rs b/bindings/python/src/asyncio.rs new file mode 100644 index 000000000..c3b3cbc52 --- /dev/null +++ b/bindings/python/src/asyncio.rs @@ -0,0 +1,51 @@ +// Copyright 2023 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use pyo3::prelude::*; + +use pyo3_asyncio::tokio::future_into_py; + +use crate::{build_connector, Connector}; + +/// `AsyncDatabendDriver` is the entry for all public async API +#[pyclass(module = "databend_driver")] +pub struct AsyncDatabendDriver(Connector); + +#[pymethods] +impl AsyncDatabendDriver { + #[new] + #[pyo3(signature = (dsn))] + pub fn new(dsn: &str) -> PyResult { + Ok(AsyncDatabendDriver(build_connector(dsn)?)) + } + + /// exec + pub fn exec<'p>(&'p self, py: Python<'p>, sql: String) -> PyResult<&'p PyAny> { + let this = self.0.clone(); + future_into_py(py, async move { + let res = this.connector.exec(&sql).await.unwrap(); + Ok(res) + }) + } + + pub fn query_row<'p>(&'p self, py: Python<'p>, sql: String) -> PyResult<&'p PyAny> { + let this = self.0.clone(); + future_into_py(py, async move { + let row = this.connector.query_row(&sql).await.unwrap(); + let row = row.unwrap(); + let res = row.is_empty(); + Ok(res) + }) + } +} diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs new file mode 100644 index 000000000..7b142e20a --- /dev/null +++ b/bindings/python/src/lib.rs @@ -0,0 +1,59 @@ +// Copyright 2023 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod asyncio; + +use crate::asyncio::*; + +use databend_driver::{new_connection, Connection}; + +use pyo3::create_exception; +use pyo3::exceptions::PyException; +use pyo3::prelude::*; +use std::sync::Arc; +create_exception!( + databend_client, + Error, + PyException, + "databend_client related errors" +); + +#[derive(Clone)] +pub struct Connector { + pub connector: FusedConnector, +} + +pub type FusedConnector = Arc; + +// For bindings +impl Connector { + pub fn new_connector(dsn: &str) -> Result, Error> { + let conn = new_connection(dsn).unwrap(); + let r = Self { + connector: FusedConnector::from(conn), + }; + Ok(Box::new(r)) + } +} + +fn build_connector(dsn: &str) -> PyResult { + let conn = Connector::new_connector(dsn).unwrap(); + Ok(*conn) +} + +#[pymodule] +fn _databend_driver(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} diff --git a/bindings/python/test.sh b/bindings/python/test.sh new file mode 100755 index 000000000..e67f97e0e --- /dev/null +++ b/bindings/python/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + + +python -m venv venv +source venv/bin/activate +pip install maturin +pip install behave +maturin develop + +behave tests \ No newline at end of file diff --git a/bindings/python/tests/binding.feature b/bindings/python/tests/binding.feature new file mode 100644 index 000000000..5f4696ed0 --- /dev/null +++ b/bindings/python/tests/binding.feature @@ -0,0 +1,22 @@ +# Copyright 2023 Datafuse Labs. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +Feature: Databend-Driver Binding + + Scenario: Databend-Driver Async Operations + Given A new Databend-Driver Async Connector + When Async exec "CREATE TABLE if not exists test_data (x Int32,y VARCHAR)" + When Async exec "INSERT INTO test_data(x,y) VALUES(1,'xx')" + Then The select "SELECT * FROM test_data" should run diff --git a/bindings/python/tests/steps/binding.py b/bindings/python/tests/steps/binding.py new file mode 100644 index 000000000..5eeba945c --- /dev/null +++ b/bindings/python/tests/steps/binding.py @@ -0,0 +1,37 @@ +# Copyright 2023 Datafuse Labs. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +from behave import given, when, then +from behave.api.async_step import async_run_until_complete +import databend_driver + + +@given("A new Databend-Driver Async Connector") +@async_run_until_complete +async def step_impl(context): + dsn = os.getenv("TEST_DATABEND_DSN", "databend+http://root:root@localhost:8000/?sslmode=disable") + context.ad = databend_driver.AsyncDatabendDriver(dsn) + + +@when('Async exec "{sql}"') +@async_run_until_complete +async def step_impl(context, sql): + await context.ad.exec(sql) + + +@then('The select "{select_sql}" should run') +@async_run_until_complete +async def step_impl(context, select_sql): + await context.ad.exec(select_sql) diff --git a/driver/src/flight_sql.rs b/driver/src/flight_sql.rs index 7e29b94d4..a52bab889 100644 --- a/driver/src/flight_sql.rs +++ b/driver/src/flight_sql.rs @@ -154,7 +154,7 @@ impl FlightSQLConnection { } } -#[derive(Clone)] +#[derive(Clone, Debug)] struct Args { uri: String, host: String, @@ -167,7 +167,8 @@ struct Args { tls: bool, connect_timeout: Duration, query_timeout: Duration, - tcp_nodelay: bool, // Disable Nagle's Algorithm since we don't want packets to wait + tcp_nodelay: bool, + // Disable Nagle's Algorithm since we don't want packets to wait tcp_keepalive: Option, http2_keep_alive_interval: Duration, keep_alive_timeout: Duration, diff --git a/driver/src/lib.rs b/driver/src/lib.rs index 62d0b2ff7..c9346bf92 100644 --- a/driver/src/lib.rs +++ b/driver/src/lib.rs @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - mod conn; mod error; #[cfg(feature = "flight-sql")] diff --git a/tests/Makefile b/tests/Makefile index 2a737616c..5e061f0da 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -21,6 +21,11 @@ test-bendsql: up cd .. && ./cli/test.sh http cd .. && ./cli/test.sh flight +test-bindings-python: up + TEST_DATABEND_DSN=databend+http://root:@localhost:8000/?sslmode=disable + cd ../bindings/python && ./test.sh + cd - + down: docker compose down