chore(Python): Centralize Python dependencies in pyproject.toml

.github/workflows/benchmark.yml

@@ -41,9 +41,21 @@ jobs:
           python -m venv .venv
           echo "$GITHUB_WORKSPACE/py-polars/.venv/bin" >> $GITHUB_PATH
 
+      - name: Set up Rust
+        run: rustup show
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: py-polars
+          save-if: ${{ github.ref_name == 'main' }}
+
       - name: Install Python dependencies
         working-directory: py-polars
-        run: pip install -r requirements-dev.txt
+        run: |
+          source activate
+          pip install maturin==1.0.1
+          maturin develop --extras dev-test,pyarrow,pandas,fsspec,timezone,connectorx,adbc,numpy,hypothesis,xlsxwriter,xlsx2csv,deltalake,pydantic
 
       - name: Load benchmark data from cache
         id: cache-data

.github/workflows/docs-python.yml

@@ -30,11 +30,28 @@ jobs:
         with:
           python-version: '3.11'
           cache: pip
-          cache-dependency-path: py-polars/docs/requirements-docs.txt
+          cache-dependency-path: py-polars/pyproject.toml
+
+      - name: Create virtual environment
+        run: |
+          python -m venv .venv
+          echo "$GITHUB_WORKSPACE/py-polars/.venv/bin" >> $GITHUB_PATH
+
+      - name: Set up Rust
+        run: rustup show
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: py-polars
+          save-if: ${{ github.ref_name == 'main' }}
 
       - name: Install Python dependencies
-        working-directory: py-polars/docs
-        run: pip install -r requirements-docs.txt
+        working-directory: py-polars
+        run: |
+          source $GITHUB_WORKSPACE/py-polars/.venv/Scripts/activate
+          pip install maturin==1.0.1
+          maturin develop --extras dev-docs
 
       - name: Build Python documentation
         working-directory: py-polars/docs

.github/workflows/lint-python.yml

@@ -25,8 +25,25 @@ jobs:
         with:
           python-version: '3.11'
 
+      - name: Create virtual environment
+        run: |
+          python -m venv .venv
+          echo "$GITHUB_WORKSPACE/py-polars/.venv/bin" >> $GITHUB_PATH
+
+      - name: Set up Rust
+        run: rustup show
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: py-polars
+          save-if: ${{ github.ref_name == 'main' }}
+
       - name: Install Python dependencies
-        run: pip install -r requirements-lint.txt
+        run: |
+          source activate
+          pip install maturin==1.0.1
+          maturin develop --extras dev-lint
 
       - name: Lint Python
         run: |
@@ -52,10 +69,25 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Create virtual environment
+        run: |
+          python -m venv .venv
+          echo "$GITHUB_WORKSPACE/py-polars/.venv/bin" >> $GITHUB_PATH
+
+      - name: Set up Rust
+        run: rustup show
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: py-polars
+          save-if: ${{ github.ref_name == 'main' }}
+
       - name: Install dependencies
         run: |
-          pip install -r requirements-dev.txt
-          pip install -r requirements-lint.txt
+          source activate
+          pip install maturin==1.0.1
+          maturin develop --extras dev-test,dev-lint,pyarrow,pandas,fsspec,timezone,connectorx,adbc,numpy,hypothesis,xlsxwriter,xlsx2csv,deltalake,pydantic
 
       # Allow untyped calls for older Python versions
       - name: Run mypy

.github/workflows/test-python.yml

@@ -46,9 +46,6 @@ jobs:
           python -m venv .venv
           echo "$GITHUB_WORKSPACE/py-polars/.venv/bin" >> $GITHUB_PATH
 
-      - name: Install dependencies
-        run: pip install -r requirements-dev.txt
-
       - name: Set up Rust
         run: rustup show
 
@@ -61,7 +58,9 @@ jobs:
       - name: Install Polars
         run: |
           source activate
-          maturin develop
+          pip install maturin==1.0.1
+          # the list of tags should be in sync with the Makefile
+          maturin develop --extras dev-test,pyarrow,pandas,fsspec,timezone,connectorx,adbc,numpy,hypothesis,xlsxwriter,xlsx2csv,deltalake,pydantic
 
       - name: Run tests and report coverage
         if: github.ref_name != 'main'
@@ -105,9 +104,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install Python dependencies
-        run: pip install -r requirements-dev.txt
-
       - name: Set up Rust
         run: rustup show
 
@@ -120,8 +116,9 @@ jobs:
       - name: Install Polars
         shell: bash
         run: |
-          maturin build
-          pip install target/wheels/polars-*.whl
+          source activate
+          pip install maturin==1.0.1
+          maturin develop --extras dev-test,pyarrow,pandas,fsspec,timezone,connectorx,adbc,numpy,hypothesis,xlsxwriter,xlsx2csv,deltalake,pydantic
 
       - name: Run tests
         if: github.ref_name != 'main'

py-polars/Makefile

@@ -17,20 +17,23 @@ endif
 .PHONY: requirements
 requirements: .venv  ## Install/refresh all project requirements
 	$(VENV_BIN)/python -m pip install --upgrade pip
-	$(VENV_BIN)/pip install -r requirements-dev.txt
-	$(VENV_BIN)/pip install -r requirements-lint.txt
-	$(VENV_BIN)/pip install -r docs/requirements-docs.txt
+	$(VENV_BIN)/python -m pip install maturin==1.0.1
 
 .PHONY: build
 build: .venv  ## Compile and install Polars for development
-	@unset CONDA_PREFIX && source $(VENV_BIN)/activate && maturin develop
+	# NOTE: ideally, we can use the all tag available in pyproject.toml, but that has
+	# the form of "polars[pandas,...]". Pip incorrectly parses that as using the released
+	# polars version, rather than self referencing. So we avoid the all tag here.
+	# @unset CONDA_PREFIX && source $(VENV_BIN)/activate && maturin develop --extras all,dev-test,dev-lint,dev-docs
+	@unset CONDA_PREFIX && source $(VENV_BIN)/activate && maturin develop --extras pyarrow,pandas,fsspec,timezone,connectorx,adbc,numpy,hypothesis,xlsxwriter,xlsx2csv,deltalake,pydantic,dev-test,dev-lint,dev-docs
+
 
 .PHONY: build-release
 build-release: .venv  ## Compile and install a faster Polars binary
 	@unset CONDA_PREFIX && source $(VENV_BIN)/activate && maturin develop --release
 
 .PHONY: fmt
-fmt: .venv  ## Run autoformatting and linting
+fmt: .venv build ## Run autoformatting and linting
 	$(VENV_BIN)/ruff .
 	$(VENV_BIN)/black .
 	$(VENV_BIN)/blackdoc .

py-polars/bla.txt

@@ -0,0 +1 @@
+hello world, oh no, polars

py-polars/polars/dataframe/frame.py

@@ -3167,12 +3167,12 @@ def write_database(
                     "'sqlalchemy' not found. Install polars with 'pip install polars[sqlalchemy]'."
                 ) from exc
 
-            engine = create_engine(connection_uri)
+            engine_sa = create_engine(connection_uri)
 
             # this conversion to pandas as zero-copy
             # so we can utilize their sql utils for free
             self.to_pandas(use_pyarrow_extension_array=True).to_sql(
-                name=table_name, con=engine, if_exists=if_exists, index=False
+                name=table_name, con=engine_sa, if_exists=if_exists, index=False
             )
 
         else:

py-polars/polars/utils/show_versions.py

@@ -67,7 +67,10 @@ def _get_dependency_info() -> dict[str, str]:
         "matplotlib",
         "xlsx2csv",
         "xlsxwriter",
+        "sqlalchemy",
+        "pydantic",
     ]
+
     return {f"{name}:": _get_dependency_version(name) for name in opt_deps}
 
 

py-polars/pyproject.toml

@@ -45,15 +45,54 @@ pyarrow = ["pyarrow>=7.0.0"]
 pandas = ["pyarrow>=7.0.0", "pandas"]
 numpy = ["numpy >= 1.16.0"]
 fsspec = ["fsspec"]
-connectorx = ["connectorx"]
+connectorx = [
+  "connectorx >= 0.3.2a5; python_version >= '3.8'",
+] # Latest full release is broken - unpin when 0.3.2 released
 xlsx2csv = ["xlsx2csv >= 0.8.0"]
-deltalake = ["deltalake >= 0.8.0"]
+deltalake = ["deltalake >= 0.10.0"]
 timezone = ["backports.zoneinfo; python_version < '3.9'", "tzdata; platform_system == 'Windows'"]
 matplotlib = ["matplotlib"]
 sqlalchemy = ["sqlalchemy", "pandas"]
 xlsxwriter = ["xlsxwriter"]
+pydantic = ["pydantic"]
+hypothesis = ["hypothesis==6.75.1"]
+adbc = ["adbc_driver_sqlite; python_version >= '3.9' and platform_system != 'Windows'"]
 all = [
-  "polars[pyarrow,pandas,numpy,fsspec,connectorx,xlsx2csv,deltalake,timezone,matplotlib,sqlalchemy,xlsxwriter]",
+  "polars[pyarrow,pandas,numpy,fsspec,connectorx,xlsx2csv,deltalake,timezone,matplotlib,sqlalchemy,adbc,xlsxwriter,pydantic]",
+]
+
+# The tags below are for development of Polars. We pin precise versions here, as we have full control on this
+dev-test = [
+  "pytest==7.3.0",
+  "pytest-cov==4.1.0",
+  "pytest-xdist==3.3.1",
+]
+dev-lint = [
+  "black==23.3.0",
+  "blackdoc==0.3.8",
+  "mypy==1.3.0",
+  "ruff==0.0.270",
+  "typos==1.15",
+  "pandas-stubs==1.2.0.62",
+]
+dev-docs = [
+  "autodocsumm==0.2.10",
+  "commonmark==0.9.1",
+  "livereload==2.6.3",
+  "numpydoc==1.5.0",
+  "pydata-sphinx-theme==0.13.3",
+  "sphinx==6.2.1",
+  "sphinx-autosummary-accessors==2023.4.0",
+  "sphinx-copybutton==0.5.1",
+  "sphinx-design==0.4.1",
+  "sphinx-favicon==1.0.1",
+  "sphinxcontrib-applehelp==1.0.4",
+  "sphinxcontrib-devhelp==1.0.2",
+  "sphinxcontrib-htmlhelp==2.0.1",
+  "sphinxcontrib-jsmath==1.0.1",
+  "sphinxcontrib-napoleon==0.7",
+  "sphinxcontrib-qthelp==1.0.3",
+  "sphinxcontrib-serializinghtml==1.1.5",
 ]
 
 [tool.mypy]