Switch to uv for Python management (#635)

* Ignore egg files

* Drop all requirements.txt files

* Add pyproject.toml files for subprojects

* Update all requirements.txt refs

* Rename optional deps block to dbt_tests

* Set project dir correctly

* Ignore shellcheck range warnings

* Separate Socrata into its own pyproject.toml

* Ignore uv/python files

* Set working dir for yaml sorter

* Bump Python version to 3.11

* Add all uv.lock files

* Swap Action to use uv

* Include .python-version files

* Install Python explicitly

* Update README to use uv

.github/actions/install_dbt_requirements/action.yaml

@@ -8,16 +8,25 @@ inputs:
 runs:
   using: composite
   steps:
-      - name: Setup python
-        uses: actions/setup-python@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
         with:
-          python-version: '3.11'
-          cache: pip
+          enable-cache: true
+          cache-dependency-glob: |
+            ${{ inputs.dbt_project_dir }}/pyproject.toml
+            ${{ inputs.dbt_project_dir }}/uv.lock
 
-      - name: Install python requirements
-        working-directory: dbt
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: ${{ inputs.dbt_project_dir }}/.python-version
+
+      - name: Install Python dependencies
+        working-directory: ${{ inputs.dbt_project_dir }}
         shell: bash
-        run: python -m pip install .
+        run: uv pip install .
+        env:
+          UV_SYSTEM_PYTHON: 1
 
       - name: Cache dbt requirements
         uses: actions/cache@v4
@@ -26,6 +35,6 @@ runs:
           key: dbt-${{ hashFiles(format('{0}/packages.yml', inputs.dbt_project_dir)) }}
 
       - name: Install dbt requirements
-        run: dbt deps
         working-directory: ${{ inputs.dbt_project_dir }}
         shell: bash
+        run: dbt deps

.github/workflows/check_sort_dbt_yaml_files.yaml

@@ -6,24 +6,34 @@ on:
   push:
     branches: [master]
 
+env:
+  UV_SYSTEM_PYTHON: 1
+
 jobs:
   check-sort-dbt-yaml-files:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v4
 
-    - name: Setup Python
-      uses: actions/setup-python@v4
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+      with:
+        enable-cache: true
+        cache-dependency-glob: |
+          dbt/pyproject.toml
+          dbt/uv.lock
+
+    - name: Install Python
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
-        cache: pip
+        python-version-file: dbt/.python-version
 
     - name: Install Python dependencies
       working-directory: dbt
       shell: bash
-      run: pip install ".[ci_checks]"
+      run: uv pip install ".[ci_checks]"
 
     - name: Run dbt yaml sort checks
+      working-directory: dbt
       run: python scripts/check_sort_dbt_yaml_files.py
-      working-directory: ./dbt

.github/workflows/socrata_upload.yaml

@@ -34,6 +34,9 @@ on:
         description: Chunk Socrata upload by township
         required: false
 
+env:
+  UV_SYSTEM_PYTHON: 1
+
 jobs:
   upload-open-data-assets:
     runs-on: ubuntu-latest
@@ -47,11 +50,18 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Setup Python
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: |
+            socrata/pyproject.toml
+            socrata/uv.lock
+
+      - name: Install Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
-          cache: pip
+          python-version-file: socrata/.python-version
 
       - name: Setup dbt
         uses: ./.github/actions/setup_dbt
@@ -60,10 +70,10 @@ jobs:
 
       # Set these env vars as secrets so they get masked in the GitHub
       # Actions logs
-      - name: Install Socrata deps
+      - name: Install Python dependencies
         working-directory: socrata
         shell: bash
-        run: pip install .
+        run: uv pip install .
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4

.github/workflows/test_dbt_models.yaml

@@ -5,6 +5,9 @@ name: test-dbt-models
 # https://github.com/ccao-data/service-sqoop-iasworld
 on: workflow_dispatch
 
+env:
+  UV_SYSTEM_PYTHON: 1
+
 jobs:
   test-dbt-models:
     runs-on: ubuntu-latest
@@ -22,14 +25,14 @@ jobs:
         with:
           role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }}
 
-      - name: Install Python requirements
+      - name: Install Python dependencies
         working-directory: ${{ env.PROJECT_DIR }}
         shell: bash
-        run: pip install ".[dbt_tests]"
+        run: uv pip install ".[dbt_tests]"
 
       - name: Run tests
         run: |
-          python3 scripts/run_iasworld_data_tests.py \
+          python scripts/run_iasworld_data_tests.py \
             --target "$TARGET" \
             --output-dir ./qc_test_results/
         working-directory: ${{ env.PROJECT_DIR }}

.github/workflows/test_open_data_assets.yaml

@@ -33,7 +33,7 @@ jobs:
         shell: bash
 
       - name: Test that open data row counts match Athena views
-        run: python3 ../.github/scripts/test_open_data_assets.py target/manifest.json
+        run: python ../.github/scripts/test_open_data_assets.py target/manifest.json
         working-directory: ${{ env.PROJECT_DIR }}
         shell: bash
 

.gitignore

@@ -5,14 +5,12 @@
 *.parquet
 *.gzip
 .DS_Store
-.python-version
 .RData
 .Rhistory
 .Rproj.user
 .Ruserdata
 package*.json
 settings.json
-uv.lock
 
 # Directories
 *.egg-info/

dbt/.python-version

@@ -0,0 +1 @@
+3.11

dbt/README.md

@@ -94,7 +94,7 @@ care of that for you.)
 
 #### Requirements
 
-* Python3 with venv installed (`sudo apt install python3-venv`)
+* Python3 with `uv` installed (pre-installed on the CCAO server)
 * [AWS CLI installed
   locally](https://github.com/ccao-data/wiki/blob/master/How-To/Connect-to-AWS-Resources.md)
   * You'll also need permissions for Athena, Glue, and S3
@@ -104,25 +104,25 @@ care of that for you.)
 
 Run the following commands in this directory:
 
-```
-python3 -m venv venv
-source venv/bin/activate
-pip install -U pip
-pip install .
+```bash
+uv venv
+source .venv/bin/activate
+uv python install
+uv pip install .
 dbt deps
 ```
 
 ### Useful commands
 
 To run dbt commands, make sure you have the virtual environment activated:
 
-```
-source venv/bin/activate
+```bash
+source .venv/bin/activate
 ```
 
 You must also authenticate with AWS using MFA if you haven't already today:
 
-```
+```bash
 aws-mfa
 ```
 
@@ -146,14 +146,14 @@ it from scratch.
 Instead, start by copying the production dbt state file (also known as the
 [manifest file](https://docs.getdbt.com/reference/artifacts/manifest-json)):
 
-```
+```bash
 aws s3 cp s3://ccao-dbt-cache-us-east-1/master-cache/manifest.json master-cache/manifest.json
 ```
 
 Then, use [`dbt clone`](https://docs.getdbt.com/reference/commands/clone) to
 clone the production tables and views into your development environment:
 
-```
+```bash
 dbt clone --state master-cache
 ```
 
@@ -697,7 +697,7 @@ a major event in the Valuations calendar like the close of a township.
 
 The [`export_models` script](./scripts/export_models.py) is the foundation for
 our QC reports. The script expects certain Python requirements, which can be installed
-by running `pip install .[dbt_tests]` in a virtual
+by running `uv pip install .[dbt_tests]` in a virtual
 environment.
 
 The script exposes a few options that help to export the right data:
@@ -735,7 +735,7 @@ We run town close reports using the [`scripts/export_qc_town_close_reports.py`
 script](./scripts/export_qc_town_close_reports.py), which builds on top of
 `export_models`. As such, `export_qc_town_close_reports` expects the same set
 of Python requirements as `export_models`, which can be installed in a virtual
-environment by running `pip install .[dbt_tests]`.
+environment by running `uv pip install .[dbt_tests]`.
 
 The script exposes the following options, many of which are the same as
 `export_models`: