Merge branch 'main' into livy_support_spark_adapter

.changes/unreleased/Dependencies-20231219-222211.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Update freezegun requirement from ~=1.3 to ~=1.4"
+time: 2023-12-19T22:22:11.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 966

.changes/unreleased/Dependencies-20240419-024814.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Bump actions/download-artifact from 3 to 4"
+time: 2024-04-19T02:48:14.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 1010

.changes/unreleased/Dependencies-20240419-024818.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Bump actions/upload-artifact from 3 to 4"
+time: 2024-04-19T02:48:18.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 1011

.changes/unreleased/Dependencies-20240419-024820.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Bump dbt-labs/actions from 1.1.0 to 1.1.1"
+time: 2024-04-19T02:48:20.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 1012

.changes/unreleased/Dependencies-20240503-224157.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Update wheel requirement from ~=0.42 to ~=0.43"
+time: 2024-05-03T22:41:57.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 1035

.changes/unreleased/Dependencies-20240718-223145.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Update pytest-xdist requirement from ~=3.5 to ~=3.6"
+time: 2024-07-18T22:31:45.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 1073

.changes/unreleased/Dependencies-20240718-223238.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Loosen pin on pre-commit from 3.7.0 to 3.7"
+time: 2024-07-18T22:32:38.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 1074

.github/dependabot.yml

@@ -5,18 +5,25 @@ updates:
     schedule:
       interval: "daily"
     rebase-strategy: "disabled"
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          - version-update:semver-patch
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "weekly"
     rebase-strategy: "disabled"
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          - version-update:semver-patch
   - package-ecosystem: "docker"
     directory: "/docker"
     schedule:
       interval: "weekly"
     rebase-strategy: "disabled"
-  - package-ecosystem: "docker"
-    directory: "/docker-dev"
-    schedule:
-      interval: "weekly"
-    rebase-strategy: "disabled"
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          - version-update:semver-patch

.github/scripts/update_dev_dependency_branches.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -e
+set -e
+
+
+dbt_adapters_branch=$1
+dbt_core_branch=$2
+dbt_common_branch=$3
+target_req_file="dev-requirements.txt"
+core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g"
+adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g"
+common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g"
+if [[ "$OSTYPE" == darwin* ]]; then
+ # mac ships with a different version of sed that requires a delimiter arg
+ sed -i "" "$adapters_req_sed_pattern" $target_req_file
+ sed -i "" "$core_req_sed_pattern" $target_req_file
+ sed -i "" "$common_req_sed_pattern" $target_req_file
+else
+ sed -i "$adapters_req_sed_pattern" $target_req_file
+ sed -i "$core_req_sed_pattern" $target_req_file
+ sed -i "$common_req_sed_pattern" $target_req_file
+fi

.github/workflows/integration.yml

@@ -10,6 +10,8 @@
 
 name: Adapter Integration Tests
 
+run-name: "${{ (contains(github.event_name, 'workflow_') && inputs.name) || github.event_name }}: ${{ (contains(github.event_name, 'workflow_') &&  inputs.adapter_branch) || github.ref_name }} by @${{ github.actor }}"
+
 on:
   push:
     branches:
@@ -24,10 +26,31 @@ on:
 
   workflow_dispatch:
     inputs:
-      dbt-core-branch:
-        description: "branch of dbt-core to use in dev-requirements.txt"
+      name:
+        description: "Name to associate with run (example: 'dbt-adapters-242')"
         required: false
         type: string
+        default: "Adapter Integration Tests"
+      adapter_branch:
+        description: "The branch of this adapter repository to use"
+        type: string
+        required: false
+        default: "main"
+      dbt_adapters_branch:
+        description: "The branch of dbt-adapters to use"
+        type: string
+        required: false
+        default: "main"
+      dbt_core_branch:
+        description: "The branch of dbt-core to use"
+        type: string
+        required: false
+        default: "main"
+      dbt_common_branch:
+        description: "The branch of dbt-common to use"
+        type: string
+        required: false
+        default: "main"
 
 # explicitly turn off permissions for `GITHUB_TOKEN`
 permissions: read-all
@@ -74,12 +97,19 @@ jobs:
       DBT_TEST_USER_3: "buildbot+dbt_test_user_3@dbtlabs.com"
 
     steps:
-      - name: Check out the repository
-        if: github.event_name != 'pull_request_target'
+      - name: Check out the repository (push)
+        if: github.event_name == 'push'
         uses: actions/checkout@v4
         with:
           persist-credentials: false
 
+      - name: Check out the repository (workflow_dispatch)
+        if: github.event_name == 'workflow_dispatch'
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          ref: ${{ inputs.adapter_branch }}
+
       # explicitly checkout the branch for the PR,
       # this is necessary for the `pull_request` event
       - name: Check out the repository (PR)
@@ -95,17 +125,20 @@ jobs:
         with:
           python-version: "3.11"
 
+      - name: Update Adapters and Core branches (update dev_requirements.txt)
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          ./.github/scripts/update_dev_dependency_branches.sh \
+            ${{ inputs.dbt_adapters_branch }} \
+            ${{ inputs.dbt_core_branch }} \
+            ${{ inputs.dbt_common_branch }}
+          cat dev-requirements.txt
+
       - name: Install python dependencies
         run: |
           python -m pip install --user --upgrade pip
           python -m pip --version
           python -m pip install -r dagger/requirements.txt
 
-      - name: Update dev_requirements.txt
-        if: inputs.dbt-core-branch != ''
-        run: |
-          pip install bumpversion
-          ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}
-
       - name: Run tests for ${{ matrix.test }}
         run: python dagger/run_dbt_spark_tests.py --profile ${{ matrix.test }}

.github/workflows/main.yml

@@ -75,7 +75,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Check out the repository
@@ -104,11 +104,12 @@ jobs:
         id: date
         run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
           path: unit_results.csv
+          overwrite: true
 
   build:
     name: build packages
@@ -153,10 +154,11 @@ jobs:
           if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi
           echo "is_alpha=$is_alpha" >> $GITHUB_OUTPUT
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: dist
           path: dist/
+          overwrite: true
 
   test-build:
     name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }}
@@ -171,7 +173,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-12, windows-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Set up Python ${{ matrix.python-version }}
@@ -184,7 +186,7 @@ jobs:
           python -m pip install --user --upgrade pip
           python -m pip install --upgrade wheel
           python -m pip --version
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: dist
           path: dist/

.github/workflows/release-prep.yml

@@ -123,7 +123,7 @@ jobs:
 
       - name: "Audit Version And Parse Into Parts"
         id: semver
-        uses: dbt-labs/actions/parse-semver@v1.1.0
+        uses: dbt-labs/actions/parse-semver@v1.1.1
         with:
           version: ${{ inputs.version_number }}
 
@@ -448,7 +448,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Check out the repository

.pre-commit-config.yaml

@@ -28,6 +28,7 @@ repos:
         -   --target-version=py39
         -   --target-version=py310
         -   --target-version=py311
+        -   --target-version=py312
         additional_dependencies: [flaky]
 
 -   repo: https://github.com/pycqa/flake8

README.md

@@ -65,6 +65,13 @@ rm -rf ./.hive-metastore/
 rm -rf ./.spark-warehouse/
 ```
 
+#### Additional Configuration for MacOS
+
+If installing on MacOS, use `homebrew` to install required dependencies.
+   ```sh
+   brew install unixodbc
+   ```
+
 ### Reporting bugs and contributing code
 
 -   Want to report a bug or request a feature? Let us know on [Slack](http://slack.getdbt.com/), or open [an issue](https://github.com/fishtown-analytics/dbt-spark/issues/new).

dagger/spark-container/install_spark.sh

@@ -4,7 +4,7 @@ SPARK_VERSION=3.1.3
 HADOOP_VERSION=3.2
 
 apt-get update && \
-apt-get install -y wget netcat procps libpostgresql-jdbc-java && \
+apt-get install -y wget netcat-openbsd procps libpostgresql-jdbc-java && \
 wget -q "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
 tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
 rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \

dev-requirements.txt

@@ -7,20 +7,15 @@ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter
 
 # dev
 ipdb~=0.13.13
-pre-commit==3.7.0;python_version >="3.9"
-pre-commit==3.5.0;python_version <"3.9"
-
-# test
-freezegun~=1.3
-mock~=5.1
+pre-commit~=3.7.0;python_version>="3.9"
+pre-commit~=3.5.0;python_version<"3.9"
 pytest~=7.4
 pytest-csv~=3.0
 pytest-dotenv~=0.5.2
 pytest-logbook~=1.2
-pytest-xdist~=3.5
-thrift_sasl~=0.4.3
+pytest-xdist~=3.6
 
 # build
 bumpversion~=0.6.0
 twine~=4.0
-wheel~=0.42
+wheel~=0.43

docker/spark.Dockerfile

@@ -13,7 +13,7 @@ ENV SPARK_HOME /usr/spark
 ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}"
 
 RUN apt-get update && \
-    apt-get install -y wget netcat procps libpostgresql-jdbc-java && \
+    apt-get install -y wget netcat-openbsd procps libpostgresql-jdbc-java && \
     wget -q "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
     tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
     rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \

requirements.txt

@@ -1,7 +1,7 @@
 pyhive[hive_pure_sasl]~=0.7.0
 requests>=2.28.1
 
-pyodbc~=4.0.39 --no-binary pyodbc
+pyodbc~=5.1.0 --no-binary pyodbc
 sqlparams>=3.0.0
 thrift>=0.13.0
 pyspark>=3.0.0,<4.0.0

setup.py

@@ -44,7 +44,7 @@ def _get_plugin_version_dict():
 package_version = "1.9.0a1"
 description = """The Apache Spark adapter plugin for dbt"""
 
-odbc_extras = ["pyodbc~=4.0.39"]
+odbc_extras = ["pyodbc~=5.1.0"]
 pyhive_extras = [
     "PyHive[hive_pure_sasl]~=0.7.0",
     "thrift>=0.11.0,<0.17.0",
@@ -87,6 +87,7 @@ def _get_plugin_version_dict():
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     python_requires=">=3.8",
 )

tests/functional/adapter/dbt_show/test_dbt_show.py

@@ -0,0 +1,22 @@
+import pytest
+
+from dbt.tests.adapter.dbt_show.test_dbt_show import (
+    BaseShowSqlHeader,
+    BaseShowLimit,
+    BaseShowDoesNotHandleDoubleLimit,
+)
+
+
+class TestSparkShowLimit(BaseShowLimit):
+    pass
+
+
+class TestSparkShowSqlHeader(BaseShowSqlHeader):
+    pass
+
+
+@pytest.mark.skip_profile("apache_spark", "spark_session", "databricks_http_cluster")
+class TestSparkShowDoesNotHandleDoubleLimit(BaseShowDoesNotHandleDoubleLimit):
+    """The syntax message is quite variable across clusters, but this hits two at once."""
+
+    DATABASE_ERROR_MESSAGE = "limit"