fix tests

sdks/python/.github/workflows/python_ci.yaml → .github/workflows/python_ci.yaml

@@ -20,13 +20,13 @@ jobs:
         uses: psf/black@stable
         with:
           options: "--check --verbose"
-          src: "buildflow"
+          src: "sdks/python"
           version: "~= 23.0"
           checkName: "test_and_lint" # NOTE: this needs to be the same as the job name
       - name: Run ruff checks
         uses: chartboost/ruff-action@v1
         with:
-          src: "buildflow"
+          src: "sdks/python"
           version: 0.0.265
           checkName: "test_and_lint"
         env:
@@ -50,11 +50,15 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
+          cd sdks/python
+          pip install poetry
           poetry install --all-extras
       - name: Test with pytest
         run: |
-          pytest --disable-warnings --cov
+          cd sdks/python
+          poetry run pytest --disable-warnings --cov
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
-        continue-on-error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.pre-commit-config.yaml

@@ -11,16 +11,14 @@ repos:
       - id: debug-statements
       - id: no-commit-to-branch
         args: ["--branch", "main"]
-  - repo: https://github.com/psf/black
-    rev: 23.3.0
-    hooks:
-      - id: black
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.265
+    rev: v0.1.14
     hooks:
       - id: ruff
+        args: ["--fix", "sdks/python"]
   - repo: https://github.com/pycqa/isort
     rev: 5.12.0
     hooks:
       - id: isort
         name: isort (python)
+        args: ["sdks/python"]

sdks/python/pyproject.toml

@@ -13,6 +13,13 @@ google-cloud-pubsub = { version = "*", extras = ["gcp"] }
 google-cloud-pubsublite = { version = "*", extras = ["gcp"] }
 cloud-sql-python-connector = { version = "*", extras = ["gcp"] }
 boto3 = { version = "*", extras = ["aws"] }
+pytest = {version = "^7.4.4", extras = ["dev"]}
+pytest-cov = {version = "^4.1.0", extras = ["dev"]}
+pg8000 = {version = "^1.30.4", extras = ["dev"]}
+python-tds = {version = "^1.14.0", extras = ["dev"]}
+pymysql = {version = "^1.1.0", extras = ["dev"]}
+sqlalchemy = {extras = ["dev"], version = "^2.0.25"}
+sqlalchemy-pytds = {version = "^1.0.0", extras = ["dev"]}
 
 
 [build-system]

sdks/python/terrabridge/gcp/cloud_sql.py

@@ -2,27 +2,37 @@
 
 from terrabridge.gcp.base import GCPResource
 
+try:
+    from google.cloud.sql.connector import Connector, IPTypes, create_async_connector
+except ImportError:
+    Connector = None
+    IPTypes = None
+    create_async_connector = None
 try:
     import asyncpg
+except ImportError:
+    asyncpg = None
+try:
     import pg8000
+except ImportError:
+    pg8000 = None
+try:
     import pymysql
+except ImportError:
+    pymysql = None
+try:
     import pytds
-    from google.cloud.sql.connector import Connector, IPTypes, create_async_connector
+except ImportError:
+    pytds = None
+try:
     from sqlalchemy import Engine, create_engine, engine
     from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
 except ImportError:
-    asyncpg = None
     create_engine = None
     Engine = None
     engine = None
-    Connector = None
-    IPTypes = None
-    create_async_connector = None
-    pytds = None
-    pg8000 = None
     create_async_engine = None
     AsyncEngine = None
-    pymysql = None
 
 
 class CloudSQLInstance(GCPResource):

sdks/python/terrabridge/gcp/gcs_bucket.py

@@ -39,7 +39,8 @@ def bucket(self) -> storage.Bucket:
         """
         if storage is None:
             raise ImportError(
-                "google-cloud-storage is not installed. Please install it with `pip install terrabridge[gcp]`."
+                "google-cloud-storage is not installed. "
+                "Please install it with `pip install terrabridge[gcp]`."
             )
         if self._client is None:
             self._client = storage.Client()

sdks/python/terrabridge/gcp/pubsub.py

@@ -34,7 +34,8 @@ def __init__(self, resource_name: str, *, state_file: str) -> None:
     def publish(self, message: bytes, ordering_key: str = "", **attributes):
         if pubsub_v1 is None:
             raise ImportError(
-                "google-cloud-pubsub is not installed. Please install it with `pip install terrabridge[gcp]`."
+                "google-cloud-pubsub is not installed. "
+                "Please install it with `pip install terrabridge[gcp]`."
             )
         if self._publisher is None:
             self._publisher = pubsub_v1.PublisherClient()

sdks/python/terrabridge/gcp/secret_manager.py

@@ -38,7 +38,8 @@ def version(self, version: str = "latest") -> bytes:
         """
         if secretmanager is None:
             raise ImportError(
-                "google-cloud-secret-manager is not installed. Please install it with `pip install terrabridge[gcp]`."
+                "google-cloud-secret-manager is not installed. "
+                "Please install it with `pip install terrabridge[gcp]`."
             )
         if self._client is None:
             self._client = secretmanager.SecretManagerServiceClient()

sdks/python/tests/gcp/bigquery_test.py

@@ -8,9 +8,8 @@ def test_bigquery_dataset():
 
     assert dataset.project == "terrabridge-testing"
     assert dataset.dataset_id == "terrabridge_testing_dataset"
-    assert (
-        dataset.id
-        == "projects/terrabridge-testing/datasets/terrabridge_testing_dataset"
+    assert dataset.id == (
+        "projects/terrabridge-testing/" "datasets/terrabridge_testing_dataset"
     )
 
 
@@ -22,9 +21,9 @@ def test_bigquery_table():
     assert table.project == "terrabridge-testing"
     assert table.dataset_id == "terrabridge_testing_dataset"
     assert table.table_id == "terrabridge-testing-table"
-    assert (
-        table.id
-        == "projects/terrabridge-testing/datasets/terrabridge_testing_dataset/tables/terrabridge-testing-table"
+    assert table.id == (
+        "projects/terrabridge-testing/datasets/"
+        "terrabridge_testing_dataset/tables/terrabridge-testing-table"
     )
     assert table.dataset.project == "terrabridge-testing"
 

sdks/python/tests/gcp/bigtable_test.py

@@ -8,9 +8,8 @@ def test_bigtable_instance():
 
     assert instance.project == "terrabridge-testing"
     assert instance.name == "terrabridge-bigtable-instance"
-    assert (
-        instance.id
-        == "projects/terrabridge-testing/instances/terrabridge-bigtable-instance"
+    assert instance.id == (
+        "projects/terrabridge-testing/" "instances/terrabridge-bigtable-instance"
     )
 
 
@@ -22,8 +21,8 @@ def test_bigtable_table():
     assert table.project == "terrabridge-testing"
     assert table.instance_name == "terrabridge-bigtable-instance"
     assert table.name == "tf-table"
-    assert (
-        table.id
-        == "projects/terrabridge-testing/instances/terrabridge-bigtable-instance/tables/tf-table"
+    assert table.id == (
+        "projects/terrabridge-testing/instances"
+        "/terrabridge-bigtable-instance/tables/tf-table"
     )
     assert table.instance.project == "terrabridge-testing"

sdks/python/tests/gcp/cloud_sql_test.py

@@ -1,4 +1,4 @@
-from mock import ANY, patch
+from unittest.mock import ANY, patch
 
 from terrabridge.gcp.cloud_sql import CloudSQLDatabase, CloudSQLInstance, CloudSQLUser
 

sdks/python/tests/gcp/cloud_tasks_test.py

@@ -8,7 +8,7 @@ def test_cloud_tasks_queue():
 
     assert queue.project == "terrabridge-testing"
     assert queue.name == "cloud-tasks-queue-test"
-    assert (
-        queue.id
-        == "projects/terrabridge-testing/locations/us-central1/queues/cloud-tasks-queue-test"
+    assert queue.id == (
+        "projects/terrabridge-testing/locations/"
+        "us-central1/queues/cloud-tasks-queue-test"
     )

sdks/python/tests/gcp/gcs_bucket_test.py

@@ -1,4 +1,4 @@
-from mock import patch
+from unittest.mock import patch
 
 from terrabridge.gcp import GCSBucket
 

sdks/python/tests/gcp/pubsub_lite_test.py

@@ -8,9 +8,9 @@ def test_pubsub_lite_topic():
 
     assert topic.project == "terrabridge-testing"
     assert topic.name == "example-lite-topic"
-    assert (
-        topic.id
-        == "projects/terrabridge-testing/locations/us-central1-a/topics/example-lite-topic"
+    assert topic.id == (
+        "projects/terrabridge-testing/locations"
+        "/us-central1-a/topics/example-lite-topic"
     )
 
 
@@ -21,7 +21,7 @@ def test_pubsub_lite_subscription():
 
     assert subscription.project == "terrabridge-testing"
     assert subscription.name == "example-lite-subscription"
-    assert (
-        subscription.id
-        == "projects/terrabridge-testing/locations/us-central1-a/subscriptions/example-lite-subscription"
+    assert subscription.id == (
+        "projects/terrabridge-testing/locations/"
+        "us-central1-a/subscriptions/example-lite-subscription"
     )

sdks/python/tests/gcp/secret_manager_test.py

@@ -1,4 +1,4 @@
-from mock import patch
+from unittest.mock import patch
 
 from terrabridge.gcp.secret_manager import SecretManagerSecret