make sure module parsing works

.gitignore

@@ -163,6 +163,6 @@ cython_debug/
 .terraform.lock.hcl
 terraform.tfstate
 terraform.tfstate.backup
-!sdks/python/tests/data/terraform.tfstate
+!sdks/python/tests/data/gcp/terraform.tfstate
 testing/
 coverage.xml

docs/source/index.rst

@@ -14,7 +14,6 @@ terrabridge
     gcp-resources/pubsub
     gcp-resources/secret_manager
 
-
 |license| |CI| |Python version| |codecov|
 
 .. |license| image:: https://img.shields.io/pypi/l/terrabridge.svg

sdks/python/examples/gcp/postgres/main.tf

@@ -11,23 +11,19 @@ provider "google" {
 
 resource "google_sql_database_instance" "postgres_sql_instance" {
     name = "terrabridge-testing-instance-mysql"
-    project = var.gcp_project_id
     database_version = "POSTGRES_15"
-    region = "us-central1"
     settings {
         tier = "db-custom-1-3840"
     }
 }
 
 resource "google_sql_database" "postgres_database" {
     name = "terrabridge-testing-database"
-    project = var.gcp_project_id
     instance = google_sql_database_instance.postgres_sql_instance.name
 }
 
 resource "google_sql_user" "postgres_user" {
     name = "terrabridge-testing-user"
-    project = var.gcp_project_id
     instance = google_sql_database_instance.postgres_sql_instance.name
     password = "terrabridge-testing-password"
 }

sdks/python/pyproject.toml

@@ -28,6 +28,7 @@ gcsfs = "*"
 s3fs = "*"
 fsspec = "*"
 asyncpg = { version = "^0.29.0", extras = ["dev"] }
+boto3-stubs = {extras = ["aws"], version = "^1.34.25"}
 
 
 [build-system]

sdks/python/terrabridge/base.py

@@ -21,13 +21,11 @@ def __init__(
             )
         self.resource_name = resource_name
         resource = get_resource(
-            resource_name, module_name, state_file or terrabridge.state_file
+            resource_name,
+            module_name,
+            state_file or terrabridge.state_file,
+            self._terraform_type,
         )
-        if resource["type"] != self._terraform_type:
-            raise ValueError(
-                f"Resource {resource_name} is of type {resource['type']}, "
-                f"but expected {self._terraform_type}."
-            )
         self._attributes = resource["attributes"]
         self._dependencies = resource["dependencies"]
 

sdks/python/terrabridge/gcp/cloud_sql.py

@@ -1,6 +1,7 @@
 from typing import Optional
 
 from terrabridge.gcp.base import GCPResource
+from terrabridge.utils import parse_dependency
 
 try:
     from google.cloud.sql.connector import Connector, IPTypes, create_async_connector
@@ -118,9 +119,12 @@ def __init__(
         self.password = self._attributes["password"]
         self.cloud_sql_instance: Optional[CloudSQLInstance] = None
         for dependency in self._dependencies:
-            if dependency.startswith(CloudSQLInstance._terraform_type):
+            dependency = parse_dependency(dependency)
+            if dependency.resource_type == CloudSQLInstance._terraform_type:
                 self.cloud_sql_instance = CloudSQLInstance(
-                    dependency.split(".")[1], state_file=state_file
+                    resource_name=dependency.resource_name,
+                    module_name=dependency.module,
+                    state_file=state_file,
                 )
 
 
@@ -168,9 +172,12 @@ def __init__(
         self.name: str = self._attributes["name"]
         self.cloud_sql_instance: Optional[CloudSQLInstance] = None
         for dependency in self._dependencies:
-            if dependency.startswith(CloudSQLInstance._terraform_type):
+            dependency = parse_dependency(dependency)
+            if dependency.resource_type == CloudSQLInstance._terraform_type:
                 self.cloud_sql_instance = CloudSQLInstance(
-                    dependency.split(".")[1], state_file=state_file
+                    resource_name=dependency.resource_name,
+                    module_name=dependency.module,
+                    state_file=state_file,
                 )
 
     def sqlalchemy_engine(
@@ -181,9 +188,9 @@ def sqlalchemy_engine(
         Requires ``terrabridge[gcp]`` and ``sqlalchemy`` to be installed, and whatever
         driver is needed for the database version.
 
-        * ``POSTGRES``: ``pg8000``
-        * ``SQLSERVER``: ``pytds``
-        * ``MYSQL``: ``pymysql``
+        * ``POSTGRES``: ``pip install pg8000``
+        * ``SQLSERVER``: ``pip install pytds``
+        * ``MYSQL``: ``pip install pymysql``
 
         Parameters:
             user: The user to connect to the database with.

sdks/python/terrabridge/gcp/cloud_sql.pyi

@@ -7,6 +7,8 @@ class CloudSQLInstance:
     connection_name: str
     database_version: str
     name: str
+    project: str
+    id: str
 
     def __init__(
         self,
@@ -20,6 +22,8 @@ class CloudSQLUser:
     name: str
     password: str
     cloud_sql_instance: Optional[CloudSQLInstance]
+    project: str
+    id: str
 
     def __init__(
         self,
@@ -32,6 +36,8 @@ class CloudSQLUser:
 class CloudSQLDatabase:
     name: str
     cloud_sql_instance: Optional[CloudSQLInstance]
+    project: str
+    id: str
 
     def __init__(
         self,

sdks/python/terrabridge/gcp/gcs_bucket.py

@@ -23,7 +23,7 @@ class GCSBucket(GCPResource):
 
         from terrabridge.gcp import GCSBucket
 
-        bucket = GCSBucket("queue", state_file="gs://my-bucket/terraform.tfstate")
+        bucket = GCSBucket("bucket", state_file="gs://my-bucket/terraform.tfstate")
         print(bucket.name)
 
         bucket.bucket().upload_from_filename("local_file.txt")

sdks/python/terrabridge/gcp/gcs_bucket.pyi

@@ -6,6 +6,8 @@ class GCSBucket:
     _client: storage.Client
     url: str
     name: str
+    project: str
+    id: str
 
     def __init__(
         self,

sdks/python/terrabridge/parser.py

@@ -16,19 +16,27 @@
 class _ResourceKey:
     resource_name: str
     module_name: Optional[str]
+    resource_type: str
 
     def __repr__(self) -> str:
         if self.module_name is None:
             return self.resource_name
         return f"{self.resource_name} in {self.module_name}"
 
 
-def get_resource(resource_name: str, module_name: Optional[str], tf_state_path: str):
+def get_resource(
+    resource_name: str,
+    module_name: Optional[str],
+    tf_state_path: str,
+    resource_type: str,
+):
     """Return the attributes of a resource."""
     if tf_state_path not in tf_state_cache:
         _parse_terraform_state(tf_state_path)
     try:
-        return tf_state_cache[tf_state_path][_ResourceKey(resource_name, module_name)]
+        return tf_state_cache[tf_state_path][
+            _ResourceKey(resource_name, module_name, resource_type)
+        ]
     except KeyError:
         raise ValueError(
             f"Resource {resource_name} not found in {tf_state_path}. "
@@ -53,7 +61,7 @@ def _parse_terraform_state(tf_state_path: str):
         tf_state_cache[tf_state_path] = {}
     for resource in tf_state["resources"]:
         tf_state_cache[tf_state_path][
-            _ResourceKey(resource["name"], resource.get("module"))
+            _ResourceKey(resource["name"], resource.get("module"), resource["type"])
         ] = {
             "attributes": resource["instances"][0].get("attributes", {}),
             "dependencies": resource["instances"][0].get("dependencies", {}),

sdks/python/terrabridge/utils.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class Dependency:
+    module: Optional[str]
+    resource_type: str
+    resource_name: str
+
+
+def parse_dependency(dependency: str) -> Dependency:
+    split = dependency.split(".")
+
+    module = None
+    if split[0] == "module":
+        module = f"{split[0]}.{split[1]}"
+        split = split[2:]
+    resource_type = split[0]
+    resource_name = split[1]
+    return Dependency(module, resource_type, resource_name)

sdks/python/tests/gcp/bigquery_test.py

@@ -3,7 +3,7 @@
 
 def test_bigquery_dataset():
     dataset = BigQueryDataset(
-        resource_name="dataset", state_file="tests/data/terraform.tfstate"
+        resource_name="dataset", state_file="tests/data/gcp/terraform.tfstate"
     )
 
     assert dataset.project == "terrabridge-testing"
@@ -15,7 +15,7 @@ def test_bigquery_dataset():
 
 def test_bigquery_table():
     table = BigQueryTable(
-        resource_name="table", state_file="tests/data/terraform.tfstate"
+        resource_name="table", state_file="tests/data/gcp/terraform.tfstate"
     )
 
     assert table.project == "terrabridge-testing"
@@ -31,7 +31,8 @@ def test_bigquery_table():
 def test_type_mismatch():
     try:
         BigQueryTable(
-            resource_name="bigtable_table", state_file="tests/data/terraform.tfstate"
+            resource_name="bigtable_table",
+            state_file="tests/data/gcp/terraform.tfstate",
         )
     except ValueError:
         return

sdks/python/tests/gcp/bigtable_test.py

@@ -3,7 +3,7 @@
 
 def test_bigtable_instance():
     instance = BigTableInstance(
-        resource_name="bigtable_instance", state_file="tests/data/terraform.tfstate"
+        resource_name="bigtable_instance", state_file="tests/data/gcp/terraform.tfstate"
     )
 
     assert instance.project == "terrabridge-testing"
@@ -15,7 +15,7 @@ def test_bigtable_instance():
 
 def test_bigtable_table():
     table = BigTableTable(
-        resource_name="bigtable_table", state_file="tests/data/terraform.tfstate"
+        resource_name="bigtable_table", state_file="tests/data/gcp/terraform.tfstate"
     )
 
     assert table.project == "terrabridge-testing"

sdks/python/tests/gcp/cloud_sql_test.py

@@ -9,7 +9,8 @@
 
 def test_cloud_sql_instance():
     instance = CloudSQLInstance(
-        resource_name="cloud_sql_instance", state_file="tests/data/terraform.tfstate"
+        resource_name="cloud_sql_instance",
+        state_file="tests/data/gcp/terraform.tfstate",
     )
 
     assert instance.project == "terrabridge-testing"
@@ -23,7 +24,7 @@ def test_cloud_sql_instance():
 
 def test_cloud_sql_database():
     database = CloudSQLDatabase(
-        resource_name="database", state_file="tests/data/terraform.tfstate"
+        resource_name="database", state_file="tests/data/gcp/terraform.tfstate"
     )
 
     assert database.project == "terrabridge-testing"
@@ -37,7 +38,9 @@ def test_cloud_sql_database():
 
 
 def test_cloud_sql_user():
-    user = CloudSQLUser(resource_name="user", state_file="tests/data/terraform.tfstate")
+    user = CloudSQLUser(
+        resource_name="user", state_file="tests/data/gcp/terraform.tfstate"
+    )
 
     assert user.project == "terrabridge-testing"
     assert user.resource_name == "user"
@@ -52,9 +55,11 @@ def test_cloud_sql_user():
 
 def test_cloud_sql_database_postgres_sqlalchemy():
     database = CloudSQLDatabase(
-        resource_name="database", state_file="tests/data/terraform.tfstate"
+        resource_name="database", state_file="tests/data/gcp/terraform.tfstate"
+    )
+    user = CloudSQLUser(
+        resource_name="user", state_file="tests/data/gcp/terraform.tfstate"
     )
-    user = CloudSQLUser(resource_name="user", state_file="tests/data/terraform.tfstate")
 
     with patch("terrabridge.gcp.cloud_sql.create_engine") as mock_create_engine:
         with patch("terrabridge.gcp.cloud_sql.Connector") as mock_connector:
@@ -76,10 +81,10 @@ def test_cloud_sql_database_postgres_sqlalchemy():
 
 def test_cloud_sql_database_mysql_sqlalchemy():
     database = CloudSQLDatabase(
-        resource_name="mysql_database", state_file="tests/data/terraform.tfstate"
+        resource_name="mysql_database", state_file="tests/data/gcp/terraform.tfstate"
     )
     user = CloudSQLUser(
-        resource_name="mysql_user1", state_file="tests/data/terraform.tfstate"
+        resource_name="mysql_user1", state_file="tests/data/gcp/terraform.tfstate"
     )
 
     with patch("terrabridge.gcp.cloud_sql.create_engine") as mock_create_engine:
@@ -100,10 +105,11 @@ def test_cloud_sql_database_mysql_sqlalchemy():
 
 def test_cloud_sql_database_sqlserver_sqlalchemy():
     database = CloudSQLDatabase(
-        resource_name="sql_server_database", state_file="tests/data/terraform.tfstate"
+        resource_name="sql_server_database",
+        state_file="tests/data/gcp/terraform.tfstate",
     )
     user = CloudSQLUser(
-        resource_name="sql_server_user", state_file="tests/data/terraform.tfstate"
+        resource_name="sql_server_user", state_file="tests/data/gcp/terraform.tfstate"
     )
 
     with patch("terrabridge.gcp.cloud_sql.create_engine") as mock_create_engine:
@@ -125,9 +131,11 @@ def test_cloud_sql_database_sqlserver_sqlalchemy():
 @pytest.mark.asyncio
 async def test_cloud_sql_database_postgres_async_sqlalchemy():
     database = CloudSQLDatabase(
-        resource_name="database", state_file="tests/data/terraform.tfstate"
+        resource_name="database", state_file="tests/data/gcp/terraform.tfstate"
+    )
+    user = CloudSQLUser(
+        resource_name="user", state_file="tests/data/gcp/terraform.tfstate"
     )
-    user = CloudSQLUser(resource_name="user", state_file="tests/data/terraform.tfstate")
 
     with patch("terrabridge.gcp.cloud_sql.create_async_engine") as mock_create_engine:
         with patch(
@@ -153,10 +161,10 @@ async def test_cloud_sql_database_postgres_async_sqlalchemy():
 @pytest.mark.asyncio
 async def test_cloud_sql_database_mysql_async_sqlalchemy():
     database = CloudSQLDatabase(
-        resource_name="mysql_database", state_file="tests/data/terraform.tfstate"
+        resource_name="mysql_database", state_file="tests/data/gcp/terraform.tfstate"
     )
     user = CloudSQLUser(
-        resource_name="mysql_user1", state_file="tests/data/terraform.tfstate"
+        resource_name="mysql_user1", state_file="tests/data/gcp/terraform.tfstate"
     )
 
     try:
@@ -169,10 +177,11 @@ async def test_cloud_sql_database_mysql_async_sqlalchemy():
 @pytest.mark.asyncio
 async def test_cloud_sql_database_sqlserver_async_sqlalchemy():
     database = CloudSQLDatabase(
-        resource_name="sql_server_database", state_file="tests/data/terraform.tfstate"
+        resource_name="sql_server_database",
+        state_file="tests/data/gcp/terraform.tfstate",
     )
     user = CloudSQLUser(
-        resource_name="sql_server_user", state_file="tests/data/terraform.tfstate"
+        resource_name="sql_server_user", state_file="tests/data/gcp/terraform.tfstate"
     )
 
     try:

sdks/python/tests/gcp/cloud_tasks_test.py

@@ -3,7 +3,7 @@
 
 def test_cloud_tasks_queue():
     queue = CloudTasksQueue(
-        resource_name="queue", state_file="tests/data/terraform.tfstate"
+        resource_name="queue", state_file="tests/data/gcp/terraform.tfstate"
     )
 
     assert queue.project == "terrabridge-testing"