sources: integrate dynamodb

bruin-data · Dec 11, 2024 · 45c1628 · 45c1628
1 parent 902d8cc
commit 45c1628
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 1 deletion.
diff --git a/ingestr/src/factory.py b/ingestr/src/factory.py
@@ -41,6 +41,7 @@
 SQL_SOURCE_SCHEMES = [
     "bigquery",
     "duckdb",
+    "dynamodb",
     "mssql",
     "mysql",
     "mysql+pymysql",

diff --git a/ingestr/src/sources.py b/ingestr/src/sources.py
@@ -55,6 +55,10 @@ def handles_incrementality(self) -> bool:
     def dlt_source(self, uri: str, table: str, **kwargs):
         table_fields = table_string_to_dataclass(table)
 
+        if uri.startswith("dynamodb://"):
+            # schema as a concept doesn't exist in dynamodb
+            table_fields.dataset = None
+
         incremental = None
         if kwargs.get("incremental_key"):
             start_value = kwargs.get("interval_start")

diff --git a/requirements.txt b/requirements.txt
@@ -29,4 +29,5 @@ pyairtable==2.3.3
 types-requests==2.32.0.20240907
 s3fs==2024.10.0
 pyarrow==18.1.0
-pyathena==3.9.0
+pyathena==3.9.0
+pydynamodb==0.7.1