Merge branch '2.0' into business_core_2.0

dyvenia · Oct 2, 2024 · 46e0f6d · 46e0f6d
2 parents 21c49d4 + be88cee
commit 46e0f6d
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 54 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "viadot2"
-version = "2.1.20"
+version = "2.1.21"
 description = "A simple data ingestion library to guide data flows from some places to other places."
 authors = [
     { name = "acivitillo", email = "acivitillo@dyvenia.com" },
@@ -32,11 +32,11 @@ dependencies = [
     "pyarrow>=10.0, <10.1.0",
     # numpy>=2.0 is not compatible with the old pyarrow v10.x.
     "numpy>=1.23.4, <2.0",
-    "paramiko==2.11.0",
     "defusedxml>=0.7.1",
     "aiohttp>=3.10.5",
     "simple-salesforce==1.12.6",
     "pandas-gbq==0.23.1",
+    "paramiko>=3.5.0",
 ]
 requires-python = ">=3.10"
 readme = "README.md"
@@ -60,7 +60,7 @@ aws = [
     "awswrangler>=2.20.1, <3.0",
     "prefect-aws>=0.4.19",
 ]
-sap = ["pyrfc<3.0"]
+sap = ["pyrfc"]
 
 [tool.rye]
 managed = true

diff --git a/requirements-dev.lock b/requirements-dev.lock
@@ -98,7 +98,6 @@ comm==0.2.2
 coolname==2.2.0
     # via prefect
 coverage==7.6.1
-    # via coverage
 croniter==2.0.7
     # via prefect
 cryptography==43.0.0
@@ -208,7 +207,6 @@ httpcore==1.0.5
     # via httpx
     # via prefect
 httpx==0.27.0
-    # via httpx
     # via neoteroi-mkdocs
     # via prefect
 humanize==4.10.0
@@ -346,13 +344,11 @@ mkdocs-include-markdown-plugin==6.2.2
 mkdocs-jupyter==0.24.8
 mkdocs-material==9.5.32
     # via mkdocs-jupyter
-    # via mkdocs-material
 mkdocs-material-extensions==1.3.1
     # via mkdocs-material
 mkdocs-mermaid2-plugin==1.1.1
 mkdocs-table-reader-plugin==3.0.1
 mkdocstrings==0.25.2
-    # via mkdocstrings
     # via mkdocstrings-python
 mkdocstrings-python==1.10.5
     # via mkdocstrings
@@ -419,7 +415,7 @@ pandas-gbq==0.23.1
     # via viadot2
 pandocfilters==1.5.1
     # via nbconvert
-paramiko==2.11.0
+paramiko==3.5.0
     # via viadot2
 parso==0.8.4
     # via jedi
@@ -510,7 +506,6 @@ pytest==8.3.2
     # via pytest-mock
 pytest-asyncio==0.23.8
 pytest-mock==3.14.0
-    # via viadot2
 python-dateutil==2.9.0.post0
     # via botocore
     # via croniter
@@ -626,6 +621,10 @@ scipy==1.14.0
     # via imagehash
 sendgrid==6.11.0
     # via viadot2
+setuptools==73.0.0
+    # via mkdocs-mermaid2-plugin
+    # via pandas-gbq
+    # via pydata-google-auth
 sgqlc==16.3
     # via prefect-github
 shapely==2.0.6
@@ -642,7 +641,6 @@ six==1.16.0
     # via isodate
     # via jsbeautifier
     # via kubernetes
-    # via paramiko
     # via python-dateutil
     # via rfc3339-validator
 smmap==5.0.1
@@ -699,7 +697,6 @@ trino==0.328.0
 typer==0.12.4
     # via lumacli
     # via prefect
-    # via typer
 typing-extensions==4.12.2
     # via aiosqlite
     # via alembic
@@ -757,7 +754,3 @@ zeep==4.2.1
     # via simple-salesforce
 zipp==3.20.0
     # via importlib-metadata
-setuptools==73.0.0
-    # via mkdocs-mermaid2-plugin
-    # via pandas-gbq
-    # via pydata-google-auth
diff --git a/requirements.lock b/requirements.lock
@@ -95,7 +95,6 @@ et-xmlfile==1.1.0
 exceptiongroup==1.2.2
     # via anyio
     # via prefect
-    # via pytest
 frozenlist==1.4.1
     # via aiohttp
     # via aiosignal
@@ -151,7 +150,6 @@ httpcore==1.0.5
     # via httpx
     # via prefect
 httpx==0.27.0
-    # via httpx
     # via prefect
 humanize==4.10.0
     # via jinja2-humanize-extension
@@ -168,8 +166,6 @@ imagehash==4.3.1
     # via viadot2
 importlib-resources==6.1.3
     # via prefect
-iniconfig==2.0.0
-    # via pytest
 isodate==0.6.1
     # via zeep
 itsdangerous==2.2.0
@@ -238,15 +234,14 @@ packaging==24.1
     # via google-cloud-bigquery
     # via pandas-gbq
     # via prefect
-    # via pytest
 pandas==2.2.2
     # via db-dtypes
     # via pandas-gbq
     # via viadot2
     # via visions
 pandas-gbq==0.23.1
     # via viadot2
-paramiko==2.11.0
+paramiko==3.5.0
     # via viadot2
 pathspec==0.12.1
     # via prefect
@@ -256,8 +251,6 @@ pillow==10.4.0
     # via imagehash
 platformdirs==4.3.6
     # via zeep
-pluggy==1.5.0
-    # via pytest
 prefect==2.20.2
     # via prefect-github
     # via prefect-sqlalchemy
@@ -304,10 +297,6 @@ pynacl==1.5.0
     # via paramiko
 pyodbc==5.1.0
     # via viadot2
-pytest==8.3.3
-    # via pytest-mock
-pytest-mock==3.14.0
-    # via viadot2
 python-dateutil==2.9.0.post0
     # via croniter
     # via dateparser
@@ -391,6 +380,9 @@ scipy==1.14.0
     # via imagehash
 sendgrid==6.11.0
     # via viadot2
+setuptools==75.1.0
+    # via pandas-gbq
+    # via pydata-google-auth
 sgqlc==16.3
     # via prefect-github
 shapely==2.0.6
@@ -404,7 +396,6 @@ simple-salesforce==1.12.6
 six==1.16.0
     # via isodate
     # via kubernetes
-    # via paramiko
     # via python-dateutil
     # via rfc3339-validator
 sniffio==1.3.1
@@ -429,14 +420,11 @@ text-unidecode==1.3
     # via python-slugify
 toml==0.10.2
     # via prefect
-tomli==2.0.1
-    # via pytest
 trino==0.328.0
     # via viadot2
 typer==0.12.4
     # via lumacli
     # via prefect
-    # via typer
 typing-extensions==4.12.2
     # via aiosqlite
     # via alembic
@@ -473,6 +461,3 @@ yarl==1.9.8
     # via aiohttp
 zeep==4.2.1
     # via simple-salesforce
-setuptools==75.1.0
-    # via pandas-gbq
-    # via pydata-google-auth
diff --git a/src/viadot/sources/epicor.py b/src/viadot/sources/epicor.py
@@ -134,7 +134,7 @@ def parse_orders_xml(xml_data: str) -> pd.DataFrame:  # noqa: C901, PLR0912
     Returns:
         pd.DataFrame: DataFrame containing parsed orders data.
     """
-    final_df = pd.DataFrame()
+    dfs = []
     ship_dict = {}
     invoice_dict = {}
     header_params_dict = {}
@@ -197,10 +197,8 @@ def parse_orders_xml(xml_data: str) -> pd.DataFrame:  # noqa: C901, PLR0912
                 line_item = LineItemDetail(**item_params_dict)
                 row = Order(HeaderInformation=header_info, LineItemDetail=line_item)
                 my_dict = row.dict()
-                final_df = final_df.append(
-                    pd.json_normalize(my_dict, max_level=2), ignore_index=True
-                )
-    return final_df
+                dfs.append(pd.json_normalize(my_dict, max_level=2))
+    return pd.concat(dfs, ignore_index=True)
 
 
 def parse_bookings_xml(xml_data: str) -> pd.DataFrame:  # noqa: C901, PLR0912
@@ -211,7 +209,7 @@ def parse_bookings_xml(xml_data: str) -> pd.DataFrame:  # noqa: C901, PLR0912
     Returns:
         pd.DataFrame: DataFrame containing parsed  data.
     """
-    final_df = pd.DataFrame()
+    dfs = []
     ship_dict = {}
     header_params_dict = {}
     item_params_dict = {}
@@ -254,10 +252,8 @@ def parse_bookings_xml(xml_data: str) -> pd.DataFrame:  # noqa: C901, PLR0912
                     HeaderInformation=header_info, LineItemDetail=line_item
                 )
                 my_dict = row.dict()
-                final_df = final_df.append(
-                    pd.json_normalize(my_dict, max_level=2), ignore_index=True
-                )
-    return final_df
+                dfs.append(pd.json_normalize(my_dict, max_level=2))
+    return pd.concat(dfs, ignore_index=True)
 
 
 def parse_open_orders_xml(xml_data: str) -> pd.DataFrame:  # noqa: C901, PLR0912
@@ -268,7 +264,7 @@ def parse_open_orders_xml(xml_data: str) -> pd.DataFrame:  # noqa: C901, PLR0912
     Returns:
         pd.DataFrame: DataFrame containing parsed  data.
     """
-    final_df = pd.DataFrame()
+    dfs = []
     ship_dict = {}
     header_params_dict = {}
     item_params_dict = {}
@@ -310,10 +306,8 @@ def parse_open_orders_xml(xml_data: str) -> pd.DataFrame:  # noqa: C901, PLR0912
                 line_item = LineItemDetail(**item_params_dict)
                 row = Order(HeaderInformation=header_info, LineItemDetail=line_item)
                 my_dict = row.dict()
-                final_df = final_df.append(
-                    pd.json_normalize(my_dict, max_level=2), ignore_index=True
-                )
-    return final_df
+                dfs.append(pd.json_normalize(my_dict, max_level=2))
+    return pd.concat(dfs, ignore_index=True)
 
 
 def parse_customer_xml(xml_data: str) -> pd.DataFrame:
@@ -324,7 +318,7 @@ def parse_customer_xml(xml_data: str) -> pd.DataFrame:
     Returns:
         pd.DataFrame: DataFrame containing parsed  data.
     """
-    final_df = pd.DataFrame()
+    dfs = []
     ship_dict = {}
     customer_params_dict = {}
 
@@ -354,10 +348,8 @@ def parse_customer_xml(xml_data: str) -> pd.DataFrame:
                     customer_params_dict.update(cust_parameter)
                 cust_info = Customer(**customer_params_dict)
                 my_dict = cust_info.dict()
-                final_df = final_df.append(
-                    pd.json_normalize(my_dict, max_level=2), ignore_index=True
-                )
-    return final_df
+                dfs.append(pd.json_normalize(my_dict, max_level=2))
+    return pd.concat(dfs, ignore_index=True)
 
 
 class EpicorCredentials(BaseModel):