From 42566b149250d5d4ba758877b2eed9d98c3a968f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zawadzki?= Date: Tue, 1 Oct 2024 15:02:37 +0200 Subject: [PATCH 1/3] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Upgrade=20`pyrfc`=20to?= =?UTF-8?q?=203.x=20(#1072)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 06eedfe98..7c7ab75eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ aws = [ "awswrangler>=2.20.1, <3.0", "prefect-aws>=0.4.19", ] -sap = ["pyrfc<3.0"] +sap = ["pyrfc"] [tool.rye] managed = true From 6a957ec59e435eaccf51b30ac3c32c135580866e Mon Sep 17 00:00:00 2001 From: Angelika Tarnawa <59344718+angelika233@users.noreply.github.com> Date: Wed, 2 Oct 2024 13:41:02 +0200 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=8E=A8=20Change=20append=20to=20conca?= =?UTF-8?q?t=20in=20Epicor=20source=20(#1075)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🎨 Change append to concat in Epicor source * ⬆️ Bump viadot * ⚡️ Optimize concat in the loop --------- Co-authored-by: angelika233 --- pyproject.toml | 2 +- src/viadot/sources/epicor.py | 32 ++++++++++++-------------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7c7ab75eb..defe5b1a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "viadot2" -version = "2.1.20" +version = "2.1.21" description = "A simple data ingestion library to guide data flows from some places to other places." authors = [ { name = "acivitillo", email = "acivitillo@dyvenia.com" }, diff --git a/src/viadot/sources/epicor.py b/src/viadot/sources/epicor.py index b17743196..1cc830575 100644 --- a/src/viadot/sources/epicor.py +++ b/src/viadot/sources/epicor.py @@ -134,7 +134,7 @@ def parse_orders_xml(xml_data: str) -> pd.DataFrame: # noqa: C901, PLR0912 Returns: pd.DataFrame: DataFrame containing parsed orders data. """ - final_df = pd.DataFrame() + dfs = [] ship_dict = {} invoice_dict = {} header_params_dict = {} @@ -197,10 +197,8 @@ def parse_orders_xml(xml_data: str) -> pd.DataFrame: # noqa: C901, PLR0912 line_item = LineItemDetail(**item_params_dict) row = Order(HeaderInformation=header_info, LineItemDetail=line_item) my_dict = row.dict() - final_df = final_df.append( - pd.json_normalize(my_dict, max_level=2), ignore_index=True - ) - return final_df + dfs.append(pd.json_normalize(my_dict, max_level=2)) + return pd.concat(dfs, ignore_index=True) def parse_bookings_xml(xml_data: str) -> pd.DataFrame: # noqa: C901, PLR0912 @@ -211,7 +209,7 @@ def parse_bookings_xml(xml_data: str) -> pd.DataFrame: # noqa: C901, PLR0912 Returns: pd.DataFrame: DataFrame containing parsed data. """ - final_df = pd.DataFrame() + dfs = [] ship_dict = {} header_params_dict = {} item_params_dict = {} @@ -254,10 +252,8 @@ def parse_bookings_xml(xml_data: str) -> pd.DataFrame: # noqa: C901, PLR0912 HeaderInformation=header_info, LineItemDetail=line_item ) my_dict = row.dict() - final_df = final_df.append( - pd.json_normalize(my_dict, max_level=2), ignore_index=True - ) - return final_df + dfs.append(pd.json_normalize(my_dict, max_level=2)) + return pd.concat(dfs, ignore_index=True) def parse_open_orders_xml(xml_data: str) -> pd.DataFrame: # noqa: C901, PLR0912 @@ -268,7 +264,7 @@ def parse_open_orders_xml(xml_data: str) -> pd.DataFrame: # noqa: C901, PLR0912 Returns: pd.DataFrame: DataFrame containing parsed data. """ - final_df = pd.DataFrame() + dfs = [] ship_dict = {} header_params_dict = {} item_params_dict = {} @@ -310,10 +306,8 @@ def parse_open_orders_xml(xml_data: str) -> pd.DataFrame: # noqa: C901, PLR0912 line_item = LineItemDetail(**item_params_dict) row = Order(HeaderInformation=header_info, LineItemDetail=line_item) my_dict = row.dict() - final_df = final_df.append( - pd.json_normalize(my_dict, max_level=2), ignore_index=True - ) - return final_df + dfs.append(pd.json_normalize(my_dict, max_level=2)) + return pd.concat(dfs, ignore_index=True) def parse_customer_xml(xml_data: str) -> pd.DataFrame: @@ -324,7 +318,7 @@ def parse_customer_xml(xml_data: str) -> pd.DataFrame: Returns: pd.DataFrame: DataFrame containing parsed data. """ - final_df = pd.DataFrame() + dfs = [] ship_dict = {} customer_params_dict = {} @@ -354,10 +348,8 @@ def parse_customer_xml(xml_data: str) -> pd.DataFrame: customer_params_dict.update(cust_parameter) cust_info = Customer(**customer_params_dict) my_dict = cust_info.dict() - final_df = final_df.append( - pd.json_normalize(my_dict, max_level=2), ignore_index=True - ) - return final_df + dfs.append(pd.json_normalize(my_dict, max_level=2)) + return pd.concat(dfs, ignore_index=True) class EpicorCredentials(BaseModel): From be88ceecfdeedff56fb6e74d71ebe84897d27faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zawadzki?= Date: Wed, 2 Oct 2024 16:37:45 +0200 Subject: [PATCH 3/3] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20`paramiko`=20to?= =?UTF-8?q?=203.x=20to=20hide=20deprecation=20warnings=20(#1077)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 2 +- requirements-dev.lock | 17 +++++------------ requirements.lock | 23 ++++------------------- 3 files changed, 10 insertions(+), 32 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index defe5b1a4..b49305191 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,11 +32,11 @@ dependencies = [ "pyarrow>=10.0, <10.1.0", # numpy>=2.0 is not compatible with the old pyarrow v10.x. "numpy>=1.23.4, <2.0", - "paramiko==2.11.0", "defusedxml>=0.7.1", "aiohttp>=3.10.5", "simple-salesforce==1.12.6", "pandas-gbq==0.23.1", + "paramiko>=3.5.0", ] requires-python = ">=3.10" readme = "README.md" diff --git a/requirements-dev.lock b/requirements-dev.lock index 75c3b9a60..cafb4443b 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -98,7 +98,6 @@ comm==0.2.2 coolname==2.2.0 # via prefect coverage==7.6.1 - # via coverage croniter==2.0.7 # via prefect cryptography==43.0.0 @@ -208,7 +207,6 @@ httpcore==1.0.5 # via httpx # via prefect httpx==0.27.0 - # via httpx # via neoteroi-mkdocs # via prefect humanize==4.10.0 @@ -346,13 +344,11 @@ mkdocs-include-markdown-plugin==6.2.2 mkdocs-jupyter==0.24.8 mkdocs-material==9.5.32 # via mkdocs-jupyter - # via mkdocs-material mkdocs-material-extensions==1.3.1 # via mkdocs-material mkdocs-mermaid2-plugin==1.1.1 mkdocs-table-reader-plugin==3.0.1 mkdocstrings==0.25.2 - # via mkdocstrings # via mkdocstrings-python mkdocstrings-python==1.10.5 # via mkdocstrings @@ -419,7 +415,7 @@ pandas-gbq==0.23.1 # via viadot2 pandocfilters==1.5.1 # via nbconvert -paramiko==2.11.0 +paramiko==3.5.0 # via viadot2 parso==0.8.4 # via jedi @@ -510,7 +506,6 @@ pytest==8.3.2 # via pytest-mock pytest-asyncio==0.23.8 pytest-mock==3.14.0 - # via viadot2 python-dateutil==2.9.0.post0 # via botocore # via croniter @@ -626,6 +621,10 @@ scipy==1.14.0 # via imagehash sendgrid==6.11.0 # via viadot2 +setuptools==73.0.0 + # via mkdocs-mermaid2-plugin + # via pandas-gbq + # via pydata-google-auth sgqlc==16.3 # via prefect-github shapely==2.0.6 @@ -642,7 +641,6 @@ six==1.16.0 # via isodate # via jsbeautifier # via kubernetes - # via paramiko # via python-dateutil # via rfc3339-validator smmap==5.0.1 @@ -699,7 +697,6 @@ trino==0.328.0 typer==0.12.4 # via lumacli # via prefect - # via typer typing-extensions==4.12.2 # via aiosqlite # via alembic @@ -757,7 +754,3 @@ zeep==4.2.1 # via simple-salesforce zipp==3.20.0 # via importlib-metadata -setuptools==73.0.0 - # via mkdocs-mermaid2-plugin - # via pandas-gbq - # via pydata-google-auth diff --git a/requirements.lock b/requirements.lock index e921189bf..5b5b76e9f 100644 --- a/requirements.lock +++ b/requirements.lock @@ -95,7 +95,6 @@ et-xmlfile==1.1.0 exceptiongroup==1.2.2 # via anyio # via prefect - # via pytest frozenlist==1.4.1 # via aiohttp # via aiosignal @@ -151,7 +150,6 @@ httpcore==1.0.5 # via httpx # via prefect httpx==0.27.0 - # via httpx # via prefect humanize==4.10.0 # via jinja2-humanize-extension @@ -168,8 +166,6 @@ imagehash==4.3.1 # via viadot2 importlib-resources==6.1.3 # via prefect -iniconfig==2.0.0 - # via pytest isodate==0.6.1 # via zeep itsdangerous==2.2.0 @@ -238,7 +234,6 @@ packaging==24.1 # via google-cloud-bigquery # via pandas-gbq # via prefect - # via pytest pandas==2.2.2 # via db-dtypes # via pandas-gbq @@ -246,7 +241,7 @@ pandas==2.2.2 # via visions pandas-gbq==0.23.1 # via viadot2 -paramiko==2.11.0 +paramiko==3.5.0 # via viadot2 pathspec==0.12.1 # via prefect @@ -256,8 +251,6 @@ pillow==10.4.0 # via imagehash platformdirs==4.3.6 # via zeep -pluggy==1.5.0 - # via pytest prefect==2.20.2 # via prefect-github # via prefect-sqlalchemy @@ -304,10 +297,6 @@ pynacl==1.5.0 # via paramiko pyodbc==5.1.0 # via viadot2 -pytest==8.3.3 - # via pytest-mock -pytest-mock==3.14.0 - # via viadot2 python-dateutil==2.9.0.post0 # via croniter # via dateparser @@ -391,6 +380,9 @@ scipy==1.14.0 # via imagehash sendgrid==6.11.0 # via viadot2 +setuptools==75.1.0 + # via pandas-gbq + # via pydata-google-auth sgqlc==16.3 # via prefect-github shapely==2.0.6 @@ -404,7 +396,6 @@ simple-salesforce==1.12.6 six==1.16.0 # via isodate # via kubernetes - # via paramiko # via python-dateutil # via rfc3339-validator sniffio==1.3.1 @@ -429,14 +420,11 @@ text-unidecode==1.3 # via python-slugify toml==0.10.2 # via prefect -tomli==2.0.1 - # via pytest trino==0.328.0 # via viadot2 typer==0.12.4 # via lumacli # via prefect - # via typer typing-extensions==4.12.2 # via aiosqlite # via alembic @@ -473,6 +461,3 @@ yarl==1.9.8 # via aiohttp zeep==4.2.1 # via simple-salesforce -setuptools==75.1.0 - # via pandas-gbq - # via pydata-google-auth