Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: explicitly using a connection with to_sql call, and reporting on… #32

Merged
merged 1 commit into from
Oct 22, 2024
Merged

fix: explicitly using a connection with to_sql call, and reporting on… #32

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 24 additions & 15 deletions data_prep/oradb_lib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,22 +285,32 @@ def load_data(
"""
# debugging to view the data before it gets loaded
pandas_df = pd.read_parquet(import_file)
tmp_file = self.get_tmp_file()
LOGGER.debug("tmp_file: %s", str(tmp_file))
pandas_df.to_csv(str(tmp_file), sep="|", index_label=None)
self.get_connection() # make sure there is an oracle connection

LOGGER.debug("table: %s", table)
LOGGER.debug("loading data for table: %s", table)

self.get_sqlalchemy_engine()
if purge:
self.truncate_table(table.lower())
pandas_df.to_sql(
table.lower(),
self.sql_alchemy_engine,
schema="THE",
if_exists="append",
index=False,
)
with self.sql_alchemy_engine.connect() as connection:
with connection.begin():
pandas_df.to_sql(
table.lower(),
con=connection,
schema="THE",
if_exists="append",
index=False,
)
# now verify data
sql = f"Select count(*) from {self.schema_2_sync}.{table}"
cur = self.connection.cursor()
cur.execute(sql)
result = cur.fetchall()
rows_loaded = result[0][0]
if not rows_loaded:
LOGGER.error("no rows loaded to table %s", table)
LOGGER.debug("rows loaded to table %s are: %s", table, rows_loaded)
cur.close()

def load_data_retry(
self,
Expand Down Expand Up @@ -413,10 +423,9 @@ def purge_data(
except ( # noqa: PERF203
sqlalchemy.exc.IntegrityError,
DatabaseError,
) as e:
LOGGER.exception(
"%s purging table %s",
e.__class__.__qualname__,
):
LOGGER.warning(
"error encountered when attempting to purge table: %s, retrying",
table,
)
failed_tables.append(table)
Expand Down
Loading