Skip to content

Commit

Permalink
fix: Remove redundant batching in PostgreSQLOnlineStore.online_write_…
Browse files Browse the repository at this point in the history
…batch and fix progress bar (feast-dev#4331)

* Remove batching and fix tqdm progress bar

Signed-off-by: TomSteenbergen <tomsteenbergen1995@gmail.com>

* Comment

Signed-off-by: TomSteenbergen <tomsteenbergen1995@gmail.com>

* Remove test changes

Signed-off-by: TomSteenbergen <tomsteenbergen1995@gmail.com>

* Update comment

Signed-off-by: TomSteenbergen <tomsteenbergen1995@gmail.com>

---------

Signed-off-by: TomSteenbergen <tomsteenbergen1995@gmail.com>
  • Loading branch information
TomSteenbergen authored Jul 8, 2024
1 parent b0dc683 commit 0d89d15
Showing 1 changed file with 5 additions and 8 deletions.
13 changes: 5 additions & 8 deletions sdk/python/feast/infra/online_stores/contrib/postgres.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ def online_write_batch(
Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]]
],
progress: Optional[Callable[[int], Any]],
batch_size: int = 5000,
) -> None:
# Format insert values
insert_values = []
Expand Down Expand Up @@ -118,15 +117,13 @@ def online_write_batch(
"""
).format(sql.Identifier(_table_id(config.project, table)))

# Push data in batches to online store
# Push data into the online store
with self._get_conn(config) as conn, conn.cursor() as cur:
for i in range(0, len(insert_values), batch_size):
cur_batch = insert_values[i : i + batch_size]
cur.executemany(sql_query, cur_batch)
conn.commit()
cur.executemany(sql_query, insert_values)
conn.commit()

if progress:
progress(len(cur_batch))
if progress:
progress(len(data))

def online_read(
self,
Expand Down

0 comments on commit 0d89d15

Please sign in to comment.