Skip to content

Commit

Permalink
Use single postgres connection, enable parallel index processing (#77)
Browse files Browse the repository at this point in the history
  • Loading branch information
droserasprout authored Jun 29, 2021
1 parent 1e135f6 commit 144db32
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 4 deletions.
4 changes: 3 additions & 1 deletion src/dipdup/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,9 @@ class PostgresDatabaseConfig:

@property
def connection_string(self) -> str:
return f'{self.kind}://{self.user}:{self.password}@{self.host}:{self.port}/{self.database}?schema={self.schema_name}'
# NOTE: `maxsize=1` is important! Concurrency will be broken otherwise.
# NOTE: https://github.com/tortoise/tortoise-orm/issues/792
return f'{self.kind}://{self.user}:{self.password}@{self.host}:{self.port}/{self.database}?schema={self.schema_name}&maxsize=1'

@validator('immune_tables')
def valid_immune_tables(cls, v):
Expand Down
4 changes: 1 addition & 3 deletions src/dipdup/dipdup.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,8 @@ async def run(self, oneshot=False) -> None:
while True:
await self.reload_config()

# FIXME: Process all indexes in parallel, blocked by https://github.com/tortoise/tortoise-orm/issues/792
async with utils.slowdown(1):
for index in self._indexes.values():
await index.process()
await asyncio.gather(*[index.process() for index in self._indexes.values()])

# TODO: Continue if new indexes are spawned from origination
if oneshot:
Expand Down

0 comments on commit 144db32

Please sign in to comment.