Skip to content

Commit

Permalink
feat: limit saveBatch and deleteByIds chunk concurrency to 16
Browse files Browse the repository at this point in the history
  • Loading branch information
kirillgroshkov committed Jun 24, 2024
1 parent a6631f2 commit a2dcd52
Showing 1 changed file with 8 additions and 1 deletion.
9 changes: 8 additions & 1 deletion src/datastore.db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ import { dbQueryToDatastoreQuery } from './query.util'

// Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
const MAX_ITEMS = 500
// It's an empyrical value, but anything less than infinity is better than infinity
const DATASTORE_RECOMMENDED_CONCURRENCY = 16

const RETRY_ON = [
'GOAWAY',
Expand Down Expand Up @@ -300,7 +302,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
// Not using pMap in hope to preserve stack trace
await save(chunks[0]!)
} else {
await pMap(chunks, async batch => await save(batch))
await pMap(chunks, async batch => await save(batch), {
concurrency: DATASTORE_RECOMMENDED_CONCURRENCY,
})
}
} catch (err) {
// console.log(`datastore.save ${kind}`, { obj, entity })
Expand Down Expand Up @@ -347,6 +351,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
_chunk(keys, MAX_ITEMS),
// eslint-disable-next-line @typescript-eslint/return-await
async batch => await ((opt.tx as DatastoreDBTransaction)?.tx || this.ds()).delete(batch),
{
concurrency: DATASTORE_RECOMMENDED_CONCURRENCY,
},
)
return ids.length
}
Expand Down

0 comments on commit a2dcd52

Please sign in to comment.