Skip to content

Commit

Permalink
add batching in blob migration
Browse files Browse the repository at this point in the history
  • Loading branch information
realbigsean committed Jan 23, 2024
1 parent d5d89f9 commit 4aa98ae
Showing 1 changed file with 17 additions and 3 deletions.
20 changes: 17 additions & 3 deletions beacon_node/beacon_chain/src/schema_change/migration_schema_v19.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,24 +12,38 @@ pub fn upgrade_to_v19<T: BeaconChainTypes>(
let column = DBColumn::BeaconBlob;

debug!(log, "Migrating from v18 to v19");
// Iterate throught the blobs on disk.
// Iterate through the blobs on disk.
for res in db.hot_db.iter_column_keys::<Vec<u8>>(column) {
let key = res?;
let key_col = get_key_for_col(column.as_str(), &key);
hot_delete_ops.push(KeyValueStoreOp::DeleteKey(key_col));
blob_keys.push(key);
}

let num_blobs = blob_keys.len();
debug!(log, "Collected {} blob lists to migrate", num_blobs);

let batch_size = 500;
let mut batch = Vec::with_capacity(batch_size);

for key in blob_keys {
let next_blob = db.hot_db.get_bytes(column.as_str(), &key)?;
if let Some(next_blob) = next_blob {
let key_col = get_key_for_col(column.as_str(), &key);
db.blobs_db
.do_atomically(vec![KeyValueStoreOp::PutKeyValue(key_col, next_blob)])?;
batch.push(KeyValueStoreOp::PutKeyValue(key_col, next_blob));

if batch.len() >= batch_size {
db.blobs_db.do_atomically(batch.clone())?;
batch.clear();
}
}
}

// Process the remaining batch if it's not empty
if !batch.is_empty() {
db.blobs_db.do_atomically(batch)?;
}

debug!(log, "Wrote {} blobs to the blobs db", num_blobs);

// Delete all the blobs
Expand Down

0 comments on commit 4aa98ae

Please sign in to comment.