Writing: Exception when exceeding the rate limit for object mutation operations

[bogovicj]

java.util.concurrent.ExecutionException: org.janelia.saalfeldlab.n5.N5Exception$N5IOException: Failed to write block [2, 2, 0] into dataset mri.zarr/s0/0/2/2
	at java.util.concurrent.FutureTask.report(FutureTask.java:122)
	at java.util.concurrent.FutureTask.get(FutureTask.java:192)
	at org.janelia.saalfeldlab.n5.imglib2.N5Utils.save(N5Utils.java:1584)
	at org.janelia.saalfeldlab.n5.ij.N5ScalePyramidExporter.write(N5ScalePyramidExporter.java:1202)
	at org.janelia.saalfeldlab.n5.ij.N5ScalePyramidExporter.processMultiscale(N5ScalePyramidExporter.java:605)
	at org.janelia.saalfeldlab.n5.ij.N5ScalePyramidExporter.run(N5ScalePyramidExporter.java:1340)
	at org.scijava.command.CommandModule.run(CommandModule.java:196)
	at org.scijava.module.ModuleRunner.run(ModuleRunner.java:165)
	at org.scijava.module.ModuleRunner.call(ModuleRunner.java:125)
	at org.scijava.module.ModuleRunner.call(ModuleRunner.java:64)
	at org.scijava.thread.DefaultThreadService.lambda$wrap$2(DefaultThreadService.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:750)
Caused by: org.janelia.saalfeldlab.n5.N5Exception$N5IOException: Failed to write block [2, 2, 0] into dataset mri.zarr/s0/0/2/2
	at org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter.writeBlock(ZarrKeyValueWriter.java:546)
	at org.janelia.saalfeldlab.n5.imglib2.N5Utils.lambda$save$13(N5Utils.java:1572)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	... 4 more
Caused by: com.google.cloud.storage.StorageException: The object <PATH> exceeded the rate limit for object mutation operations (create, update, and delete). Please reduce your request rate. See https://cloud.google.com/storage/docs/gcs429.
	at com.google.cloud.storage.StorageException.translate(StorageException.java:165)
	at com.google.cloud.storage.spi.v1.HttpStorageRpc.translate(HttpStorageRpc.java:297)
	at com.google.cloud.storage.spi.v1.HttpStorageRpc.create(HttpStorageRpc.java:379)
	at com.google.cloud.storage.StorageImpl.lambda$internalCreate$2(StorageImpl.java:208)
	at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:103)
	at com.google.cloud.RetryHelper.run(RetryHelper.java:76)
	at com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:50)
	at com.google.cloud.storage.Retrying.run(Retrying.java:60)
	at com.google.cloud.storage.StorageImpl.run(StorageImpl.java:1476)
	at com.google.cloud.storage.StorageImpl.internalCreate(StorageImpl.java:205)
	at com.google.cloud.storage.StorageImpl.create(StorageImpl.java:137)
	at org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess.createDirectories(GoogleCloudStorageKeyValueAccess.java:375)
	at org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter.writeBlock(ZarrKeyValueWriter.java:536)
	... 6 more
Caused by: com.google.api.client.googleapis.json.GoogleJsonResponseException: 429 Too Many Requests
POST https://storage.googleapis.com/upload/storage/v1/b/<BUCKET>/o?projection=full&uploadType=multipart
{
  "code" : 429,
  "errors" : [ {
    "domain" : "usageLimits",
    "message" : "The object example-multi-container-bucket/mri.zarr/ exceeded the rate limit for object mutation operations (create, update, and delete). Please reduce your request rate. See https://cloud.google.com/storage/docs/gcs429.",
    "reason" : "rateLimitExceeded"
  } ],
  "message" : "The object <PATH> exceeded the rate limit for object mutation operations (create, update, and delete). Please reduce your request rate. See https://cloud.google.com/storage/docs/gcs429."
}
	at com.google.api.client.googleapis.json.GoogleJsonResponseException.from(GoogleJsonResponseException.java:146)
	at com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:118)
	at com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:37)
	at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:570)
	at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:493)
	at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.execute(AbstractGoogleClientRequest.java:603)
	at com.google.cloud.storage.spi.v1.HttpStorageRpc.create(HttpStorageRpc.java:376)
	... 16 more