Skip to content

Commit

Permalink
[Storage][DataMovement] More GA API feedback (renames/refactors) (#47796
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@jalauzon-msft
jalauzon-msft authored Jan 14, 2025
1 parent 23c0c85 commit a5c44c2
Show file tree
Hide file tree
Showing 62 changed files with 784 additions and 665 deletions.
4 changes: 2 additions & 2 deletions sdk/storage/Azure.Storage.Blobs/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ BlobContainerClientTransferOptions options = new BlobContainerClientTransferOpti
},
TransferOptions = new TransferOptions()
{
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
CreationPreference = StorageResourceCreationMode.OverwriteIfExists,
}
};

Expand Down Expand Up @@ -249,7 +249,7 @@ BlobContainerClientTransferOptions options = new BlobContainerClientTransferOpti
},
TransferOptions = new TransferOptions()
{
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
CreationPreference = StorageResourceCreationMode.OverwriteIfExists,
}
};

Expand Down
4 changes: 2 additions & 2 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ BlobContainerClientTransferOptions options = new BlobContainerClientTransferOpti
},
TransferOptions = new TransferOptions()
{
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
CreationPreference = StorageResourceCreationMode.OverwriteIfExists,
}
};

Expand Down Expand Up @@ -305,7 +305,7 @@ BlobContainerClientTransferOptions options = new BlobContainerClientTransferOpti
},
TransferOptions = new TransferOptions()
{
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
CreationPreference = StorageResourceCreationMode.OverwriteIfExists,
}
};

Expand Down
6 changes: 3 additions & 3 deletions ....Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Infrastructure/DirectoryTransferTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ public DirectoryTransferTest(TOptions options) : base(options)

TransferManagerOptions managerOptions = new()
{
ErrorHandling = TransferErrorMode.StopOnAnyFailure,
CheckpointerOptions = Options.DisableCheckpointer ? TransferCheckpointStoreOptions.Disabled() : default,
ErrorMode = TransferErrorMode.StopOnAnyFailure,
CheckpointStoreOptions = Options.DisableCheckpointer ? TransferCheckpointStoreOptions.DisableCheckpoint() : default,
MaximumConcurrency = Options.Concurrency
};
_transferManager = new TransferManager(managerOptions);
Expand Down Expand Up @@ -85,7 +85,7 @@ protected async Task RunAndVerifyTransferAsync(
{
TransferOptions options = new()
{
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
CreationPreference = StorageResourceCreationMode.OverwriteIfExists,
InitialTransferSize = Options.InitialTransferSize,
MaximumTransferChunkSize = Options.ChunkSize,
};
Expand Down
18 changes: 9 additions & 9 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ public async Task DownloadSingle_SharedKeyAuthAsync()
// Create Blob Data Controller to skip through all failures
TransferManagerOptions options = new TransferManagerOptions()
{
ErrorHandling = TransferErrorMode.ContinueOnFailure
ErrorMode = TransferErrorMode.ContinueOnFailure
};
TransferManager transferManager = new TransferManager(options);
BlobsStorageResourceProvider blobs = new();
Expand Down Expand Up @@ -368,7 +368,7 @@ public async Task UploadDirectory_SasAsync()
TransferOptions options = new TransferOptions()
{
MaximumTransferChunkSize = 4 * Constants.MB,
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
CreationPreference = StorageResourceCreationMode.OverwriteIfExists,
};
TransferManager transferManager = new TransferManager(transferManagerOptions);

Expand Down Expand Up @@ -463,7 +463,7 @@ public async Task UploadDirectory_CompletedEventHandler()
{
using (StreamWriter logStream = File.AppendText(logFile))
{
logStream.WriteLine($"File Completed Transfer: {args.SourceResource.Uri.AbsoluteUri}");
logStream.WriteLine($"File Completed Transfer: {args.Source.Uri.AbsoluteUri}");
}
return Task.CompletedTask;
};
Expand Down Expand Up @@ -565,8 +565,8 @@ public async Task UploadDirectory_EventHandler_SasAsync()
// Specifying specific resources that failed, since its a directory transfer
// maybe only one file failed out of many
logStream.WriteLine($"Exception occurred with TransferId: {args.TransferId}," +
$"Source Resource: {args.SourceResource.Uri.AbsoluteUri}, +" +
$"Destination Resource: {args.DestinationResource.Uri.AbsoluteUri}," +
$"Source Resource: {args.Source.Uri.AbsoluteUri}, +" +
$"Destination Resource: {args.Destination.Uri.AbsoluteUri}," +
$"Exception Message: {args.Exception.Message}");
}
return Task.CompletedTask;
Expand Down Expand Up @@ -901,7 +901,7 @@ public async Task PauseAndResumeAsync_ManagerId()

// Pause from the Transfer Manager using the Transfer Id
#region Snippet:PauseFromManager
await transferManager.PauseTransferIfRunningAsync(transferId);
await transferManager.PauseTransferAsync(transferId);
#endregion

// Resume all transfers
Expand Down Expand Up @@ -1029,7 +1029,7 @@ public async Task UploadDirectory()
},
TransferOptions = new TransferOptions()
{
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
CreationPreference = StorageResourceCreationMode.OverwriteIfExists,
}
};

Expand Down Expand Up @@ -1099,7 +1099,7 @@ public async Task DownloadDirectory()
},
TransferOptions = new TransferOptions()
{
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
CreationPreference = StorageResourceCreationMode.OverwriteIfExists,
}
};

Expand Down Expand Up @@ -1152,7 +1152,7 @@ async Task<TransferOperation> ListenToTransfersAsync(TransferManager transferMan
transferOptions.ItemTransferCompleted += (TransferItemCompletedEventArgs args) =>
{
using StreamWriter logStream = File.AppendText(logFile);
logStream.WriteLine($"File Completed Transfer: {args.SourceResource.Uri.LocalPath}");
logStream.WriteLine($"File Completed Transfer: {args.Source.Uri.LocalPath}");
return Task.CompletedTask;
};
return await transferManager.StartTransferAsync(
Expand Down
54 changes: 31 additions & 23 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,13 @@ internal static StorageResourceItemProperties ToStorageResourceProperties(this B
properties.Add(DataMovementConstants.ResourceProperties.AccessTier, new AccessTier(blobProperties.AccessTier));
}

return new StorageResourceItemProperties(
resourceLength: blobProperties.ContentLength,
eTag: blobProperties.ETag,
lastModifiedTime: blobProperties.LastModified,
properties: properties);
return new StorageResourceItemProperties()
{
ResourceLength = blobProperties.ContentLength,
ETag = blobProperties.ETag,
LastModifiedTime = blobProperties.LastModified,
RawProperties = properties
};
}

internal static StorageResourceItemProperties ToStorageResourceItemProperties(this BlobDownloadStreamingResult result)
Expand Down Expand Up @@ -102,11 +104,13 @@ internal static StorageResourceItemProperties ToStorageResourceItemProperties(th
size = contentRange.Size;
}

return new StorageResourceItemProperties(
resourceLength: size,
eTag : result?.Details.ETag,
lastModifiedTime: result?.Details.LastModified,
properties: properties);
return new StorageResourceItemProperties()
{
ResourceLength = size,
ETag = result?.Details.ETag,
LastModifiedTime = result?.Details.LastModified,
RawProperties = properties
};
}

internal static StorageResourceReadStreamResult ToReadStreamStorageResourceInfo(this BlobDownloadStreamingResult result)
Expand Down Expand Up @@ -162,11 +166,13 @@ internal static StorageResourceReadStreamResult ToReadStreamStorageResourceInfo(
return new StorageResourceReadStreamResult(
content: result.Content,
range: range,
properties: new StorageResourceItemProperties(
resourceLength: size.HasValue ? size : result.Details.ContentLength,
eTag: result.Details.ETag,
lastModifiedTime: result?.Details.LastModified,
properties: properties));
properties: new StorageResourceItemProperties()
{
ResourceLength = size.HasValue ? size : result.Details.ContentLength,
ETag = result.Details.ETag,
LastModifiedTime = result?.Details.LastModified,
RawProperties = properties
});
}

/// <summary>
Expand Down Expand Up @@ -631,11 +637,13 @@ internal static StorageResourceItemProperties ToResourceProperties(this BlobItem
properties.Add(DataMovementConstants.ResourceProperties.CacheControl, blobItem.Properties.CacheControl);
}

return new StorageResourceItemProperties(
resourceLength: blobItem.Properties.ContentLength,
eTag: blobItem.Properties.ETag,
lastModifiedTime: blobItem.Properties.LastModified,
properties: properties);
return new StorageResourceItemProperties()
{
ResourceLength = blobItem.Properties.ContentLength,
ETag = blobItem.Properties.ETag,
LastModifiedTime = blobItem.Properties.LastModified,
RawProperties = properties
};
}

private static string ConvertContentPropertyObjectToString(string contentPropertyName, object contentPropertyValue)
Expand All @@ -656,7 +664,7 @@ private static string ConvertContentPropertyObjectToString(string contentPropert

private static BlobHttpHeaders GetHttpHeaders(
BlobStorageResourceOptions options,
Dictionary<string, object> properties)
IDictionary<string, object> properties)
=> new()
{
ContentType = (options?.ContentType?.Preserve ?? true)
Expand Down Expand Up @@ -689,7 +697,7 @@ private static BlobHttpHeaders GetHttpHeaders(
// Get the access tier property
private static AccessTier? GetAccessTier(
BlobStorageResourceOptions options,
Dictionary<string, object> properties)
IDictionary<string, object> properties)
=> options?.AccessTier != default
? options?.AccessTier
: properties?.TryGetValue(DataMovementConstants.ResourceProperties.AccessTier, out object accessTierObject) == true
Expand All @@ -699,7 +707,7 @@ private static BlobHttpHeaders GetHttpHeaders(
// By default we preserve the metadata
private static Metadata GetMetadata(
BlobStorageResourceOptions options,
Dictionary<string, object> properties)
IDictionary<string, object> properties)
=> (options?.Metadata?.Preserve ?? true)
? properties?.TryGetValue(DataMovementConstants.ResourceProperties.Metadata, out object metadataObject) == true
? (Metadata) metadataObject
Expand Down
12 changes: 6 additions & 6 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Infrastructure/StressEventsRaised.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,8 @@ public void AssertUnexpectedFailureCheck()
{
Assert.Fail(
$"Failure occurred at Transfer id: {failure.TransferId}.\n" +
$"Source Resource Path: {failure.SourceResource.Uri.AbsoluteUri}\n" +
$"Destination Resource Path: {failure.DestinationResource.Uri.AbsoluteUri}\n" +
$"Source Resource Path: {failure.Source.Uri.AbsoluteUri}\n" +
$"Destination Resource Path: {failure.Destination.Uri.AbsoluteUri}\n" +
$"Exception Message: {failure.Exception.Message}\n" +
$"Exception Stack: {failure.Exception.StackTrace}\n");
}
Expand Down Expand Up @@ -177,8 +177,8 @@ public async Task AssertSingleSkippedCheck()
AssertUnexpectedFailureCheck();
Assert.IsEmpty(SingleCompletedEvents);
Assert.AreEqual(1, SkippedEvents.Count);
Assert.NotNull(SkippedEvents.First().SourceResource.Uri);
Assert.NotNull(SkippedEvents.First().DestinationResource.Uri);
Assert.NotNull(SkippedEvents.First().Source.Uri);
Assert.NotNull(SkippedEvents.First().Destination.Uri);

AssertTransferStatusCollection(
new TransferStatus[] {
Expand All @@ -202,8 +202,8 @@ public async Task AssertSingleFailedCheck(int failureCount)
foreach (TransferItemFailedEventArgs args in FailedEvents)
{
Assert.NotNull(args.Exception);
Assert.NotNull(args.SourceResource.Uri);
Assert.NotNull(args.DestinationResource.Uri);
Assert.NotNull(args.Source.Uri);
Assert.NotNull(args.Destination.Uri);
}

AssertTransferStatusCollection(
Expand Down
Loading

0 comments on commit a5c44c2

Please sign in to comment.