Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Storage][DataMovement] More GA API feedback (renames/refactors) #47796

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

[Storage][DataMovement] More GA API feedback (renames/refactors) #47796

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e8ba811
More API feedback/renames
jalauzon-msft Jan 10, 2025
e04d4f4
Change to IDictionary
jalauzon-msft Jan 11, 2025
cc061ac
Export API
jalauzon-msft Jan 11, 2025
b2f38ef
A couple more
jalauzon-msft Jan 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ....Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Infrastructure/DirectoryTransferTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ public DirectoryTransferTest(TOptions options) : base(options)

TransferManagerOptions managerOptions = new()
{
ErrorHandling = TransferErrorMode.StopOnAnyFailure,
CheckpointerOptions = Options.DisableCheckpointer ? TransferCheckpointStoreOptions.Disabled() : default,
ErrorMode = TransferErrorMode.StopOnAnyFailure,
CheckpointStoreOptions = Options.DisableCheckpointer ? TransferCheckpointStoreOptions.DisableCheckpoint() : default,
MaximumConcurrency = Options.Concurrency
};
_transferManager = new TransferManager(managerOptions);
Expand Down Expand Up @@ -85,7 +85,7 @@ protected async Task RunAndVerifyTransferAsync(
{
TransferOptions options = new()
{
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
CreationPreference = StorageResourceCreationMode.OverwriteIfExists,
InitialTransferSize = Options.InitialTransferSize,
MaximumTransferChunkSize = Options.ChunkSize,
};
Expand Down
18 changes: 9 additions & 9 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ public async Task DownloadSingle_SharedKeyAuthAsync()
// Create Blob Data Controller to skip through all failures
TransferManagerOptions options = new TransferManagerOptions()
{
ErrorHandling = TransferErrorMode.ContinueOnFailure
ErrorMode = TransferErrorMode.ContinueOnFailure
};
TransferManager transferManager = new TransferManager(options);
BlobsStorageResourceProvider blobs = new();
Expand Down Expand Up @@ -368,7 +368,7 @@ public async Task UploadDirectory_SasAsync()
TransferOptions options = new TransferOptions()
{
MaximumTransferChunkSize = 4 * Constants.MB,
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
CreationPreference = StorageResourceCreationMode.OverwriteIfExists,
};
TransferManager transferManager = new TransferManager(transferManagerOptions);

Expand Down Expand Up @@ -463,7 +463,7 @@ public async Task UploadDirectory_CompletedEventHandler()
{
using (StreamWriter logStream = File.AppendText(logFile))
{
logStream.WriteLine($"File Completed Transfer: {args.SourceResource.Uri.AbsoluteUri}");
logStream.WriteLine($"File Completed Transfer: {args.Source.Uri.AbsoluteUri}");
}
return Task.CompletedTask;
};
Expand Down Expand Up @@ -565,8 +565,8 @@ public async Task UploadDirectory_EventHandler_SasAsync()
// Specifying specific resources that failed, since its a directory transfer
// maybe only one file failed out of many
logStream.WriteLine($"Exception occurred with TransferId: {args.TransferId}," +
$"Source Resource: {args.SourceResource.Uri.AbsoluteUri}, +" +
$"Destination Resource: {args.DestinationResource.Uri.AbsoluteUri}," +
$"Source Resource: {args.Source.Uri.AbsoluteUri}, +" +
$"Destination Resource: {args.Destination.Uri.AbsoluteUri}," +
$"Exception Message: {args.Exception.Message}");
}
return Task.CompletedTask;
Expand Down Expand Up @@ -901,7 +901,7 @@ public async Task PauseAndResumeAsync_ManagerId()

// Pause from the Transfer Manager using the Transfer Id
#region Snippet:PauseFromManager
await transferManager.PauseTransferIfRunningAsync(transferId);
await transferManager.PauseTransferAsync(transferId);
#endregion

// Resume all transfers
Expand Down Expand Up @@ -1029,7 +1029,7 @@ public async Task UploadDirectory()
},
TransferOptions = new TransferOptions()
{
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
CreationPreference = StorageResourceCreationMode.OverwriteIfExists,
}
};

Expand Down Expand Up @@ -1099,7 +1099,7 @@ public async Task DownloadDirectory()
},
TransferOptions = new TransferOptions()
{
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
CreationPreference = StorageResourceCreationMode.OverwriteIfExists,
}
};

Expand Down Expand Up @@ -1152,7 +1152,7 @@ async Task<TransferOperation> ListenToTransfersAsync(TransferManager transferMan
transferOptions.ItemTransferCompleted += (TransferItemCompletedEventArgs args) =>
{
using StreamWriter logStream = File.AppendText(logFile);
logStream.WriteLine($"File Completed Transfer: {args.SourceResource.Uri.LocalPath}");
logStream.WriteLine($"File Completed Transfer: {args.Source.Uri.LocalPath}");
return Task.CompletedTask;
};
return await transferManager.StartTransferAsync(
Expand Down
54 changes: 31 additions & 23 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,13 @@ internal static StorageResourceItemProperties ToStorageResourceProperties(this B
properties.Add(DataMovementConstants.ResourceProperties.AccessTier, new AccessTier(blobProperties.AccessTier));
}

return new StorageResourceItemProperties(
resourceLength: blobProperties.ContentLength,
eTag: blobProperties.ETag,
lastModifiedTime: blobProperties.LastModified,
properties: properties);
return new StorageResourceItemProperties()
{
ResourceLength = blobProperties.ContentLength,
ETag = blobProperties.ETag,
LastModifiedTime = blobProperties.LastModified,
RawProperties = properties
};
}

internal static StorageResourceItemProperties ToStorageResourceItemProperties(this BlobDownloadStreamingResult result)
Expand Down Expand Up @@ -102,11 +104,13 @@ internal static StorageResourceItemProperties ToStorageResourceItemProperties(th
size = contentRange.Size;
}

return new StorageResourceItemProperties(
resourceLength: size,
eTag : result?.Details.ETag,
lastModifiedTime: result?.Details.LastModified,
properties: properties);
return new StorageResourceItemProperties()
{
ResourceLength = size,
ETag = result?.Details.ETag,
LastModifiedTime = result?.Details.LastModified,
RawProperties = properties
};
}

internal static StorageResourceReadStreamResult ToReadStreamStorageResourceInfo(this BlobDownloadStreamingResult result)
Expand Down Expand Up @@ -162,11 +166,13 @@ internal static StorageResourceReadStreamResult ToReadStreamStorageResourceInfo(
return new StorageResourceReadStreamResult(
content: result.Content,
range: range,
properties: new StorageResourceItemProperties(
resourceLength: size.HasValue ? size : result.Details.ContentLength,
eTag: result.Details.ETag,
lastModifiedTime: result?.Details.LastModified,
properties: properties));
properties: new StorageResourceItemProperties()
{
ResourceLength = size.HasValue ? size : result.Details.ContentLength,
ETag = result.Details.ETag,
LastModifiedTime = result?.Details.LastModified,
RawProperties = properties
});
}

/// <summary>
Expand Down Expand Up @@ -631,11 +637,13 @@ internal static StorageResourceItemProperties ToResourceProperties(this BlobItem
properties.Add(DataMovementConstants.ResourceProperties.CacheControl, blobItem.Properties.CacheControl);
}

return new StorageResourceItemProperties(
resourceLength: blobItem.Properties.ContentLength,
eTag: blobItem.Properties.ETag,
lastModifiedTime: blobItem.Properties.LastModified,
properties: properties);
return new StorageResourceItemProperties()
{
ResourceLength = blobItem.Properties.ContentLength,
ETag = blobItem.Properties.ETag,
LastModifiedTime = blobItem.Properties.LastModified,
RawProperties = properties
};
}

private static string ConvertContentPropertyObjectToString(string contentPropertyName, object contentPropertyValue)
Expand All @@ -656,7 +664,7 @@ private static string ConvertContentPropertyObjectToString(string contentPropert

private static BlobHttpHeaders GetHttpHeaders(
BlobStorageResourceOptions options,
Dictionary<string, object> properties)
IDictionary<string, object> properties)
=> new()
{
ContentType = (options?.ContentType?.Preserve ?? true)
Expand Down Expand Up @@ -689,7 +697,7 @@ private static BlobHttpHeaders GetHttpHeaders(
// Get the access tier property
private static AccessTier? GetAccessTier(
BlobStorageResourceOptions options,
Dictionary<string, object> properties)
IDictionary<string, object> properties)
=> options?.AccessTier != default
? options?.AccessTier
: properties?.TryGetValue(DataMovementConstants.ResourceProperties.AccessTier, out object accessTierObject) == true
Expand All @@ -699,7 +707,7 @@ private static BlobHttpHeaders GetHttpHeaders(
// By default we preserve the metadata
private static Metadata GetMetadata(
BlobStorageResourceOptions options,
Dictionary<string, object> properties)
IDictionary<string, object> properties)
=> (options?.Metadata?.Preserve ?? true)
? properties?.TryGetValue(DataMovementConstants.ResourceProperties.Metadata, out object metadataObject) == true
? (Metadata) metadataObject
Expand Down
12 changes: 6 additions & 6 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Infrastructure/StressEventsRaised.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,8 @@ public void AssertUnexpectedFailureCheck()
{
Assert.Fail(
$"Failure occurred at Transfer id: {failure.TransferId}.\n" +
$"Source Resource Path: {failure.SourceResource.Uri.AbsoluteUri}\n" +
$"Destination Resource Path: {failure.DestinationResource.Uri.AbsoluteUri}\n" +
$"Source Resource Path: {failure.Source.Uri.AbsoluteUri}\n" +
$"Destination Resource Path: {failure.Destination.Uri.AbsoluteUri}\n" +
$"Exception Message: {failure.Exception.Message}\n" +
$"Exception Stack: {failure.Exception.StackTrace}\n");
}
Expand Down Expand Up @@ -177,8 +177,8 @@ public async Task AssertSingleSkippedCheck()
AssertUnexpectedFailureCheck();
Assert.IsEmpty(SingleCompletedEvents);
Assert.AreEqual(1, SkippedEvents.Count);
Assert.NotNull(SkippedEvents.First().SourceResource.Uri);
Assert.NotNull(SkippedEvents.First().DestinationResource.Uri);
Assert.NotNull(SkippedEvents.First().Source.Uri);
Assert.NotNull(SkippedEvents.First().Destination.Uri);

AssertTransferStatusCollection(
new TransferStatus[] {
Expand All @@ -202,8 +202,8 @@ public async Task AssertSingleFailedCheck(int failureCount)
foreach (TransferItemFailedEventArgs args in FailedEvents)
{
Assert.NotNull(args.Exception);
Assert.NotNull(args.SourceResource.Uri);
Assert.NotNull(args.DestinationResource.Uri);
Assert.NotNull(args.Source.Uri);
Assert.NotNull(args.Destination.Uri);
}

AssertTransferStatusCollection(
Expand Down
Loading
Loading