Skip to content

Commit

Permalink
[Storage] Added Fix for Creating Share File before service copy; Adde…
Browse files Browse the repository at this point in the history
…d Data Movement E2E service copy tests (#39440)

* WIP

* WIP - tests written but CopyAuthorization needs to be done first

* Added tests for single sync copy file share; Added minor fixes for creation of file beforehand

* Fix recording

* Fix to share file resource tests
  • Loading branch information
amnguye authored Oct 23, 2023
1 parent 1bc8916 commit d4e9e97
Show file tree
Hide file tree
Showing 8 changed files with 1,066 additions and 18 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "net",
"TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares",
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_29f88351cb"
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_02f417687f"
}
Original file line number Diff line number Diff line change
Expand Up @@ -158,12 +158,16 @@ protected override async Task CopyFromUriAsync(
CancellationToken cancellationToken = default)
{
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
await ShareFileClient.UploadRangeFromUriAsync(
sourceUri: sourceResource.Uri,
range: new HttpRange(0, completeLength),
sourceRange: new HttpRange(0, completeLength),
options: _options.ToShareFileUploadRangeFromUriOptions(),
cancellationToken: cancellationToken).ConfigureAwait(false);
await CreateAsync(overwrite, completeLength, cancellationToken).ConfigureAwait(false);
if (completeLength > 0)
{
await ShareFileClient.UploadRangeFromUriAsync(
sourceUri: sourceResource.Uri,
range: new HttpRange(0, completeLength),
sourceRange: new HttpRange(0, completeLength),
options: _options.ToShareFileUploadRangeFromUriOptions(),
cancellationToken: cancellationToken).ConfigureAwait(false);
}
}

protected override async Task<bool> DeleteIfExistsAsync(CancellationToken cancellationToken = default)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
<Compile Include="$(AzureStorageDataMovementTestSharedSources)TestEventsRaised.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementTestSharedSources)DisposingLocalDirectory.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferUploadTestBase.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferCopyTestBase.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferDownloadTestBase.cs" LinkBase="Shared\DataMovement" />
</ItemGroup>
<ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using SharesClientBuilder = Azure.Storage.Test.Shared.ClientBuilder<
Azure.Storage.Files.Shares.ShareServiceClient,
Azure.Storage.Files.Shares.ShareClientOptions>;
using Azure.Storage.Files.Shares.Models;

namespace Azure.Storage.DataMovement.Files.Shares.Tests
{
Expand Down Expand Up @@ -42,6 +43,17 @@ public static SharesClientBuilder GetNewShareClientBuilder(TenantConfigurationBu
(uri, azureSasCredential, clientOptions) => new ShareServiceClient(uri, azureSasCredential, clientOptions),
() => new ShareClientOptions(serviceVersion));

public static ShareServiceClient GetServiceClient_OAuthAccount_SharedKey(this SharesClientBuilder clientBuilder) =>
clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigOAuth);

public static ShareServiceClient GetServiceClient_OAuth(
this SharesClientBuilder clientBuilder, ShareClientOptions options = default)
{
options ??= clientBuilder.GetOptions();
options.ShareTokenIntent = ShareTokenIntent.Backup;
return clientBuilder.GetServiceClientFromOauthConfig(clientBuilder.Tenants.TestConfigOAuth, options);
}

public static async Task<DisposingShare> GetTestShareAsync(
this SharesClientBuilder clientBuilder,
ShareServiceClient service = default,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,22 @@ public async Task CopyFromUriAsync()
contentHash: default,
isServerEncrypted: false),
new MockResponse(200))));
mockDestination.Setup(b => b.ExistsAsync(It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(Response.FromValue(false, new MockResponse(200))));
mockDestination.Setup(b => b.CreateAsync(It.IsAny<long>(), It.IsAny<ShareFileHttpHeaders>(), It.IsAny<Dictionary<string, string>>(), It.IsAny<FileSmbProperties>(), It.IsAny<string>(), It.IsAny<ShareFileRequestConditions>(), It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(Response.FromValue(
FilesModelFactory.StorageFileInfo(
eTag: new ETag("eTag"),
lastModified: DateTimeOffset.UtcNow,
isServerEncrypted: false,
filePermissionKey: "rw",
fileAttributes: "Archive|ReadOnly",
fileCreationTime: DateTimeOffset.UtcNow,
fileLastWriteTime: DateTimeOffset.UtcNow,
fileChangeTime: DateTimeOffset.UtcNow,
fileId: "48903841",
fileParentId: "93024923"),
new MockResponse(200))));
ShareFileStorageResource destinationResource = new ShareFileStorageResource(mockDestination.Object);

int length = 1024;
Expand All @@ -327,6 +343,18 @@ public async Task CopyFromUriAsync()
It.IsAny<ShareFileUploadRangeFromUriOptions>(),
It.IsAny<CancellationToken>()),
Times.Once());
mockDestination.Verify(b => b.CreateAsync(
length,
It.IsAny<ShareFileHttpHeaders>(),
It.IsAny<Dictionary<string, string>>(),
It.IsAny<FileSmbProperties>(),
It.IsAny<string>(),
It.IsAny<ShareFileRequestConditions>(),
It.IsAny<CancellationToken>()),
Times.Once());
mockDestination.Verify(b => b.ExistsAsync(
It.IsAny<CancellationToken>()),
Times.Once());
mockDestination.VerifyNoOtherCalls();
}

Expand All @@ -342,28 +370,24 @@ public async Task CopyFromUriAsync_Error()
new Uri("https://storageaccount.file.core.windows.net/container/destinationfile"),
new ShareClientOptions());

mockDestination.Setup(b => b.UploadRangeFromUriAsync(It.IsAny<Uri>(), It.IsAny<HttpRange>(), It.IsAny<HttpRange>(), It.IsAny<ShareFileUploadRangeFromUriOptions>(), It.IsAny<CancellationToken>()))
.Throws(new RequestFailedException(status: 404, message: "The specified resource does not exist.", errorCode: "ResourceNotFound", default));
mockDestination.Setup(b => b.ExistsAsync(It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(Response.FromValue(true, new MockResponse(200))));
ShareFileStorageResource destinationResource = new ShareFileStorageResource(mockDestination.Object);

// Act
int length = 1024;
await TestHelper.AssertExpectedExceptionAsync<RequestFailedException>(
destinationResource.CopyFromUriInternalAsync(sourceResource.Object, false, length),
await TestHelper.AssertExpectedExceptionAsync<InvalidOperationException>(
destinationResource.CopyBlockFromUriInternalAsync(sourceResource.Object, new HttpRange(0, length), false, length),
e =>
{
Assert.AreEqual("ResourceNotFound", e.ErrorCode);
Assert.IsTrue(e.Message.Contains("Cannot overwrite file."));
});

sourceResource.Verify(b => b.Uri, Times.Once());
sourceResource.VerifyNoOtherCalls();
mockDestination.Verify(b => b.UploadRangeFromUriAsync(
sourceResource.Object.Uri,
new HttpRange(0, length),
new HttpRange(0, length),
It.IsAny<ShareFileUploadRangeFromUriOptions>(),
mockDestination.Verify(b => b.ExistsAsync(
It.IsAny<CancellationToken>()),
Times.Once());
mockDestination.Verify(b => b.Path, Times.Once());
mockDestination.VerifyNoOtherCalls();
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;
using System.Threading.Tasks;
using Azure.Storage.Test.Shared;
using Azure.Storage.DataMovement.Tests;
using Azure.Storage.Files.Shares;
using Azure.Storage.Files.Shares.Tests;
using System.IO;
using Azure.Core;
using Azure.Core.TestFramework;

namespace Azure.Storage.DataMovement.Files.Shares.Tests
{
[ShareClientTestFixture]
public class ShareFileStartTransferCopyTests : StartTransferCopyTestBase
<ShareServiceClient,
ShareClient,
ShareFileClient,
ShareClientOptions,
ShareServiceClient,
ShareClient,
ShareFileClient,
ShareClientOptions,
StorageTestEnvironment>
{
private const string _fileResourcePrefix = "test-file-";
private const string _expectedOverwriteExceptionMessage = "Cannot overwrite file.";
protected readonly ShareClientOptions.ServiceVersion _serviceVersion;

public ShareFileStartTransferCopyTests(bool async, ShareClientOptions.ServiceVersion serviceVersion)
: base(async, _expectedOverwriteExceptionMessage, _fileResourcePrefix, null /* RecordedTestMode.Record /* to re-record */)
{
_serviceVersion = serviceVersion;
SourceClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion);
DestinationClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion);
}

protected override async Task<bool> SourceExistsAsync(ShareFileClient objectClient)
=> await objectClient.ExistsAsync();

protected override async Task<bool> DestinationExistsAsync(ShareFileClient objectClient)
=> await objectClient.ExistsAsync();

protected override async Task<IDisposingContainer<ShareClient>> GetSourceDisposingContainerAsync(ShareServiceClient service = null, string containerName = null)
=> await SourceClientBuilder.GetTestShareAsync(service, containerName);

protected override async Task<IDisposingContainer<ShareClient>> GetDestinationDisposingContainerAsync(ShareServiceClient service = null, string containerName = null)
=> await DestinationClientBuilder.GetTestShareAsync(service, containerName);

private async Task<ShareFileClient> CreateFileClientAsync(
ShareClient container,
long? objectLength = null,
bool createResource = false,
string objectName = null,
ShareClientOptions options = null,
Stream contents = null)
{
objectName ??= GetNewObjectName();
ShareFileClient fileClient = container.GetRootDirectoryClient().GetFileClient(objectName);
if (createResource)
{
if (!objectLength.HasValue)
{
throw new InvalidOperationException($"Cannot create share file without size specified. Either set {nameof(createResource)} to false or specify a {nameof(objectLength)}.");
}
await fileClient.CreateAsync(objectLength.Value);

if (contents != default)
{
await fileClient.UploadAsync(contents);
}
}
Uri sourceUri = fileClient.GenerateSasUri(Sas.ShareFileSasPermissions.All, Recording.UtcNow.AddDays(1));
return InstrumentClient(new ShareFileClient(sourceUri, GetOptions()));
}

protected override Task<ShareFileClient> GetSourceObjectClientAsync(
ShareClient container,
long? objectLength = null,
bool createResource = false,
string objectName = null,
ShareClientOptions options = null,
Stream contents = null)
=> CreateFileClientAsync(
container,
objectLength,
createResource,
objectName,
options,
contents);

protected override StorageResourceItem GetSourceStorageResourceItem(ShareFileClient objectClient)
=> new ShareFileStorageResource(objectClient);

protected override Task<Stream> SourceOpenReadAsync(ShareFileClient objectClient)
=> objectClient.OpenReadAsync();

protected override Task<ShareFileClient> GetDestinationObjectClientAsync(
ShareClient container,
long? objectLength = null,
bool createResource = false,
string objectName = null,
ShareClientOptions options = null,
Stream contents = null)
=> CreateFileClientAsync(
container,
objectLength,
createResource,
objectName,
options,
contents);

protected override StorageResourceItem GetDestinationStorageResourceItem(ShareFileClient objectClient)
=> new ShareFileStorageResource(objectClient);

protected override Task<Stream> DestinationOpenReadAsync(ShareFileClient objectClient)
=> objectClient.OpenReadAsync();

public ShareClientOptions GetOptions()
{
var options = new ShareClientOptions(_serviceVersion)
{
Diagnostics = { IsLoggingEnabled = true },
Retry =
{
Mode = RetryMode.Exponential,
MaxRetries = Constants.MaxReliabilityRetries,
Delay = TimeSpan.FromSeconds(Mode == RecordedTestMode.Playback ? 0.01 : 1),
MaxDelay = TimeSpan.FromSeconds(Mode == RecordedTestMode.Playback ? 0.1 : 60)
},
};
if (Mode != RecordedTestMode.Live)
{
options.AddPolicy(new RecordedClientRequestIdPolicy(Recording), HttpPipelinePosition.PerCall);
}

return InstrumentClientOptions(options);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,12 @@ await _destinationResource.CopyFromUriAsync(
{
await InvokeSkippedArg().ConfigureAwait(false);
}
catch (InvalidOperationException ex)
when (_createMode == StorageResourceCreationPreference.SkipIfExists
&& ex.Message.Contains("Cannot overwrite file."))
{
await InvokeSkippedArg().ConfigureAwait(false);
}
catch (Exception ex)
{
await InvokeFailedArg(ex).ConfigureAwait(false);
Expand Down
Loading

0 comments on commit d4e9e97

Please sign in to comment.