From 8f97e2a0e9284553301a8885bdd749e0e44c12ec Mon Sep 17 00:00:00 2001 From: t-rana <145645280+t-rana@users.noreply.github.com> Date: Mon, 4 Mar 2024 15:48:00 +0530 Subject: [PATCH] OAK-10670: add support for service principal in oak-upgrade (#1329) * OAK-10670: add service principal support in oak upgrade * OAK-10670: add service principal support in oak upgrade * OAK-10670: add service principal support in oak upgrade * OAK-10670: add service principal support in oak upgrade * OAK-10670: minor change * OAK-10670: delete tempDir after closing filestore * OAK-10670: add license to new files --- .../oak/segment/azure/AzureUtilities.java | 46 +++-- .../oak/segment/azure/package-info.java | 2 +- .../oak/segment/azure/tool/ToolUtils.java | 9 +- .../upgrade/cli/node/SegmentAzureFactory.java | 52 +++-- .../oak/upgrade/cli/parser/StoreType.java | 2 + ...TarToSegmentAzureServicePrincipalTest.java | 69 +++++++ ...ureServicePrincipalNodeStoreContainer.java | 116 +++++++++++ .../cli/node/SegmentAzureFactoryTest.java | 190 ++++++++++++++++++ 8 files changed, 449 insertions(+), 37 deletions(-) create mode 100644 oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/SegmentTarToSegmentAzureServicePrincipalTest.java create mode 100644 oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java create mode 100644 oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java index 46e410e3983..5817d78a443 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java @@ -16,17 +16,6 @@ */ package org.apache.jackrabbit.oak.segment.azure; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Paths; -import java.security.InvalidKeyException; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.List; - import com.azure.core.credential.TokenRequestContext; import com.azure.identity.ClientSecretCredential; import com.azure.identity.ClientSecretCredentialBuilder; @@ -41,6 +30,7 @@ import com.microsoft.azure.storage.blob.CloudBlob; import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlobDirectory; +import com.microsoft.azure.storage.blob.LeaseStatus; import com.microsoft.azure.storage.blob.ListBlobItem; import org.apache.jackrabbit.oak.commons.Buffer; import org.apache.jackrabbit.oak.segment.spi.RepositoryNotReachableException; @@ -48,6 +38,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Paths; +import java.security.InvalidKeyException; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; + public final class AzureUtilities { public static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME"; public static final String AZURE_SECRET_KEY = "AZURE_SECRET_KEY"; @@ -110,7 +111,7 @@ public static void deleteAllEntries(CloudBlobDirectory directory) throws IOExcep } public static CloudBlobDirectory cloudBlobDirectoryFrom(StorageCredentials credentials, - String uri, String dir) throws URISyntaxException, StorageException { + String uri, String dir) throws URISyntaxException, StorageException { StorageUri storageUri = new StorageUri(new URI(uri)); CloudBlobContainer container = new CloudBlobContainer(storageUri, credentials); @@ -120,7 +121,7 @@ public static CloudBlobDirectory cloudBlobDirectoryFrom(StorageCredentials crede } public static CloudBlobDirectory cloudBlobDirectoryFrom(String connection, String containerName, - String dir) throws InvalidKeyException, URISyntaxException, StorageException { + String dir) throws InvalidKeyException, URISyntaxException, StorageException { CloudStorageAccount cloud = CloudStorageAccount.parse(connection); CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(containerName); container.createIfNotExists(); @@ -140,7 +141,7 @@ public static StorageCredentialsToken storageCredentialAccessTokenFrom(String ac } private static ResultSegment listBlobsInSegments(CloudBlobDirectory directory, - ResultContinuation token) throws IOException { + ResultContinuation token) throws IOException { ResultSegment result = null; IOException lastException = null; for (int sleep = 10; sleep <= 10000; sleep *= 10) { //increment the sleep time in steps. @@ -171,6 +172,21 @@ private static ResultSegment listBlobsInSegments(CloudBlobDirector } } + public static void deleteAllBlobs(@NotNull CloudBlobDirectory directory) throws URISyntaxException, StorageException, InterruptedException { + for (ListBlobItem blobItem : directory.listBlobs()) { + if (blobItem instanceof CloudBlob) { + CloudBlob cloudBlob = (CloudBlob) blobItem; + if (cloudBlob.getProperties().getLeaseStatus() == LeaseStatus.LOCKED) { + cloudBlob.breakLease(0); + } + cloudBlob.deleteIfExists(); + } else if (blobItem instanceof CloudBlobDirectory) { + CloudBlobDirectory cloudBlobDirectory = (CloudBlobDirectory) blobItem; + deleteAllBlobs(cloudBlobDirectory); + } + } + } + private static class ByteBufferOutputStream extends OutputStream { @NotNull @@ -182,7 +198,7 @@ public ByteBufferOutputStream(@NotNull Buffer buffer) { @Override public void write(int b) { - buffer.put((byte)b); + buffer.put((byte) b); } @Override diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java index 46bf6461768..e0547511516 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java @@ -15,7 +15,7 @@ * limitations under the License. */ @Internal(since = "1.0.0") -@Version("2.2.0") +@Version("2.3.0") package org.apache.jackrabbit.oak.segment.azure; import org.apache.jackrabbit.oak.commons.annotations.Internal; diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java index 51984f440c7..f578734bcd5 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java @@ -172,12 +172,12 @@ public static SegmentArchiveManager createArchiveManager(SegmentNodeStorePersist public static CloudBlobDirectory createCloudBlobDirectory(String path) { return createCloudBlobDirectory(path, ENVIRONMENT); } - + public static CloudBlobDirectory createCloudBlobDirectory(String path, Environment environment) { Map config = parseAzureConfigurationFromUri(path); String accountName = config.get(KEY_ACCOUNT_NAME); - + StorageCredentials credentials; if (config.containsKey(KEY_SHARED_ACCESS_SIGNATURE)) { credentials = new StorageCredentialsSharedAccessSignature(config.get(KEY_SHARED_ACCESS_SIGNATURE)); @@ -196,6 +196,11 @@ public static CloudBlobDirectory createCloudBlobDirectory(String path, Environme } } + @NotNull + public static StorageCredentials getStorageCredentialsFromAccountAndEnv(@NotNull String accountName) { + return getStorageCredentialsFromAccountAndEnv(accountName, ENVIRONMENT); + } + @NotNull private static StorageCredentials getStorageCredentialsFromAccountAndEnv(String accountName, Environment environment) { String clientId = environment.getVariable(AZURE_CLIENT_ID); diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java index 7ee319f93b0..a48ecd6de67 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java @@ -16,17 +16,17 @@ */ package org.apache.jackrabbit.oak.upgrade.cli.node; -import static org.apache.jackrabbit.oak.segment.SegmentCache.DEFAULT_SEGMENT_CACHE_MB; -import static org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils.asCloseable; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.security.InvalidKeyException; - +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.CloudBlobDirectory; +import org.apache.commons.lang3.StringUtils; +import org.apache.jackrabbit.guava.common.io.Closer; +import org.apache.jackrabbit.guava.common.io.Files; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; +import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.file.FileStore; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; @@ -35,15 +35,17 @@ import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils.NodeStoreWithFileStore; -import org.apache.jackrabbit.guava.common.io.Closer; -import org.apache.jackrabbit.guava.common.io.Files; -import com.microsoft.azure.storage.StorageCredentials; -import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.blob.CloudBlobDirectory; +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.security.InvalidKeyException; + +import static org.apache.jackrabbit.oak.segment.SegmentCache.DEFAULT_SEGMENT_CACHE_MB; +import static org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils.asCloseable; public class SegmentAzureFactory implements NodeStoreFactory { private final String accountName; + private final String sasToken; private final String uri; private final String connectionString; private final String containerName; @@ -58,6 +60,7 @@ public static class Builder { private final boolean readOnly; private String accountName; + private String sasToken; private String uri; private String connectionString; private String containerName; @@ -73,6 +76,11 @@ public Builder accountName(String accountName) { return this; } + public Builder sasToken(String sasToken) { + this.sasToken = sasToken; + return this; + } + public Builder uri(String uri) { this.uri = uri; return this; @@ -95,6 +103,7 @@ public SegmentAzureFactory build() { public SegmentAzureFactory(Builder builder) { this.accountName = builder.accountName; + this.sasToken = builder.sasToken; this.uri = builder.uri; this.connectionString = builder.connectionString; this.containerName = builder.containerName; @@ -142,12 +151,17 @@ public NodeStore create(BlobStore blobStore, Closer closer) throws IOException { private AzurePersistence createAzurePersistence() throws StorageException, URISyntaxException, InvalidKeyException { CloudBlobDirectory cloudBlobDirectory = null; - if (accountName != null && uri != null) { - String key = System.getenv("AZURE_SECRET_KEY"); - StorageCredentials credentials = new StorageCredentialsAccountAndKey(accountName, key); - cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); - } else if (connectionString != null && containerName != null) { + // connection string will take precedence over accountkey / sas / service principal + if (StringUtils.isNoneBlank(connectionString, containerName)) { cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(connectionString, containerName, dir); + } else if (StringUtils.isNoneBlank(accountName, uri)) { + StorageCredentials credentials = null; + if (StringUtils.isNotBlank(sasToken)) { + credentials = new StorageCredentialsSharedAccessSignature(sasToken); + } else { + credentials = ToolUtils.getStorageCredentialsFromAccountAndEnv(accountName); + } + cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); } if (cloudBlobDirectory == null) { diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java index 5d46078c86e..b9b961b5eb4 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java @@ -21,6 +21,7 @@ import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONNECTION_STRING; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONTAINER_NAME; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_SHARED_ACCESS_SIGNATURE; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_STORAGE_URI; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.isCustomAzureConnectionString; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.parseAzureConfigurationFromCustomConnection; @@ -175,6 +176,7 @@ public StoreFactory createFactory(String[] paths, MigrationDirection direction, return new StoreFactory(new SegmentAzureFactory.Builder(config.get(KEY_DIR), migrationOptions.getCacheSizeInMB(), direction == MigrationDirection.SRC) .accountName(config.get(KEY_ACCOUNT_NAME)) + .sasToken(config.get(KEY_SHARED_ACCESS_SIGNATURE)) .uri(config.get(KEY_STORAGE_URI)) .build() ); diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/SegmentTarToSegmentAzureServicePrincipalTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/SegmentTarToSegmentAzureServicePrincipalTest.java new file mode 100644 index 00000000000..1d42f0ee296 --- /dev/null +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/SegmentTarToSegmentAzureServicePrincipalTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.upgrade.cli; + +import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; +import org.apache.jackrabbit.oak.segment.azure.util.Environment; +import org.apache.jackrabbit.oak.upgrade.cli.container.NodeStoreContainer; +import org.apache.jackrabbit.oak.upgrade.cli.container.SegmentAzureServicePrincipalNodeStoreContainer; +import org.apache.jackrabbit.oak.upgrade.cli.container.SegmentTarNodeStoreContainer; + +import java.io.IOException; + +import static org.junit.Assume.assumeNotNull; + +public class SegmentTarToSegmentAzureServicePrincipalTest extends AbstractOak2OakTest { + private static boolean skipTest = true; + private static final Environment ENVIRONMENT = new Environment(); + private final NodeStoreContainer source; + private final NodeStoreContainer destination; + + @Override + public void prepare() throws Exception { + assumeNotNull(ENVIRONMENT.getVariable(AzureUtilities.AZURE_ACCOUNT_NAME), ENVIRONMENT.getVariable(AzureUtilities.AZURE_TENANT_ID), + ENVIRONMENT.getVariable(AzureUtilities.AZURE_CLIENT_ID), ENVIRONMENT.getVariable(AzureUtilities.AZURE_CLIENT_SECRET)); + skipTest = false; + super.prepare(); + } + + @Override + public void clean() throws IOException { + if (!skipTest) { + super.clean(); + } + } + + public SegmentTarToSegmentAzureServicePrincipalTest() throws IOException { + source = new SegmentTarNodeStoreContainer(); + destination = new SegmentAzureServicePrincipalNodeStoreContainer(); + } + + @Override + protected NodeStoreContainer getSourceContainer() { + return source; + } + + @Override + protected NodeStoreContainer getDestinationContainer() { + return destination; + } + + @Override + protected String[] getArgs() { + return new String[]{source.getDescription(), destination.getDescription()}; + } +} diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java new file mode 100644 index 00000000000..2347d3e210e --- /dev/null +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.upgrade.cli.container; + +import com.microsoft.azure.storage.blob.CloudBlobDirectory; +import org.apache.jackrabbit.guava.common.io.Files; +import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; +import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; +import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; +import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; +import org.apache.jackrabbit.oak.segment.azure.util.Environment; +import org.apache.jackrabbit.oak.segment.file.FileStore; +import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; +import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils; + +import java.io.File; +import java.io.IOException; + +public class SegmentAzureServicePrincipalNodeStoreContainer implements NodeStoreContainer { + private static final Environment ENVIRONMENT = new Environment(); + private static final String CONTAINER_NAME = "oak-migration-test"; + private static final String DIR = "repository"; + private static final String AZURE_SEGMENT_STORE_PATH = "https://%s.blob.core.windows.net/%s/%s"; + + private final BlobStore blobStore; + private FileStore fs; + private File tmpDir; + private AzurePersistence azurePersistence; + + public SegmentAzureServicePrincipalNodeStoreContainer() { + this(null); + } + + public SegmentAzureServicePrincipalNodeStoreContainer(BlobStore blobStore) { + this.blobStore = blobStore; + } + + + @Override + public NodeStore open() throws IOException { + try { + azurePersistence = createAzurePersistence(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + + tmpDir = Files.createTempDir(); + FileStoreBuilder builder = FileStoreBuilder.fileStoreBuilder(tmpDir) + .withCustomPersistence(azurePersistence).withMemoryMapping(false); + if (blobStore != null) { + builder.withBlobStore(blobStore); + } + + try { + fs = builder.build(); + } catch (InvalidFileStoreVersionException e) { + throw new IllegalStateException(e); + } + + return new FileStoreUtils.NodeStoreWithFileStore(SegmentNodeStoreBuilders.builder(fs).build(), fs); + } + + private AzurePersistence createAzurePersistence() { + if (azurePersistence != null) { + return azurePersistence; + } + String path = String.format(AZURE_SEGMENT_STORE_PATH, ENVIRONMENT.getVariable(AzureUtilities.AZURE_ACCOUNT_NAME), + CONTAINER_NAME, DIR); + CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT); + return new AzurePersistence(cloudBlobDirectory); + } + + @Override + public void close() { + if (fs != null) { + fs.close(); + fs = null; + } + if (tmpDir != null) { + tmpDir.delete(); + } + } + + @Override + public void clean() throws IOException { + AzurePersistence azurePersistence = createAzurePersistence(); + try { + AzureUtilities.deleteAllBlobs(azurePersistence.getSegmentstoreDirectory()); + } catch (Exception e) { + throw new IOException(e); + } + } + + @Override + public String getDescription() { + return "az:" + String.format(AZURE_SEGMENT_STORE_PATH, ENVIRONMENT.getVariable(AzureUtilities.AZURE_ACCOUNT_NAME), + CONTAINER_NAME, DIR); + } +} \ No newline at end of file diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java new file mode 100644 index 00000000000..a28803f5975 --- /dev/null +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.upgrade.cli.node; + +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.SharedAccessAccountPermissions; +import com.microsoft.azure.storage.SharedAccessAccountPolicy; +import com.microsoft.azure.storage.SharedAccessAccountResourceType; +import com.microsoft.azure.storage.SharedAccessAccountService; +import com.microsoft.azure.storage.blob.CloudBlobDirectory; +import org.apache.commons.lang3.StringUtils; +import org.apache.jackrabbit.guava.common.io.Closer; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; +import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; +import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; +import org.apache.jackrabbit.oak.segment.azure.util.Environment; +import org.apache.jackrabbit.oak.upgrade.cli.CliUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.ClassRule; +import org.junit.Test; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.Date; +import java.util.EnumSet; + +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_ACCOUNT_NAME; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_SECRET_KEY; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID; +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeNotNull; +import static org.junit.Assume.assumeTrue; + +public class SegmentAzureFactoryTest { + + @ClassRule + public static final AzuriteDockerRule azurite = new AzuriteDockerRule(); + + private static final Environment ENVIRONMENT = new Environment(); + private static final String CONTAINER_NAME = "oak-test"; + private static final String DIR = "repository"; + private static final String CONNECTION_URI = "https://%s.blob.core.windows.net/%s"; + + + @Test + public void testConnectionWithConnectionString_accessKey() throws IOException { + String connectionStringWithPlaceholder = "DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;BlobEndpoint=http://127.0.0.1:%s/%s;"; + String connectionString = String.format(connectionStringWithPlaceholder, AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azurite.getMappedPort(), AzuriteDockerRule.ACCOUNT_NAME); + SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder("respository", 256, + false) + .connectionString(connectionString) + .containerName(CONTAINER_NAME) + .build(); + Closer closer = Closer.create(); + CliUtils.handleSigInt(closer); + FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer); + assertEquals(1, nodeStore.getFileStore().getSegmentCount()); + closer.close(); + } + + @Test + public void testConnectionWithConnectionString_sas() throws IOException { + String sasToken = getAccountSasToken(); + String connectionStringWithPlaceholder = "DefaultEndpointsProtocol=http;AccountName=%s;SharedAccessSignature=%s;BlobEndpoint=http://127.0.0.1:%s/%s;"; + String connectionString = String.format(connectionStringWithPlaceholder, AzuriteDockerRule.ACCOUNT_NAME, sasToken, azurite.getMappedPort(), AzuriteDockerRule.ACCOUNT_NAME); + SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256, + false) + .connectionString(connectionString) + .containerName(CONTAINER_NAME) + .build(); + Closer closer = Closer.create(); + CliUtils.handleSigInt(closer); + FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer); + assertEquals(1, nodeStore.getFileStore().getSegmentCount()); + closer.close(); + } + + /* if AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET are already set in environment variables + * then they will be given preference and authentication will be done via service principals and this + * test will be skipped + * */ + @Test + public void testConnectionWithUri_accessKey() throws IOException { + assumeTrue(StringUtils.isBlank(ENVIRONMENT.getVariable(AZURE_TENANT_ID))); + assumeTrue(StringUtils.isBlank(ENVIRONMENT.getVariable(AZURE_CLIENT_ID))); + assumeTrue(StringUtils.isBlank(ENVIRONMENT.getVariable(AZURE_CLIENT_SECRET))); + + assumeNotNull(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME), ENVIRONMENT.getVariable(AZURE_SECRET_KEY)); + + final String CONTAINER_NAME = "oak-migration-test"; + + String uri = String.format(CONNECTION_URI, ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME), CONTAINER_NAME); + Closer closer = Closer.create(); + try { + SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256, + false) + .accountName(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)) + .uri(uri) + .build(); + closer = Closer.create(); + CliUtils.handleSigInt(closer); + FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer); + assertEquals(1, nodeStore.getFileStore().getSegmentCount()); + } finally { + closer.close(); + cleanup(uri); + } + } + + @Test + public void testConnectionWithUri_servicePrincipal() throws IOException, InterruptedException { + assumeNotNull(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME), ENVIRONMENT.getVariable(AZURE_TENANT_ID), + ENVIRONMENT.getVariable(AZURE_CLIENT_ID), ENVIRONMENT.getVariable(AZURE_CLIENT_SECRET)); + + final String CONTAINER_NAME = "oak-migration-test"; + + String uri = String.format(CONNECTION_URI, ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME), CONTAINER_NAME); + Closer closer = Closer.create(); + try { + SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256, + false) + .accountName(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)) + .uri(uri) + .build(); + + CliUtils.handleSigInt(closer); + FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer); + assertEquals(1, nodeStore.getFileStore().getSegmentCount()); + } finally { + closer.close(); + cleanup(uri); + } + } + + private void cleanup(String uri) { + uri = uri + "/" + DIR; + try { + CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(uri, ENVIRONMENT); + AzureUtilities.deleteAllBlobs(cloudBlobDirectory); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + + @NotNull + private String getAccountSasToken() { + try { + CloudStorageAccount cloudStorageAccount = azurite.getCloudStorageAccount(); + return cloudStorageAccount.generateSharedAccessSignature(getPolicy()); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + @NotNull + private SharedAccessAccountPolicy getPolicy() { + SharedAccessAccountPolicy sharedAccessAccountPolicy = new SharedAccessAccountPolicy(); + EnumSet sharedAccessAccountPermissions = EnumSet.of(SharedAccessAccountPermissions.CREATE, + SharedAccessAccountPermissions.DELETE, SharedAccessAccountPermissions.READ, SharedAccessAccountPermissions.UPDATE, + SharedAccessAccountPermissions.WRITE, SharedAccessAccountPermissions.LIST); + EnumSet sharedAccessAccountServices = EnumSet.of(SharedAccessAccountService.BLOB); + EnumSet sharedAccessAccountResourceTypes = EnumSet.of( + SharedAccessAccountResourceType.CONTAINER, SharedAccessAccountResourceType.OBJECT, SharedAccessAccountResourceType.SERVICE); + + sharedAccessAccountPolicy.setPermissions(sharedAccessAccountPermissions); + sharedAccessAccountPolicy.setServices(sharedAccessAccountServices); + sharedAccessAccountPolicy.setResourceTypes(sharedAccessAccountResourceTypes); + sharedAccessAccountPolicy.setSharedAccessExpiryTime(Date.from(Instant.now().plus(Duration.ofDays(7)))); + return sharedAccessAccountPolicy; + } + +}