From ee61ff4851cc5f12b872bfb5b6270dcc1efd7986 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl <4711730+kaklakariada@users.noreply.github.com> Date: Wed, 10 Apr 2024 15:35:18 +0200 Subject: [PATCH] Add missing permissions for regression tests (#54) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add missing permissions for regression tests * Verify that blob storage Gen2 is *not* enabled * Increment version * Fix compile error * Fix references * Mark test as itest * Upgrade ci-isolation * Update release date * Rename TestContainer class to avoid running it in unit tests * Update dependency * Apply suggestions from code review Co-authored-by: Sebastian Bär * Implement review finding --------- Co-authored-by: Sebastian Bär --- .vscode/settings.json | 3 + ci-isolation/pom.xml | 2 +- .../main/resources/exasol-permissions.json | 3 +- doc/changes/changelog.md | 1 + doc/changes/changes_2.0.4.md | 12 ++++ doc/user_guide/user_guide.md | 4 +- pk_generated_parent.pom | 2 +- pom.xml | 6 +- .../files/AbsDocumentFilesAdapterIT.java | 4 +- .../files/AbsRandomAccessInputStreamIT.java | 6 +- .../files/AbsRemoteFileContentIT.java | 6 +- .../document/files/AbsRemoteFileFinderIT.java | 21 +++--- .../files/AbsRemoteFileFinderOnlineIT.java | 17 ++--- ...stContainer.java => AbsTestContainer.java} | 13 ++-- .../document/files/AbsTestContainerIT.java | 64 +++++++++++++++++++ .../document/files/IntegrationTestSetup.java | 2 +- 16 files changed, 127 insertions(+), 39 deletions(-) create mode 100644 doc/changes/changes_2.0.4.md rename src/test/java/com/exasol/adapter/document/files/{TestContainer.java => AbsTestContainer.java} (71%) create mode 100644 src/test/java/com/exasol/adapter/document/files/AbsTestContainerIT.java diff --git a/.vscode/settings.json b/.vscode/settings.json index 85998b7..768b55c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -15,6 +15,9 @@ "-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl" ] }, + "[json]": { + "editor.indentSize": 2 + }, "sonarlint.connectedMode.project": { "connectionId": "exasol", "projectKey": "com.exasol:azure-blob-storage-document-files-virtual-schema" diff --git a/ci-isolation/pom.xml b/ci-isolation/pom.xml index 9b341a2..f2c9b2c 100644 --- a/ci-isolation/pom.xml +++ b/ci-isolation/pom.xml @@ -38,7 +38,7 @@ com.exasol ci-isolation-aws - 2.0.2 + 2.0.3 diff --git a/ci-isolation/src/main/resources/exasol-permissions.json b/ci-isolation/src/main/resources/exasol-permissions.json index a1be7b0..51573e8 100644 --- a/ci-isolation/src/main/resources/exasol-permissions.json +++ b/ci-isolation/src/main/resources/exasol-permissions.json @@ -46,6 +46,7 @@ "iam:GetRole", "iam:CreateRole", "iam:DeleteRole", + "iam:TagRole", "iam:PutRolePolicy", "iam:DetachRolePolicy", "iam:DeleteRolePolicy", @@ -75,4 +76,4 @@ "Resource": "*" } ] -} \ No newline at end of file +} diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 14afd3d..5fa81e0 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [2.0.4](changes_2.0.4.md) * [2.0.3](changes_2.0.3.md) * [2.0.2](changes_2.0.2.md) * [2.0.1](changes_2.0.1.md) diff --git a/doc/changes/changes_2.0.4.md b/doc/changes/changes_2.0.4.md new file mode 100644 index 0000000..ad50a88 --- /dev/null +++ b/doc/changes/changes_2.0.4.md @@ -0,0 +1,12 @@ +# Virtual Schema for Document Data in Files on Azure Blob Storage 2.0.4, released 2024-04-10 + +Code name: Verify storage generation in integration tests + +## Summary + +This release adds an integration test that verifies, that "Data Lake Gen2 upgrade" is not enabled for the Azure storage account. +## Dependency Updates + +### Test Dependency Updates + +* Updated `com.exasol:udf-debugging-java:0.6.12` to `0.6.13` diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index 886886e..0c22eae 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -17,7 +17,7 @@ Next create the Adapter Script: ```sql CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.AZURE_BLOB_STORAGE_FILES_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.0.3-azure-blob-storage-2.0.3.jar; + %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.0.3-azure-blob-storage-2.0.4.jar; / ``` @@ -30,7 +30,7 @@ CREATE OR REPLACE JAVA SET SCRIPT ADAPTER.IMPORT_FROM_AZURE_BLOB_STORAGE_DOCUMEN CONNECTION_NAME VARCHAR(500)) EMITS(...) AS %scriptclass com.exasol.adapter.document.UdfEntryPoint; - %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.0.3-azure-blob-storage-2.0.3.jar; + %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.0.3-azure-blob-storage-2.0.4.jar; / ``` diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom index e61cfc1..12e6f49 100644 --- a/pk_generated_parent.pom +++ b/pk_generated_parent.pom @@ -3,7 +3,7 @@ 4.0.0 com.exasol azure-blob-storage-document-files-virtual-schema-generated-parent - 2.0.3 + 2.0.4 pom UTF-8 diff --git a/pom.xml b/pom.xml index 4a47885..fa8f203 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 azure-blob-storage-document-files-virtual-schema - 2.0.3 + 2.0.4 Virtual Schema for document data in files on Azure Blob Storage Adapter for document data access from files from Azure Blob Storage. https://github.com/exasol/azure-blob-storage-document-files-virtual-schema/ @@ -90,7 +90,7 @@ com.exasol udf-debugging-java - 0.6.12 + 0.6.13 test @@ -188,7 +188,7 @@ azure-blob-storage-document-files-virtual-schema-generated-parent com.exasol - 2.0.3 + 2.0.4 pk_generated_parent.pom diff --git a/src/test/java/com/exasol/adapter/document/files/AbsDocumentFilesAdapterIT.java b/src/test/java/com/exasol/adapter/document/files/AbsDocumentFilesAdapterIT.java index 1af8be9..e034020 100644 --- a/src/test/java/com/exasol/adapter/document/files/AbsDocumentFilesAdapterIT.java +++ b/src/test/java/com/exasol/adapter/document/files/AbsDocumentFilesAdapterIT.java @@ -29,7 +29,7 @@ @Testcontainers class AbsDocumentFilesAdapterIT extends AbstractDocumentFilesAdapterIT { private static IntegrationTestSetup setup; - private static TestContainer testContainer; + private static AbsTestContainer testContainer; private static AbsTestSetup absTestSetup; @BeforeAll @@ -37,7 +37,7 @@ static void beforeAll() throws Exception { final ExasolTestSetup exasolTestSetup = new ExasolTestSetupFactory( Path.of("cloudSetup/generated/testConfig.json")).getTestSetup(); absTestSetup = getAbsTestSetup(exasolTestSetup); - testContainer = new TestContainer(absTestSetup); + testContainer = AbsTestContainer.create(absTestSetup); setup = new IntegrationTestSetup(exasolTestSetup, absTestSetup, testContainer.getBlobContainerClient()); } diff --git a/src/test/java/com/exasol/adapter/document/files/AbsRandomAccessInputStreamIT.java b/src/test/java/com/exasol/adapter/document/files/AbsRandomAccessInputStreamIT.java index 8dd8d0b..262c378 100644 --- a/src/test/java/com/exasol/adapter/document/files/AbsRandomAccessInputStreamIT.java +++ b/src/test/java/com/exasol/adapter/document/files/AbsRandomAccessInputStreamIT.java @@ -11,13 +11,13 @@ class AbsRandomAccessInputStreamIT extends RandomAccessInputStreamTestBase { private static final String TEST_DATA_KEY = "TEST_DATA"; long testDataLength; private AbsTestSetup testSetup; - private TestContainer testContainer; + private AbsTestContainer testContainer; private BlobClient blob; @Override protected void prepareTestSetup(final byte[] bytes) { this.testSetup = new LocalAbsTestSetup(); - this.testContainer = new TestContainer(this.testSetup); + this.testContainer = AbsTestContainer.create(this.testSetup); this.testDataLength = bytes.length; this.blob = this.testContainer.getBlobContainerClient().getBlobClient(TEST_DATA_KEY); this.blob.upload(BinaryData.fromBytes(bytes)); @@ -33,4 +33,4 @@ protected void cleanupTestSetup() { protected RandomAccessInputStream getSeekableInputStream() { return new AbsRandomAccessInputStream(this.blob, this.testDataLength); } -} \ No newline at end of file +} diff --git a/src/test/java/com/exasol/adapter/document/files/AbsRemoteFileContentIT.java b/src/test/java/com/exasol/adapter/document/files/AbsRemoteFileContentIT.java index 0e026f6..f3a5f55 100644 --- a/src/test/java/com/exasol/adapter/document/files/AbsRemoteFileContentIT.java +++ b/src/test/java/com/exasol/adapter/document/files/AbsRemoteFileContentIT.java @@ -20,13 +20,13 @@ class AbsRemoteFileContentIT { private static final String TEST_DATA_KEY = "TEST_DATA"; private static AbsTestSetup testSetup; private static RemoteFileContent remoteFileContent; - private static TestContainer testBucket; + private static AbsTestContainer testBucket; private static ExecutorServiceFactory executorServiceFactory; @BeforeAll static void beforeAll() { testSetup = new LocalAbsTestSetup(); - testBucket = new TestContainer(testSetup); + testBucket = AbsTestContainer.create(testSetup); final var blobContainerClient = testBucket.getBlobContainerClient(); blobContainerClient.getBlobClient(TEST_DATA_KEY).upload(BinaryData.fromBytes(TEST_DATA_VALUE.getBytes())); executorServiceFactory = new ExecutorServiceFactory(); @@ -71,4 +71,4 @@ void testCancelAsyncLoading() { future.cancel(true); assertTrue(future.isCancelled()); } -} \ No newline at end of file +} diff --git a/src/test/java/com/exasol/adapter/document/files/AbsRemoteFileFinderIT.java b/src/test/java/com/exasol/adapter/document/files/AbsRemoteFileFinderIT.java index dfee1f6..f7c789e 100644 --- a/src/test/java/com/exasol/adapter/document/files/AbsRemoteFileFinderIT.java +++ b/src/test/java/com/exasol/adapter/document/files/AbsRemoteFileFinderIT.java @@ -7,14 +7,14 @@ import java.util.ArrayList; import java.util.List; -import com.azure.core.util.BinaryData; import org.junit.jupiter.api.*; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import com.exasol.adapter.document.files.connection.AbsConnectionProperties; +import com.azure.core.util.BinaryData; import com.exasol.adapter.document.files.abstestsetup.AbsTestSetup; import com.exasol.adapter.document.files.abstestsetup.LocalAbsTestSetup; +import com.exasol.adapter.document.files.connection.AbsConnectionProperties; import com.exasol.adapter.document.files.stringfilter.wildcardexpression.WildcardExpression; @Tag("integration") @@ -24,19 +24,20 @@ class AbsRemoteFileFinderIT { private static final String CONTENT_2 = "content-2"; private static final String CONTENT_OTHER = "other"; private static AbsConnectionProperties connectionInformation; - private static TestContainer testContainer; + private static AbsTestContainer testContainer; @BeforeAll static void beforeAll() { - testContainer = new TestContainer(TEST_SETUP); - var blobContainerClient = testContainer.getBlobContainerClient(); - var file1= blobContainerClient.getBlobClient("file-1.json"); + testContainer = AbsTestContainer.create(TEST_SETUP); + final var blobContainerClient = testContainer.getBlobContainerClient(); + final var file1 = blobContainerClient.getBlobClient("file-1.json"); file1.upload(BinaryData.fromBytes(CONTENT_1.getBytes())); - var file2 = blobContainerClient.getBlobClient("file-2.json"); + final var file2 = blobContainerClient.getBlobClient("file-2.json"); file2.upload(BinaryData.fromBytes(CONTENT_2.getBytes())); - var file3 = blobContainerClient.getBlobClient("other.json"); + final var file3 = blobContainerClient.getBlobClient("other.json"); file3.upload(BinaryData.fromBytes(CONTENT_OTHER.getBytes())); - connectionInformation = AbsConnectionProperties.builder().absContainerName(testContainer.getBlobContainerClient().getBlobContainerName()) + connectionInformation = AbsConnectionProperties.builder() + .absContainerName(testContainer.getBlobContainerClient().getBlobContainerName()) .absStorageAccountConnectionString(TEST_SETUP.getStorageAccountConnectionString()).build(); } @@ -85,4 +86,4 @@ private String readFirstLine(final InputStream stream) { throw new IllegalArgumentException("", exception); } } -} \ No newline at end of file +} diff --git a/src/test/java/com/exasol/adapter/document/files/AbsRemoteFileFinderOnlineIT.java b/src/test/java/com/exasol/adapter/document/files/AbsRemoteFileFinderOnlineIT.java index 06c4a35..2553000 100644 --- a/src/test/java/com/exasol/adapter/document/files/AbsRemoteFileFinderOnlineIT.java +++ b/src/test/java/com/exasol/adapter/document/files/AbsRemoteFileFinderOnlineIT.java @@ -7,12 +7,12 @@ import java.util.ArrayList; import java.util.List; -import com.azure.core.util.BinaryData; import org.junit.jupiter.api.*; -import com.exasol.adapter.document.files.connection.AbsConnectionProperties; +import com.azure.core.util.BinaryData; import com.exasol.adapter.document.files.abstestsetup.AbsTestSetup; import com.exasol.adapter.document.files.abstestsetup.OnlineAbsTestSetup; +import com.exasol.adapter.document.files.connection.AbsConnectionProperties; import com.exasol.adapter.document.files.stringfilter.wildcardexpression.WildcardExpression; @Tag("integration") @@ -20,15 +20,16 @@ class AbsRemoteFileFinderOnlineIT { private static final AbsTestSetup TEST_SETUP = new OnlineAbsTestSetup(); private static final String CONTENT_1 = "content-1"; private static AbsConnectionProperties connectionInformation; - private static TestContainer testContainer; + private static AbsTestContainer testContainer; @BeforeAll static void beforeAll() { - testContainer = new TestContainer(TEST_SETUP); - var blobContainerClient = testContainer.getBlobContainerClient(); - var file1= blobContainerClient.getBlobClient("file-1.json"); + testContainer = AbsTestContainer.create(TEST_SETUP); + final var blobContainerClient = testContainer.getBlobContainerClient(); + final var file1 = blobContainerClient.getBlobClient("file-1.json"); file1.upload(BinaryData.fromBytes(CONTENT_1.getBytes())); - connectionInformation = AbsConnectionProperties.builder().absContainerName(testContainer.getBlobContainerClient().getBlobContainerName()) + connectionInformation = AbsConnectionProperties.builder() + .absContainerName(testContainer.getBlobContainerClient().getBlobContainerName()) .absStorageAccountConnectionString(TEST_SETUP.getStorageAccountConnectionString()).build(); } @@ -58,4 +59,4 @@ private String readFirstLine(final InputStream stream) { throw new IllegalArgumentException("", exception); } } -} \ No newline at end of file +} diff --git a/src/test/java/com/exasol/adapter/document/files/TestContainer.java b/src/test/java/com/exasol/adapter/document/files/AbsTestContainer.java similarity index 71% rename from src/test/java/com/exasol/adapter/document/files/TestContainer.java rename to src/test/java/com/exasol/adapter/document/files/AbsTestContainer.java index d59a871..c6816eb 100644 --- a/src/test/java/com/exasol/adapter/document/files/TestContainer.java +++ b/src/test/java/com/exasol/adapter/document/files/AbsTestContainer.java @@ -5,15 +5,20 @@ import com.azure.storage.blob.models.BlobItem; import com.exasol.adapter.document.files.abstestsetup.AbsTestSetup; -public class TestContainer implements AutoCloseable { +public class AbsTestContainer implements AutoCloseable { private final BlobContainerClient blobContainerClient; private final String blobContainerName; - public TestContainer(final AbsTestSetup testSetup) { + private AbsTestContainer(final BlobContainerClient containerClient, final String containerName) { + this.blobContainerClient = containerClient; + this.blobContainerName = containerName; + } + + public static AbsTestContainer create(final AbsTestSetup testSetup) { final String containerName = "abs-document-vs-test-" + System.currentTimeMillis(); final BlobServiceClient absClient = testSetup.getAbsClient(); - this.blobContainerClient = absClient.createBlobContainer(containerName); - this.blobContainerName = containerName; + final BlobContainerClient containerClient = absClient.createBlobContainer(containerName); + return new AbsTestContainer(containerClient, containerName); } // https://github.com/Azure/azure-sdk-for-java/issues/10180 diff --git a/src/test/java/com/exasol/adapter/document/files/AbsTestContainerIT.java b/src/test/java/com/exasol/adapter/document/files/AbsTestContainerIT.java new file mode 100644 index 0000000..74a2571 --- /dev/null +++ b/src/test/java/com/exasol/adapter/document/files/AbsTestContainerIT.java @@ -0,0 +1,64 @@ +package com.exasol.adapter.document.files; + +import static java.util.stream.Collectors.toList; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.List; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import com.azure.core.util.BinaryData; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.models.BlobItem; +import com.exasol.adapter.document.files.abstestsetup.AbsTestSetup; +import com.exasol.adapter.document.files.abstestsetup.OnlineAbsTestSetup; + +@Tag("integration") +class AbsTestContainerIT { + static AbsTestSetup absTestSetup = new OnlineAbsTestSetup(); + + @Test + void emptyFilledContainer() { + try (final AbsTestContainer testContainer = AbsTestContainer.create(absTestSetup)) { + final BlobContainerClient client = testContainer.getBlobContainerClient(); + client.getBlobClient("blobName").upload(BinaryData.fromString("blob content")); + assertThat(listBlobNames(client), contains("blobName")); + testContainer.empty(); + assertThat(listBlobNames(client), empty()); + } + } + + // Note: This test is expected to fail when Azure Blob Storage Gen2 is enabled. + @Test + void emptyFilledContainerWithDirectory() { + try (final AbsTestContainer testContainer = AbsTestContainer.create(absTestSetup)) { + final BlobContainerClient client = testContainer.getBlobContainerClient(); + client.getBlobClient("dir/blobName").upload(BinaryData.fromString("blob content")); + assertThat(listBlobNames(client), contains("dir/blobName")); + testContainer.empty(); + assertThat(listBlobNames(client), empty()); + } + } + + @Test + void checkBlobStorageVersion() { + try (final AbsTestContainer testContainer = AbsTestContainer.create(absTestSetup)) { + final BlobContainerClient client = testContainer.getBlobContainerClient(); + client.getBlobClient("dir/blobName").upload(BinaryData.fromString("blob content")); + final List blobNames = listBlobNames(client); + if (blobNames.size() > 1) { + fail("Expected exactly one blob, but found " + blobNames.size() + " blobs.: " + blobNames + + ". This usually means that Azure Blob Storage Gen2 is enabled. Please create a new storage account without Gen2 enabled."); + } + assertThat(listBlobNames(client), contains("dir/blobName")); + } + } + + private List listBlobNames(final BlobContainerClient client) { + return client.listBlobs().stream().map(BlobItem::getName).collect(toList()); + } +} diff --git a/src/test/java/com/exasol/adapter/document/files/IntegrationTestSetup.java b/src/test/java/com/exasol/adapter/document/files/IntegrationTestSetup.java index dab08a8..b01e7ae 100644 --- a/src/test/java/com/exasol/adapter/document/files/IntegrationTestSetup.java +++ b/src/test/java/com/exasol/adapter/document/files/IntegrationTestSetup.java @@ -26,7 +26,7 @@ import jakarta.json.*; public class IntegrationTestSetup implements AutoCloseable { - private static final String ADAPTER_JAR = "document-files-virtual-schema-dist-8.0.3-azure-blob-storage-2.0.3.jar"; + private static final String ADAPTER_JAR = "document-files-virtual-schema-dist-8.0.3-azure-blob-storage-2.0.4.jar"; private final ExasolTestSetup exasolTestSetup; private final Connection exasolConnection; private final Statement exasolStatement;