Nessie: Test Nessie's new storage model (#7702)

Nessie tests will only run against Java 11 or newer, because Nessie's storage implementation (server side thing) only works on Java 11. Nessie client libraries still work fine with Java 8.
apache · Jun 27, 2023 · 2fea2dd · 2fea2dd
1 parent dd0ee24
commit 2fea2dd
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 44 deletions.
diff --git a/build.gradle b/build.gradle
@@ -831,8 +831,19 @@ project(':iceberg-pig') {
 }
 
 project(':iceberg-nessie') {
-  test {
-    useJUnitPlatform()
+  if (JavaVersion.current().isJava11Compatible()) {
+    test {
+      useJUnitPlatform()
+    }
+  } else {
+    // Do not test Nessie against Java 8, because in-JVM testing requires Nessie server components,
+    // which require Java 11+.
+    test {
+      enabled = false
+    }
+    compileTestJava {
+      enabled = false
+    }
   }
 
   dependencies {
@@ -844,21 +855,25 @@ project(':iceberg-nessie') {
     implementation "com.fasterxml.jackson.core:jackson-databind"
     implementation "com.fasterxml.jackson.core:jackson-core"
 
-    testImplementation "org.projectnessie.nessie:nessie-jaxrs-testextension"
-    testImplementation "org.projectnessie.nessie:nessie-versioned-persist-in-memory"
-    testImplementation "org.projectnessie.nessie:nessie-versioned-persist-in-memory-test"
-    // Need to "pull in" el-api explicitly :(
-    testImplementation "jakarta.el:jakarta.el-api"
-
     compileOnly "org.apache.hadoop:hadoop-common"
-    testImplementation "org.apache.avro:avro"
-
-    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
-    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
-
     // Only there to prevent "warning: unknown enum constant SchemaType.OBJECT" compile messages
     compileOnly "org.eclipse.microprofile.openapi:microprofile-openapi-api:3.1"
-    testCompileOnly "org.eclipse.microprofile.openapi:microprofile-openapi-api:3.0"
+
+    if (JavaVersion.current().isJava11Compatible()) {
+      testImplementation "org.projectnessie.nessie:nessie-jaxrs-testextension"
+      testImplementation "org.projectnessie.nessie:nessie-versioned-storage-inmemory"
+      testImplementation "org.projectnessie.nessie:nessie-versioned-storage-testextension"
+      // Need to "pull in" el-api explicitly :(
+      testImplementation "jakarta.el:jakarta.el-api"
+
+      testImplementation "org.apache.avro:avro"
+
+      testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+      testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
+
+      // Only there to prevent "warning: unknown enum constant SchemaType.OBJECT" compile messages
+      testCompileOnly "org.eclipse.microprofile.openapi:microprofile-openapi-api:3.1"
+    }
   }
 }
 

diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieIcebergClient.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieIcebergClient.java
@@ -330,8 +330,8 @@ public void renameTable(TableIdentifier from, TableIdentifier to) {
                 NessieUtil.buildCommitMetadata(
                     String.format("Iceberg rename table from '%s' to '%s'", from, to),
                     catalogOptions))
-            .operation(Operation.Put.of(NessieUtil.toKey(to), existingFromTable, existingFromTable))
-            .operation(Operation.Delete.of(NessieUtil.toKey(from)));
+            .operation(Operation.Delete.of(NessieUtil.toKey(from)))
+            .operation(Operation.Put.of(NessieUtil.toKey(to), existingFromTable));
 
     try {
       Tasks.foreach(operations)

diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java b/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java
@@ -60,27 +60,23 @@
 import org.projectnessie.model.Branch;
 import org.projectnessie.model.Reference;
 import org.projectnessie.model.Tag;
-import org.projectnessie.versioned.persist.adapter.DatabaseAdapter;
-import org.projectnessie.versioned.persist.inmem.InmemoryDatabaseAdapterFactory;
-import org.projectnessie.versioned.persist.inmem.InmemoryTestConnectionProviderSource;
-import org.projectnessie.versioned.persist.tests.extension.DatabaseAdapterExtension;
-import org.projectnessie.versioned.persist.tests.extension.NessieDbAdapter;
-import org.projectnessie.versioned.persist.tests.extension.NessieDbAdapterName;
-import org.projectnessie.versioned.persist.tests.extension.NessieExternalDatabase;
+import org.projectnessie.versioned.storage.common.persist.Persist;
+import org.projectnessie.versioned.storage.inmemory.InmemoryBackendTestFactory;
+import org.projectnessie.versioned.storage.testextension.NessieBackend;
+import org.projectnessie.versioned.storage.testextension.NessiePersist;
+import org.projectnessie.versioned.storage.testextension.PersistExtension;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-@ExtendWith(DatabaseAdapterExtension.class)
-@NessieDbAdapterName(InmemoryDatabaseAdapterFactory.NAME)
-@NessieExternalDatabase(InmemoryTestConnectionProviderSource.class)
+@ExtendWith(PersistExtension.class)
+@NessieBackend(InmemoryBackendTestFactory.class)
 @NessieApiVersions // test all versions
 public abstract class BaseTestIceberg {
 
-  @NessieDbAdapter static DatabaseAdapter databaseAdapter;
+  @NessiePersist static Persist persist;
 
   @RegisterExtension
-  static NessieJaxRsExtension server =
-      NessieJaxRsExtension.jaxRsExtensionForDatabaseAdapter(() -> databaseAdapter);
+  static NessieJaxRsExtension server = NessieJaxRsExtension.jaxRsExtension(() -> persist);
 
   private static final Logger LOG = LoggerFactory.getLogger(BaseTestIceberg.class);
 

diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java
@@ -43,25 +43,21 @@
 import org.projectnessie.model.Branch;
 import org.projectnessie.model.Reference;
 import org.projectnessie.model.Tag;
-import org.projectnessie.versioned.persist.adapter.DatabaseAdapter;
-import org.projectnessie.versioned.persist.inmem.InmemoryDatabaseAdapterFactory;
-import org.projectnessie.versioned.persist.inmem.InmemoryTestConnectionProviderSource;
-import org.projectnessie.versioned.persist.tests.extension.DatabaseAdapterExtension;
-import org.projectnessie.versioned.persist.tests.extension.NessieDbAdapter;
-import org.projectnessie.versioned.persist.tests.extension.NessieDbAdapterName;
-import org.projectnessie.versioned.persist.tests.extension.NessieExternalDatabase;
-
-@ExtendWith(DatabaseAdapterExtension.class)
-@NessieDbAdapterName(InmemoryDatabaseAdapterFactory.NAME)
-@NessieExternalDatabase(InmemoryTestConnectionProviderSource.class)
+import org.projectnessie.versioned.storage.common.persist.Persist;
+import org.projectnessie.versioned.storage.inmemory.InmemoryBackendTestFactory;
+import org.projectnessie.versioned.storage.testextension.NessieBackend;
+import org.projectnessie.versioned.storage.testextension.NessiePersist;
+import org.projectnessie.versioned.storage.testextension.PersistExtension;
+
+@ExtendWith(PersistExtension.class)
+@NessieBackend(InmemoryBackendTestFactory.class)
 @NessieApiVersions // test all versions
 public class TestNessieCatalog extends CatalogTests<NessieCatalog> {
 
-  @NessieDbAdapter static DatabaseAdapter databaseAdapter;
+  @NessiePersist static Persist persist;
 
   @RegisterExtension
-  static NessieJaxRsExtension server =
-      NessieJaxRsExtension.jaxRsExtensionForDatabaseAdapter(() -> databaseAdapter);
+  static NessieJaxRsExtension server = NessieJaxRsExtension.jaxRsExtension(() -> persist);
 
   @TempDir public Path temp;
 

diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java
@@ -542,7 +542,7 @@ public void testFailure() throws NessieNotFoundException, NessieConflictExceptio
 
     IcebergTable table = getTable(BRANCH, KEY);
 
-    IcebergTable value = IcebergTable.of("dummytable.metadata.json", 42, 42, 42, 42, "cid");
+    IcebergTable value = IcebergTable.of("dummytable.metadata.json", 42, 42, 42, 42, table.getId());
     api.commitMultipleOperations()
         .branch(branch)
         .operation(Operation.Put.of(KEY, value))