Merge remote-tracking branch 'origin/master' into python/manfiest_and…

…_manifest_list_writers

.github/workflows/python-release.yml

@@ -61,7 +61,7 @@ jobs:
         working-directory: ./python
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.15.0
+        uses: pypa/cibuildwheel@v2.16.0
         with:
           package-dir: python/
           output-dir: wheelhouse

.github/workflows/spark-ci.yml

@@ -60,7 +60,7 @@ jobs:
     strategy:
       matrix:
         jvm: [8, 11]
-        spark: ['3.1', '3.2', '3.3', '3.4', '3.5']
+        spark: ['3.2', '3.3', '3.4', '3.5']
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.gitignore

@@ -28,7 +28,6 @@ lib/
 site/site
 
 # benchmark output
-spark/v3.1/spark/benchmark/*
 spark/v3.2/spark/benchmark/*
 spark/v3.3/spark/benchmark/*
 spark/v3.3/spark-extensions/benchmark/*

.palantir/revapi.yml

@@ -801,11 +801,10 @@ acceptedBreaks:
     - code: "java.method.removed"
       old: "method org.apache.iceberg.view.ViewBuilder org.apache.iceberg.view.ViewBuilder::withQueryColumnNames(java.util.List<java.lang.String>)"
       justification: "Acceptable break due to updating View APIs and the View Spec"
+    - code: "java.method.removed"
+      old: "method void org.apache.iceberg.view.ViewVersion::check()"
+      justification: "Acceptable break due to updating View APIs"
     org.apache.iceberg:iceberg-core:
-    - code: "java.method.visibilityReduced"
-      old: "method void org.apache.iceberg.encryption.Ciphers::<init>()"
-      new: "method void org.apache.iceberg.encryption.Ciphers::<init>()"
-      justification: "Static utility class - should not have public constructor"
     - code: "java.class.removed"
       old: "class org.apache.iceberg.actions.BaseDeleteOrphanFilesActionResult"
       justification: "Removing deprecated code"
@@ -833,6 +832,12 @@ acceptedBreaks:
     - code: "java.class.removed"
       old: "interface org.apache.iceberg.actions.RewritePositionDeleteStrategy"
       justification: "Removing deprecated code"
+    - code: "java.field.removedWithConstant"
+      old: "field org.apache.iceberg.TableProperties.MERGE_CARDINALITY_CHECK_ENABLED"
+      justification: "Spark 3.1 has been dropped"
+    - code: "java.field.removedWithConstant"
+      old: "field org.apache.iceberg.TableProperties.MERGE_CARDINALITY_CHECK_ENABLED_DEFAULT"
+      justification: "Spark 3.1 has been dropped"
     - code: "java.method.removed"
       old: "method java.util.List<org.apache.iceberg.DataFile> org.apache.iceberg.MergingSnapshotProducer<ThisT>::addedFiles()\
         \ @ org.apache.iceberg.BaseOverwriteFiles"
@@ -857,6 +862,10 @@ acceptedBreaks:
       old: "method void org.apache.iceberg.MergingSnapshotProducer<ThisT>::setNewFilesSequenceNumber(long)\
         \ @ org.apache.iceberg.StreamingDelete"
       justification: "Removing deprecated code"
+    - code: "java.method.visibilityReduced"
+      old: "method void org.apache.iceberg.encryption.Ciphers::<init>()"
+      new: "method void org.apache.iceberg.encryption.Ciphers::<init>()"
+      justification: "Static utility class - should not have public constructor"
   apache-iceberg-0.14.0:
     org.apache.iceberg:iceberg-api:
     - code: "java.class.defaultSerializationChanged"

api/src/main/java/org/apache/iceberg/view/ViewVersion.java

@@ -21,7 +21,6 @@
 import java.util.List;
 import java.util.Map;
 import org.apache.iceberg.catalog.Namespace;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
 
 /**
  * A version of the view at a point in time.
@@ -79,9 +78,4 @@ default String defaultCatalog() {
 
   /** The default namespace to use when the SQL does not contain a namespace. */
   Namespace defaultNamespace();
-
-  default void check() {
-    Preconditions.checkArgument(
-        summary().containsKey("operation"), "Invalid view version summary, missing operation");
-  }
 }

aws/src/integration/java/org/apache/iceberg/aws/TestDefaultAwsClientFactory.java

@@ -55,7 +55,7 @@ public void testS3FileIoEndpointOverride() {
     AssertHelpers.assertThrowsCause(
         "Should refuse connection to unknown endpoint",
         SdkClientException.class,
-        "Unable to execute HTTP request: unknown",
+        "Unable to execute HTTP request: bucket.unknown",
         () -> s3Client.getObject(GetObjectRequest.builder().bucket("bucket").key("key").build()));
   }
 

aws/src/main/java/org/apache/iceberg/aws/s3/S3OutputStream.java

@@ -22,10 +22,9 @@
 import java.io.BufferedOutputStream;
 import java.io.ByteArrayInputStream;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.SequenceInputStream;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
@@ -228,6 +227,7 @@ private void newStream() throws IOException {
     }
 
     stagingFiles.add(new FileAndDigest(currentStagingFile, currentPartMessageDigest));
+    OutputStream outputStream = Files.newOutputStream(currentStagingFile.toPath());
 
     if (isChecksumEnabled) {
       DigestOutputStream digestOutputStream;
@@ -236,22 +236,18 @@ private void newStream() throws IOException {
       if (multipartUploadId != null) {
         digestOutputStream =
             new DigestOutputStream(
-                new BufferedOutputStream(new FileOutputStream(currentStagingFile)),
-                currentPartMessageDigest);
+                new BufferedOutputStream(outputStream), currentPartMessageDigest);
       } else {
         digestOutputStream =
             new DigestOutputStream(
                 new DigestOutputStream(
-                    new BufferedOutputStream(new FileOutputStream(currentStagingFile)),
-                    currentPartMessageDigest),
+                    new BufferedOutputStream(outputStream), currentPartMessageDigest),
                 completeMessageDigest);
       }
 
       stream = new CountingOutputStream(digestOutputStream);
     } else {
-      stream =
-          new CountingOutputStream(
-              new BufferedOutputStream(new FileOutputStream(currentStagingFile)));
+      stream = new CountingOutputStream(new BufferedOutputStream(outputStream));
     }
   }
 
@@ -451,7 +447,7 @@ private void completeUploads() {
 
   private static InputStream uncheckedInputStream(File file) {
     try {
-      return new FileInputStream(file);
+      return Files.newInputStream(file.toPath());
     } catch (IOException e) {
       throw new UncheckedIOException(e);
     }

core/src/main/java/org/apache/iceberg/TableProperties.java

@@ -304,6 +304,9 @@ private TableProperties() {}
   public static final String SPARK_WRITE_ACCEPT_ANY_SCHEMA = "write.spark.accept-any-schema";
   public static final boolean SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT = false;
 
+  public static final String SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES =
+      "write.spark.advisory-partition-size-bytes";
+
   public static final String SNAPSHOT_ID_INHERITANCE_ENABLED =
       "compatibility.snapshot-id-inheritance.enabled";
   public static final boolean SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT = false;
@@ -353,19 +356,6 @@ private TableProperties() {}
   public static final String MERGE_MODE = "write.merge.mode";
   public static final String MERGE_MODE_DEFAULT = RowLevelOperationMode.COPY_ON_WRITE.modeName();
 
-  /**
-   * @deprecated will be removed once Spark 3.1 support is dropped, the cardinality check is always
-   *     performed starting from 0.13.0.
-   */
-  @Deprecated
-  public static final String MERGE_CARDINALITY_CHECK_ENABLED =
-      "write.merge.cardinality-check.enabled";
-  /**
-   * @deprecated will be removed once Spark 3.1 support is dropped, the cardinality check is always
-   *     performed starting from 0.13.0.
-   */
-  @Deprecated public static final boolean MERGE_CARDINALITY_CHECK_ENABLED_DEFAULT = true;
-
   public static final String MERGE_DISTRIBUTION_MODE = "write.merge.distribution-mode";
 
   public static final String UPSERT_ENABLED = "write.upsert.enabled";

core/src/main/java/org/apache/iceberg/actions/BinPackStrategy.java

@@ -46,7 +46,7 @@
  * RewriteDataFiles#TARGET_FILE_SIZE_BYTES}.
  *
  * @deprecated since 1.3.0, will be removed in 1.4.0; use {@link SizeBasedFileRewriter} instead.
- *     Note: This can only be removed once Spark 3.1 + 3.2 isn't using this API anymore.
+ *     Note: This can only be removed once Spark 3.2 isn't using this API anymore.
  */
 @Deprecated
 public abstract class BinPackStrategy implements RewriteStrategy {

core/src/main/java/org/apache/iceberg/actions/RewriteStrategy.java

@@ -30,7 +30,7 @@
  * A strategy for rewriting files.
  *
  * @deprecated since 1.3.0, will be removed in 1.4.0; use {@link FileRewriter} instead. Note: This
- *     can only be removed once Spark 3.1 + 3.2 isn't using this API anymore.
+ *     can only be removed once Spark 3.2 isn't using this API anymore.
  */
 @Deprecated
 public interface RewriteStrategy extends Serializable {

core/src/main/java/org/apache/iceberg/actions/SortStrategy.java

@@ -39,7 +39,7 @@
  * <p>In the future other algorithms for determining files to rewrite will be provided.
  *
  * @deprecated since 1.3.0, will be removed in 1.4.0; use {@link SizeBasedFileRewriter} instead.
- *     Note: This can only be removed once Spark 3.1 + 3.2 isn't using this API anymore.
+ *     Note: This can only be removed once Spark 3.2 isn't using this API anymore.
  */
 @Deprecated
 public abstract class SortStrategy extends BinPackStrategy {