Merge branch 'master' into master-broadcast-var

.github/labeler.yml

@@ -17,23 +17,6 @@
 # under the License.
 #
 
-#
-# Pull Request Labeler Github Action Configuration: https://github.com/marketplace/actions/labeler
-#
-# Note that we currently cannot use the negatioon operator  (i.e. `!`)  for miniglob matches as they
-# would match any file that doesn't touch them. What's needed is the concept of `any `, which takes a
-# list of constraints / globs and then matches all of the constraints for either `any` of the files or
-# `all` of the files in the change set.
-#
-# However, `any`/`all` are not supported in a released version and testing off of the `main` branch
-# resulted in some other errors when testing.
-#
-# An issue has been opened upstream requesting that a release be cut that has support for all/any:
-#   - https://github.com/actions/labeler/issues/111
-#
-# While we wait for this issue to be handled upstream, we can remove
-# the negated / `!` matches for now and at least have labels again.
-#
 INFRA:
   - ".github/**/*"
   - "appveyor.yml"
@@ -45,32 +28,24 @@ INFRA:
   - "dev/merge_spark_pr.py"
   - "dev/run-tests-jenkins*"
 BUILD:
- # Can be supported when a stable release with correct all/any is released
- #- any: ['dev/**/*', '!dev/merge_spark_pr.py', '!dev/.rat-excludes']
- - "dev/**/*"
+ - any: ['dev/**/*', '!dev/merge_spark_pr.py', '!dev/run-tests-jenkins*']
  - "build/**/*"
  - "project/**/*"
  - "assembly/**/*"
  - "**/*pom.xml"
  - "bin/docker-image-tool.sh"
  - "bin/find-spark-home*"
  - "scalastyle-config.xml"
- # These can be added in the above `any` clause (and the /dev/**/* glob removed) when
- # `any`/`all` support is released
- # - "!dev/merge_spark_pr.py"
- # - "!dev/run-tests-jenkins*"
- # - "!dev/.rat-excludes"
 DOCS:
   - "docs/**/*"
   - "**/README.md"
   - "**/CONTRIBUTING.md"
+  - "python/docs/**/*"
 EXAMPLES:
   - "examples/**/*"
   - "bin/run-example*"
-# CORE needs to be updated when all/any are released upstream.
 CORE:
-  # - any: ["core/**/*", "!**/*UI.scala", "!**/ui/**/*"] # If any file matches all of the globs defined in the list started by `any`, label is applied.
-  - "core/**/*"
+  - any: ["core/**/*", "!**/*UI.scala", "!**/ui/**/*"]
   - "common/kvstore/**/*"
   - "common/network-common/**/*"
   - "common/network-shuffle/**/*"
@@ -82,12 +57,8 @@ SPARK SHELL:
   - "repl/**/*"
   - "bin/spark-shell*"
 SQL:
-#- any: ["**/sql/**/*", "!python/pyspark/sql/avro/**/*", "!python/pyspark/sql/streaming/**/*", "!python/pyspark/sql/tests/streaming/test_streaming.py"]
-  - "**/sql/**/*"
+  - any: ["**/sql/**/*", "!python/pyspark/sql/avro/**/*", "!python/pyspark/sql/streaming/**/*", "!python/pyspark/sql/tests/streaming/test_streaming*.py"]
   - "common/unsafe/**/*"
-  #- "!python/pyspark/sql/avro/**/*"
-  #- "!python/pyspark/sql/streaming/**/*"
-  #- "!python/pyspark/sql/tests/streaming/test_streaming.py"
   - "bin/spark-sql*"
   - "bin/beeline*"
   - "sbin/*thriftserver*.sh"
@@ -123,7 +94,7 @@ STRUCTURED STREAMING:
   - "**/sql/**/streaming/**/*"
   - "connector/kafka-0-10-sql/**/*"
   - "python/pyspark/sql/streaming/**/*"
-  - "python/pyspark/sql/tests/streaming/test_streaming.py"
+  - "python/pyspark/sql/tests/streaming/test_streaming*.py"
   - "**/*streaming.R"
 PYTHON:
   - "bin/pyspark*"
@@ -148,7 +119,6 @@ DEPLOY:
   - "sbin/**/*"
 CONNECT:
   - "connector/connect/**/*"
-  - "**/sql/sparkconnect/**/*"
   - "python/pyspark/sql/**/connect/**/*"
   - "python/pyspark/ml/**/connect/**/*"
 PROTOBUF:

.github/workflows/benchmark.yml

@@ -65,7 +65,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
     steps:
       - name: Checkout Spark repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         # In order to get diff files
         with:
           fetch-depth: 0
@@ -95,7 +95,7 @@ jobs:
           key: tpcds-${{ hashFiles('.github/workflows/benchmark.yml', 'sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala') }}
       - name: Checkout tpcds-kit repository
         if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: databricks/tpcds-kit
           ref: 2a5078a782192ddb6efbcead8de9973d6ab4f069
@@ -134,7 +134,7 @@ jobs:
       SPARK_TPCDS_DATA: ${{ github.workspace }}/tpcds-sf-1
     steps:
     - name: Checkout Spark repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       # In order to get diff files
       with:
         fetch-depth: 0

.github/workflows/build_and_test.yml

@@ -64,7 +64,7 @@ jobs:
         }}
     steps:
     - name: Checkout Spark repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         repository: apache/spark
@@ -143,11 +143,11 @@ jobs:
           - >-
             core, unsafe, kvstore, avro, utils,
             network-common, network-shuffle, repl, launcher,
-            examples, sketch, graphx
+            examples, sketch
           - >-
             api, catalyst, hive-thriftserver
           - >-
-            mllib-local,mllib
+            mllib-local, mllib, graphx
           - >-
             streaming, sql-kafka-0-10, streaming-kafka-0-10,
             yarn, kubernetes, hadoop-cloud, spark-ganglia-lgpl,
@@ -202,7 +202,7 @@ jobs:
       SKIP_PACKAGING: true
     steps:
     - name: Checkout Spark repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       # In order to fetch changed files
       with:
         fetch-depth: 0
@@ -302,7 +302,7 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Checkout Spark repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         # In order to fetch changed files
         with:
           fetch-depth: 0
@@ -381,7 +381,7 @@ jobs:
       BRANCH: ${{ inputs.branch }}
     steps:
     - name: Checkout Spark repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       # In order to fetch changed files
       with:
         fetch-depth: 0
@@ -496,7 +496,7 @@ jobs:
       SKIP_PACKAGING: true
     steps:
     - name: Checkout Spark repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       # In order to fetch changed files
       with:
         fetch-depth: 0
@@ -564,7 +564,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - name: Checkout Spark repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         repository: apache/spark
@@ -604,7 +604,7 @@ jobs:
       image: ${{ needs.precondition.outputs.image_url }}
     steps:
     - name: Checkout Spark repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         repository: apache/spark
@@ -786,7 +786,7 @@ jobs:
     timeout-minutes: 300
     steps:
     - name: Checkout Spark repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         repository: apache/spark
@@ -841,7 +841,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
     steps:
     - name: Checkout Spark repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         repository: apache/spark
@@ -883,7 +883,7 @@ jobs:
         key: tpcds-${{ hashFiles('.github/workflows/build_and_test.yml', 'sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala') }}
     - name: Checkout tpcds-kit repository
       if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: databricks/tpcds-kit
         ref: 2a5078a782192ddb6efbcead8de9973d6ab4f069
@@ -946,7 +946,7 @@ jobs:
       SKIP_PACKAGING: true
     steps:
     - name: Checkout Spark repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         repository: apache/spark
@@ -1006,7 +1006,7 @@ jobs:
     timeout-minutes: 300
     steps:
       - name: Checkout Spark repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           repository: apache/spark
@@ -1063,7 +1063,7 @@ jobs:
           export PVC_TESTS_VM_PATH=$PVC_TMP_DIR
           minikube mount ${PVC_TESTS_HOST_PATH}:${PVC_TESTS_VM_PATH} --gid=0 --uid=185 &
           kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true
-          kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.8.0/installer/volcano-development.yaml || true
+          kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.8.1/installer/volcano-development.yaml || true
           eval $(minikube docker-env)
           build/sbt -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test"
       - name: Upload Spark on K8S integration tests log files

.github/workflows/build_infra_images_cache.yml

@@ -38,7 +38,7 @@ jobs:
       packages: write
     steps:
       - name: Checkout Spark repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx

.github/workflows/labeler.yml

@@ -34,19 +34,6 @@ jobs:
       contents: read
       pull-requests: write
     steps:
-    # In order to get back the negated matches like in the old config,
-    # we need the actinons/labeler concept of `all` and `any` which matches
-    # all of the given constraints / glob patterns for either `all`
-    # files or `any` file in the change set.
-    #
-    # Github issue which requests a timeline for a release with any/all support:
-    #     - https://github.com/actions/labeler/issues/111
-    # This issue also references the issue that mentioned that any/all are only
-    # supported on main branch (previously called master):
-    #    - https://github.com/actions/labeler/issues/73#issuecomment-639034278
-    #
-    # However, these are not in a published release and the current `main` branch
-    # has some issues upon testing.
     - uses: actions/labeler@v4
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/maven_test.yml

@@ -115,7 +115,7 @@ jobs:
       GITHUB_PREV_SHA: ${{ github.event.before }}
     steps:
       - name: Checkout Spark repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         # In order to fetch changed files
         with:
           fetch-depth: 0

.github/workflows/publish_snapshot.yml

@@ -41,7 +41,7 @@ jobs:
         branch: ${{ fromJSON( inputs.branch || '["master", "branch-3.5", "branch-3.4", "branch-3.3"]' ) }}
     steps:
     - name: Checkout Spark repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: ${{ matrix.branch }}
     - name: Cache Maven local repository

build/spark-build-info.ps1

@@ -41,6 +41,7 @@ user=$($Env:USERNAME)
 revision=$(git rev-parse HEAD)
 branch=$(git rev-parse --abbrev-ref HEAD)
 date=$([DateTime]::UtcNow | Get-Date -UFormat +%Y-%m-%dT%H:%M:%SZ)
-url=$(git config --get remote.origin.url)"
+url=$(git config --get remote.origin.url),
+docroot=https://spark.apache.org/docs/latest"
 
 Set-Content -Path $SparkBuildInfoPath -Value $SparkBuildInfoContent

common/network-common/src/test/java/org/apache/spark/network/ssl/ReloadingX509TrustManagerSuite.java

@@ -161,14 +161,17 @@ public void testReload() throws Exception {
       // At this point we haven't reloaded, just the initial load
       assertEquals(0, tm.reloadCount);
 
+      // Wait so that the file modification time is different
+      Thread.sleep((tm.getReloadInterval() + 1000));
+
       // Add another cert
       Map<String, X509Certificate> certs = new HashMap<String, X509Certificate>();
       certs.put("cert1", cert1);
       certs.put("cert2", cert2);
       createTrustStore(trustStore, "password", certs);
 
-      // Wait up to 5s until we reload
-      waitForReloadCount(tm, 1, 50);
+      // Wait up to 10s until we reload
+      waitForReloadCount(tm, 1, 100);
 
       assertEquals(2, tm.getAcceptedIssuers().length);
     } finally {
@@ -286,17 +289,17 @@ public void testReloadSymlink() throws Exception {
       trustStoreSymlink.delete();
       Files.createSymbolicLink(trustStoreSymlink.toPath(), trustStore2.toPath());
 
-      // Wait up to 5s until we reload
-      waitForReloadCount(tm, 1, 50);
+      // Wait up to 10s until we reload
+      waitForReloadCount(tm, 1, 100);
 
       assertEquals(2, tm.getAcceptedIssuers().length);
 
       // Add another cert
       certs.put("cert3", cert3);
       createTrustStore(trustStore2, "password", certs);
 
-      // Wait up to 5s until we reload
-      waitForReloadCount(tm, 2, 50);
+      // Wait up to 10s until we reload
+      waitForReloadCount(tm, 2, 100);
 
       assertEquals(3, tm.getAcceptedIssuers().length);
     } finally {

common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java

@@ -59,13 +59,6 @@ public final class Platform {
     // reflection to invoke it, which is not necessarily possible by default in Java 9+.
     // Code below can test for null to see whether to use it.
 
-    // The implementation of Cleaner changed from JDK 8 to 9
-    String cleanerClassName;
-    if (majorVersion < 9) {
-      cleanerClassName = "sun.misc.Cleaner";
-    } else {
-      cleanerClassName = "jdk.internal.ref.Cleaner";
-    }
     try {
       Class<?> cls = Class.forName("java.nio.DirectByteBuffer");
       Constructor<?> constructor = (majorVersion < 21) ?
@@ -84,7 +77,7 @@ public final class Platform {
 
       // no point continuing if the above failed:
       if (DBB_CONSTRUCTOR != null && DBB_CLEANER_FIELD != null) {
-        Class<?> cleanerClass = Class.forName(cleanerClassName);
+        Class<?> cleanerClass = Class.forName("jdk.internal.ref.Cleaner");
         Method createMethod = cleanerClass.getMethod("create", Object.class, Runnable.class);
         // Accessing jdk.internal.ref.Cleaner should actually fail by default in JDK 9+,
         // unfortunately, unless the user has allowed access with something like
@@ -314,7 +307,7 @@ public static void throwException(Throwable t) {
     }
   }
 
-  // This requires `majorVersion` and `_UNSAFE`.
+  // This requires `_UNSAFE`.
   static {
     boolean _unaligned;
     String arch = System.getProperty("os.arch", "");
@@ -326,10 +319,8 @@ public static void throwException(Throwable t) {
       try {
         Class<?> bitsClass =
           Class.forName("java.nio.Bits", false, ClassLoader.getSystemClassLoader());
-        if (_UNSAFE != null && majorVersion >= 9) {
-          // Java 9/10 and 11/12 have different field names.
-          Field unalignedField =
-            bitsClass.getDeclaredField(majorVersion >= 11 ? "UNALIGNED" : "unaligned");
+        if (_UNSAFE != null) {
+          Field unalignedField = bitsClass.getDeclaredField("UNALIGNED");
           _unaligned = _UNSAFE.getBoolean(
             _UNSAFE.staticFieldBase(unalignedField), _UNSAFE.staticFieldOffset(unalignedField));
         } else {