diff --git a/client-spark/shuffle-manager-3/src/main/java/org/apache/spark/shuffle/rss/HashBasedShuffleWriter.java b/client-spark/shuffle-manager-3/src/main/java/org/apache/spark/shuffle/rss/HashBasedShuffleWriter.java index 768dc0b7f2d..a4910719f86 100644 --- a/client-spark/shuffle-manager-3/src/main/java/org/apache/spark/shuffle/rss/HashBasedShuffleWriter.java +++ b/client-spark/shuffle-manager-3/src/main/java/org/apache/spark/shuffle/rss/HashBasedShuffleWriter.java @@ -361,5 +361,10 @@ public Option stop(boolean success) { essShuffleClient.cleanup(appId, shuffleId, mapId, taskContext.attemptNumber()); } } + + public long[] getPartitionLengths() { + throw new UnsupportedOperationException( + "RSS is not compatible with Spark push mode, please set spark.shuffle.push.enabled to false"); + } } diff --git a/client-spark/shuffle-manager-common/src/main/java/org/apache/spark/shuffle/rss/SortBasedShuffleWriter.java b/client-spark/shuffle-manager-common/src/main/java/org/apache/spark/shuffle/rss/SortBasedShuffleWriter.java index 947a7d2bf1b..2ce2fb9dcad 100644 --- a/client-spark/shuffle-manager-common/src/main/java/org/apache/spark/shuffle/rss/SortBasedShuffleWriter.java +++ b/client-spark/shuffle-manager-common/src/main/java/org/apache/spark/shuffle/rss/SortBasedShuffleWriter.java @@ -298,5 +298,10 @@ public Option stop(boolean success) { essShuffleClient.cleanup(appId, shuffleId, mapId, taskContext.attemptNumber()); } } + + public long[] getPartitionLengths() { + throw new UnsupportedOperationException( + "RSS is not compatible with Spark push mode, please set spark.shuffle.push.enabled to false"); + } }