From 7e5250a797e385dea65bdc3315e9bcfd827afbfb Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Wed, 23 Aug 2023 08:01:06 +0800 Subject: [PATCH] [SPARK-44907][PYTHON][CONNECT] DataFrame.join` should throw IllegalArgumentException for invalid join types ### What changes were proposed in this pull request? `DataFrame.join` should throw IllegalArgumentException for invalid join types ### Why are the changes needed? all valid join types have already been supported, for an unknown one, should throw `IllegalArgumentException` now ### Does this PR introduce _any_ user-facing change? yes ### How was this patch tested? enabled UT ### Was this patch authored or co-authored using generative AI tooling? NO Closes #42603 from zhengruifeng/test_df_join_type. Authored-by: Ruifeng Zheng Signed-off-by: Ruifeng Zheng --- python/pyspark/sql/connect/plan.py | 9 +++++++-- .../pyspark/sql/tests/connect/test_parity_dataframe.py | 5 ----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py index e21a0fab00018..4ef789e28a527 100644 --- a/python/pyspark/sql/connect/plan.py +++ b/python/pyspark/sql/connect/plan.py @@ -41,7 +41,12 @@ LiteralExpression, ) from pyspark.sql.connect.types import pyspark_types_to_proto_types, UnparsedDataType -from pyspark.errors import PySparkTypeError, PySparkNotImplementedError, PySparkRuntimeError +from pyspark.errors import ( + PySparkTypeError, + PySparkNotImplementedError, + PySparkRuntimeError, + IllegalArgumentException, +) if TYPE_CHECKING: from pyspark.sql.connect._typing import ColumnOrName @@ -854,7 +859,7 @@ def __init__( elif how == "cross": join_type = proto.Join.JoinType.JOIN_TYPE_CROSS else: - raise PySparkNotImplementedError( + raise IllegalArgumentException( error_class="UNSUPPORTED_JOIN_TYPE", message_parameters={"join_type": how}, ) diff --git a/python/pyspark/sql/tests/connect/test_parity_dataframe.py b/python/pyspark/sql/tests/connect/test_parity_dataframe.py index ccc0b997e8d4b..cc9f71f8b46f1 100644 --- a/python/pyspark/sql/tests/connect/test_parity_dataframe.py +++ b/python/pyspark/sql/tests/connect/test_parity_dataframe.py @@ -26,11 +26,6 @@ class DataFrameParityTests(DataFrameTestsMixin, ReusedConnectTestCase): def test_help_command(self): super().test_help_command() - # Spark Connect throws NotImplementedError tests expects IllegalArgumentException - @unittest.skip("Fails in Spark Connect, should enable.") - def test_invalid_join_method(self): - super().test_invalid_join_method() - # TODO(SPARK-41527): Implement DataFrame.observe @unittest.skip("Fails in Spark Connect, should enable.") def test_observe(self):