From e47643a02c5c19ba454a4968ab1d6151258f5fd8 Mon Sep 17 00:00:00 2001 From: Chenhao Li Date: Tue, 26 Mar 2024 12:09:13 -0700 Subject: [PATCH 1/2] initial --- .../sql/catalyst/analysis/CheckAnalysis.scala | 12 ++++++++++++ .../catalyst/analysis/AnalysisErrorSuite.scala | 15 +++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala index 4a979fd214aba..052b2b8a7e0dc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala @@ -537,6 +537,18 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB } } + case Window(_, partitionSpec, _, _) => + // Both `partitionSpec` and `orderSpec` must be orderable. We only need an extra check + // for `partitionSpec` here because `orderSpec` has the type check itself. + partitionSpec.foreach { p => + if (!RowOrdering.isOrderable(p.dataType)) { + p.dataTypeMismatch(p, TypeCheckResult.DataTypeMismatch( + errorSubClass = "INVALID_ORDERING_TYPE", + Map("functionName" -> toSQLId(p.prettyName), "dataType" -> toSQLType(p.dataType)) + )) + } + } + case GlobalLimit(limitExpr, _) => checkLimitLikeClause("limit", limitExpr) case LocalLimit(limitExpr, child) => diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala index 8366e8a22d428..2bc42e29319cc 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala @@ -1387,4 +1387,19 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase { ) } } + + errorClassTest( + "SPARK-47572: Enforce Window partitionSpec is orderable", + testRelation2.select( + WindowExpression( + new Rank(), + WindowSpecDefinition( + CreateMap(Literal("key") :: UnresolvedAttribute("a") :: Nil) :: Nil, + SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil, + UnspecifiedFrame)).as("window")), + errorClass = "DATATYPE_MISMATCH.INVALID_ORDERING_TYPE", + messageParameters = Map( + "functionName" -> "`attributereference`", + "dataType" -> "\"MAP\"", + "sqlExpr" -> "\"_w0\"")) } From 2231e504668a1d5e832e524f6c2dce1ab3005b78 Mon Sep 17 00:00:00 2001 From: Chenhao Li Date: Thu, 28 Mar 2024 22:53:45 -0700 Subject: [PATCH 2/2] change error class --- .../spark/sql/catalyst/analysis/CheckAnalysis.scala | 9 +++++---- .../spark/sql/catalyst/analysis/AnalysisErrorSuite.scala | 7 +++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala index 052b2b8a7e0dc..10bff5e6e59a2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala @@ -542,10 +542,11 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB // for `partitionSpec` here because `orderSpec` has the type check itself. partitionSpec.foreach { p => if (!RowOrdering.isOrderable(p.dataType)) { - p.dataTypeMismatch(p, TypeCheckResult.DataTypeMismatch( - errorSubClass = "INVALID_ORDERING_TYPE", - Map("functionName" -> toSQLId(p.prettyName), "dataType" -> toSQLType(p.dataType)) - )) + p.failAnalysis( + errorClass = "EXPRESSION_TYPE_IS_NOT_ORDERABLE", + messageParameters = Map( + "expr" -> toSQLExpr(p), + "exprType" -> toSQLType(p.dataType))) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala index cdb2574e95eca..f12d224096917 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala @@ -1326,9 +1326,8 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase { CreateMap(Literal("key") :: UnresolvedAttribute("a") :: Nil) :: Nil, SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil, UnspecifiedFrame)).as("window")), - errorClass = "DATATYPE_MISMATCH.INVALID_ORDERING_TYPE", + errorClass = "EXPRESSION_TYPE_IS_NOT_ORDERABLE", messageParameters = Map( - "functionName" -> "`attributereference`", - "dataType" -> "\"MAP\"", - "sqlExpr" -> "\"_w0\"")) + "expr" -> "\"_w0\"", + "exprType" -> "\"MAP\"")) }