Skip to content

Commit

Permalink
[SPARK-42306][SQL] Integrate _LEGACY_ERROR_TEMP_1317 into `UNRESOLV…
Browse files Browse the repository at this point in the history
…ED_COLUMN.WITH_SUGGESTION`

### What changes were proposed in this pull request?

This PR proposes to integrate `_LEGACY_ERROR_TEMP_1317` into `UNRESOLVED_COLUMN.WITH_SUGGESTION`.

**_LEGACY_ERROR_TEMP_1317**
```json
  "_LEGACY_ERROR_TEMP_1317" : {
    "message" : [
      "Cannot resolve column name \"<colName>\" among (<fieldsStr>)<extraMsg>"
    ]
  },
```

**UNRESOLVED_COLUMN.WITH_SUGGESTION**
```json
  "UNRESOLVED_COLUMN" : {
    "message" : [
      "A column or function parameter with name <objectName> cannot be resolved."
    ],
    "subClass" : {
      "WITHOUT_SUGGESTION" : {
        "message" : [
          ""
        ]
      },
      "WITH_SUGGESTION" : {
        "message" : [
          "Did you mean one of the following? [<proposal>]."
        ]
      }
    },
    "sqlState" : "42703"
  },
```

### Why are the changes needed?

We should assign proper name to _LEGACY_ERROR_TEMP_*

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

`./build/sbt "sql/testOnly org.apache.spark.sql.SQLQueryTestSuite*"`

Closes apache#39877 from itholic/LEGACY_1317.

Authored-by: itholic <haejoon.lee@databricks.com>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
  • Loading branch information
itholic authored and MaxGekk committed Feb 7, 2023
1 parent 54b5cf6 commit 6b6bb6f
Show file tree
Hide file tree
Showing 6 changed files with 51 additions and 37 deletions.
5 changes: 0 additions & 5 deletions core/src/main/resources/error/error-classes.json
Original file line number Diff line number Diff line change
Expand Up @@ -3463,11 +3463,6 @@
"Invalid partition transformation: <expr>."
]
},
"_LEGACY_ERROR_TEMP_1317" : {
"message" : [
"Cannot resolve column name \"<colName>\" among (<fieldsStr>)<extraMsg>"
]
},
"_LEGACY_ERROR_TEMP_1318" : {
"message" : [
"Unable to parse '<delayThreshold>'."
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2915,14 +2915,15 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase {
messageParameters = Map("expr" -> expr.sql))
}

def cannotResolveColumnNameAmongFieldsError(
colName: String, fieldsStr: String, extraMsg: String): AnalysisException = {
def unresolvedColumnWithSuggestionError(
objectName: String, suggestion: String): AnalysisException = {
new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_1317",
errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
messageParameters = Map(
"colName" -> colName,
"fieldsStr" -> fieldsStr,
"extraMsg" -> extraMsg))
"objectName" -> toSQLId(objectName),
"proposal" -> suggestion
)
)
}

def cannotParseIntervalError(delayThreshold: String, e: Throwable): Throwable = {
Expand Down
8 changes: 3 additions & 5 deletions sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.trees.TreeNodeTag
import org.apache.spark.sql.catalyst.util.IntervalUtils
import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
import org.apache.spark.sql.errors.QueryCompilationErrors.toSQLId
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.aggregate.TypedAggregateExpression
import org.apache.spark.sql.execution.arrow.{ArrowBatchStreamWriter, ArrowConverters}
Expand Down Expand Up @@ -250,11 +251,8 @@ class Dataset[T] private[sql](
}

private def resolveException(colName: String, fields: Array[String]): AnalysisException = {
val extraMsg = if (fields.exists(sparkSession.sessionState.analyzer.resolver(_, colName))) {
s"; did you mean to quote the `$colName` column?"
} else ""
val fieldsStr = fields.mkString(", ")
QueryCompilationErrors.cannotResolveColumnNameAmongFieldsError(colName, fieldsStr, extraMsg)
QueryCompilationErrors.unresolvedColumnWithSuggestionError(
colName, fields.map(toSQLId).mkString(", "))
}

private[sql] def numericColumns: Seq[Expression] = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -299,10 +299,13 @@ class DataFrameNaFunctionsSuite extends QueryTest with SharedSparkSession {

test("drop with col(*)") {
val df = createDF()
val exception = intercept[AnalysisException] {
df.na.drop("any", Seq("*"))
}
assert(exception.getMessage.contains("Cannot resolve column name \"*\""))
checkError(
exception = intercept[AnalysisException] {
df.na.drop("any", Seq("*"))
},
errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
parameters = Map("objectName" -> "`*`", "proposal" -> "`name`, `age`, `height`")
)
}

test("fill with nested columns") {
Expand Down Expand Up @@ -534,7 +537,11 @@ class DataFrameNaFunctionsSuite extends QueryTest with SharedSparkSession {
val exception = intercept[AnalysisException] {
df.na.replace("aa", Map( "n/a" -> "unknown"))
}
assert(exception.getMessage.equals("Cannot resolve column name \"aa\" among (Col.1, Col.2)"))
checkError(
exception = exception,
errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
parameters = Map("objectName" -> "`aa`", "proposal" -> "`Col`.`1`, `Col`.`2`")
)
}

test("SPARK-34649: replace value of a nested column") {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -752,10 +752,13 @@ class DataFrameSuite extends QueryTest
val df2 = df1.withMetadata("x", metadata)
assert(df2.schema(0).metadata === metadata)

val err = intercept[AnalysisException] {
df1.withMetadata("x1", metadata)
}
assert(err.getMessage.contains("Cannot resolve column name"))
checkError(
exception = intercept[AnalysisException] {
df1.withMetadata("x1", metadata)
},
errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
parameters = Map("objectName" -> "`x1`", "proposal" -> "`x`")
)
}

test("replace column using withColumn") {
Expand Down
34 changes: 22 additions & 12 deletions sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -354,15 +354,21 @@ class DatasetSuite extends QueryTest
start = 0,
stop = 8))

var e = intercept[AnalysisException] {
ds.select(ds("`(_1)?+.+`"))
}
assert(e.getMessage.contains("Cannot resolve column name \"`(_1)?+.+`\""))
checkError(
exception = intercept[AnalysisException] {
ds.select(ds("`(_1)?+.+`"))
},
errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
parameters = Map("objectName" -> "`(_1)?+.+`", "proposal" -> "`_1`, `_2`")
)

e = intercept[AnalysisException] {
ds.select(ds("`(_1|_2)`"))
}
assert(e.getMessage.contains("Cannot resolve column name \"`(_1|_2)`\""))
checkError(
exception = intercept[AnalysisException] {
ds.select(ds("`(_1|_2)`"))
},
errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
parameters = Map("objectName" -> "`(_1|_2)`", "proposal" -> "`_1`, `_2`")
)
}

withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "true") {
Expand Down Expand Up @@ -2166,10 +2172,14 @@ class DatasetSuite extends QueryTest
forAll(dotColumnTestModes) { (caseSensitive, colName) =>
val ds = Seq(SpecialCharClass("1", "2")).toDS
withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive) {
val errorMsg = intercept[AnalysisException] {
ds(colName)
}
assert(errorMsg.getMessage.contains(s"did you mean to quote the `$colName` column?"))
val colName = if (caseSensitive == "true") "`Field`.`1`" else "`field`.`1`"
checkError(
exception = intercept[AnalysisException] {
ds(colName)
},
errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
parameters = Map("objectName" -> colName, "proposal" -> "`field`.`1`, `field 2`")
)
}
}
}
Expand Down

0 comments on commit 6b6bb6f

Please sign in to comment.