feat(spark): enable upper/lower/concat/coalesce string functions (#308)

Add mappings for upper/lower/concat/coalesce Signed-off-by: Andrew Coleman <andrew_coleman@uk.ibm.com>
substrait-io · Oct 24, 2024 · fc8a764 · fc8a764
1 parent ac0b7d1
commit fc8a764
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 3 deletions.
diff --git a/spark/src/main/scala/io/substrait/spark/expression/FunctionMappings.scala b/spark/src/main/scala/io/substrait/spark/expression/FunctionMappings.scala
@@ -48,14 +48,17 @@ class FunctionMappings {
     s[GreaterThanOrEqual]("gte"),
     s[EqualTo]("equal"),
     s[EqualNullSafe]("is_not_distinct_from"),
-    // s[BitwiseXor]("xor"),
     s[IsNull]("is_null"),
     s[IsNotNull]("is_not_null"),
     s[EndsWith]("ends_with"),
     s[Like]("like"),
     s[Contains]("contains"),
     s[StartsWith]("starts_with"),
     s[Substring]("substring"),
+    s[Upper]("upper"),
+    s[Lower]("lower"),
+    s[Concat]("concat"),
+    s[Coalesce]("coalesce"),
     s[Year]("year"),
 
     // internal

diff --git a/spark/src/main/scala/io/substrait/spark/expression/ToSubstraitExpression.scala b/spark/src/main/scala/io/substrait/spark/expression/ToSubstraitExpression.scala
@@ -35,6 +35,8 @@ abstract class ToSubstraitExpression extends HasOutputStack[Seq[Attribute]] {
       case BinaryExpression(left, right) => Some(Seq(left, right))
       case UnaryExpression(child) => Some(Seq(child))
       case t: TernaryExpression => Some(Seq(t.first, t.second, t.third))
+      case Coalesce(children) => Some(children.toList)
+      case Concat(children) => Some(children.toList)
       case _ => None
     }
   }

diff --git a/spark/src/test/scala/io/substrait/spark/TPCDSPlan.scala b/spark/src/test/scala/io/substrait/spark/TPCDSPlan.scala
@@ -34,9 +34,9 @@ class TPCDSPlan extends TPCDSBase with SubstraitPlanTestBase {
   // spotless:off
   val successfulSQL: Set[String] = Set("q1", "q3", "q4", "q7",
     "q11", "q13", "q14b", "q15", "q16", "q18", "q19",
-    "q21", "q22", "q23a", "q23b", "q25", "q26", "q28", "q29",
+    "q21", "q22", "q23a", "q23b", "q24a", "q24b", "q25", "q26", "q28", "q29",
     "q30", "q31", "q32", "q33", "q37", "q38",
-    "q41", "q42", "q43", "q46", "q48",
+    "q40", "q41", "q42", "q43", "q46", "q48",
     "q50", "q52", "q54", "q55", "q56", "q58", "q59",
     "q60", "q61", "q62", "q65", "q66", "q68", "q69",
     "q71", "q73", "q76", "q79",