Skip to content

Commit

Permalink
[SPARK-27514] Skip collapsing windows with empty window expressions
Browse files Browse the repository at this point in the history
  • Loading branch information
yifeih committed Apr 18, 2019
1 parent 3748b38 commit 9fc933e
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -770,6 +770,7 @@ object CollapseWindow extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
case w1 @ Window(we1, ps1, os1, w2 @ Window(we2, ps2, os2, grandChild))
if ps1 == ps2 && os1 == os2 && w1.references.intersect(w2.windowOutputSet).isEmpty &&
we1.nonEmpty && we2.nonEmpty &&
// This assumes Window contains the same type of window expressions. This is ensured
// by ExtractWindowFunctions.
WindowFunctionType.functionType(we1.head) == WindowFunctionType.functionType(we2.head) =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,4 +89,15 @@ class CollapseWindowSuite extends PlanTest {
val optimized = Optimize.execute(query.analyze)
comparePlans(optimized, expected)
}

test("Skip windows with empty window expressions") {
val query = testRelation
.window(Seq(), partitionSpec1, orderSpec1)
.window(Seq(sum(a).as('sum_a)), partitionSpec1, orderSpec1)

val optimized = Optimize.execute(query.analyze)
val correctAnswer = query.analyze

comparePlans(optimized, correctAnswer)
}
}

0 comments on commit 9fc933e

Please sign in to comment.