Skip to content

Commit

Permalink
Qbeast-io#207 Fixes for NormalizedWeightIntegrationTest
Browse files Browse the repository at this point in the history
  • Loading branch information
alexeiakimov committed Sep 1, 2023
1 parent 0eb9097 commit 63f53e2
Showing 1 changed file with 8 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ package io.qbeast.spark.index
import io.qbeast.TestClasses.Client3
import io.qbeast.spark.QbeastIntegrationTestSpec
import io.qbeast.spark.delta.DeltaQbeastSnapshot
import io.qbeast.spark.utils.TagUtils
import org.apache.spark.sql.delta.DeltaLog
import org.apache.spark.sql.{Dataset, SparkSession}
import io.qbeast.spark.utils.State

class NormalizedWeightIntegrationTest extends QbeastIntegrationTestSpec {

Expand All @@ -22,7 +22,6 @@ class NormalizedWeightIntegrationTest extends QbeastIntegrationTestSpec {
"CubeNormalizedWeights" should
"write a the right Weight with a single full file" in withQbeastContextSparkAndTmpDir {
(spark, tmpDir) =>
import spark.implicits._
val cubeSize = 10000
val df = createDF(cubeSize).repartition(1)
val names = List("age", "val2")
Expand All @@ -36,11 +35,13 @@ class NormalizedWeightIntegrationTest extends QbeastIntegrationTestSpec {
spark.read.format("qbeast").load(tmpDir).count() shouldBe cubeSize

val deltaLog = DeltaLog.forTable(spark, tmpDir)
val files = deltaLog.snapshot.allFiles
files.count() shouldBe 1
files.map(_.tags(TagUtils.maxWeight).toInt).collect()(0) shouldBe <=(Int.MaxValue)
files.map(_.tags(TagUtils.state)).collect()(0) shouldBe "FLOODED"

val snapshot = DeltaQbeastSnapshot(deltaLog.snapshot)
val index = snapshot.loadLatestIndexStatus
index.cubesStatuses.size shouldBe 1
val status = index.cubesStatuses.values.head
status.files.size shouldBe 1
val block = status.files.head
block.state shouldBe State.FLOODED
}

it should
Expand Down

0 comments on commit 63f53e2

Please sign in to comment.