From 02b32b7eb187bf5005b93dec029fdbdd9bd57fd1 Mon Sep 17 00:00:00 2001 From: Alexey Kuzin Date: Mon, 30 May 2022 17:42:58 -0400 Subject: [PATCH] Add test for decimal types support --- build.sbt | 20 +++++--- .../cartridge/app/roles/api_storage.lua | 26 ++++++++++ .../integration/SharedSparkContext.scala | 2 +- .../TarantoolSparkWriteClusterTest.scala | 49 +++++++++++++++++++ 4 files changed, 89 insertions(+), 8 deletions(-) diff --git a/build.sbt b/build.sbt index c0abbb6..21978f7 100644 --- a/build.sbt +++ b/build.sbt @@ -33,15 +33,16 @@ ThisBuild / developers := List( ThisBuild / scalaVersion := scala211 val commonDependencies = Seq( - "io.tarantool" % "cartridge-driver" % "0.7.2", + "io.tarantool" % "cartridge-driver" % "0.8.0", "junit" % "junit" % "4.12" % Test, "com.github.sbt" % "junit-interface" % "0.12" % Test, - "org.testcontainers" % "testcontainers" % "1.16.0" % Test, - "io.tarantool" % "testcontainers-java-tarantool" % "0.4.7" % Test, + "org.testcontainers" % "testcontainers" % "1.17.0" % Test, + "io.tarantool" % "testcontainers-java-tarantool" % "0.5.0" % Test, "org.scalatest" %% "scalatest" % "3.2.9" % Test, "org.scalamock" %% "scalamock" % "5.1.0" % Test, "com.dimafeng" %% "testcontainers-scala-scalatest" % "0.39.5" % Test, - "ch.qos.logback" % "logback-classic" % "1.2.5" % Test + "ch.qos.logback" % "logback-classic" % "1.2.5" % Test, + "org.apache.derby" % "derby" % "10.11.1.1" % Test ) lazy val root = (project in file(".")) @@ -55,12 +56,14 @@ lazy val root = (project in file(".")) case Some((2, scalaMajor)) if scalaMajor >= 12 => Seq( "org.apache.spark" %% "spark-core" % "2.4.8" % "provided", - "org.apache.spark" %% "spark-sql" % "2.4.8" % "provided" + "org.apache.spark" %% "spark-sql" % "2.4.8" % "provided", + "org.apache.spark" %% "spark-hive" % "2.4.8" % "provided" ) case _ => Seq( "org.apache.spark" %% "spark-core" % "2.2.3" % "provided", - "org.apache.spark" %% "spark-sql" % "2.2.3" % "provided" + "org.apache.spark" %% "spark-sql" % "2.2.3" % "provided", + "org.apache.spark" %% "spark-hive" % "2.2.3" % "provided" ) } }).map( @@ -100,7 +103,10 @@ lazy val root = (project in file(".")) "UTF-8" ), // Test frameworks options - testOptions += Tests.Argument(TestFrameworks.JUnit, "-v"), + testOptions ++= Seq( + Tests.Argument(TestFrameworks.JUnit, "-v"), + Tests.Setup(() => System.setSecurityManager(null)) // SPARK-22918 + ), // Publishing settings publishTo := { val nexus = "https://oss.sonatype.org/" diff --git a/src/test/resources/cartridge/app/roles/api_storage.lua b/src/test/resources/cartridge/app/roles/api_storage.lua index 8200ec3..7f4a736 100644 --- a/src/test/resources/cartridge/app/roles/api_storage.lua +++ b/src/test/resources/cartridge/app/roles/api_storage.lua @@ -70,6 +70,32 @@ local function init_space() unique = false, if_not_exists = true, }) + + local reg_numbers = box.schema.space.create( + 'reg_numbers', + { + format = { + {name = 'bucket_id' , type = 'unsigned' , is_nullable = false}, + {name = 'idreg' , type = 'decimal' , is_nullable = false}, + {name = 'regnum' , type = 'decimal' , is_nullable = true}, + }, + if_not_exists = true, + } + ) + + reg_numbers:create_index('index_id', { + unique = true, + type = 'tree', + parts = {{2, 'decimal'}}, + if_not_exists = true, + }) + + reg_numbers:create_index('bucket_id', { + unique = false, + type = 'tree', + parts = {{1, 'unsigned'}}, + if_not_exists = true, + }) end local function init(opts) diff --git a/src/test/scala/io/tarantool/spark/connector/integration/SharedSparkContext.scala b/src/test/scala/io/tarantool/spark/connector/integration/SharedSparkContext.scala index 3b9c606..ee24776 100644 --- a/src/test/scala/io/tarantool/spark/connector/integration/SharedSparkContext.scala +++ b/src/test/scala/io/tarantool/spark/connector/integration/SharedSparkContext.scala @@ -38,7 +38,7 @@ trait SharedSparkContext extends BeforeAndAfterAll { self: Suite => sessionBuilder: SparkSession.Builder, conf: SparkConf ): SparkSession.Builder = { - sessionBuilder.config(conf) + sessionBuilder.config(conf).enableHiveSupport() sessionBuilder } diff --git a/src/test/scala/io/tarantool/spark/connector/integration/TarantoolSparkWriteClusterTest.scala b/src/test/scala/io/tarantool/spark/connector/integration/TarantoolSparkWriteClusterTest.scala index 72eb849..bff6059 100644 --- a/src/test/scala/io/tarantool/spark/connector/integration/TarantoolSparkWriteClusterTest.scala +++ b/src/test/scala/io/tarantool/spark/connector/integration/TarantoolSparkWriteClusterTest.scala @@ -258,6 +258,55 @@ class TarantoolSparkWriteClusterTest .save() } } + + test("should write a Dataset to the space with decimal values") { + val space = "reg_numbers" + + spark.sql("create database if not exists dl_raw") + spark.sql("drop table if exists DL_RAW.reg_numbers") + + spark.sql(""" + |create table if not exists DL_RAW.reg_numbers ( + | bucket_id integer + | ,idreg decimal(38,18) + | ,regnum decimal(38) + | ) stored as orc""".stripMargin) + spark.sql(""" + |insert into dl_raw.reg_numbers values + |(null, 1085529600000.13452690000413, 404503014700028), + |(null, 1086629600000.13452690000413, 404503015800028), + |(null, 1087430400000.13452690000413, 304503016900085) + |""".stripMargin) + + val ds = spark.table("dl_raw.reg_numbers") + + ds.show(false) + ds.printSchema() + + ds.write + .format("org.apache.spark.sql.tarantool") + .option("tarantool.space", space) + .mode(SaveMode.Overwrite) + .save() + + val actual = spark.sparkContext.tarantoolSpace(space, Conditions.any()).collect() + actual.length should equal(3) + + actual(0).getDecimal("idreg") should equal( + BigDecimal("1085529600000.134526900004130000").bigDecimal + ) + actual(0).getDecimal("regnum") should equal(BigDecimal("404503014700028").bigDecimal) + + actual(1).getDecimal("idreg") should equal( + BigDecimal("1086629600000.134526900004130000").bigDecimal + ) + actual(1).getDecimal("regnum") should equal(BigDecimal("404503015800028").bigDecimal) + + actual(2).getDecimal("idreg") should equal( + BigDecimal("1087430400000.134526900004130000").bigDecimal + ) + actual(2).getDecimal("regnum") should equal(BigDecimal("304503016900085").bigDecimal) + } } case class Order(