From 6a8f241f44012fde1c258b744d75607c1f77c907 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Wed, 22 Nov 2023 14:35:00 +0100 Subject: [PATCH] Update scio to 0.13.6 --- build.sbt | 8 ++++---- .../spotify/ratatool/diffy/AvroDiffyTest.scala | 16 ++++++++-------- .../spotify/ratatool/diffy/BigDiffyTest.scala | 4 ++-- .../examples/diffy/PreProcessBigDiffy.scala | 4 ++-- .../examples/PreProcessBigDiffyJobTest.scala | 4 ++-- .../ratatool/scalacheck/AvroGenerator.scala | 4 ++-- .../ratatool/scalacheck/AvroGeneratorTest.scala | 4 ++-- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/build.sbt b/build.sbt index 53d71d3b..8919b35e 100644 --- a/build.sbt +++ b/build.sbt @@ -20,9 +20,9 @@ import Keys._ val algebirdVersion = "0.13.10" val avroVersion = "1.8.2" // keep in sync with scio -val beamVersion = "2.50.0" // keep in sync with scio -val bigqueryVersion = "v2-rev20230520-2.0.0" // keep in sync with scio -val floggerVersion = "0.7.4" +val beamVersion = "2.52.0" // keep in sync with scio +val bigqueryVersion = "v2-rev20230812-2.0.0" // keep in sync with scio +val floggerVersion = "0.7.4" // keep in sync with scio + beam val guavaVersion = "32.1.2-jre" // keep in sync with scio + beam val hadoopVersion = "2.10.2" // keep in sync with scio val jodaTimeVersion = "2.10.10" // keep in sync with scio @@ -31,7 +31,7 @@ val protoBufVersion = "3.23.2" // keep in sync with scio val scalaTestVersion = "3.2.17" val scalaCheckVersion = "1.17.0" val scalaCollectionCompatVersion = "2.11.0" -val scioVersion = "0.13.5" +val scioVersion = "0.13.6" val scoptVersion = "4.1.0" val shapelessVersion = "2.3.10" // keep in sync with scio val sourcecodeVersion = "0.3.1" diff --git a/ratatool-diffy/src/test/scala/com/spotify/ratatool/diffy/AvroDiffyTest.scala b/ratatool-diffy/src/test/scala/com/spotify/ratatool/diffy/AvroDiffyTest.scala index 8addeaa8..2de6491b 100644 --- a/ratatool-diffy/src/test/scala/com/spotify/ratatool/diffy/AvroDiffyTest.scala +++ b/ratatool-diffy/src/test/scala/com/spotify/ratatool/diffy/AvroDiffyTest.scala @@ -24,7 +24,7 @@ import com.spotify.ratatool.Schemas import com.spotify.ratatool.avro.specific._ import com.spotify.ratatool.scalacheck._ import org.apache.avro.generic.{GenericRecord, GenericRecordBuilder} -import org.apache.beam.sdk.coders.AvroCoder +import org.apache.beam.sdk.extensions.avro.coders.AvroCoder import org.apache.beam.sdk.util.CoderUtils import org.scalacheck.Arbitrary @@ -48,7 +48,7 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers { } it should "support nested fields" in { - val coder = AvroCoder.of(classOf[TestRecord]) + val coder = AvroCoder.reflect(classOf[TestRecord]) val nnr = specificRecordOf[NullableNestedRecord].sample.get nnr.setIntField(10) @@ -83,7 +83,7 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers { } it should "support repeated fields" in { - val coder = AvroCoder.of(classOf[TestRecord]) + val coder = AvroCoder.reflect(classOf[TestRecord]) val x = specificRecordOf[TestRecord].sample.get x.getRepeatedFields.setIntField(jl(10, 11)) @@ -125,7 +125,7 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers { } it should "support unordered" in { - val coder = AvroCoder.of(classOf[TestRecord]) + val coder = AvroCoder.reflect(classOf[TestRecord]) val a = NullableNestedRecord.newBuilder().setIntField(10).setLongField(100L).build() val b = NullableNestedRecord.newBuilder().setIntField(20).setLongField(200L).build() @@ -146,8 +146,8 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers { } it should "support unordered nested" in { - val drnrCoder = AvroCoder.of(classOf[RepeatedRecord]) - val drrCoder = AvroCoder.of(classOf[DeeplyRepeatedRecord]) + val drnrCoder = AvroCoder.reflect(classOf[RepeatedRecord]) + val drrCoder = AvroCoder.reflect(classOf[DeeplyRepeatedRecord]) val a = avroOf[RepeatedRecord].sample.get a.setNestedRepeatedField(jl(10, 20, 30)) @@ -176,8 +176,8 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers { } it should "support unordered nested of different lengths" in { - val drnrCoder = AvroCoder.of(classOf[RepeatedRecord]) - val drrCoder = AvroCoder.of(classOf[DeeplyRepeatedRecord]) + val drnrCoder = AvroCoder.reflect(classOf[RepeatedRecord]) + val drrCoder = AvroCoder.reflect(classOf[DeeplyRepeatedRecord]) val a = avroOf[RepeatedRecord].sample.get a.setNestedRepeatedField(jl(30, 20, 10)) diff --git a/ratatool-diffy/src/test/scala/com/spotify/ratatool/diffy/BigDiffyTest.scala b/ratatool-diffy/src/test/scala/com/spotify/ratatool/diffy/BigDiffyTest.scala index f75bcf13..ad300927 100644 --- a/ratatool-diffy/src/test/scala/com/spotify/ratatool/diffy/BigDiffyTest.scala +++ b/ratatool-diffy/src/test/scala/com/spotify/ratatool/diffy/BigDiffyTest.scala @@ -30,7 +30,7 @@ import com.spotify.ratatool.io.{ParquetIO, ParquetTestData} import com.spotify.scio.ScioContext import org.apache.avro.Schema import org.apache.avro.generic.{GenericData, GenericRecord} -import org.apache.beam.sdk.coders.AvroCoder +import org.apache.beam.sdk.extensions.avro.coders.AvroCoder import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder import scala.jdk.CollectionConverters.seqAsJavaListConverter @@ -39,7 +39,7 @@ import scala.language.higherKinds class BigDiffyTest extends PipelineSpec { val keys = (1 to 1000).map(k => MultiKey("key" + k)) - val coder = AvroCoder.of(classOf[TestRecord]) + val coder = AvroCoder.reflect(classOf[TestRecord]) /** Fixed to a small range so that Std. Dev. & Variance calculations are easier to predict */ val rnr = specificRecordOf[RequiredNestedRecord] diff --git a/ratatool-examples/src/main/scala/com/spotify/ratatool/examples/diffy/PreProcessBigDiffy.scala b/ratatool-examples/src/main/scala/com/spotify/ratatool/examples/diffy/PreProcessBigDiffy.scala index 545bdd4f..7fd1a344 100644 --- a/ratatool-examples/src/main/scala/com/spotify/ratatool/examples/diffy/PreProcessBigDiffy.scala +++ b/ratatool-examples/src/main/scala/com/spotify/ratatool/examples/diffy/PreProcessBigDiffy.scala @@ -22,7 +22,7 @@ import com.spotify.ratatool.diffy.{AvroDiffy, BigDiffy, MultiKey} import com.spotify.scio.ContextAndArgs import org.apache.beam.sdk.util.CoderUtils import com.spotify.scio.avro._ -import org.apache.beam.sdk.coders.AvroCoder +import org.apache.beam.sdk.extensions.avro.coders.AvroCoder object PreProcessBigDiffy { def recordKeyFn(r: ExampleRecord): MultiKey = @@ -37,7 +37,7 @@ object PreProcessBigDiffy { } def main(cmdlineArgs: Array[String]): Unit = { - @transient lazy val coder = AvroCoder.of(classOf[ExampleRecord]) + @transient lazy val coder = AvroCoder.reflect(classOf[ExampleRecord]) val (sc, args) = ContextAndArgs(cmdlineArgs) val (lhsPath, rhsPath, output, header, ignore, unordered) = diff --git a/ratatool-examples/src/test/scala/com/spotify/ratatool/examples/PreProcessBigDiffyJobTest.scala b/ratatool-examples/src/test/scala/com/spotify/ratatool/examples/PreProcessBigDiffyJobTest.scala index fdd3b942..15e8cdb1 100644 --- a/ratatool-examples/src/test/scala/com/spotify/ratatool/examples/PreProcessBigDiffyJobTest.scala +++ b/ratatool-examples/src/test/scala/com/spotify/ratatool/examples/PreProcessBigDiffyJobTest.scala @@ -25,7 +25,7 @@ import org.apache.beam.sdk.util.CoderUtils import org.scalacheck.Gen import com.spotify.scio.avro._ import com.spotify.scio.io.TextIO -import org.apache.beam.sdk.coders.AvroCoder +import org.apache.beam.sdk.extensions.avro.coders.AvroCoder class PreProcessBigDiffyJobTest extends PipelineSpec { val lhs = Gen.listOfN(1000, ExampleAvroGen.exampleRecordGen).sample.get.map { r => @@ -33,7 +33,7 @@ class PreProcessBigDiffyJobTest extends PipelineSpec { r } - val rhs = lhs.map(CoderUtils.clone(AvroCoder.of(classOf[ExampleRecord]), _)).map { r => + val rhs = lhs.map(CoderUtils.clone(AvroCoder.reflect(classOf[ExampleRecord]), _)).map { r => r.setNullableIntField(0) r } diff --git a/ratatool-scalacheck/src/main/scala/com/spotify/ratatool/scalacheck/AvroGenerator.scala b/ratatool-scalacheck/src/main/scala/com/spotify/ratatool/scalacheck/AvroGenerator.scala index ee4241ee..2e1b40ab 100644 --- a/ratatool-scalacheck/src/main/scala/com/spotify/ratatool/scalacheck/AvroGenerator.scala +++ b/ratatool-scalacheck/src/main/scala/com/spotify/ratatool/scalacheck/AvroGenerator.scala @@ -23,7 +23,7 @@ import org.apache.avro._ import org.apache.avro.generic.{GenericData, GenericRecord} import org.apache.avro.specific.SpecificRecord import org.apache.avro.util.Utf8 -import org.apache.beam.sdk.coders.{AvroCoder, AvroGenericCoder} +import org.apache.beam.sdk.extensions.avro.coders.{AvroCoder, AvroGenericCoder} import org.apache.beam.sdk.util.CoderUtils import org.scalacheck.{Arbitrary, Gen} @@ -52,7 +52,7 @@ object AvroGeneratorOps extends AvroGeneratorOps trait AvroGeneratorOps { def specificRecordOf[A <: SpecificRecord: ClassTag]: Gen[A] = { val cls = implicitly[ClassTag[A]].runtimeClass.asInstanceOf[Class[A]] - val specificCoder = AvroCoder.of(cls) + val specificCoder = AvroCoder.reflect(cls) val genericCoder = AvroGenericCoder.of(specificCoder.getSchema) genericRecordOf(specificCoder.getSchema).map { generic => diff --git a/ratatool-scalacheck/src/test/scala/com/spotify/ratatool/scalacheck/AvroGeneratorTest.scala b/ratatool-scalacheck/src/test/scala/com/spotify/ratatool/scalacheck/AvroGeneratorTest.scala index 032bddfb..82a56c6f 100644 --- a/ratatool-scalacheck/src/test/scala/com/spotify/ratatool/scalacheck/AvroGeneratorTest.scala +++ b/ratatool-scalacheck/src/test/scala/com/spotify/ratatool/scalacheck/AvroGeneratorTest.scala @@ -18,14 +18,14 @@ package com.spotify.ratatool.scalacheck import com.spotify.ratatool.avro.specific.{RequiredNestedRecord, TestRecord} -import org.apache.beam.sdk.coders.AvroCoder +import org.apache.beam.sdk.extensions.avro.coders.AvroCoder import org.apache.beam.sdk.util.CoderUtils import org.scalacheck._ import org.scalacheck.Prop.{all, forAll, propBoolean, AnyOperators} object AvroGeneratorTest extends Properties("AvroGenerator") { property("round trips") = forAll(specificRecordOf[TestRecord]) { m => - val coder = AvroCoder.of(classOf[TestRecord]) + val coder = AvroCoder.reflect(classOf[TestRecord]) val bytes = CoderUtils.encodeToByteArray(coder, m) val decoded = CoderUtils.decodeFromByteArray(coder, bytes)