Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update scio to 0.13.6 #689

Merged
merged 1 commit into from
Nov 22, 2023
Merged

Update scio to 0.13.6 #689

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions build.sbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ import Keys._

val algebirdVersion = "0.13.10"
val avroVersion = "1.8.2" // keep in sync with scio
val beamVersion = "2.50.0" // keep in sync with scio
val bigqueryVersion = "v2-rev20230520-2.0.0" // keep in sync with scio
val floggerVersion = "0.7.4"
val beamVersion = "2.52.0" // keep in sync with scio
val bigqueryVersion = "v2-rev20230812-2.0.0" // keep in sync with scio
val floggerVersion = "0.7.4" // keep in sync with scio + beam
val guavaVersion = "32.1.2-jre" // keep in sync with scio + beam
val hadoopVersion = "2.10.2" // keep in sync with scio
val jodaTimeVersion = "2.10.10" // keep in sync with scio
Expand All @@ -31,7 +31,7 @@ val protoBufVersion = "3.23.2" // keep in sync with scio
val scalaTestVersion = "3.2.17"
val scalaCheckVersion = "1.17.0"
val scalaCollectionCompatVersion = "2.11.0"
val scioVersion = "0.13.5"
val scioVersion = "0.13.6"
val scoptVersion = "4.1.0"
val shapelessVersion = "2.3.10" // keep in sync with scio
val sourcecodeVersion = "0.3.1"
Expand Down
16 changes: 8 additions & 8 deletions ratatool-diffy/src/test/scala/com/spotify/ratatool/diffy/AvroDiffyTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import com.spotify.ratatool.Schemas
import com.spotify.ratatool.avro.specific._
import com.spotify.ratatool.scalacheck._
import org.apache.avro.generic.{GenericRecord, GenericRecordBuilder}
import org.apache.beam.sdk.coders.AvroCoder
import org.apache.beam.sdk.extensions.avro.coders.AvroCoder
import org.apache.beam.sdk.util.CoderUtils
import org.scalacheck.Arbitrary

Expand All @@ -48,7 +48,7 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers {
}

it should "support nested fields" in {
val coder = AvroCoder.of(classOf[TestRecord])
val coder = AvroCoder.reflect(classOf[TestRecord])

val nnr = specificRecordOf[NullableNestedRecord].sample.get
nnr.setIntField(10)
Expand Down Expand Up @@ -83,7 +83,7 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers {
}

it should "support repeated fields" in {
val coder = AvroCoder.of(classOf[TestRecord])
val coder = AvroCoder.reflect(classOf[TestRecord])

val x = specificRecordOf[TestRecord].sample.get
x.getRepeatedFields.setIntField(jl(10, 11))
Expand Down Expand Up @@ -125,7 +125,7 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers {
}

it should "support unordered" in {
val coder = AvroCoder.of(classOf[TestRecord])
val coder = AvroCoder.reflect(classOf[TestRecord])

val a = NullableNestedRecord.newBuilder().setIntField(10).setLongField(100L).build()
val b = NullableNestedRecord.newBuilder().setIntField(20).setLongField(200L).build()
Expand All @@ -146,8 +146,8 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers {
}

it should "support unordered nested" in {
val drnrCoder = AvroCoder.of(classOf[RepeatedRecord])
val drrCoder = AvroCoder.of(classOf[DeeplyRepeatedRecord])
val drnrCoder = AvroCoder.reflect(classOf[RepeatedRecord])
val drrCoder = AvroCoder.reflect(classOf[DeeplyRepeatedRecord])

val a = avroOf[RepeatedRecord].sample.get
a.setNestedRepeatedField(jl(10, 20, 30))
Expand Down Expand Up @@ -176,8 +176,8 @@ class AvroDiffyTest extends AnyFlatSpec with Matchers {
}

it should "support unordered nested of different lengths" in {
val drnrCoder = AvroCoder.of(classOf[RepeatedRecord])
val drrCoder = AvroCoder.of(classOf[DeeplyRepeatedRecord])
val drnrCoder = AvroCoder.reflect(classOf[RepeatedRecord])
val drrCoder = AvroCoder.reflect(classOf[DeeplyRepeatedRecord])

val a = avroOf[RepeatedRecord].sample.get
a.setNestedRepeatedField(jl(30, 20, 10))
Expand Down
4 changes: 2 additions & 2 deletions ratatool-diffy/src/test/scala/com/spotify/ratatool/diffy/BigDiffyTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import com.spotify.ratatool.io.{ParquetIO, ParquetTestData}
import com.spotify.scio.ScioContext
import org.apache.avro.Schema
import org.apache.avro.generic.{GenericData, GenericRecord}
import org.apache.beam.sdk.coders.AvroCoder
import org.apache.beam.sdk.extensions.avro.coders.AvroCoder
import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder

import scala.jdk.CollectionConverters.seqAsJavaListConverter
Expand All @@ -39,7 +39,7 @@ import scala.language.higherKinds
class BigDiffyTest extends PipelineSpec {

val keys = (1 to 1000).map(k => MultiKey("key" + k))
val coder = AvroCoder.of(classOf[TestRecord])
val coder = AvroCoder.reflect(classOf[TestRecord])

/** Fixed to a small range so that Std. Dev. & Variance calculations are easier to predict */
val rnr = specificRecordOf[RequiredNestedRecord]
Expand Down
4 changes: 2 additions & 2 deletions ...tool-examples/src/main/scala/com/spotify/ratatool/examples/diffy/PreProcessBigDiffy.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import com.spotify.ratatool.diffy.{AvroDiffy, BigDiffy, MultiKey}
import com.spotify.scio.ContextAndArgs
import org.apache.beam.sdk.util.CoderUtils
import com.spotify.scio.avro._
import org.apache.beam.sdk.coders.AvroCoder
import org.apache.beam.sdk.extensions.avro.coders.AvroCoder

object PreProcessBigDiffy {
def recordKeyFn(r: ExampleRecord): MultiKey =
Expand All @@ -37,7 +37,7 @@ object PreProcessBigDiffy {
}

def main(cmdlineArgs: Array[String]): Unit = {
@transient lazy val coder = AvroCoder.of(classOf[ExampleRecord])
@transient lazy val coder = AvroCoder.reflect(classOf[ExampleRecord])
val (sc, args) = ContextAndArgs(cmdlineArgs)

val (lhsPath, rhsPath, output, header, ignore, unordered) =
Expand Down
4 changes: 2 additions & 2 deletions ...ool-examples/src/test/scala/com/spotify/ratatool/examples/PreProcessBigDiffyJobTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,15 @@ import org.apache.beam.sdk.util.CoderUtils
import org.scalacheck.Gen
import com.spotify.scio.avro._
import com.spotify.scio.io.TextIO
import org.apache.beam.sdk.coders.AvroCoder
import org.apache.beam.sdk.extensions.avro.coders.AvroCoder

class PreProcessBigDiffyJobTest extends PipelineSpec {
val lhs = Gen.listOfN(1000, ExampleAvroGen.exampleRecordGen).sample.get.map { r =>
r.setNullableIntField(null)
r
}

val rhs = lhs.map(CoderUtils.clone(AvroCoder.of(classOf[ExampleRecord]), _)).map { r =>
val rhs = lhs.map(CoderUtils.clone(AvroCoder.reflect(classOf[ExampleRecord]), _)).map { r =>
r.setNullableIntField(0)
r
}
Expand Down
4 changes: 2 additions & 2 deletions ratatool-scalacheck/src/main/scala/com/spotify/ratatool/scalacheck/AvroGenerator.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import org.apache.avro._
import org.apache.avro.generic.{GenericData, GenericRecord}
import org.apache.avro.specific.SpecificRecord
import org.apache.avro.util.Utf8
import org.apache.beam.sdk.coders.{AvroCoder, AvroGenericCoder}
import org.apache.beam.sdk.extensions.avro.coders.{AvroCoder, AvroGenericCoder}
import org.apache.beam.sdk.util.CoderUtils
import org.scalacheck.{Arbitrary, Gen}

Expand Down Expand Up @@ -52,7 +52,7 @@ object AvroGeneratorOps extends AvroGeneratorOps
trait AvroGeneratorOps {
def specificRecordOf[A <: SpecificRecord: ClassTag]: Gen[A] = {
val cls = implicitly[ClassTag[A]].runtimeClass.asInstanceOf[Class[A]]
val specificCoder = AvroCoder.of(cls)
val specificCoder = AvroCoder.reflect(cls)
val genericCoder = AvroGenericCoder.of(specificCoder.getSchema)

genericRecordOf(specificCoder.getSchema).map { generic =>
Expand Down
4 changes: 2 additions & 2 deletions ratatool-scalacheck/src/test/scala/com/spotify/ratatool/scalacheck/AvroGeneratorTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@
package com.spotify.ratatool.scalacheck

import com.spotify.ratatool.avro.specific.{RequiredNestedRecord, TestRecord}
import org.apache.beam.sdk.coders.AvroCoder
import org.apache.beam.sdk.extensions.avro.coders.AvroCoder
import org.apache.beam.sdk.util.CoderUtils
import org.scalacheck._
import org.scalacheck.Prop.{all, forAll, propBoolean, AnyOperators}

object AvroGeneratorTest extends Properties("AvroGenerator") {
property("round trips") = forAll(specificRecordOf[TestRecord]) { m =>
val coder = AvroCoder.of(classOf[TestRecord])
val coder = AvroCoder.reflect(classOf[TestRecord])

val bytes = CoderUtils.encodeToByteArray(coder, m)
val decoded = CoderUtils.decodeFromByteArray(coder, bytes)
Expand Down