From 23813dbc7ff97e8d1c5e4ec9b904c426982e9bef Mon Sep 17 00:00:00 2001 From: dhruvdua Date: Thu, 1 Feb 2024 06:01:47 +0000 Subject: [PATCH 1/6] Update to libraries-bom 26.31.0 --- .../groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 6 +++--- sdks/java/container/license_scripts/dep_urls_java.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index f3d79c528f8c5..494c0d29c2eca 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -607,7 +607,7 @@ class BeamModulePlugin implements Plugin { def google_clients_version = "2.0.0" def google_cloud_bigdataoss_version = "2.2.16" // Try to keep google_cloud_spanner_version consistent with google_cloud_spanner_bom in google_cloud_platform_libraries_bom - def google_cloud_spanner_version = "6.56.0" + def google_cloud_spanner_version = "6.57.0" def google_code_gson_version = "2.10.1" def google_oauth_clients_version = "1.34.1" // Try to keep grpc_version consistent with gRPC version in google_cloud_platform_libraries_bom @@ -756,9 +756,9 @@ class BeamModulePlugin implements Plugin { google_cloud_pubsub : "com.google.cloud:google-cloud-pubsub", // google_cloud_platform_libraries_bom sets version google_cloud_pubsublite : "com.google.cloud:google-cloud-pubsublite", // google_cloud_platform_libraries_bom sets version // The release notes shows the versions set by the BOM: - // https://github.com/googleapis/java-cloud-bom/releases/tag/v26.30.0 + // https://github.com/googleapis/java-cloud-bom/releases/tag/v26.31.0 // Update libraries-bom version on sdks/java/container/license_scripts/dep_urls_java.yaml - google_cloud_platform_libraries_bom : "com.google.cloud:libraries-bom:26.30.0", + google_cloud_platform_libraries_bom : "com.google.cloud:libraries-bom:26.31.0", google_cloud_spanner : "com.google.cloud:google-cloud-spanner", // google_cloud_platform_libraries_bom sets version google_cloud_spanner_test : "com.google.cloud:google-cloud-spanner:$google_cloud_spanner_version:tests", google_code_gson : "com.google.code.gson:gson:$google_code_gson_version", diff --git a/sdks/java/container/license_scripts/dep_urls_java.yaml b/sdks/java/container/license_scripts/dep_urls_java.yaml index 42a7c7a0a7444..a084805c70deb 100644 --- a/sdks/java/container/license_scripts/dep_urls_java.yaml +++ b/sdks/java/container/license_scripts/dep_urls_java.yaml @@ -46,7 +46,7 @@ jaxen: '1.1.6': type: "3-Clause BSD" libraries-bom: - '26.30.0': + '26.31.0': license: "https://raw.githubusercontent.com/GoogleCloudPlatform/cloud-opensource-java/master/LICENSE" type: "Apache License 2.0" paranamer: From c012662deee2f610c53c4e7bcfb2b90ba608903a Mon Sep 17 00:00:00 2001 From: dhruvdua Date: Thu, 1 Feb 2024 07:30:28 +0000 Subject: [PATCH 2/6] Support for proto & enum types in SpannerSchema & MutationSizeEstimator --- .../io/gcp/spanner/MutationSizeEstimator.java | 6 ++- .../sdk/io/gcp/spanner/SpannerSchema.java | 11 +++++ .../spanner/MutationSizeEstimatorTest.java | 41 +++++++++++++++++++ .../sdk/io/gcp/spanner/SpannerSchemaTest.java | 6 ++- 4 files changed, 62 insertions(+), 2 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java index b0a1da5fb15e3..b2e5da83f3564 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java @@ -108,6 +108,7 @@ private static long estimatePrimitiveValue(Value v) { return 1; case INT64: case FLOAT64: + case ENUM: return 8; case DATE: case TIMESTAMP: @@ -116,6 +117,7 @@ private static long estimatePrimitiveValue(Value v) { case PG_NUMERIC: return v.isNull() ? 0 : v.getString().length(); case BYTES: + case PROTO: return v.isNull() ? 0 : v.getBytes().length(); case NUMERIC: // see @@ -141,6 +143,7 @@ private static long estimateArrayValue(Value v) { case BOOL: return v.getBoolArray().size(); case INT64: + case ENUM: return 8L * v.getInt64Array().size(); case FLOAT64: return 8L * v.getFloat64Array().size(); @@ -155,6 +158,7 @@ private static long estimateArrayValue(Value v) { } return totalLength; case BYTES: + case PROTO: totalLength = 0; for (ByteArray bytes : v.getBytesArray()) { if (bytes == null) { @@ -163,7 +167,7 @@ private static long estimateArrayValue(Value v) { totalLength += bytes.length(); } return totalLength; - case DATE: + case DATE: return 12L * v.getDateArray().size(); case TIMESTAMP: return 12L * v.getTimestampArray().size(); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java index e7587365fe12f..f295854f6c452 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java @@ -161,6 +161,7 @@ static Column create(String name, String spannerType, Dialect dialect) { public abstract Type getType(); private static Type parseSpannerType(String spannerType, Dialect dialect) { + String originalSpannerType = spannerType; spannerType = spannerType.toUpperCase(); switch (dialect) { case GOOGLE_STANDARD_SQL: @@ -197,6 +198,16 @@ private static Type parseSpannerType(String spannerType, Dialect dialect) { Type itemType = parseSpannerType(spannerArrayType, dialect); return Type.array(itemType); } + if (spannerType.startsWith("PROTO")) { + // Substring "PROTO" + String spannerProtoType = originalSpannerType.substring(6,originalSpannerType.length() - 1); + return Type.proto(spannerProtoType); + } + if (spannerType.startsWith("ENUM")) { + // Substring "ENUM" + String spannerEnumType = originalSpannerType.substring(5,originalSpannerType.length() - 1); + return Type.protoEnum(spannerEnumType); + } throw new IllegalArgumentException("Unknown spanner type " + spannerType); case POSTGRESQL: if (spannerType.endsWith("[]")) { diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java index ebabfa8b575fd..293692cb166e1 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java @@ -145,6 +145,19 @@ public void primitiveArrays() throws Exception { "{\"key123\":\"value123\", \"key321\":\"value321\"}", "{\"key456\":\"value456\", \"key789\":600}")) .build(); + Mutation protoEnum = + Mutation.newInsertOrUpdateBuilder("test") + .set("one") + .toProtoEnumArray(ImmutableList.of(1L, 2L, 3L),"customer.app.TestEnum") + .build(); + Mutation protos = + Mutation.newInsertOrUpdateBuilder("test") + .set("bytes") + .toProtoMessageArray( + ImmutableList.of( + ByteArray.copyFrom("some_bytes".getBytes(UTF_8)), + ByteArray.copyFrom("some_bytes".getBytes(UTF_8))),"customer.app.TestMessage") + .build(); assertThat(MutationSizeEstimator.sizeOf(int64), is(24L)); assertThat(MutationSizeEstimator.sizeOf(float64), is(16L)); assertThat(MutationSizeEstimator.sizeOf(bool), is(4L)); @@ -153,12 +166,16 @@ public void primitiveArrays() throws Exception { assertThat(MutationSizeEstimator.sizeOf(json), is(62L)); assertThat(MutationSizeEstimator.sizeOf(bytes), is(20L)); assertThat(MutationSizeEstimator.sizeOf(jsonb), is(77L)); + assertThat(MutationSizeEstimator.sizeOf(protoEnum),is(24L)); + assertThat(MutationSizeEstimator.sizeOf(protos),is(20L)); } @Test public void nullPrimitiveArrays() throws Exception { Mutation int64 = Mutation.newInsertOrUpdateBuilder("test").set("one").toInt64Array((long[]) null).build(); + Mutation protoEnum = + Mutation.newInsertOrUpdateBuilder("test").set("one").toProtoEnumArray(null,"customer.app.TestEnum").build(); Mutation float64 = Mutation.newInsertOrUpdateBuilder("test") .set("one") @@ -187,6 +204,7 @@ public void nullPrimitiveArrays() throws Exception { assertThat(MutationSizeEstimator.sizeOf(pgNumeric), is(0L)); assertThat(MutationSizeEstimator.sizeOf(json), is(0L)); assertThat(MutationSizeEstimator.sizeOf(jsonb), is(0L)); + assertThat(MutationSizeEstimator.sizeOf(protoEnum),is(0L)); } @Test @@ -235,6 +253,29 @@ public void bytes() throws Exception { assertThat(MutationSizeEstimator.sizeOf(deleteBytes), is(10L)); } + @Test + public void protos() throws Exception { + Mutation empty = + Mutation.newInsertOrUpdateBuilder("test").set("one").to(ByteArray.fromBase64(""),"customer.app.TestMessage").build(); + Mutation nullValue = + Mutation.newInsertOrUpdateBuilder("test").set("one").to((ByteArray) null,"customer.app.TestMessage").build(); + Mutation sample = + Mutation.newInsertOrUpdateBuilder("test") + .set("one") + .to(ByteArray.fromBase64("abcdabcd"),"customer.app.TestMessage") + .build(); + Mutation nullArray = + Mutation.newInsertOrUpdateBuilder("test").set("one").toProtoMessageArray(null,"customer.app.TestMessage").build(); + Mutation deleteBytes = + Mutation.delete("test", Key.of(ByteArray.copyFrom("some_bytes".getBytes(UTF_8)))); + + assertThat(MutationSizeEstimator.sizeOf(empty), is(0L)); + assertThat(MutationSizeEstimator.sizeOf(nullValue), is(0L)); + assertThat(MutationSizeEstimator.sizeOf(sample), is(6L)); + assertThat(MutationSizeEstimator.sizeOf(nullArray), is(0L)); + assertThat(MutationSizeEstimator.sizeOf(deleteBytes), is(10L)); + } + @Test public void jsons() throws Exception { Mutation empty = diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchemaTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchemaTest.java index 7ba345a24885d..660c2850d3b10 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchemaTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchemaTest.java @@ -38,12 +38,16 @@ public void testSingleTable() throws Exception { .addColumn("test", "maxKey", "STRING(MAX)") .addColumn("test", "numericVal", "NUMERIC") .addColumn("test", "jsonVal", "JSON") + .addColumn("test", "protoVal", "PROTO") + .addColumn("test", "enumVal", "ENUM") .build(); assertEquals(1, schema.getTables().size()); - assertEquals(4, schema.getColumns("test").size()); + assertEquals(6, schema.getColumns("test").size()); assertEquals(1, schema.getKeyParts("test").size()); assertEquals(Type.json(), schema.getColumns("test").get(3).getType()); + assertEquals(Type.proto("customer.app.TestMessage"), schema.getColumns("test").get(4).getType()); + assertEquals(Type.protoEnum("customer.app.TestEnum"), schema.getColumns("test").get(5).getType()); } @Test From 2f36a0c5cb6cc00c13218d29aed6e9f8da1ff02c Mon Sep 17 00:00:00 2001 From: dhruvdua Date: Thu, 1 Feb 2024 15:13:52 +0000 Subject: [PATCH 3/6] Support for proto & enum types in SpannerSchema & MutationSizeEstimator --- .../io/gcp/spanner/MutationSizeEstimator.java | 2 +- .../sdk/io/gcp/spanner/SpannerSchema.java | 6 ++-- .../spanner/MutationSizeEstimatorTest.java | 33 +++++++++++++------ .../sdk/io/gcp/spanner/SpannerSchemaTest.java | 6 ++-- 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java index b2e5da83f3564..83fcf026cc1a1 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimator.java @@ -167,7 +167,7 @@ private static long estimateArrayValue(Value v) { totalLength += bytes.length(); } return totalLength; - case DATE: + case DATE: return 12L * v.getDateArray().size(); case TIMESTAMP: return 12L * v.getTimestampArray().size(); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java index f295854f6c452..4390ac1a91751 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java @@ -200,12 +200,14 @@ private static Type parseSpannerType(String spannerType, Dialect dialect) { } if (spannerType.startsWith("PROTO")) { // Substring "PROTO" - String spannerProtoType = originalSpannerType.substring(6,originalSpannerType.length() - 1); + String spannerProtoType = + originalSpannerType.substring(6, originalSpannerType.length() - 1); return Type.proto(spannerProtoType); } if (spannerType.startsWith("ENUM")) { // Substring "ENUM" - String spannerEnumType = originalSpannerType.substring(5,originalSpannerType.length() - 1); + String spannerEnumType = + originalSpannerType.substring(5, originalSpannerType.length() - 1); return Type.protoEnum(spannerEnumType); } throw new IllegalArgumentException("Unknown spanner type " + spannerType); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java index 293692cb166e1..497e33d3cfc9c 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/MutationSizeEstimatorTest.java @@ -148,7 +148,7 @@ public void primitiveArrays() throws Exception { Mutation protoEnum = Mutation.newInsertOrUpdateBuilder("test") .set("one") - .toProtoEnumArray(ImmutableList.of(1L, 2L, 3L),"customer.app.TestEnum") + .toProtoEnumArray(ImmutableList.of(1L, 2L, 3L), "customer.app.TestEnum") .build(); Mutation protos = Mutation.newInsertOrUpdateBuilder("test") @@ -156,7 +156,8 @@ public void primitiveArrays() throws Exception { .toProtoMessageArray( ImmutableList.of( ByteArray.copyFrom("some_bytes".getBytes(UTF_8)), - ByteArray.copyFrom("some_bytes".getBytes(UTF_8))),"customer.app.TestMessage") + ByteArray.copyFrom("some_bytes".getBytes(UTF_8))), + "customer.app.TestMessage") .build(); assertThat(MutationSizeEstimator.sizeOf(int64), is(24L)); assertThat(MutationSizeEstimator.sizeOf(float64), is(16L)); @@ -166,8 +167,8 @@ public void primitiveArrays() throws Exception { assertThat(MutationSizeEstimator.sizeOf(json), is(62L)); assertThat(MutationSizeEstimator.sizeOf(bytes), is(20L)); assertThat(MutationSizeEstimator.sizeOf(jsonb), is(77L)); - assertThat(MutationSizeEstimator.sizeOf(protoEnum),is(24L)); - assertThat(MutationSizeEstimator.sizeOf(protos),is(20L)); + assertThat(MutationSizeEstimator.sizeOf(protoEnum), is(24L)); + assertThat(MutationSizeEstimator.sizeOf(protos), is(20L)); } @Test @@ -175,7 +176,10 @@ public void nullPrimitiveArrays() throws Exception { Mutation int64 = Mutation.newInsertOrUpdateBuilder("test").set("one").toInt64Array((long[]) null).build(); Mutation protoEnum = - Mutation.newInsertOrUpdateBuilder("test").set("one").toProtoEnumArray(null,"customer.app.TestEnum").build(); + Mutation.newInsertOrUpdateBuilder("test") + .set("one") + .toProtoEnumArray(null, "customer.app.TestEnum") + .build(); Mutation float64 = Mutation.newInsertOrUpdateBuilder("test") .set("one") @@ -204,7 +208,7 @@ public void nullPrimitiveArrays() throws Exception { assertThat(MutationSizeEstimator.sizeOf(pgNumeric), is(0L)); assertThat(MutationSizeEstimator.sizeOf(json), is(0L)); assertThat(MutationSizeEstimator.sizeOf(jsonb), is(0L)); - assertThat(MutationSizeEstimator.sizeOf(protoEnum),is(0L)); + assertThat(MutationSizeEstimator.sizeOf(protoEnum), is(0L)); } @Test @@ -256,16 +260,25 @@ public void bytes() throws Exception { @Test public void protos() throws Exception { Mutation empty = - Mutation.newInsertOrUpdateBuilder("test").set("one").to(ByteArray.fromBase64(""),"customer.app.TestMessage").build(); + Mutation.newInsertOrUpdateBuilder("test") + .set("one") + .to(ByteArray.fromBase64(""), "customer.app.TestMessage") + .build(); Mutation nullValue = - Mutation.newInsertOrUpdateBuilder("test").set("one").to((ByteArray) null,"customer.app.TestMessage").build(); + Mutation.newInsertOrUpdateBuilder("test") + .set("one") + .to((ByteArray) null, "customer.app.TestMessage") + .build(); Mutation sample = Mutation.newInsertOrUpdateBuilder("test") .set("one") - .to(ByteArray.fromBase64("abcdabcd"),"customer.app.TestMessage") + .to(ByteArray.fromBase64("abcdabcd"), "customer.app.TestMessage") .build(); Mutation nullArray = - Mutation.newInsertOrUpdateBuilder("test").set("one").toProtoMessageArray(null,"customer.app.TestMessage").build(); + Mutation.newInsertOrUpdateBuilder("test") + .set("one") + .toProtoMessageArray(null, "customer.app.TestMessage") + .build(); Mutation deleteBytes = Mutation.delete("test", Key.of(ByteArray.copyFrom("some_bytes".getBytes(UTF_8)))); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchemaTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchemaTest.java index 660c2850d3b10..166df1704ca8a 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchemaTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchemaTest.java @@ -46,8 +46,10 @@ public void testSingleTable() throws Exception { assertEquals(6, schema.getColumns("test").size()); assertEquals(1, schema.getKeyParts("test").size()); assertEquals(Type.json(), schema.getColumns("test").get(3).getType()); - assertEquals(Type.proto("customer.app.TestMessage"), schema.getColumns("test").get(4).getType()); - assertEquals(Type.protoEnum("customer.app.TestEnum"), schema.getColumns("test").get(5).getType()); + assertEquals( + Type.proto("customer.app.TestMessage"), schema.getColumns("test").get(4).getType()); + assertEquals( + Type.protoEnum("customer.app.TestEnum"), schema.getColumns("test").get(5).getType()); } @Test From af1a93d28f60de2a16e209aee732bb4519558638 Mon Sep 17 00:00:00 2001 From: dhruvdua Date: Thu, 8 Feb 2024 06:11:44 +0000 Subject: [PATCH 4/6] Upgrade dependencies for libraries bom to be consistent with libraries bom, upgrade arrow version to latest, bug fix for spanner proto type --- .../groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 6 +++--- .../org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 494c0d29c2eca..d704e9116a9e1 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -602,7 +602,7 @@ class BeamModulePlugin implements Plugin { def dbcp2_version = "2.9.0" def errorprone_version = "2.10.0" // Try to keep gax_version consistent with gax-grpc version in google_cloud_platform_libraries_bom - def gax_version = "2.39.0" + def gax_version = "2.41.0" def google_ads_version = "26.0.0" def google_clients_version = "2.0.0" def google_cloud_bigdataoss_version = "2.2.16" @@ -626,7 +626,7 @@ class BeamModulePlugin implements Plugin { def log4j2_version = "2.20.0" def nemo_version = "0.1" // Try to keep netty_version consistent with the netty version in grpc_bom (includes grpc_netty) in google_cloud_platform_libraries_bom - def netty_version = "4.1.87.Final" + def netty_version = "4.1.100.Final" def postgres_version = "42.2.16" def powermock_version = "2.0.9" // Try to keep protobuf_version consistent with the protobuf version in google_cloud_platform_libraries_bom @@ -640,7 +640,7 @@ class BeamModulePlugin implements Plugin { def spark3_version = "3.2.2" def spotbugs_version = "4.0.6" def testcontainers_version = "1.17.3" - def arrow_version = "5.0.0" + def arrow_version = "15.0.0" def jmh_version = "1.34" def jupiter_version = "5.7.0" diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java index 4390ac1a91751..1b47176f579d6 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java @@ -35,6 +35,7 @@ "nullness" // TODO(https://github.com/apache/beam/issues/20497) }) public abstract class SpannerSchema implements Serializable { + abstract ImmutableList tables(); abstract Dialect dialect(); @@ -194,7 +195,7 @@ private static Type parseSpannerType(String spannerType, Dialect dialect) { } if (spannerType.startsWith("ARRAY")) { // Substring "ARRAY" - String spannerArrayType = spannerType.substring(6, spannerType.length() - 1); + String spannerArrayType = originalSpannerType.substring(6, originalSpannerType.length() - 1); Type itemType = parseSpannerType(spannerArrayType, dialect); return Type.array(itemType); } From 10459eda8ca7984004a495fda053b318b5f2a458 Mon Sep 17 00:00:00 2001 From: dhruvdua Date: Thu, 8 Feb 2024 06:40:53 +0000 Subject: [PATCH 5/6] Spotless Apply --- .../java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java index 1b47176f579d6..1196dbe0a53c7 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java @@ -195,7 +195,8 @@ private static Type parseSpannerType(String spannerType, Dialect dialect) { } if (spannerType.startsWith("ARRAY")) { // Substring "ARRAY" - String spannerArrayType = originalSpannerType.substring(6, originalSpannerType.length() - 1); + String spannerArrayType = + originalSpannerType.substring(6, originalSpannerType.length() - 1); Type itemType = parseSpannerType(spannerArrayType, dialect); return Type.array(itemType); } From 2a73376becdaf87d51d45a8b4980cbc9b3fc385f Mon Sep 17 00:00:00 2001 From: dhruvdua Date: Fri, 9 Feb 2024 05:25:26 +0000 Subject: [PATCH 6/6] Entry in CHANGES.md for arrow version bump, comment to keep the arrow version consistent with google_cloud_bigquery --- CHANGES.md | 1 + .../main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index c63464a6e01e0..a09c51f44c5f0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -71,6 +71,7 @@ ## Breaking Changes * X behavior was changed ([#X](https://github.com/apache/beam/issues/X)). +* Arrow version was bumped to 15.0.0 from 5.0.0 ([#30181](https://github.com/apache/beam/pull/30181)). ## Deprecations diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index d704e9116a9e1..8e66f3d033485 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -640,6 +640,7 @@ class BeamModulePlugin implements Plugin { def spark3_version = "3.2.2" def spotbugs_version = "4.0.6" def testcontainers_version = "1.17.3" + // Try to keep arrow_version consistent with the arrow version in google_cloud_bigquery, managed by google_cloud_platform_libraries_bom def arrow_version = "15.0.0" def jmh_version = "1.34" def jupiter_version = "5.7.0"