Skip to content

Commit

Permalink
[BigQueryIO] Use final destination table schema and metadata when cre…
Browse files Browse the repository at this point in the history
…ating temp tables (apache#24471)

* always attempt to match tables with final destination

* do not match when schema update options exist

* spotless
  • Loading branch information
ahmedabu98 authored and lostluck committed Dec 22, 2022
1 parent ca574a9 commit 298cb59
Showing 1 changed file with 13 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -728,6 +728,18 @@ private PCollection<KV<DestinationT, WriteTables.Result>> writeTempTables(
ShardedKeyCoder.of(NullableCoder.of(destinationCoder)),
WritePartition.ResultCoder.INSTANCE);

// If the final destination table exists already (and we're appending to it), then the temp
// tables must exactly match schema, partitioning, etc. Wrap the DynamicDestinations object
// with one that makes this happen.
// In the case schemaUpdateOptions are specified by the user, matching does not occur in order
// to respect those options.
DynamicDestinations<?, DestinationT> destinations = dynamicDestinations;
if (schemaUpdateOptions.isEmpty()) {
destinations =
DynamicDestinationsHelpers.matchTableDynamicDestinations(
dynamicDestinations, bigQueryServices);
}

// If WriteBundlesToFiles produced more than DEFAULT_MAX_FILES_PER_PARTITION files or
// DEFAULT_MAX_BYTES_PER_PARTITION bytes, then
// the import needs to be split into multiple partitions, and those partitions will be
Expand All @@ -746,7 +758,7 @@ private PCollection<KV<DestinationT, WriteTables.Result>> writeTempTables(
WriteDisposition.WRITE_EMPTY,
CreateDisposition.CREATE_IF_NEEDED,
sideInputs,
dynamicDestinations,
destinations,
loadJobProjectId,
maxRetryJobs,
ignoreUnknownValues,
Expand Down

0 comments on commit 298cb59

Please sign in to comment.