From 07f0806a945c2cf0fbc431b63d9c8a30ed3a22fd Mon Sep 17 00:00:00 2001 From: shollyman Date: Wed, 17 Apr 2024 13:12:40 -0700 Subject: [PATCH] feat(bigquery): RANGE support for basic data movement (#9762) * feat(bigquery): RANGE type StandardSQLDataType support This PR augments the StandardSQLDataRepresentation(s) to support range-specific augmentations, and adds some testing. Astute observers will note that this does include mapping changes to param handling, which will be tested in a subsequent PR that expands RANGE coverage to that area of the library. * feat(bigquery): RANGE support (data) This PR adds a new RangeValue type for conveying parsed range start/end values, and adds the requisite plumbing for parameter handling (type and binding) and data result reading. * more typing work and inference * round trip * more testing, cleanup * more tests * explicit param type test * correct supported RANGE element types * more testing * add temporary arg to initQueryParameterTestCases to isolate storage work * reorder error handling --- bigquery/integration_test.go | 157 +++++++++++++++----- bigquery/params.go | 69 ++++++++- bigquery/params_test.go | 209 ++++++++++++++++++++++++++- bigquery/rangevalue.go | 28 ++++ bigquery/schema.go | 15 ++ bigquery/schema_test.go | 12 +- bigquery/storage_integration_test.go | 2 +- bigquery/value.go | 66 ++++++++- bigquery/value_test.go | 4 +- 9 files changed, 514 insertions(+), 48 deletions(-) create mode 100644 bigquery/rangevalue.go diff --git a/bigquery/integration_test.go b/bigquery/integration_test.go index 43d442fdb0ed..cca14c5115c1 100644 --- a/bigquery/integration_test.go +++ b/bigquery/integration_test.go @@ -1160,18 +1160,20 @@ type SubTestStruct struct { } type TestStruct struct { - Name string - Bytes []byte - Integer int64 - Float float64 - Boolean bool - Timestamp time.Time - Date civil.Date - Time civil.Time - DateTime civil.DateTime - Numeric *big.Rat - Geography string - + Name string + Bytes []byte + Integer int64 + Float float64 + Boolean bool + Timestamp time.Time + Date civil.Date + Time civil.Time + DateTime civil.DateTime + Numeric *big.Rat + Geography string + RangeDate *RangeValue `bigquery:"rangedate"` //TODO: remove tags when field normalization works + RangeDateTime *RangeValue `bigquery:"rangedatetime"` + RangeTimestamp *RangeValue `bigquery:"rangetimestamp"` StringArray []string IntegerArray []int64 FloatArray []float64 @@ -1200,6 +1202,19 @@ func TestIntegration_InsertAndReadStructs(t *testing.T) { t.Fatal(err) } + // Finish declaring the ambigous range element types. + for idx, typ := range map[int]FieldType{ + 11: DateFieldType, + 12: DateTimeFieldType, + 13: TimestampFieldType, + } { + if schema[idx].Type != RangeFieldType { + t.Fatalf("mismatch in expected RANGE element in schema field %d", idx) + } else { + schema[idx].RangeElementType = &RangeElementType{Type: typ} + } + } + ctx := context.Background() table := newTable(t, schema) defer table.Delete(ctx) @@ -1214,6 +1229,15 @@ func TestIntegration_InsertAndReadStructs(t *testing.T) { dtm2 := civil.DateTime{Date: d2, Time: tm2} g := "POINT(-122.350220 47.649154)" g2 := "POINT(-122.0836791 37.421827)" + rangedate := &RangeValue{Start: civil.Date{Year: 2024, Month: 04, Day: 11}} + rangedatetime := &RangeValue{ + End: civil.DateTime{ + Date: civil.Date{Year: 2024, Month: 04, Day: 11}, + Time: civil.Time{Hour: 2, Minute: 4, Second: 6, Nanosecond: 0}}, + } + rangetimestamp := &RangeValue{ + Start: time.Date(2016, 3, 20, 15, 4, 5, 6000, time.UTC), + } // Populate the table. ins := table.Inserter() @@ -1230,6 +1254,9 @@ func TestIntegration_InsertAndReadStructs(t *testing.T) { dtm, big.NewRat(57, 100), g, + rangedate, + rangedatetime, + rangetimestamp, []string{"a", "b"}, []int64{1, 2}, []float64{1, 1.41}, @@ -1255,16 +1282,19 @@ func TestIntegration_InsertAndReadStructs(t *testing.T) { }, }, { - Name: "b", - Bytes: []byte("byte2"), - Integer: 24, - Float: 4.13, - Boolean: false, - Timestamp: ts, - Date: d, - Time: tm, - DateTime: dtm, - Numeric: big.NewRat(4499, 10000), + Name: "b", + Bytes: []byte("byte2"), + Integer: 24, + Float: 4.13, + Boolean: false, + Timestamp: ts, + Date: d, + Time: tm, + DateTime: dtm, + Numeric: big.NewRat(4499, 10000), + RangeDate: rangedate, + RangeDateTime: rangedatetime, + RangeTimestamp: rangetimestamp, }, } var savers []*StructSaver @@ -1866,6 +1896,10 @@ func TestIntegration_StandardQuery(t *testing.T) { {"ArrayOfStructs", "SELECT [(1, 2, 3), (4, 5, 6)]", []Value{[]Value{ints(1, 2, 3), ints(4, 5, 6)}}}, {"ComplexNested", "SELECT [([1, 2, 3], 4), ([5, 6], 7)]", []Value{[]Value{[]Value{ints(1, 2, 3), int64(4)}, []Value{ints(5, 6), int64(7)}}}}, {"SubSelectArray", "SELECT ARRAY(SELECT STRUCT([1, 2]))", []Value{[]Value{[]Value{ints(1, 2)}}}}, + {"RangeOofDateLiteral", + "SELECT RANGE(DATE '2023-03-01', DATE '2024-04-16')", + []Value{&RangeValue{Start: civil.Date{Year: 2023, Month: 03, Day: 01}, End: civil.Date{Year: 2024, Month: 04, Day: 16}}}, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -2108,17 +2142,19 @@ func TestIntegration_QuerySessionSupport(t *testing.T) { } +type queryParameterTestCase struct { + name string + query string + parameters []QueryParameter + wantRow []Value + wantConfig interface{} +} + var ( - queryParameterTestCases = []struct { - name string - query string - parameters []QueryParameter - wantRow []Value - wantConfig interface{} - }{} + queryParameterTestCases = []queryParameterTestCase{} ) -func initQueryParameterTestCases() { +func initQueryParameterTestCases(includeRangeCases bool) { d := civil.Date{Year: 2016, Month: 3, Day: 20} tm := civil.Time{Hour: 15, Minute: 04, Second: 05, Nanosecond: 3008} rtm := tm @@ -2127,6 +2163,12 @@ func initQueryParameterTestCases() { ts := time.Date(2016, 3, 20, 15, 04, 05, 0, time.UTC) rat := big.NewRat(13, 10) bigRat := big.NewRat(12345, 10e10) + rangeTimestamp1 := &RangeValue{ + Start: time.Date(2016, 3, 20, 15, 04, 05, 0, time.UTC), + } + rangeTimestamp2 := &RangeValue{ + End: time.Date(2016, 3, 20, 15, 04, 05, 0, time.UTC), + } type ss struct { String string @@ -2139,13 +2181,7 @@ func initQueryParameterTestCases() { SubStructArray []ss } - queryParameterTestCases = []struct { - name string - query string - parameters []QueryParameter - wantRow []Value - wantConfig interface{} - }{ + queryParameterTestCases = []queryParameterTestCase{ { "Int64Param", "SELECT @val", @@ -2412,6 +2448,51 @@ func initQueryParameterTestCases() { }, }, } + + if includeRangeCases { + queryParameterTestCases = append(queryParameterTestCases, []queryParameterTestCase{ + { + "RangeUnboundedEnd", + "SELECT @val", + []QueryParameter{ + { + Name: "val", + Value: &QueryParameterValue{ + Type: StandardSQLDataType{ + TypeKind: "RANGE", + RangeElementType: &StandardSQLDataType{ + TypeKind: "TIMESTAMP", + }, + }, + Value: rangeTimestamp1, + }, + }, + }, + []Value{rangeTimestamp1}, + rangeTimestamp1, + }, + { + "RangeUnboundedStart", + "SELECT @val", + []QueryParameter{ + { + Name: "val", + Value: &QueryParameterValue{ + Type: StandardSQLDataType{ + TypeKind: "RANGE", + RangeElementType: &StandardSQLDataType{ + TypeKind: "TIMESTAMP", + }, + }, + Value: rangeTimestamp2, + }, + }, + }, + []Value{rangeTimestamp2}, + rangeTimestamp2, + }, + }...) + } } func TestIntegration_QueryParameters(t *testing.T) { @@ -2420,7 +2501,7 @@ func TestIntegration_QueryParameters(t *testing.T) { } ctx := context.Background() - initQueryParameterTestCases() + initQueryParameterTestCases(true) for _, tc := range queryParameterTestCases { t.Run(tc.name, func(t *testing.T) { diff --git a/bigquery/params.go b/bigquery/params.go index 67b0c94537d8..e256a7c3a82b 100644 --- a/bigquery/params.go +++ b/bigquery/params.go @@ -86,6 +86,7 @@ var ( geographyParamType = &bq.QueryParameterType{Type: "GEOGRAPHY"} intervalParamType = &bq.QueryParameterType{Type: "INTERVAL"} jsonParamType = &bq.QueryParameterType{Type: "JSON"} + rangeParamType = &bq.QueryParameterType{Type: "RANGE"} ) var ( @@ -95,6 +96,7 @@ var ( typeOfGoTime = reflect.TypeOf(time.Time{}) typeOfRat = reflect.TypeOf(&big.Rat{}) typeOfIntervalValue = reflect.TypeOf(&IntervalValue{}) + typeOfRangeValue = reflect.TypeOf(&RangeValue{}) typeOfQueryParameterValue = reflect.TypeOf(&QueryParameterValue{}) ) @@ -315,9 +317,11 @@ func (p QueryParameter) toBQ() (*bq.QueryParameter, error) { }, nil } +var errNilParam = fmt.Errorf("bigquery: nil parameter") + func paramType(t reflect.Type, v reflect.Value) (*bq.QueryParameterType, error) { if t == nil { - return nil, errors.New("bigquery: nil parameter") + return nil, errNilParam } switch t { case typeOfDate, typeOfNullDate: @@ -344,6 +348,25 @@ func paramType(t reflect.Type, v reflect.Value) (*bq.QueryParameterType, error) return geographyParamType, nil case typeOfNullJSON: return jsonParamType, nil + case typeOfRangeValue: + iv := v.Interface().(*RangeValue) + // In order to autodetect a Range param correctly, at least one of start,end must be populated. + // Without it, users must declare typing via using QueryParameterValue. + element := iv.Start + if element == nil { + element = iv.End + } + if element == nil { + return nil, fmt.Errorf("unable to determine range element type from RangeValue without a non-nil start or end value") + } + elet, err := paramType(reflect.TypeOf(element), reflect.ValueOf(element)) + if err != nil { + return nil, err + } + return &bq.QueryParameterType{ + Type: "RANGE", + RangeElementType: elet, + }, nil case typeOfQueryParameterValue: return v.Interface().(*QueryParameterValue).toBQParamType(), nil } @@ -410,7 +433,7 @@ func paramType(t reflect.Type, v reflect.Value) (*bq.QueryParameterType, error) func paramValue(v reflect.Value) (*bq.QueryParameterValue, error) { res := &bq.QueryParameterValue{} if !v.IsValid() { - return res, errors.New("bigquery: nil parameter") + return res, errNilParam } t := v.Type() switch t { @@ -492,6 +515,28 @@ func paramValue(v reflect.Value) (*bq.QueryParameterValue, error) { case typeOfIntervalValue: res.Value = IntervalString(v.Interface().(*IntervalValue)) return res, nil + case typeOfRangeValue: + // RangeValue is a compound type, and we must process the start/end to + // fully populate the value. + res.RangeValue = &bq.RangeValue{} + iv := v.Interface().(*RangeValue) + sVal, err := paramValue(reflect.ValueOf(iv.Start)) + if err != nil { + if !errors.Is(err, errNilParam) { + return nil, err + } + } else { + res.RangeValue.Start = sVal + } + eVal, err := paramValue(reflect.ValueOf(iv.End)) + if err != nil { + if !errors.Is(err, errNilParam) { + return nil, err + } + } else { + res.RangeValue.End = eVal + } + return res, nil case typeOfQueryParameterValue: return v.Interface().(*QueryParameterValue).toBQParamValue() } @@ -592,6 +637,26 @@ func convertParamValue(qval *bq.QueryParameterValue, qtype *bq.QueryParameterTyp return map[string]interface{}(nil), nil } return convertParamStruct(qval.StructValues, qtype.StructTypes) + case "RANGE": + rv := &RangeValue{} + if qval.RangeValue == nil { + return rv, nil + } + if qval.RangeValue.Start != nil { + startVal, err := convertParamValue(qval.RangeValue.Start, qtype.RangeElementType) + if err != nil { + return nil, err + } + rv.Start = startVal + } + if qval.RangeValue.End != nil { + endVal, err := convertParamValue(qval.RangeValue.End, qtype.RangeElementType) + if err != nil { + return nil, err + } + rv.End = endVal + } + return rv, nil case "TIMESTAMP": if isNullScalar(qval) { return NullTimestamp{Valid: false}, nil diff --git a/bigquery/params_test.go b/bigquery/params_test.go index 7754afb6b6c6..8e95313b320f 100644 --- a/bigquery/params_test.go +++ b/bigquery/params_test.go @@ -211,6 +211,175 @@ func TestParamValueScalar(t *testing.T) { } } +func TestParamValueRange(t *testing.T) { + + tTimestamp := time.Date(2016, 3, 22, 4, 22, 9, 5000, time.FixedZone("neg1-2", -3720)) + tDate := civil.Date{Year: 2016, Month: 03, Day: 22} + tDateTime := civil.DateTime{ + Date: civil.Date{Year: 2017, Month: 7, Day: 13}, + Time: civil.Time{Hour: 4, Minute: 5, Second: 6, Nanosecond: 789000000}, + } + wTimestamp := "2016-03-22 04:22:09.000005-01:02" + wDate := "2016-03-22" + wDateTime := "2017-07-13 04:05:06.789000" + + var testCases = []struct { + desc string + in interface{} + want *bq.QueryParameterValue + }{ + { + desc: "RangeValue time.Time both populated", + in: &RangeValue{ + Start: tTimestamp, + End: tTimestamp, + }, + want: &bq.QueryParameterValue{ + RangeValue: &bq.RangeValue{ + Start: &bq.QueryParameterValue{ + Value: wTimestamp, + }, + End: &bq.QueryParameterValue{ + Value: wTimestamp, + }, + }, + }, + }, + { + desc: "RangeValue time.Time start only", + in: &RangeValue{ + Start: tTimestamp, + }, + want: &bq.QueryParameterValue{ + RangeValue: &bq.RangeValue{ + Start: &bq.QueryParameterValue{ + Value: wTimestamp, + }, + }, + }, + }, + { + desc: "RangeValue time.Time end only", + in: &RangeValue{ + End: tTimestamp, + }, + want: &bq.QueryParameterValue{ + RangeValue: &bq.RangeValue{ + End: &bq.QueryParameterValue{ + Value: wTimestamp, + }, + }, + }, + }, + { + desc: "RangeValue NullTimestamp both populated", + in: &RangeValue{ + Start: NullTimestamp{Valid: true, Timestamp: tTimestamp}, + End: NullTimestamp{Valid: true, Timestamp: tTimestamp}, + }, + want: &bq.QueryParameterValue{ + RangeValue: &bq.RangeValue{ + Start: &bq.QueryParameterValue{ + Value: wTimestamp, + }, + End: &bq.QueryParameterValue{ + Value: wTimestamp, + }, + }, + }, + }, + { + desc: "RangeValue NullTimestamp start only", + in: &RangeValue{ + Start: NullTimestamp{Valid: true, Timestamp: tTimestamp}, + End: NullTimestamp{Valid: false}, + }, + want: &bq.QueryParameterValue{ + RangeValue: &bq.RangeValue{ + Start: &bq.QueryParameterValue{ + Value: wTimestamp, + }, + End: &bq.QueryParameterValue{NullFields: []string{"Value"}}, + }, + }, + }, + { + desc: "RangeValue time.Time end only", + in: &RangeValue{ + Start: NullTimestamp{Valid: false}, + End: NullTimestamp{Valid: true, Timestamp: tTimestamp}, + }, + want: &bq.QueryParameterValue{ + RangeValue: &bq.RangeValue{ + Start: &bq.QueryParameterValue{NullFields: []string{"Value"}}, + End: &bq.QueryParameterValue{ + Value: wTimestamp, + }, + }, + }, + }, + { + desc: "RangeValue civil.Date both populated", + in: &RangeValue{ + Start: tDate, + End: tDate, + }, + want: &bq.QueryParameterValue{ + RangeValue: &bq.RangeValue{ + Start: &bq.QueryParameterValue{ + Value: wDate, + }, + End: &bq.QueryParameterValue{ + Value: wDate, + }, + }, + }, + }, + { + desc: "RangeValue civil.DateTime both populated", + in: &RangeValue{ + Start: tDateTime, + End: tDateTime, + }, + want: &bq.QueryParameterValue{ + RangeValue: &bq.RangeValue{ + Start: &bq.QueryParameterValue{ + Value: wDateTime, + }, + End: &bq.QueryParameterValue{ + Value: wDateTime, + }, + }, + }, + }, + { + desc: "Unbounded Range in QueryParameterValue", + in: &QueryParameterValue{ + Type: StandardSQLDataType{ + TypeKind: "RANGE", + RangeElementType: &StandardSQLDataType{ + TypeKind: "DATETIME", + }, + }, + Value: &RangeValue{}, + }, + want: &bq.QueryParameterValue{ + RangeValue: &bq.RangeValue{}, + }, + }, + } + + for _, tc := range testCases { + got, err := paramValue(reflect.ValueOf(tc.in)) + if err != nil { + t.Errorf("%q: got error %v", tc.desc, err) + } + if d := testutil.Diff(got, tc.want); d != "" { + t.Errorf("%q: mismatch\n%s", tc.desc, d) + } + } +} + func TestParamValueArray(t *testing.T) { qpv := &bq.QueryParameterValue{ArrayValues: []*bq.QueryParameterValue{ {Value: "1"}, @@ -280,6 +449,29 @@ func TestParamType(t *testing.T) { {"intArray", []int{}, &bq.QueryParameterType{Type: "ARRAY", ArrayType: int64ParamType}}, {"boolArray", [3]bool{}, &bq.QueryParameterType{Type: "ARRAY", ArrayType: boolParamType}}, {"emptyStruct", S1{}, s1ParamType}, + {"RangeTimestampNilEnd", &RangeValue{Start: time.Now()}, &bq.QueryParameterType{Type: "RANGE", RangeElementType: timestampParamType}}, + {"RangeTimestampNilStart", &RangeValue{End: time.Now()}, &bq.QueryParameterType{Type: "RANGE", RangeElementType: timestampParamType}}, + {"RangeTimestampNullValStart", &RangeValue{Start: NullTimestamp{Valid: false}}, &bq.QueryParameterType{Type: "RANGE", RangeElementType: timestampParamType}}, + {"RangeTimestampNullValEnd", &RangeValue{End: NullTimestamp{Valid: false}}, &bq.QueryParameterType{Type: "RANGE", RangeElementType: timestampParamType}}, + {"RangeDateTimeEmptyStart", &RangeValue{Start: civil.DateTime{}}, &bq.QueryParameterType{Type: "RANGE", RangeElementType: dateTimeParamType}}, + {"RangeDateEmptyEnd", &RangeValue{End: civil.Date{}}, &bq.QueryParameterType{Type: "RANGE", RangeElementType: dateParamType}}, + {"RangeDateTimeInQPV", + &QueryParameterValue{ + Type: StandardSQLDataType{ + TypeKind: "RANGE", + RangeElementType: &StandardSQLDataType{ + TypeKind: "DATETIME", + }, + }, + Value: &RangeValue{}, + }, + &bq.QueryParameterType{ + Type: "RANGE", + RangeElementType: &bq.QueryParameterType{ + Type: "DATETIME", + }, + }, + }, } { t.Run(fmt.Sprintf("complex-%s", tc.name), func(t *testing.T) { got, err := paramType(reflect.TypeOf(tc.val), reflect.ValueOf(tc.val)) @@ -294,7 +486,7 @@ func TestParamType(t *testing.T) { } func TestParamTypeErrors(t *testing.T) { for _, val := range []interface{}{ - nil, uint(0), new([]int), make(chan int), map[int]interface{}{}, + nil, uint(0), new([]int), make(chan int), map[int]interface{}{}, &RangeValue{}, } { _, err := paramType(reflect.TypeOf(val), reflect.ValueOf(val)) if err == nil { @@ -469,6 +661,21 @@ func TestIntegration_OtherParam(t *testing.T) { []Value{int64(1), []Value{"s"}, true}, s1ParamReturnValue, }, + { + "RangeTimestamp", + &RangeValue{ + Start: time.Date(2016, 3, 22, 4, 22, 9, 5000, time.FixedZone("neg1-2", -3720)), + End: NullTimestamp{}, + }, + &RangeValue{ + Start: time.Date(2016, 3, 22, 4, 22, 9, 5000, time.FixedZone("neg1-2", -3720)), + End: nil, + }, + &RangeValue{ + Start: time.Date(2016, 3, 22, 4, 22, 9, 5000, time.FixedZone("neg1-2", -3720)), + End: NullTimestamp{}, + }, + }, } { t.Run(tc.name, func(t *testing.T) { gotData, gotParam, err := paramRoundTrip(c, tc.val) diff --git a/bigquery/rangevalue.go b/bigquery/rangevalue.go new file mode 100644 index 000000000000..63010ce7ed00 --- /dev/null +++ b/bigquery/rangevalue.go @@ -0,0 +1,28 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +// RangeValue represents a continuous RANGE of values of a given element +// type. The supported element types for RANGE are currently the BigQuery +// DATE, DATETIME, and TIMESTAMP, types. +type RangeValue struct { + // The start value of the range. A missing value represents an + // unbounded start. + Start Value `json:"start"` + + // The end value of the range. A missing value represents an + // unbounded end. + End Value `json:"end"` +} diff --git a/bigquery/schema.go b/bigquery/schema.go index 6179fdfb49dd..22bc945739a5 100644 --- a/bigquery/schema.go +++ b/bigquery/schema.go @@ -376,6 +376,15 @@ var typeOfByteSlice = reflect.TypeOf([]byte{}) // Due to lack of unique native Go type for GEOGRAPHY, there is no schema // inference to GEOGRAPHY at this time. // +// This package also provides some value types for expressing the corresponding SQL types. +// +// INTERVAL *IntervalValue +// RANGE *RangeValue +// +// In the case of RANGE types, a RANGE represents a continuous set of values of a given +// element type (DATE, DATETIME, or TIMESTAMP). InferSchema does not attempt to determine +// the element type, as it uses generic Value types to denote the start/end of the range. +// // Nullable fields are inferred from the NullXXX types, declared in this package: // // STRING NullString @@ -496,6 +505,12 @@ func inferFieldSchema(fieldName string, rt reflect.Type, nullable, json bool) (* // larger precision of BIGNUMERIC need to manipulate the inferred // schema. return &FieldSchema{Required: !nullable, Type: NumericFieldType}, nil + case typeOfIntervalValue: + return &FieldSchema{Required: !nullable, Type: IntervalFieldType}, nil + case typeOfRangeValue: + // We can't fully infer the element type of a range without additional + // information, and don't set the RangeElementType when inferred. + return &FieldSchema{Required: !nullable, Type: RangeFieldType}, nil } if ft := nullableFieldType(rt); ft != "" { return &FieldSchema{Required: false, Type: ft}, nil diff --git a/bigquery/schema_test.go b/bigquery/schema_test.go index 65bbd1e3d662..754de36dbfbb 100644 --- a/bigquery/schema_test.go +++ b/bigquery/schema_test.go @@ -485,10 +485,12 @@ type allBoolean struct { } type allTime struct { - Timestamp time.Time - Time civil.Time - Date civil.Date - DateTime civil.DateTime + Timestamp time.Time + Time civil.Time + Date civil.Date + DateTime civil.DateTime + Interval *IntervalValue + RangeGeneric *RangeValue } type allNumeric struct { @@ -566,6 +568,8 @@ func TestSimpleInference(t *testing.T) { reqField("Time", "TIME"), reqField("Date", "DATE"), reqField("DateTime", "DATETIME"), + reqField("Interval", "INTERVAL"), + reqField("RangeGeneric", "RANGE"), }, }, { diff --git a/bigquery/storage_integration_test.go b/bigquery/storage_integration_test.go index 5fcaf9b14625..6f275927c6bc 100644 --- a/bigquery/storage_integration_test.go +++ b/bigquery/storage_integration_test.go @@ -37,7 +37,7 @@ func TestIntegration_StorageReadBasicTypes(t *testing.T) { } ctx := context.Background() - initQueryParameterTestCases() + initQueryParameterTestCases(false) for _, c := range queryParameterTestCases { q := storageOptimizedClient.Query(c.query) diff --git a/bigquery/value.go b/bigquery/value.go index 34070d033f99..c178f8b819db 100644 --- a/bigquery/value.go +++ b/bigquery/value.go @@ -433,7 +433,16 @@ func determineSetFunc(ftype reflect.Type, stype FieldType) setFunc { return setNull(v, x, func() interface{} { return x.(*big.Rat) }) } } + + case RangeFieldType: + if ftype == typeOfRangeValue { + return func(v reflect.Value, x interface{}) error { + return setNull(v, x, func() interface{} { return x.(*RangeValue) }) + } + } + } + return nil } @@ -765,6 +774,8 @@ func toUploadValueReflect(v reflect.Value, fs *FieldSchema) interface{} { return formatUploadValue(v, fs, func(v reflect.Value) string { return IntervalString(v.Interface().(*IntervalValue)) }) + case RangeFieldType: + return v.Interface() default: if !fs.Repeated || v.Len() > 0 { return v.Interface() @@ -879,7 +890,17 @@ func convertRow(r *bq.TableRow, schema Schema) ([]Value, error) { var values []Value for i, cell := range r.F { fs := schema[i] - v, err := convertValue(cell.V, fs.Type, fs.Schema) + var v Value + var err error + if fs.Type == RangeFieldType { + // interception range conversion here, as we don't propagate range element type more deeply. + if fs.RangeElementType == nil { + return nil, errors.New("bigquery: incomplete range schema for conversion") + } + v, err = convertRangeValue(cell.V.(string), fs.RangeElementType.Type) + } else { + v, err = convertValue(cell.V, fs.Type, fs.Schema) + } if err != nil { return nil, err } @@ -991,3 +1012,46 @@ func convertBasicType(val string, typ FieldType) (Value, error) { return nil, fmt.Errorf("unrecognized type: %s", typ) } } + +// how BQ declares an unbounded RANGE. +var unboundedRangeSentinel = "UNBOUNDED" + +// convertRangeValue aids in parsing the compound RANGE api data representation. +// The format for a range value is: "[startval, endval)" +func convertRangeValue(val string, elementType FieldType) (Value, error) { + supported := false + for _, t := range []FieldType{DateFieldType, DateTimeFieldType, TimestampFieldType} { + if elementType == t { + supported = true + break + } + } + if !supported { + return nil, fmt.Errorf("bigquery: invalid RANGE element type %q", elementType) + } + if !strings.HasPrefix(val, "[") || !strings.HasSuffix(val, ")") { + return nil, fmt.Errorf("bigquery: invalid RANGE value %q", val) + } + // trim the leading/trailing characters + val = val[1 : len(val)-1] + parts := strings.Split(val, ", ") + if len(parts) != 2 { + return nil, fmt.Errorf("bigquery: invalid RANGE value %q", val) + } + rv := &RangeValue{} + if parts[0] != unboundedRangeSentinel { + sv, err := convertBasicType(parts[0], elementType) + if err != nil { + return nil, fmt.Errorf("bigquery: invalid RANGE start value %q", parts[0]) + } + rv.Start = sv + } + if parts[1] != unboundedRangeSentinel { + ev, err := convertBasicType(parts[1], elementType) + if err != nil { + return nil, fmt.Errorf("bigquery: invalid RANGE end value %q", parts[1]) + } + rv.End = ev + } + return rv, nil +} diff --git a/bigquery/value_test.go b/bigquery/value_test.go index affb9821e98d..7211835b3f82 100644 --- a/bigquery/value_test.go +++ b/bigquery/value_test.go @@ -74,6 +74,7 @@ func TestConvertTime(t *testing.T) { {Type: DateFieldType}, {Type: TimeFieldType}, {Type: DateTimeFieldType}, + {Type: RangeFieldType, RangeElementType: &RangeElementType{Type: TimestampFieldType}}, } ts := testTimestamp.Round(time.Millisecond) row := &bq.TableRow{ @@ -82,13 +83,14 @@ func TestConvertTime(t *testing.T) { {V: testDate.String()}, {V: testTime.String()}, {V: testDateTime.String()}, + {V: fmt.Sprintf("[UNBOUNDED, %d)", ts.UnixMicro())}, }, } got, err := convertRow(row, schema) if err != nil { t.Fatalf("error converting: %v", err) } - want := []Value{ts, testDate, testTime, testDateTime} + want := []Value{ts, testDate, testTime, testDateTime, &RangeValue{End: ts}} for i, g := range got { w := want[i] if !testutil.Equal(g, w) {