From ebc6a2d39dd7bdf53f0586c24f78079bf3ed6278 Mon Sep 17 00:00:00 2001 From: hg2990656 <346461986@qq.com> Date: Tue, 4 Feb 2020 21:41:55 +0800 Subject: [PATCH] *: add builtin aggregate function `json_objectagg` (#11154) --- ddl/ddl.go | 1 + executor/aggfuncs/aggfunc_test.go | 172 ++++++++++++++++++ executor/aggfuncs/aggfuncs.go | 3 + executor/aggfuncs/builder.go | 16 ++ executor/aggfuncs/func_json_objectagg.go | 114 ++++++++++++ executor/aggfuncs/func_json_objectagg_test.go | 104 +++++++++++ expression/aggregation/agg_to_pb.go | 2 + expression/aggregation/base_func.go | 18 +- expression/integration_test.go | 38 ++++ go.mod | 8 +- go.sum | 16 +- planner/core/rule_aggregation_push_down.go | 2 +- types/json/constants.go | 18 +- 13 files changed, 490 insertions(+), 22 deletions(-) create mode 100644 executor/aggfuncs/func_json_objectagg.go create mode 100644 executor/aggfuncs/func_json_objectagg_test.go diff --git a/ddl/ddl.go b/ddl/ddl.go index 019a6be3897e1..9aa4d2709f15c 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -705,6 +705,7 @@ func init() { mysql.ErrInvalidStoreVersion: mysql.ErrInvalidStoreVersion, mysql.ErrInvalidUseOfNull: mysql.ErrInvalidUseOfNull, mysql.ErrJSONUsedAsKey: mysql.ErrJSONUsedAsKey, + mysql.ErrJSONDocumentNULLKey: mysql.ErrJSONDocumentNULLKey, mysql.ErrKeyColumnDoesNotExits: mysql.ErrKeyColumnDoesNotExits, mysql.ErrLockWaitTimeout: mysql.ErrLockWaitTimeout, mysql.ErrNoParts: mysql.ErrNoParts, diff --git a/executor/aggfuncs/aggfunc_test.go b/executor/aggfuncs/aggfunc_test.go index c47578975654c..876cc76dbb5b2 100644 --- a/executor/aggfuncs/aggfunc_test.go +++ b/executor/aggfuncs/aggfunc_test.go @@ -70,6 +70,15 @@ type aggTest struct { results []types.Datum } +type multiArgsAggTest struct { + dataTypes []*types.FieldType + retType *types.FieldType + numRows int + dataGens []func(i int) types.Datum + funcName string + results []types.Datum +} + func (s *testSuite) testMergePartialResult(c *C, p aggTest) { srcChk := chunk.NewChunkWithCapacity([]*types.FieldType{p.dataType}, p.numRows) for i := 0; i < p.numRows; i++ { @@ -150,6 +159,99 @@ func buildAggTesterWithFieldType(funcName string, ft *types.FieldType, numRows i return pt } +func (s *testSuite) testMultiArgsMergePartialResult(c *C, p multiArgsAggTest) { + srcChk := chunk.NewChunkWithCapacity(p.dataTypes, p.numRows) + for i := 0; i < p.numRows; i++ { + for j := 0; j < len(p.dataGens); j++ { + fdt := p.dataGens[j](i) + srcChk.AppendDatum(j, &fdt) + } + } + iter := chunk.NewIterator4Chunk(srcChk) + + args := make([]expression.Expression, len(p.dataTypes)) + for k := 0; k < len(p.dataTypes); k++ { + args[k] = &expression.Column{RetType: p.dataTypes[k], Index: k} + } + + desc, err := aggregation.NewAggFuncDesc(s.ctx, p.funcName, args, false) + c.Assert(err, IsNil) + partialDesc, finalDesc := desc.Split([]int{0, 1}) + + // build partial func for partial phase. + partialFunc := aggfuncs.Build(s.ctx, partialDesc, 0) + partialResult := partialFunc.AllocPartialResult() + + // build final func for final phase. + finalFunc := aggfuncs.Build(s.ctx, finalDesc, 0) + finalPr := finalFunc.AllocPartialResult() + resultChk := chunk.NewChunkWithCapacity([]*types.FieldType{p.retType}, 1) + + // update partial result. + for row := iter.Begin(); row != iter.End(); row = iter.Next() { + partialFunc.UpdatePartialResult(s.ctx, []chunk.Row{row}, partialResult) + } + partialFunc.AppendFinalResult2Chunk(s.ctx, partialResult, resultChk) + dt := resultChk.GetRow(0).GetDatum(0, p.retType) + result, err := dt.CompareDatum(s.ctx.GetSessionVars().StmtCtx, &p.results[0]) + c.Assert(err, IsNil) + c.Assert(result, Equals, 0) + + err = finalFunc.MergePartialResult(s.ctx, partialResult, finalPr) + c.Assert(err, IsNil) + partialFunc.ResetPartialResult(partialResult) + + iter.Begin() + iter.Next() + for row := iter.Next(); row != iter.End(); row = iter.Next() { + partialFunc.UpdatePartialResult(s.ctx, []chunk.Row{row}, partialResult) + } + resultChk.Reset() + partialFunc.AppendFinalResult2Chunk(s.ctx, partialResult, resultChk) + dt = resultChk.GetRow(0).GetDatum(0, p.retType) + result, err = dt.CompareDatum(s.ctx.GetSessionVars().StmtCtx, &p.results[1]) + c.Assert(err, IsNil) + c.Assert(result, Equals, 0) + err = finalFunc.MergePartialResult(s.ctx, partialResult, finalPr) + c.Assert(err, IsNil) + + resultChk.Reset() + err = finalFunc.AppendFinalResult2Chunk(s.ctx, finalPr, resultChk) + c.Assert(err, IsNil) + + dt = resultChk.GetRow(0).GetDatum(0, p.retType) + result, err = dt.CompareDatum(s.ctx.GetSessionVars().StmtCtx, &p.results[2]) + c.Assert(err, IsNil) + c.Assert(result, Equals, 0) +} + +// for multiple args in aggfuncs such as json_objectagg(c1, c2) +func buildMultiArgsAggTester(funcName string, tps []byte, rt byte, numRows int, results ...interface{}) multiArgsAggTest { + fts := make([]*types.FieldType, len(tps)) + for i := 0; i < len(tps); i++ { + fts[i] = types.NewFieldType(tps[i]) + } + return buildMultiArgsAggTesterWithFieldType(funcName, fts, types.NewFieldType(rt), numRows, results...) +} + +func buildMultiArgsAggTesterWithFieldType(funcName string, fts []*types.FieldType, rt *types.FieldType, numRows int, results ...interface{}) multiArgsAggTest { + dataGens := make([]func(i int) types.Datum, len(fts)) + for i := 0; i < len(fts); i++ { + dataGens[i] = getDataGenFunc(fts[i]) + } + mt := multiArgsAggTest{ + dataTypes: fts, + retType: rt, + numRows: numRows, + funcName: funcName, + dataGens: dataGens, + } + for _, result := range results { + mt.results = append(mt.results, types.NewDatum(result)) + } + return mt +} + func getDataGenFunc(ft *types.FieldType) func(i int) types.Datum { switch ft.Tp { case mysql.TypeLonglong: @@ -250,3 +352,73 @@ func (s *testSuite) testAggFunc(c *C, p aggTest) { c.Assert(err, IsNil) c.Assert(result, Equals, 0) } + +func (s *testSuite) testMultiArgsAggFunc(c *C, p multiArgsAggTest) { + srcChk := chunk.NewChunkWithCapacity(p.dataTypes, p.numRows) + for i := 0; i < p.numRows; i++ { + for j := 0; j < len(p.dataGens); j++ { + fdt := p.dataGens[j](i) + srcChk.AppendDatum(j, &fdt) + } + } + srcChk.AppendDatum(0, &types.Datum{}) + + args := make([]expression.Expression, len(p.dataTypes)) + for k := 0; k < len(p.dataTypes); k++ { + args[k] = &expression.Column{RetType: p.dataTypes[k], Index: k} + } + + desc, err := aggregation.NewAggFuncDesc(s.ctx, p.funcName, args, false) + c.Assert(err, IsNil) + finalFunc := aggfuncs.Build(s.ctx, desc, 0) + finalPr := finalFunc.AllocPartialResult() + resultChk := chunk.NewChunkWithCapacity([]*types.FieldType{desc.RetTp}, 1) + + iter := chunk.NewIterator4Chunk(srcChk) + for row := iter.Begin(); row != iter.End(); row = iter.Next() { + finalFunc.UpdatePartialResult(s.ctx, []chunk.Row{row}, finalPr) + } + finalFunc.AppendFinalResult2Chunk(s.ctx, finalPr, resultChk) + dt := resultChk.GetRow(0).GetDatum(0, desc.RetTp) + result, err := dt.CompareDatum(s.ctx.GetSessionVars().StmtCtx, &p.results[1]) + c.Assert(err, IsNil) + c.Assert(result, Equals, 0) + + // test the empty input + resultChk.Reset() + finalFunc.ResetPartialResult(finalPr) + finalFunc.AppendFinalResult2Chunk(s.ctx, finalPr, resultChk) + dt = resultChk.GetRow(0).GetDatum(0, desc.RetTp) + result, err = dt.CompareDatum(s.ctx.GetSessionVars().StmtCtx, &p.results[0]) + c.Assert(err, IsNil) + c.Assert(result, Equals, 0) + + // test the agg func with distinct + desc, err = aggregation.NewAggFuncDesc(s.ctx, p.funcName, args, true) + c.Assert(err, IsNil) + finalFunc = aggfuncs.Build(s.ctx, desc, 0) + finalPr = finalFunc.AllocPartialResult() + + resultChk.Reset() + iter = chunk.NewIterator4Chunk(srcChk) + for row := iter.Begin(); row != iter.End(); row = iter.Next() { + finalFunc.UpdatePartialResult(s.ctx, []chunk.Row{row}, finalPr) + } + for row := iter.Begin(); row != iter.End(); row = iter.Next() { + finalFunc.UpdatePartialResult(s.ctx, []chunk.Row{row}, finalPr) + } + finalFunc.AppendFinalResult2Chunk(s.ctx, finalPr, resultChk) + dt = resultChk.GetRow(0).GetDatum(0, desc.RetTp) + result, err = dt.CompareDatum(s.ctx.GetSessionVars().StmtCtx, &p.results[1]) + c.Assert(err, IsNil) + c.Assert(result, Equals, 0) + + // test the empty input + resultChk.Reset() + finalFunc.ResetPartialResult(finalPr) + finalFunc.AppendFinalResult2Chunk(s.ctx, finalPr, resultChk) + dt = resultChk.GetRow(0).GetDatum(0, desc.RetTp) + result, err = dt.CompareDatum(s.ctx.GetSessionVars().StmtCtx, &p.results[0]) + c.Assert(err, IsNil) + c.Assert(result, Equals, 0) +} diff --git a/executor/aggfuncs/aggfuncs.go b/executor/aggfuncs/aggfuncs.go index e5db2b389d345..74c2b79c2e2e2 100644 --- a/executor/aggfuncs/aggfuncs.go +++ b/executor/aggfuncs/aggfuncs.go @@ -83,6 +83,9 @@ var ( // All the AggFunc implementations for "BIT_AND" are listed here. _ AggFunc = (*bitAndUint64)(nil) + + // All the AggFunc implementations for "JSON_OBJECTAGG" are listed here + _ AggFunc = (*jsonObjectAgg)(nil) ) // PartialResult represents data structure to store the partial result for the diff --git a/executor/aggfuncs/builder.go b/executor/aggfuncs/builder.go index 93083688f8b7c..21de0ef2120a7 100644 --- a/executor/aggfuncs/builder.go +++ b/executor/aggfuncs/builder.go @@ -53,6 +53,8 @@ func Build(ctx sessionctx.Context, aggFuncDesc *aggregation.AggFuncDesc, ordinal return buildBitAnd(aggFuncDesc, ordinal) case ast.AggFuncVarPop: return buildVarPop(aggFuncDesc, ordinal) + case ast.AggFuncJsonObjectAgg: + return buildJSONObjectAgg(aggFuncDesc, ordinal) } return nil } @@ -371,6 +373,20 @@ func buildVarPop(aggFuncDesc *aggregation.AggFuncDesc, ordinal int) AggFunc { } } +// buildJSONObjectAgg builds the AggFunc implementation for function "json_objectagg". +func buildJSONObjectAgg(aggFuncDesc *aggregation.AggFuncDesc, ordinal int) AggFunc { + base := baseAggFunc{ + args: aggFuncDesc.Args, + ordinal: ordinal, + } + switch aggFuncDesc.Mode { + case aggregation.DedupMode: + return nil + default: + return &jsonObjectAgg{base} + } +} + // buildRowNumber builds the AggFunc implementation for function "ROW_NUMBER". func buildRowNumber(aggFuncDesc *aggregation.AggFuncDesc, ordinal int) AggFunc { base := baseAggFunc{ diff --git a/executor/aggfuncs/func_json_objectagg.go b/executor/aggfuncs/func_json_objectagg.go new file mode 100644 index 0000000000000..48bd1d8e80a6d --- /dev/null +++ b/executor/aggfuncs/func_json_objectagg.go @@ -0,0 +1,114 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package aggfuncs + +import ( + "github.com/pingcap/errors" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/types/json" + "github.com/pingcap/tidb/util/chunk" +) + +type jsonObjectAgg struct { + baseAggFunc +} + +type partialResult4JsonObjectAgg struct { + entries map[string]interface{} +} + +func (e *jsonObjectAgg) AllocPartialResult() PartialResult { + p := partialResult4JsonObjectAgg{} + p.entries = make(map[string]interface{}) + return PartialResult(&p) +} + +func (e *jsonObjectAgg) ResetPartialResult(pr PartialResult) { + p := (*partialResult4JsonObjectAgg)(pr) + p.entries = make(map[string]interface{}) +} + +func (e *jsonObjectAgg) AppendFinalResult2Chunk(sctx sessionctx.Context, pr PartialResult, chk *chunk.Chunk) error { + p := (*partialResult4JsonObjectAgg)(pr) + if len(p.entries) == 0 { + chk.AppendNull(e.ordinal) + return nil + } + + // appendBinary does not support some type such as uint8、types.time，so convert is needed here + for key, val := range p.entries { + switch x := val.(type) { + case *types.MyDecimal: + float64Val, err := x.ToFloat64() + if err != nil { + return errors.Trace(err) + } + p.entries[key] = float64Val + case []uint8, types.Time, types.Duration: + strVal, err := types.ToString(x) + if err != nil { + return errors.Trace(err) + } + p.entries[key] = strVal + } + } + + chk.AppendJSON(e.ordinal, json.CreateBinary(p.entries)) + return nil +} + +func (e *jsonObjectAgg) UpdatePartialResult(sctx sessionctx.Context, rowsInGroup []chunk.Row, pr PartialResult) error { + p := (*partialResult4JsonObjectAgg)(pr) + for _, row := range rowsInGroup { + key, err := e.args[0].Eval(row) + if err != nil { + return errors.Trace(err) + } + + value, err := e.args[1].Eval(row) + if err != nil { + return errors.Trace(err) + } + + if key.IsNull() { + return json.ErrJSONDocumentNULLKey + } + + // the result json's key is string, so it needs to convert the first arg to string + keyString, err := key.ToString() + if err != nil { + return errors.Trace(err) + } + + realVal := value.GetValue() + switch x := realVal.(type) { + case nil, bool, int64, uint64, float64, string, json.BinaryJSON, *types.MyDecimal, []uint8, types.Time, types.Duration: + p.entries[keyString] = realVal + default: + return json.ErrUnsupportedSecondArgumentType.GenWithStackByArgs(x) + } + } + return nil +} + +func (e *jsonObjectAgg) MergePartialResult(sctx sessionctx.Context, src PartialResult, dst PartialResult) error { + p1, p2 := (*partialResult4JsonObjectAgg)(src), (*partialResult4JsonObjectAgg)(dst) + // When the result of this function is normalized, values having duplicate keys are discarded, + // and only the last value encountered is used with that key in the returned object + for k, v := range p1.entries { + p2.entries[k] = v + } + return nil +} diff --git a/executor/aggfuncs/func_json_objectagg_test.go b/executor/aggfuncs/func_json_objectagg_test.go new file mode 100644 index 0000000000000..82354a858eae7 --- /dev/null +++ b/executor/aggfuncs/func_json_objectagg_test.go @@ -0,0 +1,104 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package aggfuncs_test + +import ( + . "github.com/pingcap/check" + "github.com/pingcap/parser/ast" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/types/json" +) + +func (s *testSuite) TestMergePartialResult4JsonObjectagg(c *C) { + typeList := []byte{mysql.TypeLonglong, mysql.TypeDouble, mysql.TypeString, mysql.TypeJSON} + var argCombines [][]byte + for i := 0; i < len(typeList); i++ { + for j := 0; j < len(typeList); j++ { + argTypes := []byte{typeList[i], typeList[j]} + argCombines = append(argCombines, argTypes) + } + } + + var tests []multiArgsAggTest + numRows := 5 + + for k := 0; k < len(argCombines); k++ { + entries1 := make(map[string]interface{}) + entries2 := make(map[string]interface{}) + + argTypes := argCombines[k] + fGenFunc := getDataGenFunc(types.NewFieldType(argTypes[0])) + sGenFunc := getDataGenFunc(types.NewFieldType(argTypes[1])) + + for m := 0; m < numRows; m++ { + firstArg := fGenFunc(m) + secondArg := sGenFunc(m) + keyString, _ := firstArg.ToString() + entries1[keyString] = secondArg.GetValue() + } + + for m := 2; m < numRows; m++ { + firstArg := fGenFunc(m) + secondArg := sGenFunc(m) + keyString, _ := firstArg.ToString() + entries2[keyString] = secondArg.GetValue() + } + + aggTest := buildMultiArgsAggTester(ast.AggFuncJsonObjectAgg, argTypes, mysql.TypeJSON, numRows, json.CreateBinary(entries1), json.CreateBinary(entries2), json.CreateBinary(entries1)) + + tests = append(tests, aggTest) + } + + for _, test := range tests { + s.testMultiArgsMergePartialResult(c, test) + } +} + +func (s *testSuite) TestJsonObjectagg(c *C) { + typeList := []byte{mysql.TypeLonglong, mysql.TypeDouble, mysql.TypeString, mysql.TypeJSON} + var argCombines [][]byte + for i := 0; i < len(typeList); i++ { + for j := 0; j < len(typeList); j++ { + argTypes := []byte{typeList[i], typeList[j]} + argCombines = append(argCombines, argTypes) + } + } + + var tests []multiArgsAggTest + numRows := 5 + + for k := 0; k < len(argCombines); k++ { + entries := make(map[string]interface{}) + + argTypes := argCombines[k] + fGenFunc := getDataGenFunc(types.NewFieldType(argTypes[0])) + sGenFunc := getDataGenFunc(types.NewFieldType(argTypes[1])) + + for m := 0; m < numRows; m++ { + firstArg := fGenFunc(m) + secondArg := sGenFunc(m) + keyString, _ := firstArg.ToString() + entries[keyString] = secondArg.GetValue() + } + + aggTest := buildMultiArgsAggTester(ast.AggFuncJsonObjectAgg, argTypes, mysql.TypeJSON, numRows, nil, json.CreateBinary(entries)) + + tests = append(tests, aggTest) + } + + for _, test := range tests { + s.testMultiArgsAggFunc(c, test) + } +} diff --git a/expression/aggregation/agg_to_pb.go b/expression/aggregation/agg_to_pb.go index dfff74c0b9918..6ae07153690bc 100644 --- a/expression/aggregation/agg_to_pb.go +++ b/expression/aggregation/agg_to_pb.go @@ -53,6 +53,8 @@ func AggFuncToPBExpr(sc *stmtctx.StatementContext, client kv.Client, aggFunc *Ag tp = tipb.ExprType_Agg_BitAnd case ast.AggFuncVarPop: tp = tipb.ExprType_VarPop + case ast.AggFuncJsonObjectAgg: + tp = tipb.ExprType_JsonObjectAgg } if !client.IsRequestTypeSupported(kv.ReqTypeSelect, int64(tp)) { return nil diff --git a/expression/aggregation/base_func.go b/expression/aggregation/base_func.go index efad3651809a3..9179a87fab211 100644 --- a/expression/aggregation/base_func.go +++ b/expression/aggregation/base_func.go @@ -109,6 +109,8 @@ func (a *baseFuncDesc) typeInfer(ctx sessionctx.Context) error { a.typeInfer4LeadLag(ctx) case ast.AggFuncVarPop: a.typeInfer4VarPop(ctx) + case ast.AggFuncJsonObjectAgg: + a.typeInfer4JsonFuncs(ctx) default: return errors.Errorf("unsupported agg function: %s", a.Name) } @@ -204,6 +206,11 @@ func (a *baseFuncDesc) typeInfer4BitFuncs(ctx sessionctx.Context) { // TODO: a.Args[0] = expression.WrapWithCastAsInt(ctx, a.Args[0]) } +func (a *baseFuncDesc) typeInfer4JsonFuncs(ctx sessionctx.Context) { + a.RetTp = types.NewFieldType(mysql.TypeJSON) + types.SetBinChsClnFlag(a.RetTp) +} + func (a *baseFuncDesc) typeInfer4NumberFuncs() { a.RetTp = types.NewFieldType(mysql.TypeLonglong) a.RetTp.Flen = 21 @@ -274,11 +281,12 @@ func (a *baseFuncDesc) GetDefaultValue() (v types.Datum) { // We do not need to wrap cast upon these functions, // since the EvalXXX method called by the arg is determined by the corresponding arg type. var noNeedCastAggFuncs = map[string]struct{}{ - ast.AggFuncCount: {}, - ast.AggFuncMax: {}, - ast.AggFuncMin: {}, - ast.AggFuncFirstRow: {}, - ast.WindowFuncNtile: {}, + ast.AggFuncCount: {}, + ast.AggFuncMax: {}, + ast.AggFuncMin: {}, + ast.AggFuncFirstRow: {}, + ast.WindowFuncNtile: {}, + ast.AggFuncJsonObjectAgg: {}, } // WrapCastForAggArgs wraps the args of an aggregate function with a cast function. diff --git a/expression/integration_test.go b/expression/integration_test.go index 5b57c3c4e0f89..a7d6274f76dc6 100755 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -3590,6 +3590,44 @@ func (s *testIntegrationSuite) TestAggregationBuiltinGroupConcat(c *C) { tk.MustQuery("select * from d").Check(testkit.Rows("hello,h")) } +func (s *testIntegrationSuite) TestAggregationBuiltinJSONObjectAgg(c *C) { + defer s.cleanEnv(c) + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + tk.MustExec("drop table if exists t;") + tk.MustExec(`CREATE TABLE t ( + a int(11), + b varchar(100), + c decimal(3,2), + d json, + e date, + f time, + g datetime DEFAULT '2012-01-01', + h timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + i char(36), + j text(50));`) + + tk.MustExec(`insert into t values(1, 'ab', 5.5, '{"id": 1}', '2020-01-10', '11:12:13', '2020-01-11', '0000-00-00 00:00:00', 'first', 'json_objectagg_test');`) + + result := tk.MustQuery("select json_objectagg(a, b) from t group by a order by a;") + result.Check(testkit.Rows(`{"1": "ab"}`)) + result = tk.MustQuery("select json_objectagg(b, c) from t group by b order by b;") + result.Check(testkit.Rows(`{"ab": 5.5}`)) + result = tk.MustQuery("select json_objectagg(e, f) from t group by e order by e;") + result.Check(testkit.Rows(`{"2020-01-10": "11:12:13"}`)) + result = tk.MustQuery("select json_objectagg(f, g) from t group by f order by f;") + result.Check(testkit.Rows(`{"11:12:13": "2020-01-11 00:00:00"}`)) + result = tk.MustQuery("select json_objectagg(g, h) from t group by g order by g;") + result.Check(testkit.Rows(`{"2020-01-11 00:00:00": "0000-00-00 00:00:00"}`)) + result = tk.MustQuery("select json_objectagg(h, i) from t group by h order by h;") + result.Check(testkit.Rows(`{"0000-00-00 00:00:00": "first"}`)) + result = tk.MustQuery("select json_objectagg(i, j) from t group by i order by i;") + result.Check(testkit.Rows(`{"first": "json_objectagg_test"}`)) + result = tk.MustQuery("select json_objectagg(a, null) from t group by a order by a;") + result.Check(testkit.Rows(`{"1": null}`)) +} + func (s *testIntegrationSuite2) TestOtherBuiltin(c *C) { defer s.cleanEnv(c) tk := testkit.NewTestKit(c, s.store) diff --git a/go.mod b/go.mod index 5d9d3f7a9d32b..b6288f53bb111 100644 --- a/go.mod +++ b/go.mod @@ -37,12 +37,12 @@ require ( github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 github.com/pingcap/kvproto v0.0.0-20200108025604-a4dc183d2af5 - github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 - github.com/pingcap/parser v0.0.0-20200109073933-a9496438d77d + github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd + github.com/pingcap/parser v0.0.0-20200120100653-1d87b3907217 github.com/pingcap/pd v1.1.0-beta.0.20191219054547-4d65bbefbc6d github.com/pingcap/sysutil v0.0.0-20191216090214-5f9620d22b3b github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible - github.com/pingcap/tipb v0.0.0-20191227083941-3996eff010dc + github.com/pingcap/tipb v0.0.0-20200103084511-1d37e605f65d github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 github.com/prometheus/common v0.4.1 @@ -67,7 +67,7 @@ require ( golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 golang.org/x/text v0.3.2 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect - golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 + golang.org/x/tools v0.0.0-20200119215504-eb0d8dd85bcc google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514 // indirect google.golang.org/grpc v1.25.1 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect diff --git a/go.sum b/go.sum index 346b203aa3a20..4cc0e9f9a9d56 100644 --- a/go.sum +++ b/go.sum @@ -205,8 +205,10 @@ github.com/pingcap/kvproto v0.0.0-20200108025604-a4dc183d2af5 h1:RUxQExD5yubAjWG github.com/pingcap/kvproto v0.0.0-20200108025604-a4dc183d2af5/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/parser v0.0.0-20200109073933-a9496438d77d h1:4QwSJRxmBjTB9ssJNWg2f2bDm5rqnHCUUjMh4N1QOOY= -github.com/pingcap/parser v0.0.0-20200109073933-a9496438d77d/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= +github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM= +github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= +github.com/pingcap/parser v0.0.0-20200120100653-1d87b3907217 h1:UtieYveNGV84dIdb01UIXMQzGIyGLRiAoGXgLe9rJws= +github.com/pingcap/parser v0.0.0-20200120100653-1d87b3907217/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= github.com/pingcap/pd v1.1.0-beta.0.20191219054547-4d65bbefbc6d h1:Ui80aiLTyd0EZD56o2tjFRYpHfhazBjtBdKeR8UoTFY= github.com/pingcap/pd v1.1.0-beta.0.20191219054547-4d65bbefbc6d/go.mod h1:CML+b1JVjN+VbDijaIcUSmuPgpDjXEY7UiOx5yDP8eE= github.com/pingcap/sysutil v0.0.0-20191216090214-5f9620d22b3b h1:EEyo/SCRswLGuSk+7SB86Ak1p8bS6HL1Mi4Dhyuv6zg= @@ -214,8 +216,8 @@ github.com/pingcap/sysutil v0.0.0-20191216090214-5f9620d22b3b/go.mod h1:EB/852NM github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible h1:H1jg0aDWz2SLRh3hNBo2HFtnuHtudIUvBumU7syRkic= github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= -github.com/pingcap/tipb v0.0.0-20191227083941-3996eff010dc h1:IOKsFObJ4GZwAgyuhdJKg3oKCzWcoBFfHhpq2TOn5H0= -github.com/pingcap/tipb v0.0.0-20191227083941-3996eff010dc/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tipb v0.0.0-20200103084511-1d37e605f65d h1:ohGnm9xZ7pIysk7quOC7lZa8kOm9Pl5TMyjBThXqy2U= +github.com/pingcap/tipb v0.0.0-20200103084511-1d37e605f65d/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -382,8 +384,10 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200114235610-7ae403b6b589 h1:rjUrONFu4kLchcZTfp3/96bR8bW8dIa8uz3cR5n0cgM= +golang.org/x/tools v0.0.0-20200114235610-7ae403b6b589/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200119215504-eb0d8dd85bcc h1:ZA7KFRdqWZkBr0/YbHm1h08vDJ5gQdjVG/8L153z5c4= +golang.org/x/tools v0.0.0-20200119215504-eb0d8dd85bcc/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/planner/core/rule_aggregation_push_down.go b/planner/core/rule_aggregation_push_down.go index 7e0a4933dc434..d5bb65361d6e5 100644 --- a/planner/core/rule_aggregation_push_down.go +++ b/planner/core/rule_aggregation_push_down.go @@ -35,7 +35,7 @@ type aggregationPushDownSolver struct { // Currently we don't support avg and concat. func (a *aggregationPushDownSolver) isDecomposable(fun *aggregation.AggFuncDesc) bool { switch fun.Name { - case ast.AggFuncAvg, ast.AggFuncGroupConcat, ast.AggFuncVarPop: + case ast.AggFuncAvg, ast.AggFuncGroupConcat, ast.AggFuncVarPop, ast.AggFuncJsonObjectAgg: // TODO: Support avg push down. return false case ast.AggFuncMax, ast.AggFuncMin, ast.AggFuncFirstRow: diff --git a/types/json/constants.go b/types/json/constants.go index 1f8186fb807e2..36988b980c54e 100644 --- a/types/json/constants.go +++ b/types/json/constants.go @@ -216,18 +216,24 @@ var ( ErrInvalidJSONPathWildcard = terror.ClassJSON.New(mysql.ErrInvalidJSONPathWildcard, mysql.MySQLErrName[mysql.ErrInvalidJSONPathWildcard]) // ErrInvalidJSONContainsPathType means invalid JSON contains path type. ErrInvalidJSONContainsPathType = terror.ClassJSON.New(mysql.ErrInvalidJSONContainsPathType, mysql.MySQLErrName[mysql.ErrInvalidJSONContainsPathType]) + // ErrJSONDocumentNULLKey means that json's key is null + ErrJSONDocumentNULLKey = terror.ClassJSON.New(mysql.ErrJSONDocumentNULLKey, mysql.MySQLErrName[mysql.ErrJSONDocumentNULLKey]) // ErrInvalidJSONPathArrayCell means invalid JSON path for an array cell. ErrInvalidJSONPathArrayCell = terror.ClassJSON.New(mysql.ErrInvalidJSONPathArrayCell, mysql.MySQLErrName[mysql.ErrInvalidJSONPathArrayCell]) + // ErrUnsupportedSecondArgumentType means unsupported second argument type in json_objectagg + ErrUnsupportedSecondArgumentType = terror.ClassJSON.New(mysql.ErrUnsupportedSecondArgumentType, mysql.MySQLErrName[mysql.ErrUnsupportedSecondArgumentType]) ) func init() { terror.ErrClassToMySQLCodes[terror.ClassJSON] = map[terror.ErrCode]uint16{ - mysql.ErrInvalidJSONText: mysql.ErrInvalidJSONText, - mysql.ErrInvalidJSONPath: mysql.ErrInvalidJSONPath, - mysql.ErrInvalidJSONData: mysql.ErrInvalidJSONData, - mysql.ErrInvalidJSONPathWildcard: mysql.ErrInvalidJSONPathWildcard, - mysql.ErrInvalidJSONContainsPathType: mysql.ErrInvalidJSONContainsPathType, - mysql.ErrInvalidJSONPathArrayCell: mysql.ErrInvalidJSONPathArrayCell, + mysql.ErrInvalidJSONText: mysql.ErrInvalidJSONText, + mysql.ErrInvalidJSONPath: mysql.ErrInvalidJSONPath, + mysql.ErrInvalidJSONData: mysql.ErrInvalidJSONData, + mysql.ErrInvalidJSONPathWildcard: mysql.ErrInvalidJSONPathWildcard, + mysql.ErrInvalidJSONContainsPathType: mysql.ErrInvalidJSONContainsPathType, + mysql.ErrJSONDocumentNULLKey: mysql.ErrJSONDocumentNULLKey, + mysql.ErrInvalidJSONPathArrayCell: mysql.ErrInvalidJSONPathArrayCell, + mysql.ErrUnsupportedSecondArgumentType: mysql.ErrUnsupportedSecondArgumentType, } }