From e07afc2dbc10ac279e3ac43f0b8ea1f501dfc0e9 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 12 Oct 2023 20:59:51 +0800 Subject: [PATCH 01/49] fix: join groupBy, project aggr, joined field --- packages/core/src/driver.ts | 4 +-- packages/mongo/src/utils.ts | 6 +++-- packages/mysql/src/index.ts | 2 +- packages/sql-utils/src/index.ts | 6 ++++- packages/sqlite/src/index.ts | 2 +- packages/tests/src/selection.ts | 48 ++++++++++++++++++++++++++++++++- 6 files changed, 60 insertions(+), 8 deletions(-) diff --git a/packages/core/src/driver.ts b/packages/core/src/driver.ts index 8694b1e4..b4df705a 100644 --- a/packages/core/src/driver.ts +++ b/packages/core/src/driver.ts @@ -119,14 +119,14 @@ export class Database { sel.args[0].having = Eval.and(query(...tables.map(name => sel.row[name]))) } sel.args[0].optional = Object.fromEntries(tables.map((name, index) => [name, optional?.[index]])) - return sel + return new Selection(this.getDriver(sel), sel) } else { const sel = new Selection(this.getDriver(tables[0]), tables) if (typeof query === 'function') { sel.args[0].having = Eval.and(query(sel.row)) } sel.args[0].optional = optional - return sel + return new Selection(this.getDriver(sel), sel) } } diff --git a/packages/mongo/src/utils.ts b/packages/mongo/src/utils.ts index 9f638224..f8b0852c 100644 --- a/packages/mongo/src/utils.ts +++ b/packages/mongo/src/utils.ts @@ -240,10 +240,12 @@ export class Transformer { stages.push({ $match: { $expr } }) } stages.push({ $project }) + $group['_id'] = model.parse($group['_id'], false) } else if (fields) { - const $project = valueMap(fields, (expr) => this.eval(expr)) + const $group: Dict = { _id: null } + const $project = valueMap(fields, (expr) => this.eval(expr, $group)) $project._id = 0 - stages.push({ $project }) + stages.push(...Object.keys($group).length === 1 ? [] : [{ $group }], { $project }) } else { const $project: Dict = { _id: 0 } for (const key in model.fields) { diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index 5571c9e2..ac798fb8 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -440,7 +440,7 @@ export class MySQLDriver extends Driver { const builder = new MySQLBuilder(sel.tables) const output = builder.parseEval(expr) const inner = builder.get(sel.table as Selection, true) - const [data] = await this.queue(`SELECT ${output} AS value FROM ${inner} ${sel.ref}`) + const [data] = await this.queue(`SELECT ${output} AS value FROM ${inner}`) return data.value } diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index b17490ed..f93c03a7 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -304,11 +304,15 @@ export class Builder { if (filter !== '1') { suffix = ` WHERE ${filter}` + suffix } + + if (inline && !args[0].fields && !suffix) { + return (prefix.startsWith('(') && prefix.endsWith(')')) ? `${prefix} ${ref}` : prefix + } + if (!prefix.includes(' ') || prefix.startsWith('(')) { suffix = ` ${ref}` + suffix } - if (inline && !args[0].fields && !suffix) return prefix const result = `SELECT ${keys} FROM ${prefix}${suffix}` return inline ? `(${result})` : result } diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index 07694b3a..dc457227 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -318,7 +318,7 @@ export class SQLiteDriver extends Driver { const builder = new SQLiteBuilder(sel.tables) const output = builder.parseEval(expr) const inner = builder.get(sel.table as Selection, true) - const { value } = this.#get(`SELECT ${output} AS value FROM ${inner} ${sel.ref}`) + const { value } = this.#get(`SELECT ${output} AS value FROM ${inner}`) return value } diff --git a/packages/tests/src/selection.ts b/packages/tests/src/selection.ts index 3a86d8e2..be00402d 100644 --- a/packages/tests/src/selection.ts +++ b/packages/tests/src/selection.ts @@ -1,6 +1,7 @@ import { $, Database } from '@minatojs/core' import { expect } from 'chai' import { setup } from './utils' +import MongoDriver from '@minatojs/driver-mongo' interface Foo { id: number @@ -26,7 +27,7 @@ function SelectionTests(database: Database) { value: 'integer', }) - database.migrate('foo', { deprecated: 'unsigned' }, async () => {}) + database.migrate('foo', { deprecated: 'unsigned' }, async () => { }) database.extend('bar', { id: 'unsigned', @@ -139,6 +140,33 @@ namespace SelectionTests { { id: 10 }, ]) }) + + it('aggregate', async () => { + await expect(database + .select('foo') + .project({ + count: row => $.count(row.id), + max: row => $.max(row.id), + min: row => $.min(row.id), + avg: row => $.avg(row.id), + }) + .execute() + ).to.eventually.deep.equal([ + { avg: 2, count: 3, max: 3, min: 1 }, + ]) + + await expect(database.select('foo') + .groupBy({}, { + count: row => $.count(row.id), + max: row => $.max(row.id), + min: row => $.min(row.id), + avg: row => $.avg(row.id), + }) + .execute() + ).to.eventually.deep.equal([ + { avg: 2, count: 3, max: 3, min: 1 }, + ]) + }) } export function aggregate(database: Database) { @@ -258,6 +286,24 @@ namespace SelectionTests { .execute() ).to.eventually.have.length(2) }) + + it('group', async () => { + await expect(database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) + .groupBy('foo', { count: row => $.sum(row.bar.uid) }) + .orderBy(row => row.foo.id) + .execute()).to.eventually.deep.equal([ + { foo: { id: 1, value: 0 }, count: 6 }, + { foo: { id: 2, value: 2 }, count: 1 }, + { foo: { id: 3, value: 2 }, count: 1 }, + ]) + }) + + it('aggregate', async () => { + await expect(database + .join(['foo', 'bar'] as const) + .execute(row => $.count(row.bar.id)) + ).to.eventually.equal(6) + }) } } From 74570231a14b562b0e42bd80acb4a48ce3cf6925 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Sat, 14 Oct 2023 21:52:29 +0800 Subject: [PATCH 02/49] stage: add runtimeType --- packages/core/src/driver.ts | 8 ++-- packages/core/src/eval.ts | 78 +++++++++++++++++++++------------ packages/core/src/model.ts | 22 +++++++++- packages/core/src/runtime.ts | 30 +++++++++++++ packages/core/src/selection.ts | 16 ++++--- packages/sql-utils/src/index.ts | 2 +- 6 files changed, 116 insertions(+), 40 deletions(-) create mode 100644 packages/core/src/runtime.ts diff --git a/packages/core/src/driver.ts b/packages/core/src/driver.ts index b4df705a..21053795 100644 --- a/packages/core/src/driver.ts +++ b/packages/core/src/driver.ts @@ -1,5 +1,5 @@ import { Awaitable, Dict, Intersect, makeArray, MaybeArray, valueMap } from 'cosmokit' -import { Eval, Update } from './eval' +import { Eval, getExprRuntimeType, Update } from './eval' import { Field, Model } from './model' import { Query } from './query' import { Flatten, Indexable, Keys, Row } from './utils' @@ -227,8 +227,9 @@ export abstract class Driver { if (table instanceof Selection) { if (!table.args[0].fields) return table.model const model = new Model('temp') - model.fields = valueMap(table.args[0].fields, (_, key) => ({ + model.fields = valueMap(table.args[0].fields, (expr, key) => ({ type: 'expr', + runtimeType: getExprRuntimeType(expr), })) return model } @@ -240,7 +241,8 @@ export abstract class Driver { if (submodel.fields[field]!.deprecated) continue model.fields[`${key}.${field}`] = { type: 'expr', - expr: { $: [key, field] } as any, + expr: Eval('', [key, field], Field.getRuntimeType(submodel.fields[field]!)), + runtimeType: Field.getRuntimeType(submodel.fields[field]!), } } } diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index c27a22a3..d3c2003f 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -1,10 +1,24 @@ -import { defineProperty, isNullable } from 'cosmokit' +import { defineProperty, isNullable, valueMap } from 'cosmokit' +import { RuntimeType } from './runtime' import { Comparable, Flatten, isComparable, makeRegExp } from './utils' export function isEvalExpr(value: any): value is Eval.Expr { return value && Object.keys(value).some(key => key.startsWith('$')) } +export function getExprRuntimeType(value: any): RuntimeType { + if (isNullable(value)) return RuntimeType.any + if (RuntimeType.test(value)) return value + if (isEvalExpr(value)) return value[kRuntimeType] + else if (typeof value === 'string') return RuntimeType.create('string') + else if (typeof value === 'number') return RuntimeType.create('number') + else if (typeof value === 'boolean') return RuntimeType.create('boolean') + else if (value instanceof Date) return RuntimeType.create('date') + else if (value instanceof RegExp) return RuntimeType.create('regexp') + else if (Array.isArray(value)) return RuntimeType.create(RuntimeType.merge(...value), true) + else return RuntimeType.create(valueMap(value, getExprRuntimeType)) +} + type $Date = Date type $RegExp = RegExp @@ -24,12 +38,14 @@ export type Eval = const kExpr = Symbol('expr') const kType = Symbol('type') const kAggr = Symbol('aggr') +const kRuntimeType = Symbol('RuntimeType') export namespace Eval { export interface Expr { [kExpr]: true [kType]?: T [kAggr]?: A + [kRuntimeType]: RuntimeType } export type Number = number | Expr @@ -48,7 +64,7 @@ export namespace Eval { } export interface Static { - (key: string, value: any): Eval.Expr + (key: string, value: any, type: RuntimeType): Eval.Expr // univeral if(cond: Any, vThen: T | Expr, vElse: T | Expr): Expr @@ -96,22 +112,22 @@ export namespace Eval { } } -export const Eval = ((key, value) => defineProperty({ ['$' + key]: value }, kExpr, true)) as Eval.Static +export const Eval = ((key, value, type) => defineProperty({ ['$' + key]: value, [kRuntimeType]: type }, kExpr, true)) as Eval.Static const operators = {} as Record<`$${keyof Eval.Static}`, (args: any, data: any) => any> operators['$'] = getRecursive type UnaryCallback = T extends (value: infer R) => Eval.Expr ? (value: R, data: any[]) => S : never -function unary(key: K, callback: UnaryCallback): Eval.Static[K] { +function unary(key: K, callback: UnaryCallback, type: RuntimeType): Eval.Static[K] { operators[`$${key}`] = callback - return (value: any) => Eval(key, value) as any + return (value: any) => Eval(key, value, type) as any } type MultivariateCallback = T extends (...args: infer R) => Eval.Expr ? (args: R, data: any) => S : never -function multary(key: K, callback: MultivariateCallback): Eval.Static[K] { +function multary(key: K, callback: MultivariateCallback, type: RuntimeType): Eval.Static[K] { operators[`$${key}`] = callback - return (...args: any) => Eval(key, args) as any + return (...args: any[]) => Eval(key, args, type) as any } type BinaryCallback = T extends (...args: any[]) => Eval.Expr ? (...args: any[]) => S : never @@ -122,10 +138,10 @@ function comparator(key: K, callback: BinaryCallbac if (isNullable(left) || isNullable(right)) return true return callback(left.valueOf(), right.valueOf()) } - return (...args: any) => Eval(key, args) as any + return (...args: any[]) => Eval(key, args, RuntimeType.create('boolean')) as any } -Eval.switch = (branches, vDefault) => Eval('switch', { branches, default: vDefault }) +Eval.switch = (branches, vDefault) => Eval('switch', { branches, default: vDefault }, getExprRuntimeType(vDefault)) operators.$switch = (args, data) => { for (const branch of args.branches) { if (executeEval(data, branch.case)) return executeEval(data, branch.then) @@ -134,14 +150,22 @@ operators.$switch = (args, data) => { } // univeral -Eval.if = multary('if', ([cond, vThen, vElse], data) => executeEval(data, cond) ? executeEval(data, vThen) : executeEval(data, vElse)) -Eval.ifNull = multary('ifNull', ([value, fallback], data) => executeEval(data, value) ?? executeEval(data, fallback)) +operators.$if = ([cond, vThen, vElse], data) => executeEval(data, cond) ? executeEval(data, vThen) : executeEval(data, vElse) +Eval.if = (...args: any[]) => { + return Eval('if', args, RuntimeType.merge(...(args.slice(1)))) +} + +operators.$ifNull = ([value, fallback], data) => executeEval(data, value) ?? executeEval(data, fallback) +Eval.ifNull = (...args: any[]) => { + return Eval('ifNull', args, RuntimeType.merge(...args)) +} // arithmetic -Eval.add = multary('add', (args, data) => args.reduce((prev, curr) => prev + executeEval(data, curr), 0)) -Eval.mul = Eval.multiply = multary('multiply', (args, data) => args.reduce((prev, curr) => prev * executeEval(data, curr), 1)) -Eval.sub = Eval.subtract = multary('subtract', ([left, right], data) => executeEval(data, left) - executeEval(data, right)) -Eval.div = Eval.divide = multary('divide', ([left, right], data) => executeEval(data, left) / executeEval(data, right)) +Eval.add = multary('add', (args, data) => args.reduce((prev, curr) => prev + executeEval(data, curr), 0), RuntimeType.create('number')) +Eval.mul = Eval.multiply = multary( + 'multiply', (args, data) => args.reduce((prev, curr) => prev * executeEval(data, curr), 1), RuntimeType.create('number')) +Eval.sub = Eval.subtract = multary('subtract', ([left, right], data) => executeEval(data, left) - executeEval(data, right), RuntimeType.create('number')) +Eval.div = Eval.divide = multary('divide', ([left, right], data) => executeEval(data, left) / executeEval(data, right), RuntimeType.create('number')) // comparison Eval.eq = comparator('eq', (left, right) => left === right) @@ -152,24 +176,24 @@ Eval.lt = comparator('lt', (left, right) => left < right) Eval.le = Eval.lte = comparator('lte', (left, right) => left <= right) // element -Eval.in = multary('in', ([value, array], data) => executeEval(data, array).includes(executeEval(data, value))) -Eval.nin = multary('nin', ([value, array], data) => !executeEval(data, array).includes(executeEval(data, value))) +Eval.in = multary('in', ([value, array], data) => executeEval(data, array).includes(executeEval(data, value)), RuntimeType.create('boolean')) +Eval.nin = multary('nin', ([value, array], data) => !executeEval(data, array).includes(executeEval(data, value)), RuntimeType.create('boolean')) // string -Eval.concat = multary('concat', (args, data) => args.map(arg => executeEval(data, arg)).join('')) -Eval.regex = multary('regex', ([value, regex], data) => makeRegExp(executeEval(data, regex)).test(executeEval(data, value))) +Eval.concat = multary('concat', (args, data) => args.map(arg => executeEval(data, arg)).join(''), RuntimeType.create('string')) +Eval.regex = multary('regex', ([value, regex], data) => makeRegExp(executeEval(data, regex)).test(executeEval(data, value)), RuntimeType.create('boolean')) // logical -Eval.and = multary('and', (args, data) => args.every(arg => executeEval(data, arg))) -Eval.or = multary('or', (args, data) => args.some(arg => executeEval(data, arg))) -Eval.not = unary('not', (value, data) => !executeEval(data, value)) +Eval.and = multary('and', (args, data) => args.every(arg => executeEval(data, arg)), RuntimeType.create('boolean')) +Eval.or = multary('or', (args, data) => args.some(arg => executeEval(data, arg)), RuntimeType.create('boolean')) +Eval.not = unary('not', (value, data) => !executeEval(data, value), RuntimeType.create('boolean')) // aggregation -Eval.sum = unary('sum', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0)) -Eval.avg = unary('avg', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length) -Eval.max = unary('max', (expr, table) => Math.max(...table.map(data => executeAggr(expr, data)))) -Eval.min = unary('min', (expr, table) => Math.min(...table.map(data => executeAggr(expr, data)))) -Eval.count = unary('count', (expr, table) => new Set(table.map(data => executeAggr(expr, data))).size) +Eval.sum = unary('sum', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0), RuntimeType.create('number')) +Eval.avg = unary('avg', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length, RuntimeType.create('number')) +Eval.max = unary('max', (expr, table) => Math.max(...table.map(data => executeAggr(expr, data))), RuntimeType.create('number')) +Eval.min = unary('min', (expr, table) => Math.min(...table.map(data => executeAggr(expr, data))), RuntimeType.create('number')) +Eval.count = unary('count', (expr, table) => new Set(table.map(data => executeAggr(expr, data))).size, RuntimeType.create('number')) export { Eval as $ } diff --git a/packages/core/src/model.ts b/packages/core/src/model.ts index 07dd6324..8befc378 100644 --- a/packages/core/src/model.ts +++ b/packages/core/src/model.ts @@ -1,6 +1,7 @@ -import { clone, isNullable, makeArray, MaybeArray } from 'cosmokit' +import { clone, isNullable, makeArray, MaybeArray, valueMap } from 'cosmokit' import { Database } from './driver' -import { Eval, isEvalExpr } from './eval' +import { Eval, getExprRuntimeType, isEvalExpr } from './eval' +import { RuntimeType } from './runtime' import { Selection } from './selection' import { Flatten, Keys } from './utils' @@ -17,6 +18,7 @@ export interface Field { expr?: Eval.Expr legacy?: string[] deprecated?: boolean + runtimeType?: RuntimeType } export namespace Field { @@ -79,6 +81,17 @@ export namespace Field { return field } + + export function getRuntimeType(field: Field): RuntimeType { + if (number.includes(field.type)) return RuntimeType.create('number') + else if (string.includes(field.type)) return RuntimeType.create('string') + else if (boolean.includes(field.type)) return RuntimeType.create('boolean') + else if (date.includes(field.type)) return RuntimeType.create('date') + else if (field.type === 'list') return RuntimeType.create('string', true) + else if (field.type === 'json' || field.type === 'primary') return RuntimeType.any + else if (field.type === 'expr') return getExprRuntimeType(field.expr) + else throw new Error(`No runtime type for ${field}`) + } } export namespace Model { @@ -123,6 +136,7 @@ export class Model { for (const key in fields) { this.fields[key] = Field.parse(fields[key]) this.fields[key].deprecated = !!callback + this.fields[key].runtimeType = Field.getRuntimeType(this.fields[key]) } if (typeof this.primary === 'string' && this.fields[this.primary]?.type === 'primary') { @@ -214,4 +228,8 @@ export class Model { } return this.parse({ ...result, ...data }) } + + getRuntimeType(): RuntimeType { + return RuntimeType.create(valueMap(this.fields, Field.getRuntimeType)) + } } diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts new file mode 100644 index 00000000..aa4c4e9a --- /dev/null +++ b/packages/core/src/runtime.ts @@ -0,0 +1,30 @@ +import { getExprRuntimeType } from './eval' + +const kRuntime = Symbol('Runtime') + +export interface RuntimeType { + [kRuntime]: true + primitive: RuntimeType.Primitive + list: A +} + +export namespace RuntimeType { + export type Primitive = 'any' | 'number' | 'string' | 'boolean' | 'date' | 'regexp' | RuntimeType + | (T extends object ? { [key in keyof T]: RuntimeType } : never) + + export function create>(primitive: P): RuntimeType + export function create, A extends boolean>(primitive: P, list: A): RuntimeType + export function create(primitive: any, list: boolean = false) { + return { [kRuntime]: true, primitive, list } + } + + export function merge(...types: any[]): RuntimeType { + return types.map(x => getExprRuntimeType(x)).find(x => x.primitive !== 'any') ?? RuntimeType.any + } + + export function test(type: any): type is RuntimeType { + return type && type[kRuntime] + } + + export const any = RuntimeType.create('any') +} diff --git a/packages/core/src/selection.ts b/packages/core/src/selection.ts index 05972387..1bddd927 100644 --- a/packages/core/src/selection.ts +++ b/packages/core/src/selection.ts @@ -4,6 +4,7 @@ import { Eval, executeEval } from './eval' import { Model } from './model' import { Query } from './query' import { Keys, randomId, Row } from './utils' +import { RuntimeType } from './runtime' export type Direction = 'asc' | 'desc' @@ -29,10 +30,11 @@ namespace Executable { } } -const createRow = (ref: string, expr = {}, prefix = '') => new Proxy(expr, { +const createRow = (ref: string, expr = {}, prefix = '', model?: Model) => new Proxy(expr, { get(target, key) { - if (typeof key === 'symbol' || key in target || key.startsWith('$')) return Reflect.get(target, key) - return createRow(ref, Eval('', [ref, `${prefix}${key}`]), `${prefix}${key}.`) + if (typeof key === 'symbol' || key in target || key === 'toJSON' || key.startsWith('$')) return Reflect.get(target, key) + const fullKey = `${prefix}${key}` + return createRow(ref, Eval('', [ref, fullKey], model?.fields?.[fullKey]?.runtimeType ?? RuntimeType.any), `${fullKey}.`, model) }, }) @@ -45,15 +47,15 @@ class Executable { constructor(driver: Driver, payload: Executable.Payload) { Object.assign(this, payload) + defineProperty(this, 'driver', driver) + defineProperty(this, 'model', driver.model(this.table)) const expr = {} if (typeof payload.table !== 'string' && !(payload.table instanceof Selection)) { for (const key in payload.table) { - expr[key] = createRow(key) + expr[key] = createRow(key, {}, '', this.model) } } - defineProperty(this, 'driver', driver) - defineProperty(this, 'row', createRow(this.ref, expr)) - defineProperty(this, 'model', driver.model(this.table)) + defineProperty(this, 'row', createRow(this.ref, {}, '', this.model)) } protected resolveQuery(query?: Query): Query.Expr diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index f93c03a7..43d13a5c 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -275,7 +275,7 @@ export class Builder { const fields = args[0].fields ?? Object.fromEntries(Object .entries(model.fields) .filter(([, field]) => !field!.deprecated) - .map(([key]) => [key, { $: [ref, key] }])) + .map(([key, field]) => [key, Eval('', [ref, key], field!.runtimeType!)])) const keys = Object.entries(fields).map(([key, value]) => { key = escapeId(key) value = this.parseEval(value) From 2e89aee11db007af55ebe1edf7dfdc958c1f4893 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Wed, 18 Oct 2023 23:29:20 +0800 Subject: [PATCH 03/49] fix: mongo join nested row --- packages/core/src/selection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/selection.ts b/packages/core/src/selection.ts index 1bddd927..4f9d3869 100644 --- a/packages/core/src/selection.ts +++ b/packages/core/src/selection.ts @@ -55,7 +55,7 @@ class Executable { expr[key] = createRow(key, {}, '', this.model) } } - defineProperty(this, 'row', createRow(this.ref, {}, '', this.model)) + defineProperty(this, 'row', createRow(this.ref, expr, '', this.model)) } protected resolveQuery(query?: Query): Query.Expr From d8dd3a33679664fcfc245a7ced57917ac2bf5e03 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Sun, 22 Oct 2023 01:13:26 +0800 Subject: [PATCH 04/49] stage: move to callback --- packages/core/src/driver.ts | 2 +- packages/core/src/eval.ts | 4 + packages/core/src/model.ts | 4 +- packages/core/src/selection.ts | 2 +- packages/mysql/src/index.ts | 10 ++- packages/sql-utils/src/index.ts | 26 ++++++- packages/sqlite/src/index.ts | 11 ++- packages/tests/src/experimental.ts | 121 +++++++++++++++++++++++++++++ packages/tests/src/index.ts | 2 + packages/tests/src/selection.ts | 17 ++++ 10 files changed, 187 insertions(+), 12 deletions(-) create mode 100644 packages/tests/src/experimental.ts diff --git a/packages/core/src/driver.ts b/packages/core/src/driver.ts index 21053795..ee8b4c86 100644 --- a/packages/core/src/driver.ts +++ b/packages/core/src/driver.ts @@ -241,7 +241,7 @@ export abstract class Driver { if (submodel.fields[field]!.deprecated) continue model.fields[`${key}.${field}`] = { type: 'expr', - expr: Eval('', [key, field], Field.getRuntimeType(submodel.fields[field]!)), + expr: () => Eval('', [key, field], Field.getRuntimeType(submodel.fields[field]!)), runtimeType: Field.getRuntimeType(submodel.fields[field]!), } } diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index d3c2003f..a666caa3 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -109,6 +109,8 @@ export namespace Eval { max(value: Number): Expr min(value: Number): Expr count(value: Any): Expr + + aggr(value: Expr, aggr?: A): Expr } } @@ -195,6 +197,8 @@ Eval.max = unary('max', (expr, table) => Math.max(...table.map(data => executeAg Eval.min = unary('min', (expr, table) => Math.min(...table.map(data => executeAggr(expr, data))), RuntimeType.create('number')) Eval.count = unary('count', (expr, table) => new Set(table.map(data => executeAggr(expr, data))).size, RuntimeType.create('number')) +Eval.aggr = (value, aggr) => Eval('aggr', [value, aggr ?? true], getExprRuntimeType(value)) + export { Eval as $ } type MapUneval = { diff --git a/packages/core/src/model.ts b/packages/core/src/model.ts index 8befc378..f5274328 100644 --- a/packages/core/src/model.ts +++ b/packages/core/src/model.ts @@ -1,6 +1,6 @@ import { clone, isNullable, makeArray, MaybeArray, valueMap } from 'cosmokit' import { Database } from './driver' -import { Eval, getExprRuntimeType, isEvalExpr } from './eval' +import { getExprRuntimeType, isEvalExpr } from './eval' import { RuntimeType } from './runtime' import { Selection } from './selection' import { Flatten, Keys } from './utils' @@ -15,7 +15,7 @@ export interface Field { initial?: T precision?: number scale?: number - expr?: Eval.Expr + expr?: Selection.Callback legacy?: string[] deprecated?: boolean runtimeType?: RuntimeType diff --git a/packages/core/src/selection.ts b/packages/core/src/selection.ts index 4f9d3869..124a8bea 100644 --- a/packages/core/src/selection.ts +++ b/packages/core/src/selection.ts @@ -30,7 +30,7 @@ namespace Executable { } } -const createRow = (ref: string, expr = {}, prefix = '', model?: Model) => new Proxy(expr, { +export const createRow = (ref: string, expr = {}, prefix = '', model?: Model) => new Proxy(expr, { get(target, key) { if (typeof key === 'symbol' || key in target || key === 'toJSON' || key.startsWith('$')) return Reflect.get(target, key) const fullKey = `${prefix}${key}` diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index ac798fb8..4798cefc 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -253,8 +253,8 @@ export class MySQLDriver extends Driver { // field definitions for (const key in fields) { - const { deprecated, initial, nullable = true } = fields[key]! - if (deprecated) continue + const { deprecated, expr, initial, nullable = true } = fields[key]! + if (deprecated || expr) continue const legacy = [key, ...fields[key]!.legacy || []] const column = columns.find(info => legacy.includes(info.COLUMN_NAME)) let shouldUpdate = column?.COLUMN_NAME !== key @@ -528,7 +528,11 @@ export class MySQLDriver extends Driver { } return `${escaped} = ${value}` }).join(', ') - + console.log([ + `INSERT INTO ${escapeId(table)} (${initFields.map(escapeId).join(', ')})`, + `VALUES (${insertion.map(item => this._formatValues(table, item, initFields)).join('), (')})`, + `ON DUPLICATE KEY UPDATE ${update}`, + ].join(' ')) await this.query([ `INSERT INTO ${escapeId(table)} (${initFields.map(escapeId).join(', ')})`, `VALUES (${insertion.map(item => this._formatValues(table, item, initFields)).join('), (')})`, diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 43d13a5c..464c1044 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -1,5 +1,5 @@ import { Dict, isNullable } from 'cosmokit' -import { Eval, Field, isComparable, Model, Modifier, Query, Selection } from '@minatojs/core' +import { createRow, Eval, Field, getExprRuntimeType, isComparable, Model, Modifier, Query, Selection } from '@minatojs/core' export function escapeId(value: string) { return '`' + value + '`' @@ -111,6 +111,8 @@ export class Builder { $min: (expr) => `min(${this.parseAggr(expr)})`, $max: (expr) => `max(${this.parseAggr(expr)})`, $count: (expr) => `count(distinct ${this.parseAggr(expr)})`, + + $aggr: ([value, aggr]) => aggr ? this.groupJson(this.parseEvalExpr(value)) : this.parseEvalExpr(value), } } @@ -159,6 +161,10 @@ export class Builder { return `NOT(${condition})` } + protected groupJson(key: string) { + return `concat('[', group_concat(${key}), ']')` + } + protected parseFieldQuery(key: string, query: Query.FieldExpr) { const conditions: string[] = [] @@ -219,11 +225,15 @@ export class Builder { return this.parseEvalExpr(expr) } + protected transformJsonField(obj: string, path: string) { + return `json_unquote(json_extract(${obj}, '$${path}'))` + } + private transformKey(key: string, fields: {}, prefix: string) { if (key in fields || !key.includes('.')) return prefix + escapeId(key) const field = Object.keys(fields).find(k => key.startsWith(k + '.')) || key.split('.')[0] const rest = key.slice(field.length + 1).split('.') - return `json_unquote(json_extract(${prefix} ${escapeId(field)}, '$${rest.map(key => `."${key}"`).join('')}'))` + return this.transformJsonField(`${prefix} ${escapeId(field)}`, rest.map(key => `."${key}"`).join('')) } private getRecursive(args: string | string[]) { @@ -232,8 +242,16 @@ export class Builder { } const [table, key] = args const fields = this.tables?.[table]?.fields || {} - if (fields[key]?.expr) { - return this.parseEvalExpr(fields[key]?.expr) + const fkey = Object.keys(fields).find(field => key === field || key.startsWith(field + '.')) + if (fkey && fields[fkey]?.expr) { + if (key === fkey) { + return this.parseEvalExpr(fields[fkey]?.expr!(createRow(table))) + } else { + const field = this.parseEvalExpr(fields[fkey]?.expr!(createRow(table))) + const rest = key.slice(fkey.length + 1).split('.') + console.log(getExprRuntimeType(field)) + return this.transformJsonField(`${field}`, rest.map(key => `."${key}"`).join('')) + } } const prefix = !this.tables || table === '_' || key in fields // the only table must be the main table diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index dc457227..acfd0e5e 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -85,6 +85,14 @@ class SQLiteBuilder extends Builder { protected createElementQuery(key: string, value: any) { return `(',' || ${key} || ',') LIKE ${this.escape('%,' + value + ',%')}` } + + protected groupJson(key: string) { + return `('[' || group_concat(${key}) || ']')` + } + + protected transformJsonField(obj: string, path: string) { + return `json_extract(${obj}, '$${path}')` + } } export class SQLiteDriver extends Driver { @@ -118,7 +126,8 @@ export class SQLiteDriver extends Driver { const legacy = [key, ...model.fields[key]!.legacy || []] const column = columns.find(({ name }) => legacy.includes(name)) - const { initial, nullable = true } = model.fields[key]! + const { expr, initial, nullable = true } = model.fields[key]! + if (expr) continue const typedef = getTypeDef(model.fields[key]!) let def = `${escapeId(key)} ${typedef}` if (key === model.primary && model.autoInc) { diff --git a/packages/tests/src/experimental.ts b/packages/tests/src/experimental.ts new file mode 100644 index 00000000..6bf421eb --- /dev/null +++ b/packages/tests/src/experimental.ts @@ -0,0 +1,121 @@ +import { $, Database } from '@minatojs/core' +import { expect } from 'chai' +import { setup } from './utils' + +interface Foo { + id: number + value: number + bars: Bar[] +} + +interface Bar { + id: number + uid: number + pid: number + value: number + obj: { + x: string + y: string + } + l: string[] +} + +interface Tables { + foo: Foo + bar: Bar +} + +function ExperimentalTests(database: Database) { + database.extend('foo', { + id: 'unsigned', + value: 'integer', + }) + + database.extend('bar', { + id: 'unsigned', + uid: 'unsigned', + pid: 'unsigned', + value: 'integer', + obj: 'json', + l: 'list', + }, { + autoInc: true, + }) + + before(async () => { + + database.extend('foo', { + id: 'unsigned', + value: 'integer', + // bars: row => database.select('bar').where(r => $.eq(r.pid, row.id)).evaluate(r => r.id) + }) + + await setup(database, 'foo', [ + { id: 1, value: 0 }, + { id: 2, value: 2 }, + { id: 3, value: 2 }, + ]) + + await setup(database, 'bar', [ + { uid: 1, pid: 1, value: 0, obj: { x: '1', y: 'a' }, l: ['a,b', 'c'] }, + { uid: 1, pid: 1, value: 1, obj: { x: '2', y: 'b' }, }, + { uid: 1, pid: 2, value: 0, obj: { x: '3', y: 'c' }, }, + ]) + }) +} + +namespace ExperimentalTests { + export function computed(database: Database) { + // it('strlist', async () => { + // const res = await database.get('bar', {}) + // console.log('res', res) + // }) + + // it('get', async () => { + // await expect(database.get('foo', {})).to.eventually.deep.equal([ + // { id: 2, pid: 1, uid: 1, value: 1, id2: 2 }, + // ]) + // }) + + it('project', async () => { + const res = await database.select('bar') + .project({ + count: row => (row.obj), + count2: row => (row.obj.x) + }) + // .orderBy(row => row.foo.id) + .execute() + console.log('res', res) + }) + + it('group', async () => { + const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) + .groupBy('foo', { + // count: row => $.aggr(row.bar.obj), + count2: row => $.aggr(row.bar.obj.y) + }) + // .orderBy(row => row.foo.id) + .execute() + console.log('res', res) + }) + + // it('raw', async () => { + // const driver = Object.values(database.drivers)[0] + // const res = await driver.query( + // "SELECT `foo.id`, `foo.value`, `count` FROM (SELECT `foo`.`id` AS `foo.id`, `foo`.`value` AS `foo.value`, concat('[', group_concat(json_unquote(json_extract(`niormjql`. `bar.obj`, '$.x'))), ']') AS `count` FROM `foo` JOIN `bar` ON (`foo`.`id` = `bar`.`pid`) GROUP BY `foo.id`, `foo.value`) wvmceoou" + // ) + + + // console.log('res', res) + // }) + + // it('raw2', async () => { + // const driver = Object.values(database.drivers)[0] + // const res = await driver.query("SELECT `foo.id`, `foo.value`, `count` FROM (SELECT `foo`.`id` AS `foo.id`, `foo`.`value` AS `foo.value`, group_concat(distinct `bar`.`id`) AS `count` FROM `foo` JOIN `bar` ON (`foo`.`id` = `bar`.`pid`) GROUP BY `foo.id`, `foo.value`) xlziynlx") + + // console.log('res', res) + // }) + } +} + +export default ExperimentalTests diff --git a/packages/tests/src/index.ts b/packages/tests/src/index.ts index b493d387..5deaf473 100644 --- a/packages/tests/src/index.ts +++ b/packages/tests/src/index.ts @@ -4,6 +4,7 @@ import UpdateOperators from './update' import ObjectOperations from './object' import Migration from './migration' import Selection from './selection' +import Experimental from './experimental' import './setup' const Keywords = ['name'] @@ -52,6 +53,7 @@ namespace Tests { export const object = ObjectOperations export const selection = Selection export const migration = Migration + export const experimental = Experimental } export default createUnit(Tests, true) diff --git a/packages/tests/src/selection.ts b/packages/tests/src/selection.ts index be00402d..b1d505ca 100644 --- a/packages/tests/src/selection.ts +++ b/packages/tests/src/selection.ts @@ -142,6 +142,23 @@ namespace SelectionTests { }) it('aggregate', async () => { + await database + .select('foo') + .project({ + count: row => row.id, + }) + .execute() + + await expect(database + .select('foo') + .project({ + count: row => $.count(row.id), + }) + .execute() + ).to.eventually.deep.equal([ + { count: 3 }, + ]) + await expect(database .select('foo') .project({ From 7f80a130fb57f0a3228e63b608220a0ee2f6c35a Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 26 Oct 2023 03:15:33 +0800 Subject: [PATCH 05/49] feat: $.object, $.array --- packages/core/src/eval.ts | 79 +++++++-------- packages/core/src/index.ts | 1 + packages/core/src/model.ts | 12 +-- packages/core/src/runtime.ts | 32 ++++-- packages/core/src/selection.ts | 2 +- packages/memory/src/index.ts | 19 +++- packages/mongo/src/utils.ts | 21 +++- packages/mysql/src/index.ts | 6 ++ packages/mysql/tests/maria.spec.ts | 33 +++++++ packages/sql-utils/src/index.ts | 34 +++++-- packages/sqlite/src/index.ts | 4 +- packages/tests/src/experimental.ts | 151 ++++++++++++++++++++++------- 12 files changed, 286 insertions(+), 108 deletions(-) create mode 100644 packages/mysql/tests/maria.spec.ts diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index a666caa3..d0438dc3 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -1,4 +1,4 @@ -import { defineProperty, isNullable, valueMap } from 'cosmokit' +import { defineProperty, Dict, isNullable, valueMap } from 'cosmokit' import { RuntimeType } from './runtime' import { Comparable, Flatten, isComparable, makeRegExp } from './utils' @@ -10,12 +10,12 @@ export function getExprRuntimeType(value: any): RuntimeType { if (isNullable(value)) return RuntimeType.any if (RuntimeType.test(value)) return value if (isEvalExpr(value)) return value[kRuntimeType] - else if (typeof value === 'string') return RuntimeType.create('string') - else if (typeof value === 'number') return RuntimeType.create('number') - else if (typeof value === 'boolean') return RuntimeType.create('boolean') - else if (value instanceof Date) return RuntimeType.create('date') - else if (value instanceof RegExp) return RuntimeType.create('regexp') - else if (Array.isArray(value)) return RuntimeType.create(RuntimeType.merge(...value), true) + else if (typeof value === 'string') return RuntimeType.string + else if (typeof value === 'number') return RuntimeType.number + else if (typeof value === 'boolean') return RuntimeType.boolean + else if (value instanceof Date) return RuntimeType.date + else if (value instanceof RegExp) return RuntimeType.regexp + else if (Array.isArray(value)) return RuntimeType.list(RuntimeType.merge(...value)) else return RuntimeType.create(valueMap(value, getExprRuntimeType)) } @@ -93,6 +93,7 @@ export namespace Eval { // element in(x: T | Expr, array: (T | Expr)[] | Expr): Expr nin(x: T | Expr, array: (T | Expr)[] | Expr): Expr + at(array: (T | Expr)[] | Expr, index: Number): Expr // string concat: Multi @@ -110,7 +111,8 @@ export namespace Eval { min(value: Number): Expr count(value: Any): Expr - aggr(value: Expr, aggr?: A): Expr + object>(fields: T): Expr + array(value: Expr): Expr } } @@ -121,15 +123,17 @@ const operators = {} as Record<`$${keyof Eval.Static}`, (args: any, data: any) = operators['$'] = getRecursive type UnaryCallback = T extends (value: infer R) => Eval.Expr ? (value: R, data: any[]) => S : never -function unary(key: K, callback: UnaryCallback, type: RuntimeType): Eval.Static[K] { +function unary(key: K, callback: UnaryCallback, + type: RuntimeType | ((value) => RuntimeType)): Eval.Static[K] { operators[`$${key}`] = callback - return (value: any) => Eval(key, value, type) as any + return (value: any) => Eval(key, value, typeof type === 'function' ? type(value) : type) as any } type MultivariateCallback = T extends (...args: infer R) => Eval.Expr ? (args: R, data: any) => S : never -function multary(key: K, callback: MultivariateCallback, type: RuntimeType): Eval.Static[K] { +function multary( + key: K, callback: MultivariateCallback, type: RuntimeType | ((...args) => RuntimeType)): Eval.Static[K] { operators[`$${key}`] = callback - return (...args: any[]) => Eval(key, args, type) as any + return (...args: any[]) => Eval(key, args, typeof type === 'function' ? type(...args) : type) as any } type BinaryCallback = T extends (...args: any[]) => Eval.Expr ? (...args: any[]) => S : never @@ -152,22 +156,17 @@ operators.$switch = (args, data) => { } // univeral -operators.$if = ([cond, vThen, vElse], data) => executeEval(data, cond) ? executeEval(data, vThen) : executeEval(data, vElse) -Eval.if = (...args: any[]) => { - return Eval('if', args, RuntimeType.merge(...(args.slice(1)))) -} - -operators.$ifNull = ([value, fallback], data) => executeEval(data, value) ?? executeEval(data, fallback) -Eval.ifNull = (...args: any[]) => { - return Eval('ifNull', args, RuntimeType.merge(...args)) -} +Eval.if = multary('if', ([cond, vThen, vElse], data) => executeEval(data, cond) ? executeEval(data, vThen) : executeEval(data, vElse), + (_, vThen, vElse) => RuntimeType.merge(vThen, vElse)) +Eval.ifNull = multary('ifNull', ([value, fallback], data) => executeEval(data, value) ?? executeEval(data, fallback), + (value, fallback) => RuntimeType.merge(value, fallback)) // arithmetic -Eval.add = multary('add', (args, data) => args.reduce((prev, curr) => prev + executeEval(data, curr), 0), RuntimeType.create('number')) +Eval.add = multary('add', (args, data) => args.reduce((prev, curr) => prev + executeEval(data, curr), 0), RuntimeType.number) Eval.mul = Eval.multiply = multary( - 'multiply', (args, data) => args.reduce((prev, curr) => prev * executeEval(data, curr), 1), RuntimeType.create('number')) -Eval.sub = Eval.subtract = multary('subtract', ([left, right], data) => executeEval(data, left) - executeEval(data, right), RuntimeType.create('number')) -Eval.div = Eval.divide = multary('divide', ([left, right], data) => executeEval(data, left) / executeEval(data, right), RuntimeType.create('number')) + 'multiply', (args, data) => args.reduce((prev, curr) => prev * executeEval(data, curr), 1), RuntimeType.number) +Eval.sub = Eval.subtract = multary('subtract', ([left, right], data) => executeEval(data, left) - executeEval(data, right), RuntimeType.number) +Eval.div = Eval.divide = multary('divide', ([left, right], data) => executeEval(data, left) / executeEval(data, right), RuntimeType.number) // comparison Eval.eq = comparator('eq', (left, right) => left === right) @@ -178,26 +177,28 @@ Eval.lt = comparator('lt', (left, right) => left < right) Eval.le = Eval.lte = comparator('lte', (left, right) => left <= right) // element -Eval.in = multary('in', ([value, array], data) => executeEval(data, array).includes(executeEval(data, value)), RuntimeType.create('boolean')) -Eval.nin = multary('nin', ([value, array], data) => !executeEval(data, array).includes(executeEval(data, value)), RuntimeType.create('boolean')) +Eval.in = multary('in', ([value, array], data) => executeEval(data, array).includes(executeEval(data, value)), RuntimeType.boolean) +Eval.nin = multary('nin', ([value, array], data) => !executeEval(data, array).includes(executeEval(data, value)), RuntimeType.boolean) // string -Eval.concat = multary('concat', (args, data) => args.map(arg => executeEval(data, arg)).join(''), RuntimeType.create('string')) -Eval.regex = multary('regex', ([value, regex], data) => makeRegExp(executeEval(data, regex)).test(executeEval(data, value)), RuntimeType.create('boolean')) +Eval.concat = multary('concat', (args, data) => args.map(arg => executeEval(data, arg)).join(''), RuntimeType.string) +Eval.regex = multary('regex', ([value, regex], data) => makeRegExp(executeEval(data, regex)).test(executeEval(data, value)), RuntimeType.boolean) // logical -Eval.and = multary('and', (args, data) => args.every(arg => executeEval(data, arg)), RuntimeType.create('boolean')) -Eval.or = multary('or', (args, data) => args.some(arg => executeEval(data, arg)), RuntimeType.create('boolean')) -Eval.not = unary('not', (value, data) => !executeEval(data, value), RuntimeType.create('boolean')) +Eval.and = multary('and', (args, data) => args.every(arg => executeEval(data, arg)), RuntimeType.boolean) +Eval.or = multary('or', (args, data) => args.some(arg => executeEval(data, arg)), RuntimeType.boolean) +Eval.not = unary('not', (value, data) => !executeEval(data, value), RuntimeType.boolean) // aggregation -Eval.sum = unary('sum', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0), RuntimeType.create('number')) -Eval.avg = unary('avg', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length, RuntimeType.create('number')) -Eval.max = unary('max', (expr, table) => Math.max(...table.map(data => executeAggr(expr, data))), RuntimeType.create('number')) -Eval.min = unary('min', (expr, table) => Math.min(...table.map(data => executeAggr(expr, data))), RuntimeType.create('number')) -Eval.count = unary('count', (expr, table) => new Set(table.map(data => executeAggr(expr, data))).size, RuntimeType.create('number')) - -Eval.aggr = (value, aggr) => Eval('aggr', [value, aggr ?? true], getExprRuntimeType(value)) +Eval.sum = unary('sum', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0), RuntimeType.number) +Eval.avg = unary('avg', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length, RuntimeType.number) +Eval.max = unary('max', (expr, table) => Math.max(...table.map(data => executeAggr(expr, data))), RuntimeType.number) +Eval.min = unary('min', (expr, table) => Math.min(...table.map(data => executeAggr(expr, data))), RuntimeType.number) +Eval.count = unary('count', (expr, table) => new Set(table.map(data => executeAggr(expr, data))).size, RuntimeType.number) + +Eval.object = unary('object', (field, table) => valueMap(field, value => executeAggr(value, table)), + fields => RuntimeType.json(RuntimeType.create(valueMap(fields, getExprRuntimeType)))) +Eval.array = unary('array', (expr, table) => table.map(data => executeAggr(expr, data)), expr => RuntimeType.json(RuntimeType.list(getExprRuntimeType(expr)))) export { Eval as $ } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5aeb810b..ba5225b2 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -5,3 +5,4 @@ export * from './model' export * from './query' export * from './selection' export * from './utils' +export * from './runtime' diff --git a/packages/core/src/model.ts b/packages/core/src/model.ts index f5274328..cdccf5fe 100644 --- a/packages/core/src/model.ts +++ b/packages/core/src/model.ts @@ -83,11 +83,11 @@ export namespace Field { } export function getRuntimeType(field: Field): RuntimeType { - if (number.includes(field.type)) return RuntimeType.create('number') - else if (string.includes(field.type)) return RuntimeType.create('string') - else if (boolean.includes(field.type)) return RuntimeType.create('boolean') - else if (date.includes(field.type)) return RuntimeType.create('date') - else if (field.type === 'list') return RuntimeType.create('string', true) + if (number.includes(field.type)) return RuntimeType.number + else if (string.includes(field.type)) return RuntimeType.string + else if (boolean.includes(field.type)) return RuntimeType.boolean + else if (date.includes(field.type)) return RuntimeType.date + else if (field.type === 'list') return RuntimeType.list(RuntimeType.string) else if (field.type === 'json' || field.type === 'primary') return RuntimeType.any else if (field.type === 'expr') return getExprRuntimeType(field.expr) else throw new Error(`No runtime type for ${field}`) @@ -202,7 +202,7 @@ export class Model { const field = fields.find(field => fullKey === field || fullKey.startsWith(field + '.')) if (field) { node[segments[0]] = this.resolveValue(key, value) - } else if (!value || typeof value !== 'object' || isEvalExpr(value) || Object.keys(value).length === 0) { + } else if (!value || typeof value !== 'object' || isEvalExpr(value) || Array.isArray(value) || Object.keys(value).length === 0) { if (strict) { throw new TypeError(`unknown field "${fullKey}" in model ${this.name}`) } else { diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index aa4c4e9a..15ec70a9 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -2,29 +2,45 @@ import { getExprRuntimeType } from './eval' const kRuntime = Symbol('Runtime') -export interface RuntimeType { +export interface RuntimeType { [kRuntime]: true primitive: RuntimeType.Primitive - list: A + list?: boolean + json?: boolean } export namespace RuntimeType { - export type Primitive = 'any' | 'number' | 'string' | 'boolean' | 'date' | 'regexp' | RuntimeType - | (T extends object ? { [key in keyof T]: RuntimeType } : never) + export type Primitive = 'any' | 'number' | 'string' | 'boolean' | 'date' | 'regexp' | RuntimeType | { [key in keyof T]: RuntimeType } - export function create>(primitive: P): RuntimeType - export function create, A extends boolean>(primitive: P, list: A): RuntimeType - export function create(primitive: any, list: boolean = false) { - return { [kRuntime]: true, primitive, list } + export function create = RuntimeType.Primitive>(primitive: P, extra: Partial> = {}): RuntimeType { + if (Object.keys(extra).length === 0 && test(primitive)) return primitive + return { [kRuntime]: true, primitive, ...extra } } export function merge(...types: any[]): RuntimeType { return types.map(x => getExprRuntimeType(x)).find(x => x.primitive !== 'any') ?? RuntimeType.any } + export function list(type: any): RuntimeType { + const primitive = getExprRuntimeType(type) + if (!primitive.list) return { ...primitive, list: true } + else return create(primitive, { list: true }) + } + + export function json(type: any): RuntimeType { + const primitive = getExprRuntimeType(type) + if (!primitive.json) return { ...primitive, json: true } + else return create(primitive, { json: true }) + } + export function test(type: any): type is RuntimeType { return type && type[kRuntime] } export const any = RuntimeType.create('any') + export const number = RuntimeType.create('number') + export const string = RuntimeType.create('string') + export const boolean = RuntimeType.create('boolean') + export const date = RuntimeType.create('date') + export const regexp = RuntimeType.create('regexp') } diff --git a/packages/core/src/selection.ts b/packages/core/src/selection.ts index 124a8bea..25869364 100644 --- a/packages/core/src/selection.ts +++ b/packages/core/src/selection.ts @@ -52,7 +52,7 @@ class Executable { const expr = {} if (typeof payload.table !== 'string' && !(payload.table instanceof Selection)) { for (const key in payload.table) { - expr[key] = createRow(key, {}, '', this.model) + expr[key] = createRow(key, { $ref: this.ref }, '', this.model) } } defineProperty(this, 'row', createRow(this.ref, expr, '', this.model)) diff --git a/packages/memory/src/index.ts b/packages/memory/src/index.ts index 4ee9f048..2ca7eaad 100644 --- a/packages/memory/src/index.ts +++ b/packages/memory/src/index.ts @@ -1,5 +1,5 @@ import { clone, Dict, makeArray, noop, omit, pick, valueMap } from 'cosmokit' -import { Database, Driver, Eval, executeEval, executeQuery, executeSort, executeUpdate, RuntimeError, Selection } from '@minatojs/core' +import { Database, Driver, Eval, executeEval, executeQuery, executeSort, executeUpdate, isEvalExpr, RuntimeError, Selection } from '@minatojs/core' export namespace MemoryDriver { export interface Config {} @@ -47,6 +47,10 @@ export class MemoryDriver extends Driver { const { ref, query, table, args, model } = sel const { fields, group, having } = sel.args[0] const data = this.table(table, having).filter(row => executeQuery(row, query, ref)) + if (!group.length && fields && Object.values(args[0].fields ?? {}).some(x => isAggrExpr(x))) { + return [valueMap(fields!, (expr) => executeEval(data.map(row => ({ [ref]: row })), expr))] + } + const branches: { index: Dict; table: any[] }[] = [] const groupFields = group.length ? pick(fields!, group) : fields for (let row of executeSort(data, args[0], ref)) { @@ -164,4 +168,17 @@ export class MemoryDriver extends Driver { } } +const nonAggrKeys = ['$'] +const aggrKeys = ['$sum', '$avg', '$min', '$max', '$count', '$array'] + +function isAggrExpr(value: any) { + if (!isEvalExpr(value)) return false + for (const [key, args] of Object.entries(value)) { + if (!key.startsWith('$')) continue + if (nonAggrKeys.includes(key)) return false + if (aggrKeys.includes(key) || ((Array.isArray(args) ? args : [args]).some(x => isAggrExpr(x)))) return true + } + return false +} + export default MemoryDriver diff --git a/packages/mongo/src/utils.ts b/packages/mongo/src/utils.ts index f8b0852c..623c67c6 100644 --- a/packages/mongo/src/utils.ts +++ b/packages/mongo/src/utils.ts @@ -72,7 +72,7 @@ function transformFieldQuery(query: Query.FieldQuery, key: string, filters: Filt return result } -const aggrKeys = ['$sum', '$avg', '$min', '$max', '$count'] +const aggrKeys = ['$sum', '$avg', '$min', '$max', '$count', '$array'] export class Transformer { private counter = 0 @@ -114,6 +114,14 @@ export class Transformer { return { $cond: expr.$if.map(val => this.eval(val, group)) } } + if (expr.$object) { + return this.transformEvalExpr(expr.$object) + } + + // if (expr.$array) { + // return {} + // } + return valueMap(expr as any, (value) => { if (Array.isArray(value)) { return value.map(val => this.eval(val, group)) @@ -145,12 +153,15 @@ export class Transformer { if (!expr[type]) continue const key = this.createKey() const value = this.transformAggr(expr[type]) - if (type !== '$count') { - group![key] = { [type]: value } - return '$' + key - } else { + if (type === '$count') { group![key] = { $addToSet: value } return { $size: '$' + key } + } else if (type === '$array') { + group![key] = { $push: value } + return '$' + key + } else { + group![key] = { [type]: value } + return '$' + key } } diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index 4798cefc..c7b05fe9 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -112,6 +112,12 @@ class MySQLBuilder extends Builder { dump: value => value.join(','), load: value => value ? value.split(',') : [], }) + + this.define({ + types: ['json'], + dump: value => JSON.stringify(value), + load: value => typeof value === 'string' ? JSON.parse(value) : value, + }) } escape(value: any, field?: Field) { diff --git a/packages/mysql/tests/maria.spec.ts b/packages/mysql/tests/maria.spec.ts new file mode 100644 index 00000000..76f4782b --- /dev/null +++ b/packages/mysql/tests/maria.spec.ts @@ -0,0 +1,33 @@ +import { Database } from 'minato' +import Logger from 'reggol' +import test from '@minatojs/tests' + +const logger = new Logger('maria') + +describe('@minatojs/driver-mysql/maria', () => { + const database = new Database() + + before(async () => { + logger.level = 3 + await database.connect('mysql', { + port: 3307, + user: 'koishi', + password: 'koishi@114514', + database: 'test', + }) + }) + + after(async () => { + await database.dropAll() + await database.stopAll() + logger.level = 2 + }) + + test(database, { + query: { + list: { + elementQuery: false, + }, + }, + }) +}) diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 464c1044..c1460d4e 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -1,5 +1,5 @@ import { Dict, isNullable } from 'cosmokit' -import { createRow, Eval, Field, getExprRuntimeType, isComparable, Model, Modifier, Query, Selection } from '@minatojs/core' +import { createRow, Eval, Field, isComparable, Model, Modifier, Query, Selection } from '@minatojs/core' export function escapeId(value: string) { return '`' + value + '`' @@ -28,6 +28,7 @@ export class Builder { protected createEqualQuery = this.comparator('=') protected queryOperators: QueryOperators protected evalOperators: EvalOperators + protected jsonQuoteMode = false constructor(public tables?: Dict) { this.queryOperators = { @@ -112,7 +113,8 @@ export class Builder { $max: (expr) => `max(${this.parseAggr(expr)})`, $count: (expr) => `count(distinct ${this.parseAggr(expr)})`, - $aggr: ([value, aggr]) => aggr ? this.groupJson(this.parseEvalExpr(value)) : this.parseEvalExpr(value), + $object: (fields) => this.groupObject(fields), + $array: (expr) => this.groupArray(expr), } } @@ -161,8 +163,21 @@ export class Builder { return `NOT(${condition})` } - protected groupJson(key: string) { - return `concat('[', group_concat(${key}), ']')` + protected groupObject(fields: any) { + this.jsonQuoteMode = true + const ret = `json_object(` + Object.entries(fields).map(([key, expr]) => `'${key}', ${this.parseAggr(expr)}`).join(',') + `)` + this.jsonQuoteMode = false + return ret + } + + protected groupArray(expr: any) { + // this.jsonQuoteMode = true + // const ret = `cast(concat('[', group_concat(${this.parseAggr(expr)}), ']') as json)` + // this.jsonQuoteMode = false + this.jsonQuoteMode = true + const ret = `json_arrayagg(${this.parseAggr(expr)})` + this.jsonQuoteMode = false + return ret } protected parseFieldQuery(key: string, query: Query.FieldExpr) { @@ -218,7 +233,7 @@ export class Builder { return this.escape(expr) } - private parseAggr(expr: any) { + protected parseAggr(expr: any) { if (typeof expr === 'string') { return this.getRecursive(expr) } @@ -226,7 +241,8 @@ export class Builder { } protected transformJsonField(obj: string, path: string) { - return `json_unquote(json_extract(${obj}, '$${path}'))` + if (this.jsonQuoteMode) return `json_extract(${obj}, '$${path}')` + else return `json_unquote(json_extract(${obj}, '$${path}'))` } private transformKey(key: string, fields: {}, prefix: string) { @@ -249,7 +265,6 @@ export class Builder { } else { const field = this.parseEvalExpr(fields[fkey]?.expr!(createRow(table))) const rest = key.slice(fkey.length + 1).split('.') - console.log(getExprRuntimeType(field)) return this.transformJsonField(`${field}`, rest.map(key => `."${key}"`).join('')) } } @@ -352,8 +367,9 @@ export class Builder { const result = {} for (const key in obj) { if (!(key in model.fields)) continue - const { type, initial } = model.fields[key]! - const converter = this.types[type] + const { type, initial, runtimeType } = model.fields[key]! + const converter = runtimeType?.json ? this.types['json'] : this.types[type] + // const converter = this.types[type] result[key] = converter ? converter.load(obj[key], initial) : obj[key] } return model.parse(result) diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index acfd0e5e..8d7381dc 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -86,8 +86,8 @@ class SQLiteBuilder extends Builder { return `(',' || ${key} || ',') LIKE ${this.escape('%,' + value + ',%')}` } - protected groupJson(key: string) { - return `('[' || group_concat(${key}) || ']')` + protected groupArray(expr: any) { + return `('[' || group_concat(json_quote(${this.parseAggr(expr)})) || ']')` } protected transformJsonField(obj: string, path: string) { diff --git a/packages/tests/src/experimental.ts b/packages/tests/src/experimental.ts index 6bf421eb..852f37c4 100644 --- a/packages/tests/src/experimental.ts +++ b/packages/tests/src/experimental.ts @@ -5,7 +5,6 @@ import { setup } from './utils' interface Foo { id: number value: number - bars: Bar[] } interface Bar { @@ -13,9 +12,15 @@ interface Bar { uid: number pid: number value: number + s: string obj: { - x: string + x: number y: string + z: string + o: { + a: number + b: string + } } l: string[] } @@ -37,6 +42,7 @@ function ExperimentalTests(database: Database) { pid: 'unsigned', value: 'integer', obj: 'json', + s: 'string', l: 'list', }, { autoInc: true, @@ -47,7 +53,6 @@ function ExperimentalTests(database: Database) { database.extend('foo', { id: 'unsigned', value: 'integer', - // bars: row => database.select('bar').where(r => $.eq(r.pid, row.id)).evaluate(r => r.id) }) await setup(database, 'foo', [ @@ -57,64 +62,136 @@ function ExperimentalTests(database: Database) { ]) await setup(database, 'bar', [ - { uid: 1, pid: 1, value: 0, obj: { x: '1', y: 'a' }, l: ['a,b', 'c'] }, - { uid: 1, pid: 1, value: 1, obj: { x: '2', y: 'b' }, }, - { uid: 1, pid: 2, value: 0, obj: { x: '3', y: 'c' }, }, + { uid: 1, pid: 1, value: 0, obj: { x: 1, y: 'a', z: '1', o: { a: 1, b: '1' } }, s: '1', l: ['1', '2'] }, + { uid: 1, pid: 1, value: 1, obj: { x: 2, y: 'b', z: '2', o: { a: 2, b: '2' } }, s: '2' }, + { uid: 1, pid: 2, value: 0, obj: { x: 3, y: 'c', z: '3', o: { a: 3, b: '3' } }, s: '3' }, ]) }) } namespace ExperimentalTests { export function computed(database: Database) { - // it('strlist', async () => { - // const res = await database.get('bar', {}) + // it('project', async () => { + // const res = await database.select('bar') + // .project({ + // obj: row => (row.obj), + // objX: row => row.obj.x, + // objX2: row => $.add(row.obj.x, 2), + // objY: row => (row.obj.y), + // }) + // // .orderBy(row => row.foo.id) + // .execute() // console.log('res', res) // }) - // it('get', async () => { - // await expect(database.get('foo', {})).to.eventually.deep.equal([ - // { id: 2, pid: 1, uid: 1, value: 1, id2: 2 }, - // ]) - // }) + it('$.object', async () => { + const res = await database.select('foo') + .project({ + obj: row => $.object({ + id: row.id, + value: row.value, + }) + }) + .orderBy(row => row.obj.id) + .execute() + + expect(res).to.deep.equal([ + { obj: { id: 1, value: 0 } }, + { obj: { id: 2, value: 2 } }, + { obj: { id: 3, value: 2 } } + ]) + }) + + it('$.object in json', async () => { - it('project', async () => { const res = await database.select('bar') .project({ - count: row => (row.obj), - count2: row => (row.obj.x) + obj: row => $.object({ + num: row.obj.x, + str: row.obj.y, + str2: row.obj.z, + obj: row.obj.o, + a: row.obj.o.a, + }) }) - // .orderBy(row => row.foo.id) .execute() - console.log('res', res) + + expect(res).to.deep.equal([ + { obj: { a: 1, num: 1, obj: { a: 1, b: "1" }, str: 'a', str2: '1' } }, + { obj: { a: 2, num: 2, obj: { a: 2, b: "2" }, str: 'b', str2: '2' } }, + { obj: { a: 3, num: 3, obj: { a: 3, b: "3" }, str: 'c', str2: '3' } } + ]) }) - it('group', async () => { + it('$.array groupBy', async () => { const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) - .groupBy('foo', { - // count: row => $.aggr(row.bar.obj), - count2: row => $.aggr(row.bar.obj.y) + .groupBy(['foo'], { + x: row => $.array(row.bar.obj.x), + y: row => $.array(row.bar.obj.y), }) - // .orderBy(row => row.foo.id) + .orderBy(row => row.foo.id) .execute() - console.log('res', res) - }) - // it('raw', async () => { - // const driver = Object.values(database.drivers)[0] - // const res = await driver.query( - // "SELECT `foo.id`, `foo.value`, `count` FROM (SELECT `foo`.`id` AS `foo.id`, `foo`.`value` AS `foo.value`, concat('[', group_concat(json_unquote(json_extract(`niormjql`. `bar.obj`, '$.x'))), ']') AS `count` FROM `foo` JOIN `bar` ON (`foo`.`id` = `bar`.`pid`) GROUP BY `foo.id`, `foo.value`) wvmceoou" - // ) + expect(res).to.deep.equal([ + { foo: { id: 1, value: 0 }, x: [1, 2], y: ['a', 'b'] }, + { foo: { id: 2, value: 2 }, x: [3], y: ['c'] } + ]) + // console.log('res', res) + }) + it('$.array groupFull', async () => { + const res = await database.select('bar') + .project({ + // count: row => $.aggr(row.bar.obj), + count2: row => $.array(row.s), + countnumber: row => $.array(row.value), + x: row => $.array(row.obj.x), + y: row => $.array(row.obj.y), + }) + .execute() - // console.log('res', res) - // }) + expect(res).to.deep.equal([ + { + count2: ["1", "2", "3"], + countnumber: [0, 1, 0], + x: [1, 2, 3], + y: ['a', 'b', 'c'] + } + ]) + }) - // it('raw2', async () => { - // const driver = Object.values(database.drivers)[0] - // const res = await driver.query("SELECT `foo.id`, `foo.value`, `count` FROM (SELECT `foo`.`id` AS `foo.id`, `foo`.`value` AS `foo.value`, group_concat(distinct `bar`.`id`) AS `count` FROM `foo` JOIN `bar` ON (`foo`.`id` = `bar`.`pid`) GROUP BY `foo.id`, `foo.value`) xlziynlx") + it('$.array groupBy in json', async () => { + const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) + .groupBy('foo', { + bars: row => $.array($.object({ + value: row.bar.value + })), + x: row => $.array(row.bar.obj.x), + y: row => $.array(row.bar.obj.y), + z: row => $.array(row.bar.obj.z), + o: row => $.array(row.bar.obj.o), + }) + .orderBy(row => row.foo.id) + .execute() - // console.log('res', res) - // }) + expect(res).to.deep.equal([{ + foo: { id: 1, value: 0 }, + bars: [{ value: 0 }, { value: 1 }], + x: [1, 2], + y: ['a', 'b'], + z: ['1', '2'], + o: [{ a: 1, b: '1' }, { a: 2, b: '2' }] + }, + { + foo: { id: 2, value: 2 }, + bars: [{ value: 0 }], + x: [3], + y: ['c'], + z: ['3'], + o: [{ a: 3, b: '3' }] + } + ]) + }) } } From 731787149d2ba2664ec4f50dbf2e3d351f4615fa Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 26 Oct 2023 03:22:43 +0800 Subject: [PATCH 06/49] chore: clean --- packages/core/src/eval.ts | 1 - packages/mongo/src/utils.ts | 4 ---- packages/mysql/src/index.ts | 6 +----- packages/mysql/tests/maria.spec.ts | 33 ------------------------------ packages/sql-utils/src/index.ts | 4 ---- 5 files changed, 1 insertion(+), 47 deletions(-) delete mode 100644 packages/mysql/tests/maria.spec.ts diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index d0438dc3..e81ccb9b 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -93,7 +93,6 @@ export namespace Eval { // element in(x: T | Expr, array: (T | Expr)[] | Expr): Expr nin(x: T | Expr, array: (T | Expr)[] | Expr): Expr - at(array: (T | Expr)[] | Expr, index: Number): Expr // string concat: Multi diff --git a/packages/mongo/src/utils.ts b/packages/mongo/src/utils.ts index 623c67c6..ed8545c1 100644 --- a/packages/mongo/src/utils.ts +++ b/packages/mongo/src/utils.ts @@ -118,10 +118,6 @@ export class Transformer { return this.transformEvalExpr(expr.$object) } - // if (expr.$array) { - // return {} - // } - return valueMap(expr as any, (value) => { if (Array.isArray(value)) { return value.map(val => this.eval(val, group)) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index c7b05fe9..a42b17b6 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -534,11 +534,7 @@ export class MySQLDriver extends Driver { } return `${escaped} = ${value}` }).join(', ') - console.log([ - `INSERT INTO ${escapeId(table)} (${initFields.map(escapeId).join(', ')})`, - `VALUES (${insertion.map(item => this._formatValues(table, item, initFields)).join('), (')})`, - `ON DUPLICATE KEY UPDATE ${update}`, - ].join(' ')) + await this.query([ `INSERT INTO ${escapeId(table)} (${initFields.map(escapeId).join(', ')})`, `VALUES (${insertion.map(item => this._formatValues(table, item, initFields)).join('), (')})`, diff --git a/packages/mysql/tests/maria.spec.ts b/packages/mysql/tests/maria.spec.ts deleted file mode 100644 index 76f4782b..00000000 --- a/packages/mysql/tests/maria.spec.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Database } from 'minato' -import Logger from 'reggol' -import test from '@minatojs/tests' - -const logger = new Logger('maria') - -describe('@minatojs/driver-mysql/maria', () => { - const database = new Database() - - before(async () => { - logger.level = 3 - await database.connect('mysql', { - port: 3307, - user: 'koishi', - password: 'koishi@114514', - database: 'test', - }) - }) - - after(async () => { - await database.dropAll() - await database.stopAll() - logger.level = 2 - }) - - test(database, { - query: { - list: { - elementQuery: false, - }, - }, - }) -}) diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index c1460d4e..38e13cf9 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -171,9 +171,6 @@ export class Builder { } protected groupArray(expr: any) { - // this.jsonQuoteMode = true - // const ret = `cast(concat('[', group_concat(${this.parseAggr(expr)}), ']') as json)` - // this.jsonQuoteMode = false this.jsonQuoteMode = true const ret = `json_arrayagg(${this.parseAggr(expr)})` this.jsonQuoteMode = false @@ -369,7 +366,6 @@ export class Builder { if (!(key in model.fields)) continue const { type, initial, runtimeType } = model.fields[key]! const converter = runtimeType?.json ? this.types['json'] : this.types[type] - // const converter = this.types[type] result[key] = converter ? converter.load(obj[key], initial) : obj[key] } return model.parse(result) From 8844364a491454279c0334e37628197f3538dab6 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 26 Oct 2023 03:34:12 +0800 Subject: [PATCH 07/49] test for maria 10.5 --- packages/sql-utils/src/index.ts | 3 ++- packages/tests/src/selection.ts | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 38e13cf9..1262280c 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -172,7 +172,8 @@ export class Builder { protected groupArray(expr: any) { this.jsonQuoteMode = true - const ret = `json_arrayagg(${this.parseAggr(expr)})` + const ret = `json_arrayagg(json_unquote(${this.parseAggr(expr)}))` + // const ret = `concat('[', group_concat(${this.parseAggr(expr)}), ']')` this.jsonQuoteMode = false return ret } diff --git a/packages/tests/src/selection.ts b/packages/tests/src/selection.ts index b1d505ca..d2c1e7cf 100644 --- a/packages/tests/src/selection.ts +++ b/packages/tests/src/selection.ts @@ -1,7 +1,6 @@ import { $, Database } from '@minatojs/core' import { expect } from 'chai' import { setup } from './utils' -import MongoDriver from '@minatojs/driver-mongo' interface Foo { id: number From 55b78179b684db6166ab610829f5b85489dd5b48 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 26 Oct 2023 03:48:00 +0800 Subject: [PATCH 08/49] stage: wait for runtimeType for callback expr --- packages/sql-utils/src/index.ts | 3 +-- packages/tests/src/experimental.ts | 35 +++++++++++++++--------------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 1262280c..38e13cf9 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -172,8 +172,7 @@ export class Builder { protected groupArray(expr: any) { this.jsonQuoteMode = true - const ret = `json_arrayagg(json_unquote(${this.parseAggr(expr)}))` - // const ret = `concat('[', group_concat(${this.parseAggr(expr)}), ']')` + const ret = `json_arrayagg(${this.parseAggr(expr)})` this.jsonQuoteMode = false return ret } diff --git a/packages/tests/src/experimental.ts b/packages/tests/src/experimental.ts index 852f37c4..cbe171f8 100644 --- a/packages/tests/src/experimental.ts +++ b/packages/tests/src/experimental.ts @@ -136,13 +136,11 @@ namespace ExperimentalTests { { foo: { id: 1, value: 0 }, x: [1, 2], y: ['a', 'b'] }, { foo: { id: 2, value: 2 }, x: [3], y: ['c'] } ]) - // console.log('res', res) }) it('$.array groupFull', async () => { const res = await database.select('bar') .project({ - // count: row => $.aggr(row.bar.obj), count2: row => $.array(row.s), countnumber: row => $.array(row.value), x: row => $.array(row.obj.x), @@ -174,22 +172,23 @@ namespace ExperimentalTests { .orderBy(row => row.foo.id) .execute() - expect(res).to.deep.equal([{ - foo: { id: 1, value: 0 }, - bars: [{ value: 0 }, { value: 1 }], - x: [1, 2], - y: ['a', 'b'], - z: ['1', '2'], - o: [{ a: 1, b: '1' }, { a: 2, b: '2' }] - }, - { - foo: { id: 2, value: 2 }, - bars: [{ value: 0 }], - x: [3], - y: ['c'], - z: ['3'], - o: [{ a: 3, b: '3' }] - } + expect(res).to.deep.equal([ + { + foo: { id: 1, value: 0 }, + bars: [{ value: 0 }, { value: 1 }], + x: [1, 2], + y: ['a', 'b'], + z: ['1', '2'], + o: [{ a: 1, b: '1' }, { a: 2, b: '2' }] + }, + { + foo: { id: 2, value: 2 }, + bars: [{ value: 0 }], + x: [3], + y: ['c'], + z: ['3'], + o: [{ a: 3, b: '3' }] + } ]) }) } From 1f23cb7bbd7b5622195cd4a78e907dee65be5f22 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 26 Oct 2023 13:49:39 +0800 Subject: [PATCH 09/49] revert fieldType::expr --- packages/core/src/driver.ts | 2 +- packages/core/src/eval.ts | 4 ++-- packages/core/src/model.ts | 5 +++-- packages/core/src/selection.ts | 2 +- packages/sql-utils/src/index.ts | 6 +++--- packages/tests/src/experimental.ts | 20 -------------------- 6 files changed, 10 insertions(+), 29 deletions(-) diff --git a/packages/core/src/driver.ts b/packages/core/src/driver.ts index ee8b4c86..21053795 100644 --- a/packages/core/src/driver.ts +++ b/packages/core/src/driver.ts @@ -241,7 +241,7 @@ export abstract class Driver { if (submodel.fields[field]!.deprecated) continue model.fields[`${key}.${field}`] = { type: 'expr', - expr: () => Eval('', [key, field], Field.getRuntimeType(submodel.fields[field]!)), + expr: Eval('', [key, field], Field.getRuntimeType(submodel.fields[field]!)), runtimeType: Field.getRuntimeType(submodel.fields[field]!), } } diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index e81ccb9b..ace07eb7 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -132,7 +132,7 @@ type MultivariateCallback = T extends (...args: infer R) => Eval.Expr( key: K, callback: MultivariateCallback, type: RuntimeType | ((...args) => RuntimeType)): Eval.Static[K] { operators[`$${key}`] = callback - return (...args: any[]) => Eval(key, args, typeof type === 'function' ? type(...args) : type) as any + return (...args: any) => Eval(key, args, typeof type === 'function' ? type(...args) : type) as any } type BinaryCallback = T extends (...args: any[]) => Eval.Expr ? (...args: any[]) => S : never @@ -143,7 +143,7 @@ function comparator(key: K, callback: BinaryCallbac if (isNullable(left) || isNullable(right)) return true return callback(left.valueOf(), right.valueOf()) } - return (...args: any[]) => Eval(key, args, RuntimeType.create('boolean')) as any + return (...args: any) => Eval(key, args, RuntimeType.create('boolean')) as any } Eval.switch = (branches, vDefault) => Eval('switch', { branches, default: vDefault }, getExprRuntimeType(vDefault)) diff --git a/packages/core/src/model.ts b/packages/core/src/model.ts index cdccf5fe..fc4a86d9 100644 --- a/packages/core/src/model.ts +++ b/packages/core/src/model.ts @@ -1,6 +1,6 @@ import { clone, isNullable, makeArray, MaybeArray, valueMap } from 'cosmokit' import { Database } from './driver' -import { getExprRuntimeType, isEvalExpr } from './eval' +import { Eval, getExprRuntimeType, isEvalExpr } from './eval' import { RuntimeType } from './runtime' import { Selection } from './selection' import { Flatten, Keys } from './utils' @@ -15,7 +15,7 @@ export interface Field { initial?: T precision?: number scale?: number - expr?: Selection.Callback + expr?: Eval.Expr legacy?: string[] deprecated?: boolean runtimeType?: RuntimeType @@ -83,6 +83,7 @@ export namespace Field { } export function getRuntimeType(field: Field): RuntimeType { + if (field.runtimeType) return field.runtimeType if (number.includes(field.type)) return RuntimeType.number else if (string.includes(field.type)) return RuntimeType.string else if (boolean.includes(field.type)) return RuntimeType.boolean diff --git a/packages/core/src/selection.ts b/packages/core/src/selection.ts index 25869364..92c588e1 100644 --- a/packages/core/src/selection.ts +++ b/packages/core/src/selection.ts @@ -30,7 +30,7 @@ namespace Executable { } } -export const createRow = (ref: string, expr = {}, prefix = '', model?: Model) => new Proxy(expr, { +const createRow = (ref: string, expr = {}, prefix = '', model?: Model) => new Proxy(expr, { get(target, key) { if (typeof key === 'symbol' || key in target || key === 'toJSON' || key.startsWith('$')) return Reflect.get(target, key) const fullKey = `${prefix}${key}` diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 38e13cf9..a6d25fe7 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -1,5 +1,5 @@ import { Dict, isNullable } from 'cosmokit' -import { createRow, Eval, Field, isComparable, Model, Modifier, Query, Selection } from '@minatojs/core' +import { Eval, Field, isComparable, Model, Modifier, Query, Selection } from '@minatojs/core' export function escapeId(value: string) { return '`' + value + '`' @@ -258,9 +258,9 @@ export class Builder { const fkey = Object.keys(fields).find(field => key === field || key.startsWith(field + '.')) if (fkey && fields[fkey]?.expr) { if (key === fkey) { - return this.parseEvalExpr(fields[fkey]?.expr!(createRow(table))) + return this.parseEvalExpr(fields[fkey]?.expr) } else { - const field = this.parseEvalExpr(fields[fkey]?.expr!(createRow(table))) + const field = this.parseEvalExpr(fields[fkey]?.expr) const rest = key.slice(fkey.length + 1).split('.') return this.transformJsonField(`${field}`, rest.map(key => `."${key}"`).join('')) } diff --git a/packages/tests/src/experimental.ts b/packages/tests/src/experimental.ts index cbe171f8..758cdab7 100644 --- a/packages/tests/src/experimental.ts +++ b/packages/tests/src/experimental.ts @@ -49,12 +49,6 @@ function ExperimentalTests(database: Database) { }) before(async () => { - - database.extend('foo', { - id: 'unsigned', - value: 'integer', - }) - await setup(database, 'foo', [ { id: 1, value: 0 }, { id: 2, value: 2 }, @@ -71,19 +65,6 @@ function ExperimentalTests(database: Database) { namespace ExperimentalTests { export function computed(database: Database) { - // it('project', async () => { - // const res = await database.select('bar') - // .project({ - // obj: row => (row.obj), - // objX: row => row.obj.x, - // objX2: row => $.add(row.obj.x, 2), - // objY: row => (row.obj.y), - // }) - // // .orderBy(row => row.foo.id) - // .execute() - // console.log('res', res) - // }) - it('$.object', async () => { const res = await database.select('foo') .project({ @@ -103,7 +84,6 @@ namespace ExperimentalTests { }) it('$.object in json', async () => { - const res = await database.select('bar') .project({ obj: row => $.object({ From 1cb33df38dd138271280491ba5f155a7c18f259d Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 26 Oct 2023 14:13:35 +0800 Subject: [PATCH 10/49] chore: clean --- packages/core/src/eval.ts | 2 +- packages/core/src/model.ts | 3 ++- packages/core/src/selection.ts | 2 +- packages/mysql/src/index.ts | 4 ++-- packages/sqlite/src/index.ts | 3 +-- packages/tests/src/selection.ts | 17 ----------------- 6 files changed, 7 insertions(+), 24 deletions(-) diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index ace07eb7..20e7d6de 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -143,7 +143,7 @@ function comparator(key: K, callback: BinaryCallbac if (isNullable(left) || isNullable(right)) return true return callback(left.valueOf(), right.valueOf()) } - return (...args: any) => Eval(key, args, RuntimeType.create('boolean')) as any + return (...args: any) => Eval(key, args, RuntimeType.boolean) as any } Eval.switch = (branches, vDefault) => Eval('switch', { branches, default: vDefault }, getExprRuntimeType(vDefault)) diff --git a/packages/core/src/model.ts b/packages/core/src/model.ts index fc4a86d9..9453a816 100644 --- a/packages/core/src/model.ts +++ b/packages/core/src/model.ts @@ -89,7 +89,8 @@ export namespace Field { else if (boolean.includes(field.type)) return RuntimeType.boolean else if (date.includes(field.type)) return RuntimeType.date else if (field.type === 'list') return RuntimeType.list(RuntimeType.string) - else if (field.type === 'json' || field.type === 'primary') return RuntimeType.any + else if (field.type === 'json') return RuntimeType.json(RuntimeType.any) + else if (field.type === 'primary') return RuntimeType.any else if (field.type === 'expr') return getExprRuntimeType(field.expr) else throw new Error(`No runtime type for ${field}`) } diff --git a/packages/core/src/selection.ts b/packages/core/src/selection.ts index 92c588e1..4f9d3869 100644 --- a/packages/core/src/selection.ts +++ b/packages/core/src/selection.ts @@ -52,7 +52,7 @@ class Executable { const expr = {} if (typeof payload.table !== 'string' && !(payload.table instanceof Selection)) { for (const key in payload.table) { - expr[key] = createRow(key, { $ref: this.ref }, '', this.model) + expr[key] = createRow(key, {}, '', this.model) } } defineProperty(this, 'row', createRow(this.ref, expr, '', this.model)) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index a42b17b6..3aff8a8f 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -259,8 +259,8 @@ export class MySQLDriver extends Driver { // field definitions for (const key in fields) { - const { deprecated, expr, initial, nullable = true } = fields[key]! - if (deprecated || expr) continue + const { deprecated, initial, nullable = true } = fields[key]! + if (deprecated) continue const legacy = [key, ...fields[key]!.legacy || []] const column = columns.find(info => legacy.includes(info.COLUMN_NAME)) let shouldUpdate = column?.COLUMN_NAME !== key diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index 8d7381dc..ec37db64 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -126,8 +126,7 @@ export class SQLiteDriver extends Driver { const legacy = [key, ...model.fields[key]!.legacy || []] const column = columns.find(({ name }) => legacy.includes(name)) - const { expr, initial, nullable = true } = model.fields[key]! - if (expr) continue + const { initial, nullable = true } = model.fields[key]! const typedef = getTypeDef(model.fields[key]!) let def = `${escapeId(key)} ${typedef}` if (key === model.primary && model.autoInc) { diff --git a/packages/tests/src/selection.ts b/packages/tests/src/selection.ts index d2c1e7cf..dc34bf63 100644 --- a/packages/tests/src/selection.ts +++ b/packages/tests/src/selection.ts @@ -141,23 +141,6 @@ namespace SelectionTests { }) it('aggregate', async () => { - await database - .select('foo') - .project({ - count: row => row.id, - }) - .execute() - - await expect(database - .select('foo') - .project({ - count: row => $.count(row.id), - }) - .execute() - ).to.eventually.deep.equal([ - { count: 3 }, - ]) - await expect(database .select('foo') .project({ From 6de2c05b8f1fd31ea4640a18da74123fb7da1d5c Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 26 Oct 2023 14:18:55 +0800 Subject: [PATCH 11/49] fix issue for maria 10.5 --- packages/sql-utils/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index a6d25fe7..35741966 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -172,7 +172,7 @@ export class Builder { protected groupArray(expr: any) { this.jsonQuoteMode = true - const ret = `json_arrayagg(${this.parseAggr(expr)})` + const ret = `json_merge('[]', json_arrayagg(${this.parseAggr(expr)}))` this.jsonQuoteMode = false return ret } From 5141424cf42a64a1c03ae0d4bbe658f6728732ac Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 26 Oct 2023 14:23:01 +0800 Subject: [PATCH 12/49] test ci maria 10.6 --- .github/workflows/test.yaml | 2 +- packages/sql-utils/src/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5d979ece..3a7b627e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,7 +15,7 @@ jobs: mysql-image: - mysql:5.7 - mysql:8.0 - - mariadb:10.5 + - mariadb:10.6 node-version: [16, 18, 20] services: diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 35741966..a6d25fe7 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -172,7 +172,7 @@ export class Builder { protected groupArray(expr: any) { this.jsonQuoteMode = true - const ret = `json_merge('[]', json_arrayagg(${this.parseAggr(expr)}))` + const ret = `json_arrayagg(${this.parseAggr(expr)})` this.jsonQuoteMode = false return ret } From 508311cdc834046b34faab265542afe8c008c551 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 26 Oct 2023 14:37:39 +0800 Subject: [PATCH 13/49] revert --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3a7b627e..5d979ece 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,7 +15,7 @@ jobs: mysql-image: - mysql:5.7 - mysql:8.0 - - mariadb:10.6 + - mariadb:10.5 node-version: [16, 18, 20] services: From 87f0eff6f5d0ca3813640479337a24468b9ea8bb Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Sun, 29 Oct 2023 02:39:49 +0800 Subject: [PATCH 14/49] refa: json return type --- packages/sql-utils/src/index.ts | 41 ++++++++++++++++++++++----------- packages/sqlite/src/index.ts | 4 ++++ 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index a6d25fe7..56a25318 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -28,7 +28,7 @@ export class Builder { protected createEqualQuery = this.comparator('=') protected queryOperators: QueryOperators protected evalOperators: EvalOperators - protected jsonQuoteMode = false + protected jsonQuoted = false constructor(public tables?: Dict) { this.queryOperators = { @@ -163,17 +163,20 @@ export class Builder { return `NOT(${condition})` } + protected unquoteJson(value: string) { + return this.jsonQuoted ? `json_unquote(${value})` : value + } + protected groupObject(fields: any) { - this.jsonQuoteMode = true const ret = `json_object(` + Object.entries(fields).map(([key, expr]) => `'${key}', ${this.parseAggr(expr)}`).join(',') + `)` - this.jsonQuoteMode = false + this.jsonQuoted = true return ret } protected groupArray(expr: any) { - this.jsonQuoteMode = true - const ret = `json_arrayagg(${this.parseAggr(expr)})` - this.jsonQuoteMode = false + const aggr = this.parseAggr(expr) + const ret = this.jsonQuoted ? `concat('[', group_concat(${aggr}), ']')` : `json_arrayagg(${aggr})` + this.jsonQuoted = true return ret } @@ -222,6 +225,7 @@ export class Builder { } private parseEvalExpr(expr: any) { + // this.jsonQuoted = false for (const key in expr) { if (key in this.evalOperators) { return this.evalOperators[key](expr[key]) @@ -230,16 +234,20 @@ export class Builder { return this.escape(expr) } - protected parseAggr(expr: any) { - if (typeof expr === 'string') { - return this.getRecursive(expr) + protected parseAggr(expr: any, unquote: boolean = true) { + const ret = typeof expr === 'string' ? this.getRecursive(expr) : this.parseEvalExpr(expr) + if (unquote) { + this.jsonQuoted = false + return this.unquoteJson(ret) + } else { + this.jsonQuoted = true + return ret } - return this.parseEvalExpr(expr) } protected transformJsonField(obj: string, path: string) { - if (this.jsonQuoteMode) return `json_extract(${obj}, '$${path}')` - else return `json_unquote(json_extract(${obj}, '$${path}'))` + this.jsonQuoted = true + return `json_extract(${obj}, '$${path}')` } private transformKey(key: string, fields: {}, prefix: string) { @@ -271,11 +279,16 @@ export class Builder { return this.transformKey(key, fields, prefix) } - parseEval(expr: any): string { + parseEval(expr: any, unquote: boolean = true): string { if (typeof expr === 'string' || typeof expr === 'number' || typeof expr === 'boolean' || expr instanceof Date) { return this.escape(expr) + } else if (unquote) { + this.jsonQuoted = false + return this.unquoteJson(this.parseEvalExpr(expr)) + } else { + this.jsonQuoted = true + return this.parseEvalExpr(expr) } - return this.parseEvalExpr(expr) } suffix(modifier: Modifier) { diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index ec37db64..99845920 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -86,6 +86,10 @@ class SQLiteBuilder extends Builder { return `(',' || ${key} || ',') LIKE ${this.escape('%,' + value + ',%')}` } + protected unquoteJson(value: string) { + return value + } + protected groupArray(expr: any) { return `('[' || group_concat(json_quote(${this.parseAggr(expr)})) || ']')` } From 12b8343135087dab9a07e5e446443ec727da351f Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Sun, 29 Oct 2023 08:11:27 +0800 Subject: [PATCH 15/49] stage: double unquote for maria10.5 --- package.json | 12 ++++++------ packages/mysql/src/index.ts | 17 ++++++++++++++--- packages/tests/src/experimental.ts | 29 +++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index fcbc8c99..2b7eec9d 100644 --- a/package.json +++ b/package.json @@ -23,15 +23,15 @@ "devDependencies": { "@koishijs/eslint-config": "^1.0.4", "@types/mocha": "^9.1.1", - "@types/node": "^20.4.2", + "@types/node": "^20.8.9", "c8": "^7.14.0", - "esbuild": "^0.18.14", - "esbuild-register": "^3.4.2", - "eslint": "^8.45.0", - "eslint-plugin-mocha": "^10.1.0", + "esbuild": "^0.18.20", + "esbuild-register": "^3.5.0", + "eslint": "^8.52.0", + "eslint-plugin-mocha": "^10.2.0", "mocha": "^9.2.2", "shx": "^0.3.4", - "typescript": "^5.1.6", + "typescript": "^5.2.2", "yakumo": "^0.3.13", "yakumo-esbuild": "^0.3.26", "yakumo-mocha": "^0.3.1", diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index b5277aa4..5b90407d 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -104,7 +104,7 @@ class MySQLBuilder extends Builder { '\\': '\\\\', } - constructor(tables?: Dict) { + constructor(tables?: Dict, issueUnquote = false) { super(tables) this.define({ @@ -116,7 +116,10 @@ class MySQLBuilder extends Builder { this.define({ types: ['json'], dump: value => JSON.stringify(value), - load: value => typeof value === 'string' ? JSON.parse(value) : value, + load: value => { + const obj = typeof value === 'string' ? JSON.parse(value) : value + return Array.isArray(obj) && issueUnquote ? obj.map(x => JSON.parse(x)) : obj + }, }) } @@ -187,6 +190,8 @@ export class MySQLDriver extends Driver { private _queryTasks: QueryTask[] = [] + private _fixMariaIssue: boolean = false + constructor(database: Database, config?: MySQLDriver.Config) { super(database) @@ -237,6 +242,12 @@ export class MySQLDriver extends Driver { /** synchronize table schema */ async prepare(name: string) { + const version = Object.values((await this.query(`SELECT version()`))[0])[0] as string + if (version.match(/10.5.\d+-MariaDB/)) { + logger.warn('MariaDB 10.5 will be depracated in the future, better move to LTS version.') + this._fixMariaIssue = true + } + const [columns, indexes] = await Promise.all([ this.queue([ `SELECT *`, @@ -434,7 +445,7 @@ export class MySQLDriver extends Driver { async get(sel: Selection.Immutable) { const { model, tables } = sel - const builder = new MySQLBuilder(tables) + const builder = new MySQLBuilder(tables, this._fixMariaIssue) const sql = builder.get(sel) if (!sql) return [] return this.queue(sql).then((data) => { diff --git a/packages/tests/src/experimental.ts b/packages/tests/src/experimental.ts index 758cdab7..9fcd2392 100644 --- a/packages/tests/src/experimental.ts +++ b/packages/tests/src/experimental.ts @@ -171,6 +171,35 @@ namespace ExperimentalTests { } ]) }) + + it('$.array groupBy with expressions', async () => { + const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) + .groupBy('foo', { + bars: row => $.array($.object({ + value: row.bar.value, + value2: $.add(row.bar.value, row.foo.value), + })), + x: row => $.array($.add(1, row.bar.obj.x)), + y: row => $.array(row.bar.obj.y), + }) + .orderBy(row => row.foo.id) + .execute() + + expect(res).to.deep.equal([ + { + foo: { id: 1, value: 0 }, + bars: [{ value: 0, value2: 0 }, { value: 1, value2: 1 }], + x: [2, 3], + y: ['a', 'b'], + }, + { + foo: { id: 2, value: 2 }, + bars: [{ value: 0, value2: 2 }], + x: [4], + y: ['c'], + } + ]) + }) } } From e93d879b756cd90ca98525633efed9c083dfe104 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Sun, 29 Oct 2023 08:18:24 +0800 Subject: [PATCH 16/49] test --- packages/mysql/src/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index 5b90407d..6d1f2a84 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -118,7 +118,10 @@ class MySQLBuilder extends Builder { dump: value => JSON.stringify(value), load: value => { const obj = typeof value === 'string' ? JSON.parse(value) : value - return Array.isArray(obj) && issueUnquote ? obj.map(x => JSON.parse(x)) : obj + if (Array.isArray(obj) && issueUnquote) { + logger.debug('unquote: ', obj) + return obj.map(x => JSON.parse(x)) + } else return obj }, }) } From 893bf27127a51e943a9aad430658551b7ed5acc2 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Sun, 29 Oct 2023 16:54:04 +0800 Subject: [PATCH 17/49] workaround --- packages/mysql/src/index.ts | 2 +- packages/sql-utils/src/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index 6d1f2a84..ae58452e 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -248,7 +248,7 @@ export class MySQLDriver extends Driver { const version = Object.values((await this.query(`SELECT version()`))[0])[0] as string if (version.match(/10.5.\d+-MariaDB/)) { logger.warn('MariaDB 10.5 will be depracated in the future, better move to LTS version.') - this._fixMariaIssue = true + // this._fixMariaIssue = true } const [columns, indexes] = await Promise.all([ diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 56a25318..4188b906 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -175,7 +175,7 @@ export class Builder { protected groupArray(expr: any) { const aggr = this.parseAggr(expr) - const ret = this.jsonQuoted ? `concat('[', group_concat(${aggr}), ']')` : `json_arrayagg(${aggr})` + const ret = this.jsonQuoted ? `concat('[', group_concat(${aggr}), ']')` : `concat('[', group_concat(json_extract(json_object('f', ${aggr}), '$.f')), ']')` this.jsonQuoted = true return ret } From f7c251acf117281fe13e47e6e01b4efa27233bee Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Mon, 30 Oct 2023 00:39:00 +0800 Subject: [PATCH 18/49] refa: ready to remove runtimeType --- packages/sql-utils/src/index.ts | 80 +++++++++++++++++++----------- packages/sqlite/src/index.ts | 15 ++++-- packages/tests/src/experimental.ts | 25 ++++++++++ 3 files changed, 88 insertions(+), 32 deletions(-) diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 4188b906..233d5bd0 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -1,6 +1,16 @@ import { Dict, isNullable } from 'cosmokit' import { Eval, Field, isComparable, Model, Modifier, Query, Selection } from '@minatojs/core' +export type SQLType = 'raw' | 'json' | 'list' + +// declare module '@minatojs/core' { +// namespace Selection { +// interface Immutable { + +// } +// } +// } + export function escapeId(value: string) { return '`' + value + '`' } @@ -29,6 +39,8 @@ export class Builder { protected queryOperators: QueryOperators protected evalOperators: EvalOperators protected jsonQuoted = false + protected workaroundArrayagg = false + protected sqlTypes: Dict = {} constructor(public tables?: Dict) { this.queryOperators = { @@ -164,7 +176,9 @@ export class Builder { } protected unquoteJson(value: string) { - return this.jsonQuoted ? `json_unquote(${value})` : value + const ret = this.jsonQuoted ? `json_unquote(${value})` : value + this.jsonQuoted = false + return ret } protected groupObject(fields: any) { @@ -175,7 +189,9 @@ export class Builder { protected groupArray(expr: any) { const aggr = this.parseAggr(expr) - const ret = this.jsonQuoted ? `concat('[', group_concat(${aggr}), ']')` : `concat('[', group_concat(json_extract(json_object('f', ${aggr}), '$.f')), ']')` + // const ret = this.jsonQuoted ? `concat('[', group_concat(${aggr}), ']')` + // : this.workaroundArrayagg ? `concat('[', group_concat(json_extract(json_object('f', ${aggr}), '$.f')), ']')` : `json_arrayagg(${aggr})` + const ret = this.workaroundArrayagg ? `concat('[', group_concat(json_extract(json_object('v', ${aggr}), '$.v')), ']')` : `json_arrayagg(${aggr})` this.jsonQuoted = true return ret } @@ -225,7 +241,7 @@ export class Builder { } private parseEvalExpr(expr: any) { - // this.jsonQuoted = false + this.jsonQuoted = false for (const key in expr) { if (key in this.evalOperators) { return this.evalOperators[key](expr[key]) @@ -234,15 +250,10 @@ export class Builder { return this.escape(expr) } - protected parseAggr(expr: any, unquote: boolean = true) { + protected parseAggr(expr: any) { + this.jsonQuoted = false const ret = typeof expr === 'string' ? this.getRecursive(expr) : this.parseEvalExpr(expr) - if (unquote) { - this.jsonQuoted = false - return this.unquoteJson(ret) - } else { - this.jsonQuoted = true - return ret - } + return ret } protected transformJsonField(obj: string, path: string) { @@ -251,7 +262,12 @@ export class Builder { } private transformKey(key: string, fields: {}, prefix: string) { - if (key in fields || !key.includes('.')) return prefix + escapeId(key) + if (key in fields || !key.includes('.')) { + if (this.sqlTypes[key]) this.jsonQuoted = this.sqlTypes[key] === 'json' + // this.jsonQuoted = fields[key]?.runtimeType?.json ?? true + // console.log(key, this.jsonQuoted, this.sqlTypes) + return prefix + escapeId(key) + } const field = Object.keys(fields).find(k => key.startsWith(k + '.')) || key.split('.')[0] const rest = key.slice(field.length + 1).split('.') return this.transformJsonField(`${prefix} ${escapeId(field)}`, rest.map(key => `."${key}"`).join('')) @@ -282,12 +298,12 @@ export class Builder { parseEval(expr: any, unquote: boolean = true): string { if (typeof expr === 'string' || typeof expr === 'number' || typeof expr === 'boolean' || expr instanceof Date) { return this.escape(expr) - } else if (unquote) { - this.jsonQuoted = false - return this.unquoteJson(this.parseEvalExpr(expr)) + } + const res = this.parseEvalExpr(expr) + if (unquote && this.jsonQuoted) { + return this.unquoteJson(res) } else { - this.jsonQuoted = true - return this.parseEvalExpr(expr) + return res } } @@ -314,16 +330,6 @@ export class Builder { const filter = this.parseQuery(query) if (filter === '0') return - // get prefix - const fields = args[0].fields ?? Object.fromEntries(Object - .entries(model.fields) - .filter(([, field]) => !field!.deprecated) - .map(([key, field]) => [key, Eval('', [ref, key], field!.runtimeType!)])) - const keys = Object.entries(fields).map(([key, value]) => { - key = escapeId(key) - value = this.parseEval(value) - return key === value ? key : `${value} AS ${key}` - }).join(', ') let prefix: string | undefined if (typeof table === 'string') { prefix = escapeId(table) @@ -342,6 +348,21 @@ export class Builder { if (filter !== '1') prefix += ` ON ${filter}` } + const sqlTypes = {} + // get prefix + const fields = args[0].fields ?? Object.fromEntries(Object + .entries(model.fields) + .filter(([, field]) => !field!.deprecated) + .map(([key, field]) => [key, Eval('', [ref, key], field!.runtimeType!)])) + const keys = Object.entries(fields).map(([key, value]) => { + value = this.parseEval(value, false) + sqlTypes[key] = this.jsonQuoted ? 'json' : 'raw' + return escapeId(key) === value ? escapeId(key) : `${value} AS ${escapeId(key)}` + }).join(', ') + + this.sqlTypes = sqlTypes + // console.log('field sqlTypes:', Object.keys(fields), sqlTypes) + // get suffix let suffix = this.suffix(args[0]) if (filter !== '1') { @@ -375,10 +396,11 @@ export class Builder { load(model: Model, obj: any): any { const result = {} + // console.log('sql', this.sqlTypes, Object.keys(obj)) for (const key in obj) { if (!(key in model.fields)) continue - const { type, initial, runtimeType } = model.fields[key]! - const converter = runtimeType?.json ? this.types['json'] : this.types[type] + const { type, initial } = model.fields[key]! + const converter = this.sqlTypes[key] === 'json' ? this.types['json'] : this.types[type] result[key] = converter ? converter.load(obj[key], initial) : obj[key] } return model.parse(result) diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index 8cc64c91..c26063b2 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -61,7 +61,10 @@ class SQLiteBuilder extends Builder { this.define({ types: ['json'], dump: value => JSON.stringify(value), - load: (value, initial) => value ? JSON.parse(value) : initial, + load: (value, initial) => { + // console.log('load json', value) + return value ? JSON.parse(value) : initial + }, }) this.define({ @@ -91,11 +94,17 @@ class SQLiteBuilder extends Builder { } protected groupArray(expr: any) { - return `('[' || group_concat(json_quote(${this.parseAggr(expr)})) || ']')` + // console.log('1') + const aggr = this.parseAggr(expr) + // console.log('2', expr, aggr, this.jsonQuoted) + const ret = this.jsonQuoted ? `('[' || group_concat(${aggr}) || ']')` : `('[' || group_concat(json_quote(${aggr})) || ']')` + this.jsonQuoted = true + return ret } protected transformJsonField(obj: string, path: string) { - return `json_extract(${obj}, '$${path}')` + this.jsonQuoted = true + return `json_quote(json_extract(${obj}, '$${path}'))` } } diff --git a/packages/tests/src/experimental.ts b/packages/tests/src/experimental.ts index 9fcd2392..b9b5ad54 100644 --- a/packages/tests/src/experimental.ts +++ b/packages/tests/src/experimental.ts @@ -200,6 +200,31 @@ namespace ExperimentalTests { } ]) }) + + + it('$.array groupBy with expressions 2', async () => { + const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) + .groupBy('foo', { + y: row => $.array(row.bar.obj.x), + }) + .orderBy(row => row.foo.id) + .project({ + z: row => $.array(row.y) + }) + .execute() + + expect(res).to.deep.equal([ + { + z: [[1, 2], [3]], + }, + ]) + }) + + // it('raw', async () => { + // const driver = Object.values(database.drivers)[0] + // const ret = await driver.query("SELECT `x` FROM (SELECT json_unquote(json_arrayagg(`x`)) AS `x` FROM (SELECT `foo`.`id` AS `foo.id`, `foo`.`value` AS `foo.value`, (json_arrayagg(json_extract(`bar`.`obj`, '$.x'))) AS `x` FROM `foo` JOIN `bar` ON (`foo`.`id` = `bar`.`pid`) GROUP BY `foo.id`, `foo.value`) hjwlkwov) ydqtdvlu") + // console.log(ret) + // }) } } From a7341e458cc2d109bc39c1fde93bf69c045c03d9 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Mon, 30 Oct 2023 00:59:53 +0800 Subject: [PATCH 19/49] fix: prev --- packages/mysql/src/index.ts | 10 +++------- packages/sql-utils/src/index.ts | 7 ++++++- packages/tests/src/experimental.ts | 10 +++++----- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index ae58452e..11750099 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -107,6 +107,8 @@ class MySQLBuilder extends Builder { constructor(tables?: Dict, issueUnquote = false) { super(tables) + this.workaroundArrayagg = issueUnquote + this.define({ types: ['list'], dump: value => value.join(','), @@ -116,13 +118,7 @@ class MySQLBuilder extends Builder { this.define({ types: ['json'], dump: value => JSON.stringify(value), - load: value => { - const obj = typeof value === 'string' ? JSON.parse(value) : value - if (Array.isArray(obj) && issueUnquote) { - logger.debug('unquote: ', obj) - return obj.map(x => JSON.parse(x)) - } else return obj - }, + load: value => typeof value === 'string' ? JSON.parse(value) : value, }) } diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 233d5bd0..0d567f7d 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -191,7 +191,12 @@ export class Builder { const aggr = this.parseAggr(expr) // const ret = this.jsonQuoted ? `concat('[', group_concat(${aggr}), ']')` // : this.workaroundArrayagg ? `concat('[', group_concat(json_extract(json_object('f', ${aggr}), '$.f')), ']')` : `json_arrayagg(${aggr})` - const ret = this.workaroundArrayagg ? `concat('[', group_concat(json_extract(json_object('v', ${aggr}), '$.v')), ']')` : `json_arrayagg(${aggr})` + // console.log(this.jsonQuoted, aggr) + + const ret = this.workaroundArrayagg + ? (this.jsonQuoted ? `concat('[', group_concat(${aggr}), ']')` : `concat('[', group_concat(json_extract(json_object('f', ${aggr}), '$.f')), ']')`) + : `json_arrayagg(${aggr})` + // const ret = this.workaroundArrayagg ? `concat('[', group_concat(json_extract(json_object('v', ${aggr}), '$.v')), ']')` : `json_arrayagg(${aggr})` this.jsonQuoted = true return ret } diff --git a/packages/tests/src/experimental.ts b/packages/tests/src/experimental.ts index b9b5ad54..6426180d 100644 --- a/packages/tests/src/experimental.ts +++ b/packages/tests/src/experimental.ts @@ -220,11 +220,11 @@ namespace ExperimentalTests { ]) }) - // it('raw', async () => { - // const driver = Object.values(database.drivers)[0] - // const ret = await driver.query("SELECT `x` FROM (SELECT json_unquote(json_arrayagg(`x`)) AS `x` FROM (SELECT `foo`.`id` AS `foo.id`, `foo`.`value` AS `foo.value`, (json_arrayagg(json_extract(`bar`.`obj`, '$.x'))) AS `x` FROM `foo` JOIN `bar` ON (`foo`.`id` = `bar`.`pid`) GROUP BY `foo.id`, `foo.value`) hjwlkwov) ydqtdvlu") - // console.log(ret) - // }) + it('raw', async () => { + const driver = Object.values(database.drivers)[0] + const ret = await driver.query("SELECT `z` FROM (SELECT concat('[', group_concat(json_extract(json_object('v', `y`), '$.v')), ']') AS `z` FROM (SELECT `foo`.`id` AS `foo.id`, `foo`.`value` AS `foo.value`, concat('[', group_concat(json_extract(json_object('v', json_extract(`bar`.`obj`, '$.x')), '$.v')), ']') AS `y` FROM `foo` JOIN `bar` ON (`foo`.`id` = `bar`.`pid`) GROUP BY `foo.id`, `foo.value`) opgaymzt ORDER BY `foo.id` ASC) iebqysix") + console.log(ret) + }) } } From d0d1b9db68b1e3fabe393431318972d784e5e01f Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Mon, 30 Oct 2023 01:01:50 +0800 Subject: [PATCH 20/49] fix: prev2 --- packages/mysql/src/index.ts | 2 +- packages/tests/src/experimental.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index 11750099..982d4aa1 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -244,7 +244,7 @@ export class MySQLDriver extends Driver { const version = Object.values((await this.query(`SELECT version()`))[0])[0] as string if (version.match(/10.5.\d+-MariaDB/)) { logger.warn('MariaDB 10.5 will be depracated in the future, better move to LTS version.') - // this._fixMariaIssue = true + this._fixMariaIssue = true } const [columns, indexes] = await Promise.all([ diff --git a/packages/tests/src/experimental.ts b/packages/tests/src/experimental.ts index 6426180d..5069243b 100644 --- a/packages/tests/src/experimental.ts +++ b/packages/tests/src/experimental.ts @@ -220,11 +220,11 @@ namespace ExperimentalTests { ]) }) - it('raw', async () => { - const driver = Object.values(database.drivers)[0] - const ret = await driver.query("SELECT `z` FROM (SELECT concat('[', group_concat(json_extract(json_object('v', `y`), '$.v')), ']') AS `z` FROM (SELECT `foo`.`id` AS `foo.id`, `foo`.`value` AS `foo.value`, concat('[', group_concat(json_extract(json_object('v', json_extract(`bar`.`obj`, '$.x')), '$.v')), ']') AS `y` FROM `foo` JOIN `bar` ON (`foo`.`id` = `bar`.`pid`) GROUP BY `foo.id`, `foo.value`) opgaymzt ORDER BY `foo.id` ASC) iebqysix") - console.log(ret) - }) + // it('raw', async () => { + // const driver = Object.values(database.drivers)[0] + // const ret = await driver.query("SELECT `z` FROM (SELECT concat('[', group_concat(json_extract(json_object('v', `y`), '$.v')), ']') AS `z` FROM (SELECT `foo`.`id` AS `foo.id`, `foo`.`value` AS `foo.value`, concat('[', group_concat(json_extract(json_object('v', json_extract(`bar`.`obj`, '$.x')), '$.v')), ']') AS `y` FROM `foo` JOIN `bar` ON (`foo`.`id` = `bar`.`pid`) GROUP BY `foo.id`, `foo.value`) opgaymzt ORDER BY `foo.id` ASC) iebqysix") + // console.log(ret) + // }) } } From 8af05b3cd31cd4758fcface55a25111c2ad7105b Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Mon, 30 Oct 2023 01:24:41 +0800 Subject: [PATCH 21/49] chore: clean code --- packages/core/src/driver.ts | 8 ++- packages/core/src/eval.ts | 80 +++++++++++------------------- packages/core/src/index.ts | 1 - packages/core/src/model.ts | 24 +-------- packages/core/src/runtime.ts | 46 ----------------- packages/core/src/selection.ts | 16 +++--- packages/mysql/src/index.ts | 15 +++--- packages/sql-utils/src/index.ts | 31 ++++-------- packages/sqlite/src/index.ts | 16 +++--- packages/tests/src/experimental.ts | 6 --- 10 files changed, 65 insertions(+), 178 deletions(-) delete mode 100644 packages/core/src/runtime.ts diff --git a/packages/core/src/driver.ts b/packages/core/src/driver.ts index 21053795..b4df705a 100644 --- a/packages/core/src/driver.ts +++ b/packages/core/src/driver.ts @@ -1,5 +1,5 @@ import { Awaitable, Dict, Intersect, makeArray, MaybeArray, valueMap } from 'cosmokit' -import { Eval, getExprRuntimeType, Update } from './eval' +import { Eval, Update } from './eval' import { Field, Model } from './model' import { Query } from './query' import { Flatten, Indexable, Keys, Row } from './utils' @@ -227,9 +227,8 @@ export abstract class Driver { if (table instanceof Selection) { if (!table.args[0].fields) return table.model const model = new Model('temp') - model.fields = valueMap(table.args[0].fields, (expr, key) => ({ + model.fields = valueMap(table.args[0].fields, (_, key) => ({ type: 'expr', - runtimeType: getExprRuntimeType(expr), })) return model } @@ -241,8 +240,7 @@ export abstract class Driver { if (submodel.fields[field]!.deprecated) continue model.fields[`${key}.${field}`] = { type: 'expr', - expr: Eval('', [key, field], Field.getRuntimeType(submodel.fields[field]!)), - runtimeType: Field.getRuntimeType(submodel.fields[field]!), + expr: { $: [key, field] } as any, } } } diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index 20e7d6de..2ef176ec 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -1,24 +1,10 @@ import { defineProperty, Dict, isNullable, valueMap } from 'cosmokit' -import { RuntimeType } from './runtime' import { Comparable, Flatten, isComparable, makeRegExp } from './utils' export function isEvalExpr(value: any): value is Eval.Expr { return value && Object.keys(value).some(key => key.startsWith('$')) } -export function getExprRuntimeType(value: any): RuntimeType { - if (isNullable(value)) return RuntimeType.any - if (RuntimeType.test(value)) return value - if (isEvalExpr(value)) return value[kRuntimeType] - else if (typeof value === 'string') return RuntimeType.string - else if (typeof value === 'number') return RuntimeType.number - else if (typeof value === 'boolean') return RuntimeType.boolean - else if (value instanceof Date) return RuntimeType.date - else if (value instanceof RegExp) return RuntimeType.regexp - else if (Array.isArray(value)) return RuntimeType.list(RuntimeType.merge(...value)) - else return RuntimeType.create(valueMap(value, getExprRuntimeType)) -} - type $Date = Date type $RegExp = RegExp @@ -38,14 +24,12 @@ export type Eval = const kExpr = Symbol('expr') const kType = Symbol('type') const kAggr = Symbol('aggr') -const kRuntimeType = Symbol('RuntimeType') export namespace Eval { export interface Expr { [kExpr]: true [kType]?: T [kAggr]?: A - [kRuntimeType]: RuntimeType } export type Number = number | Expr @@ -64,7 +48,7 @@ export namespace Eval { } export interface Static { - (key: string, value: any, type: RuntimeType): Eval.Expr + (key: string, value: any): Eval.Expr // univeral if(cond: Any, vThen: T | Expr, vElse: T | Expr): Expr @@ -115,24 +99,22 @@ export namespace Eval { } } -export const Eval = ((key, value, type) => defineProperty({ ['$' + key]: value, [kRuntimeType]: type }, kExpr, true)) as Eval.Static +export const Eval = ((key, value) => defineProperty({ ['$' + key]: value }, kExpr, true)) as Eval.Static const operators = {} as Record<`$${keyof Eval.Static}`, (args: any, data: any) => any> operators['$'] = getRecursive type UnaryCallback = T extends (value: infer R) => Eval.Expr ? (value: R, data: any[]) => S : never -function unary(key: K, callback: UnaryCallback, - type: RuntimeType | ((value) => RuntimeType)): Eval.Static[K] { +function unary(key: K, callback: UnaryCallback): Eval.Static[K] { operators[`$${key}`] = callback - return (value: any) => Eval(key, value, typeof type === 'function' ? type(value) : type) as any + return (value: any) => Eval(key, value) as any } type MultivariateCallback = T extends (...args: infer R) => Eval.Expr ? (args: R, data: any) => S : never -function multary( - key: K, callback: MultivariateCallback, type: RuntimeType | ((...args) => RuntimeType)): Eval.Static[K] { +function multary(key: K, callback: MultivariateCallback): Eval.Static[K] { operators[`$${key}`] = callback - return (...args: any) => Eval(key, args, typeof type === 'function' ? type(...args) : type) as any + return (...args: any) => Eval(key, args) as any } type BinaryCallback = T extends (...args: any[]) => Eval.Expr ? (...args: any[]) => S : never @@ -143,10 +125,10 @@ function comparator(key: K, callback: BinaryCallbac if (isNullable(left) || isNullable(right)) return true return callback(left.valueOf(), right.valueOf()) } - return (...args: any) => Eval(key, args, RuntimeType.boolean) as any + return (...args: any) => Eval(key, args) as any } -Eval.switch = (branches, vDefault) => Eval('switch', { branches, default: vDefault }, getExprRuntimeType(vDefault)) +Eval.switch = (branches, vDefault) => Eval('switch', { branches, default: vDefault }) operators.$switch = (args, data) => { for (const branch of args.branches) { if (executeEval(data, branch.case)) return executeEval(data, branch.then) @@ -155,17 +137,14 @@ operators.$switch = (args, data) => { } // univeral -Eval.if = multary('if', ([cond, vThen, vElse], data) => executeEval(data, cond) ? executeEval(data, vThen) : executeEval(data, vElse), - (_, vThen, vElse) => RuntimeType.merge(vThen, vElse)) -Eval.ifNull = multary('ifNull', ([value, fallback], data) => executeEval(data, value) ?? executeEval(data, fallback), - (value, fallback) => RuntimeType.merge(value, fallback)) +Eval.if = multary('if', ([cond, vThen, vElse], data) => executeEval(data, cond) ? executeEval(data, vThen) : executeEval(data, vElse)) +Eval.ifNull = multary('ifNull', ([value, fallback], data) => executeEval(data, value) ?? executeEval(data, fallback)) // arithmetic -Eval.add = multary('add', (args, data) => args.reduce((prev, curr) => prev + executeEval(data, curr), 0), RuntimeType.number) -Eval.mul = Eval.multiply = multary( - 'multiply', (args, data) => args.reduce((prev, curr) => prev * executeEval(data, curr), 1), RuntimeType.number) -Eval.sub = Eval.subtract = multary('subtract', ([left, right], data) => executeEval(data, left) - executeEval(data, right), RuntimeType.number) -Eval.div = Eval.divide = multary('divide', ([left, right], data) => executeEval(data, left) / executeEval(data, right), RuntimeType.number) +Eval.add = multary('add', (args, data) => args.reduce((prev, curr) => prev + executeEval(data, curr), 0)) +Eval.mul = Eval.multiply = multary('multiply', (args, data) => args.reduce((prev, curr) => prev * executeEval(data, curr), 1)) +Eval.sub = Eval.subtract = multary('subtract', ([left, right], data) => executeEval(data, left) - executeEval(data, right)) +Eval.div = Eval.divide = multary('divide', ([left, right], data) => executeEval(data, left) / executeEval(data, right)) // comparison Eval.eq = comparator('eq', (left, right) => left === right) @@ -176,28 +155,27 @@ Eval.lt = comparator('lt', (left, right) => left < right) Eval.le = Eval.lte = comparator('lte', (left, right) => left <= right) // element -Eval.in = multary('in', ([value, array], data) => executeEval(data, array).includes(executeEval(data, value)), RuntimeType.boolean) -Eval.nin = multary('nin', ([value, array], data) => !executeEval(data, array).includes(executeEval(data, value)), RuntimeType.boolean) +Eval.in = multary('in', ([value, array], data) => executeEval(data, array).includes(executeEval(data, value))) +Eval.nin = multary('nin', ([value, array], data) => !executeEval(data, array).includes(executeEval(data, value))) // string -Eval.concat = multary('concat', (args, data) => args.map(arg => executeEval(data, arg)).join(''), RuntimeType.string) -Eval.regex = multary('regex', ([value, regex], data) => makeRegExp(executeEval(data, regex)).test(executeEval(data, value)), RuntimeType.boolean) +Eval.concat = multary('concat', (args, data) => args.map(arg => executeEval(data, arg)).join('')) +Eval.regex = multary('regex', ([value, regex], data) => makeRegExp(executeEval(data, regex)).test(executeEval(data, value))) // logical -Eval.and = multary('and', (args, data) => args.every(arg => executeEval(data, arg)), RuntimeType.boolean) -Eval.or = multary('or', (args, data) => args.some(arg => executeEval(data, arg)), RuntimeType.boolean) -Eval.not = unary('not', (value, data) => !executeEval(data, value), RuntimeType.boolean) +Eval.and = multary('and', (args, data) => args.every(arg => executeEval(data, arg))) +Eval.or = multary('or', (args, data) => args.some(arg => executeEval(data, arg))) +Eval.not = unary('not', (value, data) => !executeEval(data, value)) // aggregation -Eval.sum = unary('sum', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0), RuntimeType.number) -Eval.avg = unary('avg', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length, RuntimeType.number) -Eval.max = unary('max', (expr, table) => Math.max(...table.map(data => executeAggr(expr, data))), RuntimeType.number) -Eval.min = unary('min', (expr, table) => Math.min(...table.map(data => executeAggr(expr, data))), RuntimeType.number) -Eval.count = unary('count', (expr, table) => new Set(table.map(data => executeAggr(expr, data))).size, RuntimeType.number) - -Eval.object = unary('object', (field, table) => valueMap(field, value => executeAggr(value, table)), - fields => RuntimeType.json(RuntimeType.create(valueMap(fields, getExprRuntimeType)))) -Eval.array = unary('array', (expr, table) => table.map(data => executeAggr(expr, data)), expr => RuntimeType.json(RuntimeType.list(getExprRuntimeType(expr)))) +Eval.sum = unary('sum', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0)) +Eval.avg = unary('avg', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length) +Eval.max = unary('max', (expr, table) => Math.max(...table.map(data => executeAggr(expr, data)))) +Eval.min = unary('min', (expr, table) => Math.min(...table.map(data => executeAggr(expr, data)))) +Eval.count = unary('count', (expr, table) => new Set(table.map(data => executeAggr(expr, data))).size) + +Eval.object = unary('object', (field, table) => valueMap(field, value => executeAggr(value, table))) +Eval.array = unary('array', (expr, table) => table.map(data => executeAggr(expr, data))) export { Eval as $ } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ba5225b2..5aeb810b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -5,4 +5,3 @@ export * from './model' export * from './query' export * from './selection' export * from './utils' -export * from './runtime' diff --git a/packages/core/src/model.ts b/packages/core/src/model.ts index 9453a816..55d6ad49 100644 --- a/packages/core/src/model.ts +++ b/packages/core/src/model.ts @@ -1,7 +1,6 @@ -import { clone, isNullable, makeArray, MaybeArray, valueMap } from 'cosmokit' +import { clone, isNullable, makeArray, MaybeArray } from 'cosmokit' import { Database } from './driver' -import { Eval, getExprRuntimeType, isEvalExpr } from './eval' -import { RuntimeType } from './runtime' +import { Eval, isEvalExpr } from './eval' import { Selection } from './selection' import { Flatten, Keys } from './utils' @@ -18,7 +17,6 @@ export interface Field { expr?: Eval.Expr legacy?: string[] deprecated?: boolean - runtimeType?: RuntimeType } export namespace Field { @@ -81,19 +79,6 @@ export namespace Field { return field } - - export function getRuntimeType(field: Field): RuntimeType { - if (field.runtimeType) return field.runtimeType - if (number.includes(field.type)) return RuntimeType.number - else if (string.includes(field.type)) return RuntimeType.string - else if (boolean.includes(field.type)) return RuntimeType.boolean - else if (date.includes(field.type)) return RuntimeType.date - else if (field.type === 'list') return RuntimeType.list(RuntimeType.string) - else if (field.type === 'json') return RuntimeType.json(RuntimeType.any) - else if (field.type === 'primary') return RuntimeType.any - else if (field.type === 'expr') return getExprRuntimeType(field.expr) - else throw new Error(`No runtime type for ${field}`) - } } export namespace Model { @@ -138,7 +123,6 @@ export class Model { for (const key in fields) { this.fields[key] = Field.parse(fields[key]) this.fields[key].deprecated = !!callback - this.fields[key].runtimeType = Field.getRuntimeType(this.fields[key]) } if (typeof this.primary === 'string' && this.fields[this.primary]?.type === 'primary') { @@ -230,8 +214,4 @@ export class Model { } return this.parse({ ...result, ...data }) } - - getRuntimeType(): RuntimeType { - return RuntimeType.create(valueMap(this.fields, Field.getRuntimeType)) - } } diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts deleted file mode 100644 index 15ec70a9..00000000 --- a/packages/core/src/runtime.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { getExprRuntimeType } from './eval' - -const kRuntime = Symbol('Runtime') - -export interface RuntimeType { - [kRuntime]: true - primitive: RuntimeType.Primitive - list?: boolean - json?: boolean -} - -export namespace RuntimeType { - export type Primitive = 'any' | 'number' | 'string' | 'boolean' | 'date' | 'regexp' | RuntimeType | { [key in keyof T]: RuntimeType } - - export function create = RuntimeType.Primitive>(primitive: P, extra: Partial> = {}): RuntimeType { - if (Object.keys(extra).length === 0 && test(primitive)) return primitive - return { [kRuntime]: true, primitive, ...extra } - } - - export function merge(...types: any[]): RuntimeType { - return types.map(x => getExprRuntimeType(x)).find(x => x.primitive !== 'any') ?? RuntimeType.any - } - - export function list(type: any): RuntimeType { - const primitive = getExprRuntimeType(type) - if (!primitive.list) return { ...primitive, list: true } - else return create(primitive, { list: true }) - } - - export function json(type: any): RuntimeType { - const primitive = getExprRuntimeType(type) - if (!primitive.json) return { ...primitive, json: true } - else return create(primitive, { json: true }) - } - - export function test(type: any): type is RuntimeType { - return type && type[kRuntime] - } - - export const any = RuntimeType.create('any') - export const number = RuntimeType.create('number') - export const string = RuntimeType.create('string') - export const boolean = RuntimeType.create('boolean') - export const date = RuntimeType.create('date') - export const regexp = RuntimeType.create('regexp') -} diff --git a/packages/core/src/selection.ts b/packages/core/src/selection.ts index 4f9d3869..05972387 100644 --- a/packages/core/src/selection.ts +++ b/packages/core/src/selection.ts @@ -4,7 +4,6 @@ import { Eval, executeEval } from './eval' import { Model } from './model' import { Query } from './query' import { Keys, randomId, Row } from './utils' -import { RuntimeType } from './runtime' export type Direction = 'asc' | 'desc' @@ -30,11 +29,10 @@ namespace Executable { } } -const createRow = (ref: string, expr = {}, prefix = '', model?: Model) => new Proxy(expr, { +const createRow = (ref: string, expr = {}, prefix = '') => new Proxy(expr, { get(target, key) { - if (typeof key === 'symbol' || key in target || key === 'toJSON' || key.startsWith('$')) return Reflect.get(target, key) - const fullKey = `${prefix}${key}` - return createRow(ref, Eval('', [ref, fullKey], model?.fields?.[fullKey]?.runtimeType ?? RuntimeType.any), `${fullKey}.`, model) + if (typeof key === 'symbol' || key in target || key.startsWith('$')) return Reflect.get(target, key) + return createRow(ref, Eval('', [ref, `${prefix}${key}`]), `${prefix}${key}.`) }, }) @@ -47,15 +45,15 @@ class Executable { constructor(driver: Driver, payload: Executable.Payload) { Object.assign(this, payload) - defineProperty(this, 'driver', driver) - defineProperty(this, 'model', driver.model(this.table)) const expr = {} if (typeof payload.table !== 'string' && !(payload.table instanceof Selection)) { for (const key in payload.table) { - expr[key] = createRow(key, {}, '', this.model) + expr[key] = createRow(key) } } - defineProperty(this, 'row', createRow(this.ref, expr, '', this.model)) + defineProperty(this, 'driver', driver) + defineProperty(this, 'row', createRow(this.ref, expr)) + defineProperty(this, 'model', driver.model(this.table)) } protected resolveQuery(query?: Query): Query.Expr diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index 982d4aa1..0764cee9 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -104,10 +104,9 @@ class MySQLBuilder extends Builder { '\\': '\\\\', } - constructor(tables?: Dict, issueUnquote = false) { + constructor(tables?: Dict, workaroundArrayagg = false) { super(tables) - - this.workaroundArrayagg = issueUnquote + this.workaroundArrayagg = workaroundArrayagg this.define({ types: ['list'], @@ -243,7 +242,7 @@ export class MySQLDriver extends Driver { async prepare(name: string) { const version = Object.values((await this.query(`SELECT version()`))[0])[0] as string if (version.match(/10.5.\d+-MariaDB/)) { - logger.warn('MariaDB 10.5 will be depracated in the future, better move to LTS version.') + // https://jira.mariadb.org/browse/MDEV-26506 this._fixMariaIssue = true } @@ -453,7 +452,7 @@ export class MySQLDriver extends Driver { } async eval(sel: Selection.Immutable, expr: Eval.Expr) { - const builder = new MySQLBuilder(sel.tables) + const builder = new MySQLBuilder(sel.tables, this._fixMariaIssue) const output = builder.parseEval(expr) const inner = builder.get(sel.table as Selection, true) const [data] = await this.queue(`SELECT ${output} AS value FROM ${inner}`) @@ -462,7 +461,7 @@ export class MySQLDriver extends Driver { async set(sel: Selection.Mutable, data: {}) { const { model, query, table, tables, ref } = sel - const builder = new MySQLBuilder(tables) + const builder = new MySQLBuilder(tables, this._fixMariaIssue) const filter = builder.parseQuery(query) const { fields } = model if (filter === '0') return @@ -479,7 +478,7 @@ export class MySQLDriver extends Driver { async remove(sel: Selection.Mutable) { const { query, table, tables } = sel - const builder = new MySQLBuilder(tables) + const builder = new MySQLBuilder(tables, this._fixMariaIssue) const filter = builder.parseQuery(query) if (filter === '0') return await this.query(`DELETE FROM ${escapeId(table)} WHERE ` + filter) @@ -501,7 +500,7 @@ export class MySQLDriver extends Driver { async upsert(sel: Selection.Mutable, data: any[], keys: string[]) { if (!data.length) return const { model, table, tables, ref } = sel - const builder = new MySQLBuilder(tables) + const builder = new MySQLBuilder(tables, this._fixMariaIssue) const merged = {} const insertion = data.map((item) => { diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 0d567f7d..eaa82c22 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -176,29 +176,24 @@ export class Builder { } protected unquoteJson(value: string) { - const ret = this.jsonQuoted ? `json_unquote(${value})` : value + const res = this.jsonQuoted ? `json_unquote(${value})` : value this.jsonQuoted = false - return ret + return res } protected groupObject(fields: any) { - const ret = `json_object(` + Object.entries(fields).map(([key, expr]) => `'${key}', ${this.parseAggr(expr)}`).join(',') + `)` + const res = `json_object(` + Object.entries(fields).map(([key, expr]) => `'${key}', ${this.parseEval(expr, false)}`).join(',') + `)` this.jsonQuoted = true - return ret + return res } protected groupArray(expr: any) { const aggr = this.parseAggr(expr) - // const ret = this.jsonQuoted ? `concat('[', group_concat(${aggr}), ']')` - // : this.workaroundArrayagg ? `concat('[', group_concat(json_extract(json_object('f', ${aggr}), '$.f')), ']')` : `json_arrayagg(${aggr})` - // console.log(this.jsonQuoted, aggr) - - const ret = this.workaroundArrayagg + const res = this.workaroundArrayagg ? (this.jsonQuoted ? `concat('[', group_concat(${aggr}), ']')` : `concat('[', group_concat(json_extract(json_object('f', ${aggr}), '$.f')), ']')`) : `json_arrayagg(${aggr})` - // const ret = this.workaroundArrayagg ? `concat('[', group_concat(json_extract(json_object('v', ${aggr}), '$.v')), ']')` : `json_arrayagg(${aggr})` this.jsonQuoted = true - return ret + return res } protected parseFieldQuery(key: string, query: Query.FieldExpr) { @@ -257,8 +252,8 @@ export class Builder { protected parseAggr(expr: any) { this.jsonQuoted = false - const ret = typeof expr === 'string' ? this.getRecursive(expr) : this.parseEvalExpr(expr) - return ret + const res = typeof expr === 'string' ? this.getRecursive(expr) : this.parseEvalExpr(expr) + return res } protected transformJsonField(obj: string, path: string) { @@ -269,8 +264,6 @@ export class Builder { private transformKey(key: string, fields: {}, prefix: string) { if (key in fields || !key.includes('.')) { if (this.sqlTypes[key]) this.jsonQuoted = this.sqlTypes[key] === 'json' - // this.jsonQuoted = fields[key]?.runtimeType?.json ?? true - // console.log(key, this.jsonQuoted, this.sqlTypes) return prefix + escapeId(key) } const field = Object.keys(fields).find(k => key.startsWith(k + '.')) || key.split('.')[0] @@ -301,6 +294,7 @@ export class Builder { } parseEval(expr: any, unquote: boolean = true): string { + this.jsonQuoted = false if (typeof expr === 'string' || typeof expr === 'number' || typeof expr === 'boolean' || expr instanceof Date) { return this.escape(expr) } @@ -353,20 +347,18 @@ export class Builder { if (filter !== '1') prefix += ` ON ${filter}` } - const sqlTypes = {} // get prefix + const sqlTypes = {} const fields = args[0].fields ?? Object.fromEntries(Object .entries(model.fields) .filter(([, field]) => !field!.deprecated) - .map(([key, field]) => [key, Eval('', [ref, key], field!.runtimeType!)])) + .map(([key]) => [key, { $: [ref, key] }])) const keys = Object.entries(fields).map(([key, value]) => { value = this.parseEval(value, false) sqlTypes[key] = this.jsonQuoted ? 'json' : 'raw' return escapeId(key) === value ? escapeId(key) : `${value} AS ${escapeId(key)}` }).join(', ') - this.sqlTypes = sqlTypes - // console.log('field sqlTypes:', Object.keys(fields), sqlTypes) // get suffix let suffix = this.suffix(args[0]) @@ -401,7 +393,6 @@ export class Builder { load(model: Model, obj: any): any { const result = {} - // console.log('sql', this.sqlTypes, Object.keys(obj)) for (const key in obj) { if (!(key in model.fields)) continue const { type, initial } = model.fields[key]! diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index c26063b2..be5aa6a8 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -51,6 +51,7 @@ class SQLiteBuilder extends Builder { super(tables) this.evalOperators.$if = (args) => `iif(${args.map(arg => this.parseEval(arg)).join(', ')})` + this.evalOperators.$concat = (args) => `(${args.map(arg => this.parseEval(arg)).join('||')})` this.define({ types: ['boolean'], @@ -61,10 +62,7 @@ class SQLiteBuilder extends Builder { this.define({ types: ['json'], dump: value => JSON.stringify(value), - load: (value, initial) => { - // console.log('load json', value) - return value ? JSON.parse(value) : initial - }, + load: (value, initial) => value ? JSON.parse(value) : initial, }) this.define({ @@ -94,17 +92,15 @@ class SQLiteBuilder extends Builder { } protected groupArray(expr: any) { - // console.log('1') const aggr = this.parseAggr(expr) - // console.log('2', expr, aggr, this.jsonQuoted) - const ret = this.jsonQuoted ? `('[' || group_concat(${aggr}) || ']')` : `('[' || group_concat(json_quote(${aggr})) || ']')` + const res = this.jsonQuoted ? `('[' || group_concat(${aggr}) || ']')` : `('[' || group_concat(json_quote(${aggr})) || ']')` this.jsonQuoted = true - return ret + return res } protected transformJsonField(obj: string, path: string) { - this.jsonQuoted = true - return `json_quote(json_extract(${obj}, '$${path}'))` + this.jsonQuoted = false + return `json_extract(${obj}, '$${path}')` } } diff --git a/packages/tests/src/experimental.ts b/packages/tests/src/experimental.ts index 5069243b..61cf905a 100644 --- a/packages/tests/src/experimental.ts +++ b/packages/tests/src/experimental.ts @@ -219,12 +219,6 @@ namespace ExperimentalTests { }, ]) }) - - // it('raw', async () => { - // const driver = Object.values(database.drivers)[0] - // const ret = await driver.query("SELECT `z` FROM (SELECT concat('[', group_concat(json_extract(json_object('v', `y`), '$.v')), ']') AS `z` FROM (SELECT `foo`.`id` AS `foo.id`, `foo`.`value` AS `foo.value`, concat('[', group_concat(json_extract(json_object('v', json_extract(`bar`.`obj`, '$.x')), '$.v')), ']') AS `y` FROM `foo` JOIN `bar` ON (`foo`.`id` = `bar`.`pid`) GROUP BY `foo.id`, `foo.value`) opgaymzt ORDER BY `foo.id` ASC) iebqysix") - // console.log(ret) - // }) } } From d756631ec7236e4265f9980065a278808c2f36bb Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Mon, 30 Oct 2023 02:07:13 +0800 Subject: [PATCH 22/49] refa: rename currentSQLType --- packages/sql-utils/src/index.ts | 59 +++++++++++++----------------- packages/sqlite/src/index.ts | 6 +-- packages/tests/src/experimental.ts | 8 ++-- 3 files changed, 32 insertions(+), 41 deletions(-) diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index eaa82c22..bd638d3b 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -1,16 +1,6 @@ import { Dict, isNullable } from 'cosmokit' import { Eval, Field, isComparable, Model, Modifier, Query, Selection } from '@minatojs/core' -export type SQLType = 'raw' | 'json' | 'list' - -// declare module '@minatojs/core' { -// namespace Selection { -// interface Immutable { - -// } -// } -// } - export function escapeId(value: string) { return '`' + value + '`' } @@ -31,6 +21,8 @@ export interface Transformer { load: (value: T, initial?: S) => S | null } +export type SQLType = 'raw' | 'json' + export class Builder { protected escapeMap = {} protected escapeRegExp?: RegExp @@ -38,9 +30,9 @@ export class Builder { protected createEqualQuery = this.comparator('=') protected queryOperators: QueryOperators protected evalOperators: EvalOperators - protected jsonQuoted = false - protected workaroundArrayagg = false + protected currentSQLType: SQLType = 'raw' protected sqlTypes: Dict = {} + protected workaroundArrayagg = false constructor(public tables?: Dict) { this.queryOperators = { @@ -176,23 +168,23 @@ export class Builder { } protected unquoteJson(value: string) { - const res = this.jsonQuoted ? `json_unquote(${value})` : value - this.jsonQuoted = false + const res = this.currentSQLType === 'json' ? `json_unquote(${value})` : value + this.currentSQLType = 'raw' return res } protected groupObject(fields: any) { const res = `json_object(` + Object.entries(fields).map(([key, expr]) => `'${key}', ${this.parseEval(expr, false)}`).join(',') + `)` - this.jsonQuoted = true + this.currentSQLType = 'json' return res } protected groupArray(expr: any) { const aggr = this.parseAggr(expr) - const res = this.workaroundArrayagg - ? (this.jsonQuoted ? `concat('[', group_concat(${aggr}), ']')` : `concat('[', group_concat(json_extract(json_object('f', ${aggr}), '$.f')), ']')`) + const res = this.workaroundArrayagg ? (this.currentSQLType === 'json' ? `concat('[', group_concat(${aggr}), ']')` + : `concat('[', group_concat(json_extract(json_object('f', ${aggr}), '$.f')), ']')`) : `json_arrayagg(${aggr})` - this.jsonQuoted = true + this.currentSQLType = 'json' return res } @@ -241,7 +233,7 @@ export class Builder { } private parseEvalExpr(expr: any) { - this.jsonQuoted = false + this.currentSQLType = 'raw' for (const key in expr) { if (key in this.evalOperators) { return this.evalOperators[key](expr[key]) @@ -251,19 +243,23 @@ export class Builder { } protected parseAggr(expr: any) { - this.jsonQuoted = false - const res = typeof expr === 'string' ? this.getRecursive(expr) : this.parseEvalExpr(expr) - return res + this.currentSQLType = 'raw' + if (typeof expr === 'string') { + return this.getRecursive(expr) + } + return this.parseEvalExpr(expr) } protected transformJsonField(obj: string, path: string) { - this.jsonQuoted = true + this.currentSQLType = 'json' return `json_extract(${obj}, '$${path}')` } private transformKey(key: string, fields: {}, prefix: string) { if (key in fields || !key.includes('.')) { - if (this.sqlTypes[key]) this.jsonQuoted = this.sqlTypes[key] === 'json' + if (this.sqlTypes[key]) { + this.currentSQLType = this.sqlTypes[key] + } return prefix + escapeId(key) } const field = Object.keys(fields).find(k => key.startsWith(k + '.')) || key.split('.')[0] @@ -294,16 +290,11 @@ export class Builder { } parseEval(expr: any, unquote: boolean = true): string { - this.jsonQuoted = false + this.currentSQLType = 'raw' if (typeof expr === 'string' || typeof expr === 'number' || typeof expr === 'boolean' || expr instanceof Date) { return this.escape(expr) } - const res = this.parseEvalExpr(expr) - if (unquote && this.jsonQuoted) { - return this.unquoteJson(res) - } else { - return res - } + return unquote ? this.unquoteJson(this.parseEvalExpr(expr)) : this.parseEvalExpr(expr) } suffix(modifier: Modifier) { @@ -348,14 +339,14 @@ export class Builder { } // get prefix - const sqlTypes = {} + const sqlTypes: Dict = {} const fields = args[0].fields ?? Object.fromEntries(Object .entries(model.fields) .filter(([, field]) => !field!.deprecated) .map(([key]) => [key, { $: [ref, key] }])) const keys = Object.entries(fields).map(([key, value]) => { value = this.parseEval(value, false) - sqlTypes[key] = this.jsonQuoted ? 'json' : 'raw' + sqlTypes[key] = this.currentSQLType return escapeId(key) === value ? escapeId(key) : `${value} AS ${escapeId(key)}` }).join(', ') this.sqlTypes = sqlTypes @@ -396,7 +387,7 @@ export class Builder { for (const key in obj) { if (!(key in model.fields)) continue const { type, initial } = model.fields[key]! - const converter = this.sqlTypes[key] === 'json' ? this.types['json'] : this.types[type] + const converter = this.sqlTypes[key] === 'raw' ? this.types[type] : this.types[this.sqlTypes[key]] result[key] = converter ? converter.load(obj[key], initial) : obj[key] } return model.parse(result) diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index be5aa6a8..10971f7b 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -93,13 +93,13 @@ class SQLiteBuilder extends Builder { protected groupArray(expr: any) { const aggr = this.parseAggr(expr) - const res = this.jsonQuoted ? `('[' || group_concat(${aggr}) || ']')` : `('[' || group_concat(json_quote(${aggr})) || ']')` - this.jsonQuoted = true + const res = this.currentSQLType === 'json' ? `('[' || group_concat(${aggr}) || ']')` : `('[' || group_concat(json_quote(${aggr})) || ']')` + this.currentSQLType = 'json' return res } protected transformJsonField(obj: string, path: string) { - this.jsonQuoted = false + this.currentSQLType = 'raw' return `json_extract(${obj}, '$${path}')` } } diff --git a/packages/tests/src/experimental.ts b/packages/tests/src/experimental.ts index 61cf905a..5ca9c1e0 100644 --- a/packages/tests/src/experimental.ts +++ b/packages/tests/src/experimental.ts @@ -64,7 +64,7 @@ function ExperimentalTests(database: Database) { } namespace ExperimentalTests { - export function computed(database: Database) { + export function jsontype(database: Database) { it('$.object', async () => { const res = await database.select('foo') .project({ @@ -138,7 +138,7 @@ namespace ExperimentalTests { ]) }) - it('$.array groupBy in json', async () => { + it('$.array in json', async () => { const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) .groupBy('foo', { bars: row => $.array($.object({ @@ -172,7 +172,7 @@ namespace ExperimentalTests { ]) }) - it('$.array groupBy with expressions', async () => { + it('$.array with expressions', async () => { const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) .groupBy('foo', { bars: row => $.array($.object({ @@ -202,7 +202,7 @@ namespace ExperimentalTests { }) - it('$.array groupBy with expressions 2', async () => { + it('$.array nested', async () => { const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) .groupBy('foo', { y: row => $.array(row.bar.obj.x), From 3e1107d12008af5da92e9b3c20e69a268bbdf905 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Mon, 30 Oct 2023 17:40:57 +0800 Subject: [PATCH 23/49] feat: split group & project --- packages/core/src/eval.ts | 35 ++++++-- packages/core/src/selection.ts | 4 +- packages/memory/src/index.ts | 24 +----- packages/mongo/src/utils.ts | 43 +++++----- packages/mysql/src/index.ts | 1 + packages/sql-utils/src/index.ts | 81 +++++++++++-------- packages/sqlite/src/index.ts | 23 ++++-- packages/tests/src/index.ts | 4 +- .../tests/src/{experimental.ts => json.ts} | 38 ++++++--- packages/tests/src/selection.ts | 13 --- 10 files changed, 154 insertions(+), 112 deletions(-) rename packages/tests/src/{experimental.ts => json.ts} (86%) diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index 2ef176ec..d5d5852c 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -94,6 +94,13 @@ export namespace Eval { min(value: Number): Expr count(value: Any): Expr + // json + sum(value: (Number | Expr)[] | Expr): Expr + avg(value: (Number | Expr)[] | Expr): Expr + max(value: (Number | Expr)[] | Expr): Expr + min(value: (Number | Expr)[] | Expr): Expr + count(value: (Any | Expr)[] | Expr): Expr + object>(fields: T): Expr array(value: Expr): Expr } @@ -168,14 +175,30 @@ Eval.or = multary('or', (args, data) => args.some(arg => executeEval(data, arg)) Eval.not = unary('not', (value, data) => !executeEval(data, value)) // aggregation -Eval.sum = unary('sum', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0)) -Eval.avg = unary('avg', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length) -Eval.max = unary('max', (expr, table) => Math.max(...table.map(data => executeAggr(expr, data)))) -Eval.min = unary('min', (expr, table) => Math.min(...table.map(data => executeAggr(expr, data)))) -Eval.count = unary('count', (expr, table) => new Set(table.map(data => executeAggr(expr, data))).size) +Eval.sum = unary('sum', (expr, table) => Array.isArray(table) + ? table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) + : Array.from(executeEval(table, expr)).reduce((prev, curr) => prev + curr, 0)) +Eval.avg = unary('avg', (expr, table) => { + if (Array.isArray(table)) return table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length + else { + const array = Array.from(executeEval(table, expr)) + return array.reduce((prev, curr) => prev + curr, 0) / array.length + } +}) +Eval.max = unary('max', (expr, table) => Array.isArray(table) + ? Math.max(...table.map(data => executeAggr(expr, data))) + : Math.max(...Array.from(executeEval(table, expr)))) +Eval.min = unary('min', (expr, table) => Array.isArray(table) + ? Math.min(...table.map(data => executeAggr(expr, data))) + : Math.min(...Array.from(executeEval(table, expr)))) +Eval.count = unary('count', (expr, table) => Array.isArray(table) + ? new Set(table.map(data => executeAggr(expr, data))).size + : new Set(Array.from(executeEval(table, expr))).size) Eval.object = unary('object', (field, table) => valueMap(field, value => executeAggr(value, table))) -Eval.array = unary('array', (expr, table) => table.map(data => executeAggr(expr, data))) +Eval.array = unary('array', (expr, table) => Array.isArray(table) + ? table.map(data => executeAggr(expr, data)) + : Array.from(executeEval(table, expr))) export { Eval as $ } diff --git a/packages/core/src/selection.ts b/packages/core/src/selection.ts index 05972387..79f10dc5 100644 --- a/packages/core/src/selection.ts +++ b/packages/core/src/selection.ts @@ -11,7 +11,7 @@ export interface Modifier { limit: number offset: number sort: [Eval.Expr, Direction][] - group: string[] + group?: string[] having: Eval.Expr fields?: Dict optional: Dict @@ -136,7 +136,7 @@ export class Selection extends Executable { ref: randomId(), table, query: null as never, - args: [{ sort: [], limit: Infinity, offset: 0, group: [], having: Eval.and(), optional: {} }], + args: [{ sort: [], limit: Infinity, offset: 0, group: undefined, having: Eval.and(), optional: {} }], }) this.tables[this.ref] = this.model this.query = this.resolveQuery(query) diff --git a/packages/memory/src/index.ts b/packages/memory/src/index.ts index 2ca7eaad..430bf48a 100644 --- a/packages/memory/src/index.ts +++ b/packages/memory/src/index.ts @@ -1,5 +1,5 @@ import { clone, Dict, makeArray, noop, omit, pick, valueMap } from 'cosmokit' -import { Database, Driver, Eval, executeEval, executeQuery, executeSort, executeUpdate, isEvalExpr, RuntimeError, Selection } from '@minatojs/core' +import { Database, Driver, Eval, executeEval, executeQuery, executeSort, executeUpdate, RuntimeError, Selection } from '@minatojs/core' export namespace MemoryDriver { export interface Config {} @@ -47,12 +47,9 @@ export class MemoryDriver extends Driver { const { ref, query, table, args, model } = sel const { fields, group, having } = sel.args[0] const data = this.table(table, having).filter(row => executeQuery(row, query, ref)) - if (!group.length && fields && Object.values(args[0].fields ?? {}).some(x => isAggrExpr(x))) { - return [valueMap(fields!, (expr) => executeEval(data.map(row => ({ [ref]: row })), expr))] - } const branches: { index: Dict; table: any[] }[] = [] - const groupFields = group.length ? pick(fields!, group) : fields + const groupFields = group ? pick(fields!, group) : fields for (let row of executeSort(data, args[0], ref)) { row = model.format(row, false) for (const key in model.fields) { @@ -64,7 +61,7 @@ export class MemoryDriver extends Driver { index = valueMap(groupFields!, (expr) => executeEval({ [ref]: row }, expr)) } let branch = branches.find((branch) => { - if (!groupFields) return false + if (!group || !groupFields) return false for (const key in groupFields) { if (branch.index[key] !== index[key]) return false } @@ -77,7 +74,7 @@ export class MemoryDriver extends Driver { branch.table.push(row) } return branches.map(({ index, table }) => { - if (group.length) { + if (group) { if (having) { const value = executeEval(table.map(row => ({ [ref]: row, _: row })), having) if (!value) return @@ -168,17 +165,4 @@ export class MemoryDriver extends Driver { } } -const nonAggrKeys = ['$'] -const aggrKeys = ['$sum', '$avg', '$min', '$max', '$count', '$array'] - -function isAggrExpr(value: any) { - if (!isEvalExpr(value)) return false - for (const [key, args] of Object.entries(value)) { - if (!key.startsWith('$')) continue - if (nonAggrKeys.includes(key)) return false - if (aggrKeys.includes(key) || ((Array.isArray(args) ? args : [args]).some(x => isAggrExpr(x)))) return true - } - return false -} - export default MemoryDriver diff --git a/packages/mongo/src/utils.ts b/packages/mongo/src/utils.ts index ed8545c1..0bc50aa7 100644 --- a/packages/mongo/src/utils.ts +++ b/packages/mongo/src/utils.ts @@ -114,8 +114,12 @@ export class Transformer { return { $cond: expr.$if.map(val => this.eval(val, group)) } } - if (expr.$object) { - return this.transformEvalExpr(expr.$object) + if (expr.$object || expr.$array) { + return this.transformEvalExpr(expr.$object || expr.$array) + } + + if (expr.$count) { + return { $size: this.transformEvalExpr(expr.$count) } } return valueMap(expr as any, (value) => { @@ -145,19 +149,21 @@ export class Transformer { return expr } - for (const type of aggrKeys) { - if (!expr[type]) continue - const key = this.createKey() - const value = this.transformAggr(expr[type]) - if (type === '$count') { - group![key] = { $addToSet: value } - return { $size: '$' + key } - } else if (type === '$array') { - group![key] = { $push: value } - return '$' + key - } else { - group![key] = { [type]: value } - return '$' + key + if (group) { + for (const type of aggrKeys) { + if (!expr[type]) continue + const key = this.createKey() + const value = this.transformAggr(expr[type]) + if (type === '$count') { + group![key] = { $addToSet: value } + return { $size: '$' + key } + } else if (type === '$array') { + group![key] = { $push: value } + return '$' + key + } else { + group![key] = { [type]: value } + return '$' + key + } } } @@ -229,7 +235,7 @@ export class Transformer { } // groupBy, having, fields - if (group.length) { + if (group) { const $group: Dict = { _id: {} } const $project: Dict = { _id: 0 } stages.push({ $group }) @@ -249,10 +255,9 @@ export class Transformer { stages.push({ $project }) $group['_id'] = model.parse($group['_id'], false) } else if (fields) { - const $group: Dict = { _id: null } - const $project = valueMap(fields, (expr) => this.eval(expr, $group)) + const $project = valueMap(fields, (expr) => this.eval(expr)) $project._id = 0 - stages.push(...Object.keys($group).length === 1 ? [] : [{ $group }], { $project }) + stages.push({ $project }) } else { const $project: Dict = { _id: 0 } for (const key in model.fields) { diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index 0764cee9..ba2b2a0b 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -453,6 +453,7 @@ export class MySQLDriver extends Driver { async eval(sel: Selection.Immutable, expr: Eval.Expr) { const builder = new MySQLBuilder(sel.tables, this._fixMariaIssue) + builder.state.group = true const output = builder.parseEval(expr) const inner = builder.get(sel.table as Selection, true) const [data] = await this.queue(`SELECT ${output} AS value FROM ${inner}`) diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index bd638d3b..333e9898 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -1,5 +1,5 @@ import { Dict, isNullable } from 'cosmokit' -import { Eval, Field, isComparable, Model, Modifier, Query, Selection } from '@minatojs/core' +import { Eval, Field, isComparable, Model, Modifier, Query, randomId, Selection } from '@minatojs/core' export function escapeId(value: string) { return '`' + value + '`' @@ -21,7 +21,13 @@ export interface Transformer { load: (value: T, initial?: S) => S | null } -export type SQLType = 'raw' | 'json' +type SQLType = 'raw' | 'json' + +interface State { + sqlType?: SQLType + sqlTypes?: Dict + group?: boolean +} export class Builder { protected escapeMap = {} @@ -30,9 +36,8 @@ export class Builder { protected createEqualQuery = this.comparator('=') protected queryOperators: QueryOperators protected evalOperators: EvalOperators - protected currentSQLType: SQLType = 'raw' - protected sqlTypes: Dict = {} protected workaroundArrayagg = false + state: State = {} constructor(public tables?: Dict) { this.queryOperators = { @@ -111,14 +116,14 @@ export class Builder { $lte: this.binary('<='), // aggregation - $sum: (expr) => `ifnull(sum(${this.parseAggr(expr)}), 0)`, - $avg: (expr) => `avg(${this.parseAggr(expr)})`, - $min: (expr) => `min(${this.parseAggr(expr)})`, - $max: (expr) => `max(${this.parseAggr(expr)})`, - $count: (expr) => `count(distinct ${this.parseAggr(expr)})`, + $sum: (expr) => this.createAggr(expr, value => `ifnull(sum(${value}), 0)`), + $avg: (expr) => this.createAggr(expr, value => `avg(${value})`), + $min: (expr) => this.createAggr(expr, value => `(0+min(${value}))`), + $max: (expr) => this.createAggr(expr, value => `(0+max(${value}))`), + $count: (expr) => this.createAggr(expr, value => `count(distinct ${value})`), $object: (fields) => this.groupObject(fields), - $array: (expr) => this.groupArray(expr), + $array: (expr) => this.createAggr(expr, value => this.groupArray(value)), } } @@ -168,23 +173,33 @@ export class Builder { } protected unquoteJson(value: string) { - const res = this.currentSQLType === 'json' ? `json_unquote(${value})` : value - this.currentSQLType = 'raw' + const res = this.state.sqlType === 'json' ? `json_unquote(${value})` : value + this.state.sqlType = 'raw' return res } + protected createAggr(expr: any, aggrfunc: (value: string) => string) { + if (this.state.group) { + return aggrfunc(this.parseAggr(expr)) + } else { + this.state.group = true + const aggr = this.parseAggr(expr) + this.state.group = false + return `(select ${aggrfunc(`json_unquote(${escapeId('value')})`)} from json_table(${aggr}, '$[*]' columns (value json path '$')) ${randomId()})` + } + } + protected groupObject(fields: any) { const res = `json_object(` + Object.entries(fields).map(([key, expr]) => `'${key}', ${this.parseEval(expr, false)}`).join(',') + `)` - this.currentSQLType = 'json' + this.state.sqlType = 'json' return res } - protected groupArray(expr: any) { - const aggr = this.parseAggr(expr) - const res = this.workaroundArrayagg ? (this.currentSQLType === 'json' ? `concat('[', group_concat(${aggr}), ']')` - : `concat('[', group_concat(json_extract(json_object('f', ${aggr}), '$.f')), ']')`) - : `json_arrayagg(${aggr})` - this.currentSQLType = 'json' + protected groupArray(value: string) { + const res = this.workaroundArrayagg ? (this.state.sqlType === 'json' ? `concat('[', group_concat(${value}), ']')` + : `concat('[', group_concat(json_extract(json_object('v', ${value}), '$.v')), ']')`) + : `json_arrayagg(${value})` + this.state.sqlType = 'json' return res } @@ -233,7 +248,7 @@ export class Builder { } private parseEvalExpr(expr: any) { - this.currentSQLType = 'raw' + this.state.sqlType = 'raw' for (const key in expr) { if (key in this.evalOperators) { return this.evalOperators[key](expr[key]) @@ -243,22 +258,19 @@ export class Builder { } protected parseAggr(expr: any) { - this.currentSQLType = 'raw' - if (typeof expr === 'string') { - return this.getRecursive(expr) - } - return this.parseEvalExpr(expr) + this.state.sqlType = 'raw' + return typeof expr === 'string' ? this.getRecursive(expr) : this.parseEvalExpr(expr) } protected transformJsonField(obj: string, path: string) { - this.currentSQLType = 'json' + this.state.sqlType = 'json' return `json_extract(${obj}, '$${path}')` } private transformKey(key: string, fields: {}, prefix: string) { if (key in fields || !key.includes('.')) { - if (this.sqlTypes[key]) { - this.currentSQLType = this.sqlTypes[key] + if (this.state.sqlTypes?.[key]) { + this.state.sqlType = this.state.sqlTypes[key] } return prefix + escapeId(key) } @@ -290,7 +302,7 @@ export class Builder { } parseEval(expr: any, unquote: boolean = true): string { - this.currentSQLType = 'raw' + this.state.sqlType = 'raw' if (typeof expr === 'string' || typeof expr === 'number' || typeof expr === 'boolean' || expr instanceof Date) { return this.escape(expr) } @@ -300,7 +312,7 @@ export class Builder { suffix(modifier: Modifier) { const { limit, offset, sort, group, having } = modifier let sql = '' - if (group.length) { + if (group?.length) { sql += ` GROUP BY ${group.map(escapeId).join(', ')}` const filter = this.parseEval(having) if (filter !== '1') sql += ` HAVING ${filter}` @@ -320,6 +332,7 @@ export class Builder { const filter = this.parseQuery(query) if (filter === '0') return + // get prefix let prefix: string | undefined if (typeof table === 'string') { prefix = escapeId(table) @@ -338,7 +351,7 @@ export class Builder { if (filter !== '1') prefix += ` ON ${filter}` } - // get prefix + this.state.group = !!args[0].group const sqlTypes: Dict = {} const fields = args[0].fields ?? Object.fromEntries(Object .entries(model.fields) @@ -346,10 +359,10 @@ export class Builder { .map(([key]) => [key, { $: [ref, key] }])) const keys = Object.entries(fields).map(([key, value]) => { value = this.parseEval(value, false) - sqlTypes[key] = this.currentSQLType + sqlTypes[key] = this.state.sqlType! return escapeId(key) === value ? escapeId(key) : `${value} AS ${escapeId(key)}` }).join(', ') - this.sqlTypes = sqlTypes + this.state.sqlTypes = sqlTypes // get suffix let suffix = this.suffix(args[0]) @@ -387,7 +400,7 @@ export class Builder { for (const key in obj) { if (!(key in model.fields)) continue const { type, initial } = model.fields[key]! - const converter = this.sqlTypes[key] === 'raw' ? this.types[type] : this.types[this.sqlTypes[key]] + const converter = this.state.sqlTypes?.[key] === 'raw' ? this.types[type] : this.types[this.state.sqlTypes![key]] result[key] = converter ? converter.load(obj[key], initial) : obj[key] } return model.parse(result) diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index 10971f7b..b822295a 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -1,5 +1,5 @@ import { deepEqual, Dict, difference, isNullable, makeArray } from 'cosmokit' -import { Database, Driver, Eval, executeUpdate, Field, Model, Selection } from '@minatojs/core' +import { Database, Driver, Eval, executeUpdate, Field, Model, randomId, Selection } from '@minatojs/core' import { Builder, escapeId } from '@minatojs/sql-utils' import { promises as fs } from 'fs' import init from '@minatojs/sql.js' @@ -91,15 +91,25 @@ class SQLiteBuilder extends Builder { return value } - protected groupArray(expr: any) { - const aggr = this.parseAggr(expr) - const res = this.currentSQLType === 'json' ? `('[' || group_concat(${aggr}) || ']')` : `('[' || group_concat(json_quote(${aggr})) || ']')` - this.currentSQLType = 'json' + protected createAggr(expr: any, aggrfunc: (value: string) => string) { + if (this.state.group) { + return aggrfunc(this.parseAggr(expr)) + } else { + this.state.group = true + const aggr = this.parseAggr(expr) + this.state.group = false + return `(select ${aggrfunc(escapeId('value'))} from json_each(${aggr}) ${randomId()})` + } + } + + protected groupArray(value: string) { + const res = this.state.sqlType === 'json' ? `('[' || group_concat(${value}) || ']')` : `('[' || group_concat(json_quote(${value})) || ']')` + this.state.sqlType = 'json' return res } protected transformJsonField(obj: string, path: string) { - this.currentSQLType = 'raw' + this.state.sqlType = 'raw' return `json_extract(${obj}, '$${path}')` } } @@ -333,6 +343,7 @@ export class SQLiteDriver extends Driver { async eval(sel: Selection.Immutable, expr: Eval.Expr) { const builder = new SQLiteBuilder(sel.tables) + builder.state.group = true const output = builder.parseEval(expr) const inner = builder.get(sel.table as Selection, true) const { value } = this.#get(`SELECT ${output} AS value FROM ${inner}`) diff --git a/packages/tests/src/index.ts b/packages/tests/src/index.ts index 5deaf473..615e23dd 100644 --- a/packages/tests/src/index.ts +++ b/packages/tests/src/index.ts @@ -4,7 +4,7 @@ import UpdateOperators from './update' import ObjectOperations from './object' import Migration from './migration' import Selection from './selection' -import Experimental from './experimental' +import Json from './json' import './setup' const Keywords = ['name'] @@ -53,7 +53,7 @@ namespace Tests { export const object = ObjectOperations export const selection = Selection export const migration = Migration - export const experimental = Experimental + export const json = Json } export default createUnit(Tests, true) diff --git a/packages/tests/src/experimental.ts b/packages/tests/src/json.ts similarity index 86% rename from packages/tests/src/experimental.ts rename to packages/tests/src/json.ts index 5ca9c1e0..bb692d47 100644 --- a/packages/tests/src/experimental.ts +++ b/packages/tests/src/json.ts @@ -22,7 +22,7 @@ interface Bar { b: string } } - l: string[] + l: number[] } interface Tables { @@ -30,7 +30,7 @@ interface Tables { bar: Bar } -function ExperimentalTests(database: Database) { +function JsonTests(database: Database) { database.extend('foo', { id: 'unsigned', value: 'integer', @@ -43,7 +43,7 @@ function ExperimentalTests(database: Database) { value: 'integer', obj: 'json', s: 'string', - l: 'list', + l: { type: 'json', initial: [] } }, { autoInc: true, }) @@ -56,14 +56,14 @@ function ExperimentalTests(database: Database) { ]) await setup(database, 'bar', [ - { uid: 1, pid: 1, value: 0, obj: { x: 1, y: 'a', z: '1', o: { a: 1, b: '1' } }, s: '1', l: ['1', '2'] }, - { uid: 1, pid: 1, value: 1, obj: { x: 2, y: 'b', z: '2', o: { a: 2, b: '2' } }, s: '2' }, - { uid: 1, pid: 2, value: 0, obj: { x: 3, y: 'c', z: '3', o: { a: 3, b: '3' } }, s: '3' }, + { uid: 1, pid: 1, value: 0, obj: { x: 1, y: 'a', z: '1', o: { a: 1, b: '1' } }, s: '1', l: [1, 2] }, + { uid: 1, pid: 1, value: 1, obj: { x: 2, y: 'b', z: '2', o: { a: 2, b: '2' } }, s: '2', l: [5, 3, 4] }, + { uid: 1, pid: 2, value: 0, obj: { x: 3, y: 'c', z: '3', o: { a: 3, b: '3' } }, s: '3', l: [2] }, ]) }) } -namespace ExperimentalTests { +namespace JsonTests { export function jsontype(database: Database) { it('$.object', async () => { const res = await database.select('foo') @@ -120,7 +120,7 @@ namespace ExperimentalTests { it('$.array groupFull', async () => { const res = await database.select('bar') - .project({ + .groupBy({}, { count2: row => $.array(row.s), countnumber: row => $.array(row.value), x: row => $.array(row.obj.x), @@ -208,7 +208,7 @@ namespace ExperimentalTests { y: row => $.array(row.bar.obj.x), }) .orderBy(row => row.foo.id) - .project({ + .groupBy({}, { z: row => $.array(row.y) }) .execute() @@ -219,7 +219,25 @@ namespace ExperimentalTests { }, ]) }) + + it('non-aggr functions', async () => { + const res = await database.select('bar') + .project({ + sum: row => $.sum(row.l), + avg: row => $.avg(row.l), + max: row => $.max(row.l), + min: row => $.min(row.l), + count: row => $.count(row.l), + }) + .execute() + + expect(res).to.deep.equal([ + { sum: 3, avg: 1.5, max: 2, min: 1, count: 2 }, + { sum: 12, avg: 4, max: 5, min: 3, count: 3 }, + { sum: 2, avg: 2, max: 2, min: 2, count: 1 }, + ]) + }) } } -export default ExperimentalTests +export default JsonTests diff --git a/packages/tests/src/selection.ts b/packages/tests/src/selection.ts index dc34bf63..3a771eb7 100644 --- a/packages/tests/src/selection.ts +++ b/packages/tests/src/selection.ts @@ -141,19 +141,6 @@ namespace SelectionTests { }) it('aggregate', async () => { - await expect(database - .select('foo') - .project({ - count: row => $.count(row.id), - max: row => $.max(row.id), - min: row => $.min(row.id), - avg: row => $.avg(row.id), - }) - .execute() - ).to.eventually.deep.equal([ - { avg: 2, count: 3, max: 3, min: 1 }, - ]) - await expect(database.select('foo') .groupBy({}, { count: row => $.count(row.id), From c14b0d082288abb118a113677b52cc568ae7937e Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Mon, 30 Oct 2023 18:46:50 +0800 Subject: [PATCH 24/49] revert non-aggr aggr due to unsupport table function (mysql5.7) --- packages/core/src/eval.ts | 28 ++++++---------------------- packages/sql-utils/src/index.ts | 30 +++++++++++------------------- packages/sqlite/src/index.ts | 13 +------------ packages/tests/src/json.ts | 33 +++++++++++++++------------------ 4 files changed, 33 insertions(+), 71 deletions(-) diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index d5d5852c..afaa646c 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -175,30 +175,14 @@ Eval.or = multary('or', (args, data) => args.some(arg => executeEval(data, arg)) Eval.not = unary('not', (value, data) => !executeEval(data, value)) // aggregation -Eval.sum = unary('sum', (expr, table) => Array.isArray(table) - ? table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) - : Array.from(executeEval(table, expr)).reduce((prev, curr) => prev + curr, 0)) -Eval.avg = unary('avg', (expr, table) => { - if (Array.isArray(table)) return table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length - else { - const array = Array.from(executeEval(table, expr)) - return array.reduce((prev, curr) => prev + curr, 0) / array.length - } -}) -Eval.max = unary('max', (expr, table) => Array.isArray(table) - ? Math.max(...table.map(data => executeAggr(expr, data))) - : Math.max(...Array.from(executeEval(table, expr)))) -Eval.min = unary('min', (expr, table) => Array.isArray(table) - ? Math.min(...table.map(data => executeAggr(expr, data))) - : Math.min(...Array.from(executeEval(table, expr)))) -Eval.count = unary('count', (expr, table) => Array.isArray(table) - ? new Set(table.map(data => executeAggr(expr, data))).size - : new Set(Array.from(executeEval(table, expr))).size) +Eval.sum = unary('sum', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0)) +Eval.avg = unary('avg', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length) +Eval.max = unary('max', (expr, table) => Math.max(...table.map(data => executeAggr(expr, data)))) +Eval.min = unary('min', (expr, table) => Math.min(...table.map(data => executeAggr(expr, data)))) +Eval.count = unary('count', (expr, table) => new Set(table.map(data => executeAggr(expr, data))).size) Eval.object = unary('object', (field, table) => valueMap(field, value => executeAggr(value, table))) -Eval.array = unary('array', (expr, table) => Array.isArray(table) - ? table.map(data => executeAggr(expr, data)) - : Array.from(executeEval(table, expr))) +Eval.array = unary('array', (expr, table) => table.map(data => executeAggr(expr, data))) export { Eval as $ } diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 333e9898..cab2a217 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -1,5 +1,5 @@ import { Dict, isNullable } from 'cosmokit' -import { Eval, Field, isComparable, Model, Modifier, Query, randomId, Selection } from '@minatojs/core' +import { Eval, Field, isComparable, Model, Modifier, Query, Selection } from '@minatojs/core' export function escapeId(value: string) { return '`' + value + '`' @@ -21,7 +21,7 @@ export interface Transformer { load: (value: T, initial?: S) => S | null } -type SQLType = 'raw' | 'json' +type SQLType = 'raw' | 'json' | 'list' interface State { sqlType?: SQLType @@ -116,14 +116,14 @@ export class Builder { $lte: this.binary('<='), // aggregation - $sum: (expr) => this.createAggr(expr, value => `ifnull(sum(${value}), 0)`), - $avg: (expr) => this.createAggr(expr, value => `avg(${value})`), - $min: (expr) => this.createAggr(expr, value => `(0+min(${value}))`), - $max: (expr) => this.createAggr(expr, value => `(0+max(${value}))`), - $count: (expr) => this.createAggr(expr, value => `count(distinct ${value})`), + $sum: (expr) => `ifnull(sum(${this.parseAggr(expr)}), 0)`, + $avg: (expr) => `avg(${this.parseAggr(expr)})`, + $min: (expr) => `min(${this.parseAggr(expr)})`, + $max: (expr) => `max(${this.parseAggr(expr)})`, + $count: (expr) => `count(distinct ${this.parseAggr(expr)})`, $object: (fields) => this.groupObject(fields), - $array: (expr) => this.createAggr(expr, value => this.groupArray(value)), + $array: (expr) => this.groupArray(this.parseAggr(expr)), } } @@ -178,17 +178,6 @@ export class Builder { return res } - protected createAggr(expr: any, aggrfunc: (value: string) => string) { - if (this.state.group) { - return aggrfunc(this.parseAggr(expr)) - } else { - this.state.group = true - const aggr = this.parseAggr(expr) - this.state.group = false - return `(select ${aggrfunc(`json_unquote(${escapeId('value')})`)} from json_table(${aggr}, '$[*]' columns (value json path '$')) ${randomId()})` - } - } - protected groupObject(fields: any) { const res = `json_object(` + Object.entries(fields).map(([key, expr]) => `'${key}', ${this.parseEval(expr, false)}`).join(',') + `)` this.state.sqlType = 'json' @@ -336,6 +325,9 @@ export class Builder { let prefix: string | undefined if (typeof table === 'string') { prefix = escapeId(table) + this.state.sqlTypes = Object.fromEntries(Object.entries(model.fields).map(([key, field]) => { + return [key, field!.type === 'json' ? 'json' : field!.type === 'list' ? 'list' : 'raw'] + })) } else if (table instanceof Selection) { prefix = this.get(table, true) if (!prefix) return diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index b822295a..bb148cb8 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -1,5 +1,5 @@ import { deepEqual, Dict, difference, isNullable, makeArray } from 'cosmokit' -import { Database, Driver, Eval, executeUpdate, Field, Model, randomId, Selection } from '@minatojs/core' +import { Database, Driver, Eval, executeUpdate, Field, Model, Selection } from '@minatojs/core' import { Builder, escapeId } from '@minatojs/sql-utils' import { promises as fs } from 'fs' import init from '@minatojs/sql.js' @@ -91,17 +91,6 @@ class SQLiteBuilder extends Builder { return value } - protected createAggr(expr: any, aggrfunc: (value: string) => string) { - if (this.state.group) { - return aggrfunc(this.parseAggr(expr)) - } else { - this.state.group = true - const aggr = this.parseAggr(expr) - this.state.group = false - return `(select ${aggrfunc(escapeId('value'))} from json_each(${aggr}) ${randomId()})` - } - } - protected groupArray(value: string) { const res = this.state.sqlType === 'json' ? `('[' || group_concat(${value}) || ']')` : `('[' || group_concat(json_quote(${value})) || ']')` this.state.sqlType = 'json' diff --git a/packages/tests/src/json.ts b/packages/tests/src/json.ts index bb692d47..c560e4a3 100644 --- a/packages/tests/src/json.ts +++ b/packages/tests/src/json.ts @@ -22,7 +22,7 @@ interface Bar { b: string } } - l: number[] + l: string[] } interface Tables { @@ -43,7 +43,7 @@ function JsonTests(database: Database) { value: 'integer', obj: 'json', s: 'string', - l: { type: 'json', initial: [] } + l: { type: 'list', initial: [] } }, { autoInc: true, }) @@ -56,9 +56,9 @@ function JsonTests(database: Database) { ]) await setup(database, 'bar', [ - { uid: 1, pid: 1, value: 0, obj: { x: 1, y: 'a', z: '1', o: { a: 1, b: '1' } }, s: '1', l: [1, 2] }, - { uid: 1, pid: 1, value: 1, obj: { x: 2, y: 'b', z: '2', o: { a: 2, b: '2' } }, s: '2', l: [5, 3, 4] }, - { uid: 1, pid: 2, value: 0, obj: { x: 3, y: 'c', z: '3', o: { a: 3, b: '3' } }, s: '3', l: [2] }, + { uid: 1, pid: 1, value: 0, obj: { x: 1, y: 'a', z: '1', o: { a: 1, b: '1' } }, s: '1', l: ['1', '2'] }, + { uid: 1, pid: 1, value: 1, obj: { x: 2, y: 'b', z: '2', o: { a: 2, b: '2' } }, s: '2', l: ['5', '3', '4'] }, + { uid: 1, pid: 2, value: 0, obj: { x: 3, y: 'c', z: '3', o: { a: 3, b: '3' } }, s: '3', l: ['2'] }, ]) }) } @@ -97,9 +97,9 @@ namespace JsonTests { .execute() expect(res).to.deep.equal([ - { obj: { a: 1, num: 1, obj: { a: 1, b: "1" }, str: 'a', str2: '1' } }, - { obj: { a: 2, num: 2, obj: { a: 2, b: "2" }, str: 'b', str2: '2' } }, - { obj: { a: 3, num: 3, obj: { a: 3, b: "3" }, str: 'c', str2: '3' } } + { obj: { a: 1, num: 1, obj: { a: 1, b: '1' }, str: 'a', str2: '1' } }, + { obj: { a: 2, num: 2, obj: { a: 2, b: '2' }, str: 'b', str2: '2' } }, + { obj: { a: 3, num: 3, obj: { a: 3, b: '3' }, str: 'c', str2: '3' } } ]) }) @@ -130,7 +130,7 @@ namespace JsonTests { expect(res).to.deep.equal([ { - count2: ["1", "2", "3"], + count2: ['1', '2', '3'], countnumber: [0, 1, 0], x: [1, 2, 3], y: ['a', 'b', 'c'] @@ -220,21 +220,18 @@ namespace JsonTests { ]) }) - it('non-aggr functions', async () => { + it('pass sqlType', async () => { const res = await database.select('bar') .project({ - sum: row => $.sum(row.l), - avg: row => $.avg(row.l), - max: row => $.max(row.l), - min: row => $.min(row.l), - count: row => $.count(row.l), + x: row => row.l, + y: row => row.obj, }) .execute() expect(res).to.deep.equal([ - { sum: 3, avg: 1.5, max: 2, min: 1, count: 2 }, - { sum: 12, avg: 4, max: 5, min: 3, count: 3 }, - { sum: 2, avg: 2, max: 2, min: 2, count: 1 }, + { x: ['1', '2'], y: { x: 1, y: 'a', z: '1', o: { a: 1, b: '1' } } }, + { x: ['5', '3', '4'], y: { x: 2, y: 'b', z: '2', o: { a: 2, b: '2' } } }, + { x: ['2'], y: { x: 3, y: 'c', z: '3', o: { a: 3, b: '3' } } } ]) }) } From b6e88f1b081fa0651c2dc6578a9d87ef6252d5d9 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Mon, 30 Oct 2023 20:01:53 +0800 Subject: [PATCH 25/49] fix: join pass selection --- packages/core/src/driver.ts | 4 ++-- packages/sql-utils/src/index.ts | 16 +++++++++++----- packages/tests/src/json.ts | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/packages/core/src/driver.ts b/packages/core/src/driver.ts index b4df705a..d69b8e77 100644 --- a/packages/core/src/driver.ts +++ b/packages/core/src/driver.ts @@ -114,14 +114,14 @@ export class Database { join>>(tables: U, callback?: JoinCallback2, optional?: Dict>): Selection> join(tables: any, query?: any, optional?: any) { if (Array.isArray(tables)) { - const sel = new Selection(this.getDriver(tables[0]), Object.fromEntries(tables.map((name) => [name, name]))) + const sel = new Selection(this.getDriver(tables[0]), Object.fromEntries(tables.map((name) => [name, this.select(name)]))) if (typeof query === 'function') { sel.args[0].having = Eval.and(query(...tables.map(name => sel.row[name]))) } sel.args[0].optional = Object.fromEntries(tables.map((name, index) => [name, optional?.[index]])) return new Selection(this.getDriver(sel), sel) } else { - const sel = new Selection(this.getDriver(tables[0]), tables) + const sel = new Selection(this.getDriver(Object.values(tables)[0]), valueMap(tables, (t: TableLike) => typeof t === 'string' ? this.select(t) : t)) if (typeof query === 'function') { sel.args[0].having = Eval.and(query(sel.row)) } diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index cab2a217..7b805f94 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -256,10 +256,10 @@ export class Builder { return `json_extract(${obj}, '$${path}')` } - private transformKey(key: string, fields: {}, prefix: string) { + private transformKey(key: string, fields: {}, prefix: string, fullKey: string) { if (key in fields || !key.includes('.')) { - if (this.state.sqlTypes?.[key]) { - this.state.sqlType = this.state.sqlTypes[key] + if (this.state.sqlTypes?.[key] || this.state.sqlTypes?.[fullKey]) { + this.state.sqlType = this.state.sqlTypes[key] || this.state.sqlTypes[fullKey] } return prefix + escapeId(key) } @@ -287,7 +287,7 @@ export class Builder { const prefix = !this.tables || table === '_' || key in fields // the only table must be the main table || (Object.keys(this.tables).length === 1 && table in this.tables) ? '' : `${escapeId(table)}.` - return this.transformKey(key, fields, prefix) + return this.transformKey(key, fields, prefix, `${table}.${key}`) } parseEval(expr: any, unquote: boolean = true): string { @@ -332,13 +332,19 @@ export class Builder { prefix = this.get(table, true) if (!prefix) return } else { + const sqlTypes: Dict = {} prefix = Object.entries(table).map(([key, table]) => { if (typeof table !== 'string') { - return `${this.get(table, true)} AS ${escapeId(key)}` + const t = `${this.get(table, true)} AS ${escapeId(key)}` + for (const [fieldKey, fieldType] of Object.entries(this.state.sqlTypes!)) { + sqlTypes[`${key}.${fieldKey}`] = fieldType + } + return t } else { return key === table ? escapeId(table) : `${escapeId(table)} AS ${escapeId(key)}` } }).join(' JOIN ') + this.state.sqlTypes = sqlTypes const filter = this.parseEval(args[0].having) if (filter !== '1') prefix += ` ON ${filter}` } diff --git a/packages/tests/src/json.ts b/packages/tests/src/json.ts index c560e4a3..26499bf4 100644 --- a/packages/tests/src/json.ts +++ b/packages/tests/src/json.ts @@ -234,6 +234,24 @@ namespace JsonTests { { x: ['2'], y: { x: 3, y: 'c', z: '3', o: { a: 3, b: '3' } } } ]) }) + + it('pass sqlType in join', async () => { + const res = await database.join({ + foo: 'foo', + bar: 'bar', + }, ({foo, bar}) => $.eq(foo.id, bar.pid)) + .project({ + x: row => row.bar.l, + y: row => row.bar.obj, + }) + .execute() + + expect(res).to.deep.equal([ + { x: ['1', '2'], y: { x: 1, y: 'a', z: '1', o: { a: 1, b: '1' } } }, + { x: ['5', '3', '4'], y: { x: 2, y: 'b', z: '2', o: { a: 2, b: '2' } } }, + { x: ['2'], y: { x: 3, y: 'c', z: '3', o: { a: 3, b: '3' } } } + ]) + }) } } From 0902edbdd2b2260f7a7b27ac7a59ca0651433bde Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Tue, 31 Oct 2023 17:55:33 +0800 Subject: [PATCH 26/49] test: mj_sum --- packages/mysql/src/index.ts | 40 +++++++++++++++++++++++++++++++++ packages/sql-utils/src/index.ts | 2 +- packages/tests/src/json.ts | 17 ++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index ba2b2a0b..ad49b46a 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -246,6 +246,46 @@ export class MySQLDriver extends Driver { this._fixMariaIssue = true } + try { + await this.query(` + CREATE FUNCTION mj_sum (j JSON) + RETURNS NUMERIC DETERMINISTIC + BEGIN + DECLARE n int; + DECLARE i int; + DECLARE r NUMERIC; + DROP TEMPORARY TABLE IF EXISTS mtt; + CREATE TEMPORARY TABLE mtt (value JSON); + SELECT json_length(j) into n; + set i = 0; + WHILE i([ `SELECT *`, diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 7b805f94..fc1ed349 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -116,7 +116,7 @@ export class Builder { $lte: this.binary('<='), // aggregation - $sum: (expr) => `ifnull(sum(${this.parseAggr(expr)}), 0)`, + $sum: (expr) => this.state.group ? `ifnull(sum(${this.parseAggr(expr)}), 0)` : `ifnull(mj_sum(${this.parseAggr(expr)}), 0)`, $avg: (expr) => `avg(${this.parseAggr(expr)})`, $min: (expr) => `min(${this.parseAggr(expr)})`, $max: (expr) => `max(${this.parseAggr(expr)})`, diff --git a/packages/tests/src/json.ts b/packages/tests/src/json.ts index 26499bf4..3c188fa2 100644 --- a/packages/tests/src/json.ts +++ b/packages/tests/src/json.ts @@ -220,6 +220,23 @@ namespace JsonTests { ]) }) + it('non-aggr func', async () => { + const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) + .groupBy('foo', { + y: row => $.array(row.bar.obj.x), + }) + .orderBy(row => row.foo.id) + .project({ + sum: row => $.sum(row.y) + }) + .execute() + + expect(res).to.deep.equal([ + { sum: 3 }, + { sum: 3 }, + ]) + }) + it('pass sqlType', async () => { const res = await database.select('bar') .project({ From f5dd4bbf6e0f33fc184abe9448797cd816e8869e Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Tue, 31 Oct 2023 18:51:54 +0800 Subject: [PATCH 27/49] feat: add all non-aggr and tests --- packages/core/src/eval.ts | 41 +++++++---- packages/mysql/src/index.ts | 116 ++++++++++++++++++-------------- packages/sql-utils/src/index.ts | 26 +++++-- packages/sqlite/src/index.ts | 14 +++- packages/tests/src/json.ts | 31 +++++++-- 5 files changed, 152 insertions(+), 76 deletions(-) diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index afaa646c..c1b77ed6 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -95,14 +95,14 @@ export namespace Eval { count(value: Any): Expr // json - sum(value: (Number | Expr)[] | Expr): Expr - avg(value: (Number | Expr)[] | Expr): Expr - max(value: (Number | Expr)[] | Expr): Expr - min(value: (Number | Expr)[] | Expr): Expr - count(value: (Any | Expr)[] | Expr): Expr + sum(value: (Number | Expr)[] | Expr): Expr + avg(value: (Number | Expr)[] | Expr): Expr + max(value: (Number | Expr)[] | Expr): Expr + min(value: (Number | Expr)[] | Expr): Expr + count(value: (Any | Expr)[] | Expr): Expr object>(fields: T): Expr - array(value: Expr): Expr + array(value: Expr): Expr } } @@ -175,15 +175,30 @@ Eval.or = multary('or', (args, data) => args.some(arg => executeEval(data, arg)) Eval.not = unary('not', (value, data) => !executeEval(data, value)) // aggregation -Eval.sum = unary('sum', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0)) -Eval.avg = unary('avg', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length) -Eval.max = unary('max', (expr, table) => Math.max(...table.map(data => executeAggr(expr, data)))) -Eval.min = unary('min', (expr, table) => Math.min(...table.map(data => executeAggr(expr, data)))) -Eval.count = unary('count', (expr, table) => new Set(table.map(data => executeAggr(expr, data))).size) +Eval.sum = unary('sum', (expr, table) => Array.isArray(table) + ? table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) + : Array.from(executeEval(table, expr)).reduce((prev, curr) => prev + curr, 0)) +Eval.avg = unary('avg', (expr, table) => { + if (Array.isArray(table)) return table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length + else { + const array = Array.from(executeEval(table, expr)) + return array.reduce((prev, curr) => prev + curr, 0) / array.length + } +}) +Eval.max = unary('max', (expr, table) => Array.isArray(table) + ? Math.max(...table.map(data => executeAggr(expr, data))) + : Math.max(...Array.from(executeEval(table, expr)))) +Eval.min = unary('min', (expr, table) => Array.isArray(table) + ? Math.min(...table.map(data => executeAggr(expr, data))) + : Math.min(...Array.from(executeEval(table, expr)))) +Eval.count = unary('count', (expr, table) => Array.isArray(table) + ? new Set(table.map(data => executeAggr(expr, data))).size + : new Set(Array.from(executeEval(table, expr))).size) Eval.object = unary('object', (field, table) => valueMap(field, value => executeAggr(value, table))) -Eval.array = unary('array', (expr, table) => table.map(data => executeAggr(expr, data))) - +Eval.array = unary('array', (expr, table) => Array.isArray(table) + ? table.map(data => executeAggr(expr, data)) + : Array.from(executeEval(table, expr))) export { Eval as $ } type MapUneval = { diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index ad49b46a..e7fe9964 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -68,6 +68,11 @@ function createIndex(keys: string | string[]) { return makeArray(keys).map(escapeId).join(', ') } +interface Compat { + maria105?: boolean + mysql57?: boolean +} + interface ColumnInfo { COLUMN_NAME: string IS_NULLABLE: 'YES' | 'NO' @@ -104,9 +109,13 @@ class MySQLBuilder extends Builder { '\\': '\\\\', } - constructor(tables?: Dict, workaroundArrayagg = false) { + constructor(tables?: Dict, private compat: Compat = {}) { super(tables) - this.workaroundArrayagg = workaroundArrayagg + + this.evalOperators.$sum = (expr) => this.createAggr(expr, value => `ifnull(sum(${value}), 0)`, value => `ifnull(mj_sum(${value}), 0)`) + this.evalOperators.$avg = (expr) => this.createAggr(expr, value => `avg(${value})`, value => `mj_avg(${value})`) + this.evalOperators.$min = (expr) => this.createAggr(expr, value => `(0+min(${value}))`, value => `(0+mj_min(${value}))`) + this.evalOperators.$max = (expr) => this.createAggr(expr, value => `(0+max(${value}))`, value => `(0+mj_max(${value}))`) this.define({ types: ['list'], @@ -130,6 +139,16 @@ class MySQLBuilder extends Builder { return super.escape(value, field) } + protected createAggr(expr: any, aggrfunc: (value: string) => string, compatfunc?: (value: string) => string) { + if (!this.state.group && compatfunc && (this.compat.mysql57 || this.compat.maria105)) { + const aggr = compatfunc(this.parseAggr(expr)) + this.state.sqlType = 'raw' + return aggr + } else { + return super.createAggr(expr, aggrfunc) + } + } + toUpdateExpr(item: any, key: string, field?: Field, upsert?: boolean) { const escaped = escapeId(key) @@ -186,10 +205,9 @@ export class MySQLDriver extends Driver { public config: MySQLDriver.Config public sql: MySQLBuilder + private _compat: Compat = {} private _queryTasks: QueryTask[] = [] - private _fixMariaIssue: boolean = false - constructor(database: Database, config?: MySQLDriver.Config) { super(database) @@ -241,50 +259,14 @@ export class MySQLDriver extends Driver { /** synchronize table schema */ async prepare(name: string) { const version = Object.values((await this.query(`SELECT version()`))[0])[0] as string - if (version.match(/10.5.\d+-MariaDB/)) { - // https://jira.mariadb.org/browse/MDEV-26506 - this._fixMariaIssue = true - } + // https://jira.mariadb.org/browse/MDEV-26506 + this._compat.maria105 = !!version.match(/10.5.\d+-MariaDB/) + // For json_table + this._compat.mysql57 = !!version.match(/5.7.\d+/) - try { - await this.query(` - CREATE FUNCTION mj_sum (j JSON) - RETURNS NUMERIC DETERMINISTIC - BEGIN - DECLARE n int; - DECLARE i int; - DECLARE r NUMERIC; - DROP TEMPORARY TABLE IF EXISTS mtt; - CREATE TEMPORARY TABLE mtt (value JSON); - SELECT json_length(j) into n; - set i = 0; - WHILE i([ @@ -412,6 +394,36 @@ export class MySQLDriver extends Driver { }).join(', ') } + async _setupCompatFunctions() { + try { + await this.query(`DROP FUNCTION IF EXISTS mj_sum`) + await this.query(`CREATE FUNCTION mj_sum (j JSON) RETURNS DOUBLE DETERMINISTIC BEGIN DECLARE n int; DECLARE i int; DECLARE r DOUBLE; +DROP TEMPORARY TABLE IF EXISTS mtt; CREATE TEMPORARY TABLE mtt (value JSON); SELECT json_length(j) into n; set i = 0; +WHILE i(sql: string, debug = true): Promise { const error = new Error() return new Promise((resolve, reject) => { @@ -483,7 +495,7 @@ export class MySQLDriver extends Driver { async get(sel: Selection.Immutable) { const { model, tables } = sel - const builder = new MySQLBuilder(tables, this._fixMariaIssue) + const builder = new MySQLBuilder(tables, this._compat) const sql = builder.get(sel) if (!sql) return [] return this.queue(sql).then((data) => { @@ -492,7 +504,7 @@ export class MySQLDriver extends Driver { } async eval(sel: Selection.Immutable, expr: Eval.Expr) { - const builder = new MySQLBuilder(sel.tables, this._fixMariaIssue) + const builder = new MySQLBuilder(sel.tables, this._compat) builder.state.group = true const output = builder.parseEval(expr) const inner = builder.get(sel.table as Selection, true) @@ -502,7 +514,7 @@ export class MySQLDriver extends Driver { async set(sel: Selection.Mutable, data: {}) { const { model, query, table, tables, ref } = sel - const builder = new MySQLBuilder(tables, this._fixMariaIssue) + const builder = new MySQLBuilder(tables, this._compat) const filter = builder.parseQuery(query) const { fields } = model if (filter === '0') return @@ -519,7 +531,7 @@ export class MySQLDriver extends Driver { async remove(sel: Selection.Mutable) { const { query, table, tables } = sel - const builder = new MySQLBuilder(tables, this._fixMariaIssue) + const builder = new MySQLBuilder(tables, this._compat) const filter = builder.parseQuery(query) if (filter === '0') return await this.query(`DELETE FROM ${escapeId(table)} WHERE ` + filter) @@ -541,7 +553,7 @@ export class MySQLDriver extends Driver { async upsert(sel: Selection.Mutable, data: any[], keys: string[]) { if (!data.length) return const { model, table, tables, ref } = sel - const builder = new MySQLBuilder(tables, this._fixMariaIssue) + const builder = new MySQLBuilder(tables, this._compat) const merged = {} const insertion = data.map((item) => { diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index fc1ed349..8fc8adfb 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -1,5 +1,5 @@ import { Dict, isNullable } from 'cosmokit' -import { Eval, Field, isComparable, Model, Modifier, Query, Selection } from '@minatojs/core' +import { Eval, Field, isComparable, Model, Modifier, Query, randomId, Selection } from '@minatojs/core' export function escapeId(value: string) { return '`' + value + '`' @@ -116,11 +116,11 @@ export class Builder { $lte: this.binary('<='), // aggregation - $sum: (expr) => this.state.group ? `ifnull(sum(${this.parseAggr(expr)}), 0)` : `ifnull(mj_sum(${this.parseAggr(expr)}), 0)`, - $avg: (expr) => `avg(${this.parseAggr(expr)})`, - $min: (expr) => `min(${this.parseAggr(expr)})`, - $max: (expr) => `max(${this.parseAggr(expr)})`, - $count: (expr) => `count(distinct ${this.parseAggr(expr)})`, + $sum: (expr) => this.createAggr(expr, value => `ifnull(sum(${value}), 0)`), + $avg: (expr) => this.createAggr(expr, value => `avg(${value})`), + $min: (expr) => this.createAggr(expr, value => `(0+min(${value}))`), + $max: (expr) => this.createAggr(expr, value => `(0+max(${value}))`), + $count: (expr) => this.createAggr(expr, value => `count(distinct ${value})`), $object: (fields) => this.groupObject(fields), $array: (expr) => this.groupArray(this.parseAggr(expr)), @@ -178,6 +178,20 @@ export class Builder { return res } + protected createAggr(expr: any, aggrfunc: (value: string) => string) { + if (this.state.group) { + this.state.group = false + const aggr = aggrfunc(this.parseAggr(expr)) + this.state.group = true + this.state.sqlType = 'raw' + return aggr + } else { + const aggr = this.parseAggr(expr) + this.state.sqlType = 'raw' + return `(select ${aggrfunc(`json_unquote(${escapeId('value')})`)} from json_table(${aggr}, '$[*]' columns (value json path '$')) ${randomId()})` + } + } + protected groupObject(fields: any) { const res = `json_object(` + Object.entries(fields).map(([key, expr]) => `'${key}', ${this.parseEval(expr, false)}`).join(',') + `)` this.state.sqlType = 'json' diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index bb148cb8..f736df9d 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -1,5 +1,5 @@ import { deepEqual, Dict, difference, isNullable, makeArray } from 'cosmokit' -import { Database, Driver, Eval, executeUpdate, Field, Model, Selection } from '@minatojs/core' +import { Database, Driver, Eval, executeUpdate, Field, Model, randomId, Selection } from '@minatojs/core' import { Builder, escapeId } from '@minatojs/sql-utils' import { promises as fs } from 'fs' import init from '@minatojs/sql.js' @@ -91,6 +91,18 @@ class SQLiteBuilder extends Builder { return value } + protected createAggr(expr: any, aggrfunc: (value: string) => string) { + if (this.state.group) { + this.state.group = false + const ret = aggrfunc(this.parseAggr(expr)) + this.state.group = true + return ret + } else { + const aggr = this.parseAggr(expr) + return `(select ${aggrfunc(escapeId('value'))} from json_each(${aggr}) ${randomId()})` + } + } + protected groupArray(value: string) { const res = this.state.sqlType === 'json' ? `('[' || group_concat(${value}) || ']')` : `('[' || group_concat(json_quote(${value})) || ']')` this.state.sqlType = 'json' diff --git a/packages/tests/src/json.ts b/packages/tests/src/json.ts index 3c188fa2..ed538c94 100644 --- a/packages/tests/src/json.ts +++ b/packages/tests/src/json.ts @@ -225,18 +225,41 @@ namespace JsonTests { .groupBy('foo', { y: row => $.array(row.bar.obj.x), }) - .orderBy(row => row.foo.id) .project({ - sum: row => $.sum(row.y) + sum: row => $.sum(row.y), + avg: row => $.avg(row.y), + min: row => $.min(row.y), + max: row => $.max(row.y), + count: row => $.count(row.y), }) + .orderBy(row => row.count) .execute() + expect(res).to.deep.equal([ + { sum: 3, avg: 3, min: 3, max: 3, count: 1 }, + { sum: 3, avg: 1.5, min: 1, max: 2, count: 2 }, + ]) + }) + + it('non-aggr func inside aggr', async () => { + const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) + .groupBy('foo', { + y: row => $.array(row.bar.obj.x), + }) + .orderBy(row => row.foo.id) + .groupBy({}, { + sum: row => $.avg($.sum(row.y)), + avg: row => $.avg($.avg(row.y)), + min: row => $.min($.min(row.y)), + max: row => $.max($.max(row.y)), + }) + .execute() expect(res).to.deep.equal([ - { sum: 3 }, - { sum: 3 }, + { sum: 3, avg: 2.25, min: 1, max: 3 } ]) }) + it('pass sqlType', async () => { const res = await database.select('bar') .project({ From 98fac384fa37c6d0683afa2dc801cae388d52826 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Tue, 31 Oct 2023 19:00:44 +0800 Subject: [PATCH 28/49] fix: count --- packages/mysql/src/index.ts | 21 ++++++++++++++++++--- packages/sql-utils/src/index.ts | 5 +---- packages/tests/src/json.ts | 1 - 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index e7fe9964..7a53dbc9 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -149,6 +149,14 @@ class MySQLBuilder extends Builder { } } + protected groupArray(value: string) { + const res = this.compat.maria105 ? (this.state.sqlType === 'json' ? `concat('[', group_concat(${value}), ']')` + : `concat('[', group_concat(json_extract(json_object('v', ${value}), '$.v')), ']')`) + : super.groupArray(value) + this.state.sqlType = 'json' + return res + } + toUpdateExpr(item: any, key: string, field?: Field, upsert?: boolean) { const escaped = escapeId(key) @@ -409,6 +417,13 @@ DROP TEMPORARY TABLE IF EXISTS mtt; CREATE TEMPORARY TABLE mtt (value JSON); SEL WHILE i { const res = await database.select('bar') .project({ From 1caf77d260362f411647280d400d7d027b5e4243 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Tue, 31 Oct 2023 19:04:52 +0800 Subject: [PATCH 29/49] fix --- packages/mysql/src/index.ts | 1 + packages/sql-utils/src/index.ts | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index 7a53dbc9..7fee0ce0 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -116,6 +116,7 @@ class MySQLBuilder extends Builder { this.evalOperators.$avg = (expr) => this.createAggr(expr, value => `avg(${value})`, value => `mj_avg(${value})`) this.evalOperators.$min = (expr) => this.createAggr(expr, value => `(0+min(${value}))`, value => `(0+mj_min(${value}))`) this.evalOperators.$max = (expr) => this.createAggr(expr, value => `(0+max(${value}))`, value => `(0+mj_max(${value}))`) + this.evalOperators.$count = (expr) => this.createAggr(expr, value => `count(distinct ${value})`, value => `mj_count(${value})`) this.define({ types: ['list'], diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 1ae8f660..825b7b34 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -36,7 +36,6 @@ export class Builder { protected createEqualQuery = this.comparator('=') protected queryOperators: QueryOperators protected evalOperators: EvalOperators - protected workaroundArrayagg = false state: State = {} constructor(public tables?: Dict) { From 6b5a24b96c1f3d76908c7815a79547f651b6d4b7 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Tue, 31 Oct 2023 19:08:33 +0800 Subject: [PATCH 30/49] ci: try cov --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5d979ece..7d1ad795 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -42,7 +42,7 @@ jobs: - name: Unit Test run: yarn test:json mysql - name: Report Coverage - if: ${{ matrix.node-version == 16 && matrix.mysql-image == 'mysql:5.7' }} + if: ${{ matrix.node-version == 16 }} uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} From 7773dcf69921af92e5d1db14a4a00162caccb5c3 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Tue, 31 Oct 2023 22:03:39 +0800 Subject: [PATCH 31/49] feat: $.object on cell --- packages/core/src/eval.ts | 17 ++++++++-- packages/core/src/selection.ts | 18 ++++++----- packages/mysql/src/index.ts | 55 +++++++++++++-------------------- packages/sql-utils/src/index.ts | 6 +++- packages/tests/src/json.ts | 39 ++++++++++++++++++----- 5 files changed, 83 insertions(+), 52 deletions(-) diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index c1b77ed6..054499d6 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -1,5 +1,5 @@ import { defineProperty, Dict, isNullable, valueMap } from 'cosmokit' -import { Comparable, Flatten, isComparable, makeRegExp } from './utils' +import { Comparable, Flatten, isComparable, makeRegExp, Row } from './utils' export function isEvalExpr(value: any): value is Eval.Expr { return value && Object.keys(value).some(key => key.startsWith('$')) @@ -102,6 +102,7 @@ export namespace Eval { count(value: (Any | Expr)[] | Expr): Expr object>(fields: T): Expr + object(row: Row.Cell): Expr array(value: Expr): Expr } } @@ -195,7 +196,19 @@ Eval.count = unary('count', (expr, table) => Array.isArray(table) ? new Set(table.map(data => executeAggr(expr, data))).size : new Set(Array.from(executeEval(table, expr))).size) -Eval.object = unary('object', (field, table) => valueMap(field, value => executeAggr(value, table))) +operators.$object = (field, table) => valueMap(field, value => executeAggr(value, table)) +Eval.object = (fields) => { + if (fields.$model) { + const modelFields = Object.keys(fields.$model.fields) + const prefix: string = fields.$prefix + return Eval('object', Object.fromEntries(modelFields + .filter(path => path.startsWith(prefix)) + .map(k => [k.slice(prefix.length), fields[k.slice(prefix.length)]]), + )) + } + return Eval('object', fields) as any +} +// Eval.object = unary('object', (field, table) => valueMap(field, value => executeAggr(value, table))) Eval.array = unary('array', (expr, table) => Array.isArray(table) ? table.map(data => executeAggr(expr, data)) : Array.from(executeEval(table, expr))) diff --git a/packages/core/src/selection.ts b/packages/core/src/selection.ts index 79f10dc5..0c773fc9 100644 --- a/packages/core/src/selection.ts +++ b/packages/core/src/selection.ts @@ -22,17 +22,19 @@ namespace Executable { export interface Payload { type: Action - table: string | Selection | Dict + table: string | Selection | Dict ref: string query: Query.Expr args: any[] } } -const createRow = (ref: string, expr = {}, prefix = '') => new Proxy(expr, { +const createRow = (ref: string, expr = {}, prefix = '', model?: Model) => new Proxy(expr, { get(target, key) { + if (key === '$prefix') return prefix + if (key === '$model') return model if (typeof key === 'symbol' || key in target || key.startsWith('$')) return Reflect.get(target, key) - return createRow(ref, Eval('', [ref, `${prefix}${key}`]), `${prefix}${key}.`) + return createRow(ref, Eval('', [ref, `${prefix}${key}`]), `${prefix}${key}.`, model) }, }) @@ -45,15 +47,15 @@ class Executable { constructor(driver: Driver, payload: Executable.Payload) { Object.assign(this, payload) - const expr = {} + defineProperty(this, 'model', driver.model(this.table)) + const expr = { $model: this.model } if (typeof payload.table !== 'string' && !(payload.table instanceof Selection)) { for (const key in payload.table) { - expr[key] = createRow(key) + expr[key] = createRow(key, {}, '', this.model) } } defineProperty(this, 'driver', driver) - defineProperty(this, 'row', createRow(this.ref, expr)) - defineProperty(this, 'model', driver.model(this.table)) + defineProperty(this, 'row', createRow(this.ref, expr, '', this.model)) } protected resolveQuery(query?: Query): Query.Expr @@ -130,7 +132,7 @@ export interface Selection extends Executable.Payload { export class Selection extends Executable { public tables: Dict = {} - constructor(driver: Driver, table: string | Selection | Dict, query?: Query) { + constructor(driver: Driver, table: string | Selection | Dict, query?: Query) { super(driver, { type: 'get', ref: randomId(), diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index 7fee0ce0..069352b1 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -404,40 +404,27 @@ export class MySQLDriver extends Driver { } async _setupCompatFunctions() { - try { - await this.query(`DROP FUNCTION IF EXISTS mj_sum`) - await this.query(`CREATE FUNCTION mj_sum (j JSON) RETURNS DOUBLE DETERMINISTIC BEGIN DECLARE n int; DECLARE i int; DECLARE r DOUBLE; -DROP TEMPORARY TABLE IF EXISTS mtt; CREATE TEMPORARY TABLE mtt (value JSON); SELECT json_length(j) into n; set i = 0; -WHILE i logger.debug('Failed to setup compact functions') + await this.query(`DROP FUNCTION IF EXISTS mj_sum`).catch(log) + await this.query(`CREATE FUNCTION mj_sum (j JSON) RETURNS DOUBLE DETERMINISTIC BEGIN DECLARE n int; DECLARE i int; DECLARE r DOUBLE; +DROP TEMPORARY TABLE IF EXISTS mtt; CREATE TEMPORARY TABLE mtt (value JSON); SELECT json_length(j) into n; set i = 0; WHILE i(sql: string, debug = true): Promise { diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 825b7b34..185f7c52 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -192,7 +192,11 @@ export class Builder { } protected groupObject(fields: any) { - const res = `json_object(` + Object.entries(fields).map(([key, expr]) => `'${key}', ${this.parseEval(expr, false)}`).join(',') + `)` + const parse = (expr) => { + const value = this.parseEval(expr, false) + return this.state.sqlType === 'json' ? `json_extract(${value}, '$')` : `${value}` + } + const res = `json_object(` + Object.entries(fields).map(([key, expr]) => `'${key}', ${parse(expr)}`).join(',') + `)` this.state.sqlType = 'json' return res } diff --git a/packages/tests/src/json.ts b/packages/tests/src/json.ts index 0d0ac64f..8b2de2fe 100644 --- a/packages/tests/src/json.ts +++ b/packages/tests/src/json.ts @@ -103,6 +103,20 @@ namespace JsonTests { ]) }) + it('$.object on cell', async () => { + const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) + .groupBy('bar', { + x: row => $.array($.object(row.foo)) + }) + .execute(['x']) + + expect(res).to.have.deep.members([ + { x: [{ id: 1, value: 0 }] }, + { x: [{ id: 1, value: 0 }] }, + { x: [{ id: 2, value: 2 }] }, + ]) + }) + it('$.array groupBy', async () => { const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) .groupBy(['foo'], { @@ -142,7 +156,8 @@ namespace JsonTests { const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) .groupBy('foo', { bars: row => $.array($.object({ - value: row.bar.value + value: row.bar.value, + obj: row.bar.obj, })), x: row => $.array(row.bar.obj.x), y: row => $.array(row.bar.obj.y), @@ -155,19 +170,28 @@ namespace JsonTests { expect(res).to.deep.equal([ { foo: { id: 1, value: 0 }, - bars: [{ value: 0 }, { value: 1 }], + bars: [{ + obj: { o: { a: 1, b: '1' }, x: 1, y: 'a', z: '1' }, + value: 0, + }, { + obj: { o: { a: 2, b: '2' }, x: 2, y: 'b', z: '2' }, + value: 1, + }], x: [1, 2], y: ['a', 'b'], z: ['1', '2'], - o: [{ a: 1, b: '1' }, { a: 2, b: '2' }] + o: [{ a: 1, b: '1' }, { a: 2, b: '2' }], }, { foo: { id: 2, value: 2 }, - bars: [{ value: 0 }], + bars: [{ + obj: { o: { a: 3, b: '3' }, x: 3, y: 'c', z: '3' }, + value: 0, + }], x: [3], y: ['c'], z: ['3'], - o: [{ a: 3, b: '3' }] + o: [{ a: 3, b: '3' }], } ]) }) @@ -234,13 +258,13 @@ namespace JsonTests { }) .orderBy(row => row.count) .execute() + expect(res).to.deep.equal([ { sum: 3, avg: 3, min: 3, max: 3, count: 1 }, { sum: 3, avg: 1.5, min: 1, max: 2, count: 2 }, ]) }) - it('non-aggr func inside aggr', async () => { const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) .groupBy('foo', { @@ -254,6 +278,7 @@ namespace JsonTests { max: row => $.max($.max(row.y)), }) .execute() + expect(res).to.deep.equal([ { sum: 3, avg: 2.25, min: 1, max: 3 } ]) @@ -278,7 +303,7 @@ namespace JsonTests { const res = await database.join({ foo: 'foo', bar: 'bar', - }, ({foo, bar}) => $.eq(foo.id, bar.pid)) + }, ({ foo, bar }) => $.eq(foo.id, bar.pid)) .project({ x: row => row.bar.l, y: row => row.bar.obj, From 8e9868c74b4a4efbc42ff9c63c4df6884e76786b Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Tue, 31 Oct 2023 22:40:21 +0800 Subject: [PATCH 32/49] chore: clean --- package.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 2b7eec9d..fcbc8c99 100644 --- a/package.json +++ b/package.json @@ -23,15 +23,15 @@ "devDependencies": { "@koishijs/eslint-config": "^1.0.4", "@types/mocha": "^9.1.1", - "@types/node": "^20.8.9", + "@types/node": "^20.4.2", "c8": "^7.14.0", - "esbuild": "^0.18.20", - "esbuild-register": "^3.5.0", - "eslint": "^8.52.0", - "eslint-plugin-mocha": "^10.2.0", + "esbuild": "^0.18.14", + "esbuild-register": "^3.4.2", + "eslint": "^8.45.0", + "eslint-plugin-mocha": "^10.1.0", "mocha": "^9.2.2", "shx": "^0.3.4", - "typescript": "^5.2.2", + "typescript": "^5.1.6", "yakumo": "^0.3.13", "yakumo-esbuild": "^0.3.26", "yakumo-mocha": "^0.3.1", From 32b2a5f8b7ff59660bf2fa0d338f758b211367a2 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Wed, 1 Nov 2023 02:08:12 +0800 Subject: [PATCH 33/49] chore: clean --- packages/core/src/eval.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index 054499d6..894d08f3 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -208,7 +208,6 @@ Eval.object = (fields) => { } return Eval('object', fields) as any } -// Eval.object = unary('object', (field, table) => valueMap(field, value => executeAggr(value, table))) Eval.array = unary('array', (expr, table) => Array.isArray(table) ? table.map(data => executeAggr(expr, data)) : Array.from(executeEval(table, expr))) From 429ed7a0e5625ffede3e62f38ebc17db9af4c41b Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Wed, 1 Nov 2023 02:49:07 +0800 Subject: [PATCH 34/49] feat: support query ops for json array --- packages/sql-utils/src/index.ts | 21 ++++++++++++--- packages/sqlite/src/index.ts | 15 ++++++++++- packages/tests/src/json.ts | 47 +++++++++++++++++++++++++++++---- 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 185f7c52..6ad9c9e6 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -82,7 +82,11 @@ export class Builder { }, $size: (key, value) => { if (!value) return this.logicalNot(key) - return `${key} AND LENGTH(${key}) - LENGTH(REPLACE(${key}, ${this.escape(',')}, ${this.escape('')})) = ${this.escape(value)} - 1` + if (this.state.sqlTypes?.[this.unescapeId(key)] === 'json') { + return `json_length(${key}) = ${this.escape(value)}` + } else { + return `${key} AND LENGTH(${key}) - LENGTH(REPLACE(${key}, ${this.escape(',')}, ${this.escape('')})) = ${this.escape(value)} - 1` + } }, } @@ -126,6 +130,10 @@ export class Builder { } } + protected unescapeId(value: string) { + return value.slice(1, value.length - 1) + } + protected createNullQuery(key: string, value: boolean) { return `${key} is ${value ? 'not ' : ''}null` } @@ -140,7 +148,11 @@ export class Builder { } protected createElementQuery(key: string, value: any) { - return `find_in_set(${this.escape(value)}, ${key})` + if (this.state.sqlTypes?.[this.unescapeId(key)] === 'json') { + return `json_contains(${key}, ${this.quote(JSON.stringify(value))})` + } else { + return `find_in_set(${this.escape(value)}, ${key})` + } } protected comparator(operator: string) { @@ -332,8 +344,6 @@ export class Builder { get(sel: Selection.Immutable, inline = false) { const { args, table, query, ref, model } = sel - const filter = this.parseQuery(query) - if (filter === '0') return // get prefix let prefix: string | undefined @@ -376,6 +386,9 @@ export class Builder { }).join(', ') this.state.sqlTypes = sqlTypes + const filter = this.parseQuery(query) + if (filter === '0') return + // get suffix let suffix = this.suffix(args[0]) if (filter !== '1') { diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index f736df9d..7caefab3 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -52,6 +52,14 @@ class SQLiteBuilder extends Builder { this.evalOperators.$if = (args) => `iif(${args.map(arg => this.parseEval(arg)).join(', ')})` this.evalOperators.$concat = (args) => `(${args.map(arg => this.parseEval(arg)).join('||')})` + this.queryOperators.$size = (key, value) => { + if (!value) return this.logicalNot(key) + if (this.state.sqlTypes?.[this.unescapeId(key)] === 'json') { + return `json_array_length(${key}) = ${this.escape(value)}` + } else { + return `${key} AND LENGTH(${key}) - LENGTH(REPLACE(${key}, ${this.escape(',')}, ${this.escape('')})) = ${this.escape(value)} - 1` + } + } this.define({ types: ['boolean'], @@ -84,7 +92,11 @@ class SQLiteBuilder extends Builder { } protected createElementQuery(key: string, value: any) { - return `(',' || ${key} || ',') LIKE ${this.escape('%,' + value + ',%')}` + if (this.state.sqlTypes?.[this.unescapeId(key)] === 'json') { + return `json_array_contains(${key}, ${this.quote(JSON.stringify(value))})` + } else { + return `(',' || ${key} || ',') LIKE ${this.escape('%,' + value + ',%')}` + } } protected unquoteJson(value: string) { @@ -230,6 +242,7 @@ export class SQLiteDriver extends Driver { init(buffer: ArrayLike | null) { this.db = new this.sqlite.Database(buffer) this.db.create_function('regexp', (pattern, str) => +new RegExp(pattern).test(str)) + this.db.create_function('json_array_contains', (array, value) => +(JSON.parse(array) as any[]).includes(JSON.parse(value))) } async load() { diff --git a/packages/tests/src/json.ts b/packages/tests/src/json.ts index 8b2de2fe..ed5ac3b1 100644 --- a/packages/tests/src/json.ts +++ b/packages/tests/src/json.ts @@ -25,9 +25,15 @@ interface Bar { l: string[] } +interface Baz { + id: number + nums: number[] +} + interface Tables { foo: Foo bar: Bar + baz: Baz } function JsonTests(database: Database) { @@ -43,11 +49,16 @@ function JsonTests(database: Database) { value: 'integer', obj: 'json', s: 'string', - l: { type: 'list', initial: [] } + l: 'list', }, { autoInc: true, }) + database.extend('baz', { + id: 'unsigned', + nums: { type: 'json', initial: [] }, + }) + before(async () => { await setup(database, 'foo', [ { id: 1, value: 0 }, @@ -60,11 +71,37 @@ function JsonTests(database: Database) { { uid: 1, pid: 1, value: 1, obj: { x: 2, y: 'b', z: '2', o: { a: 2, b: '2' } }, s: '2', l: ['5', '3', '4'] }, { uid: 1, pid: 2, value: 0, obj: { x: 3, y: 'c', z: '3', o: { a: 3, b: '3' } }, s: '3', l: ['2'] }, ]) + + await setup(database, 'baz', [ + { id: 1, nums: [4, 5, 6] }, + { id: 2, nums: [5, 6, 7] }, + { id: 3, nums: [6, 8] }, + ]) }) } namespace JsonTests { - export function jsontype(database: Database) { + export function query(database: Database) { + it('$size', async () => { + await expect(database.get('baz', { + nums: { $size: 3 }, + })).to.eventually.deep.equal([ + { id: 1, nums: [4, 5, 6] }, + { id: 2, nums: [5, 6, 7] }, + ]) + }) + + it('$el', async () => { + await expect(database.get('baz', { + nums: { $el: 5 }, + })).to.eventually.deep.equal([ + { id: 1, nums: [4, 5, 6] }, + { id: 2, nums: [5, 6, 7] }, + ]) + }) + } + + export function selection(database: Database) { it('$.object', async () => { const res = await database.select('foo') .project({ @@ -233,7 +270,7 @@ namespace JsonTests { }) .orderBy(row => row.foo.id) .groupBy({}, { - z: row => $.array(row.y) + z: row => $.array(row.y), }) .execute() @@ -295,7 +332,7 @@ namespace JsonTests { expect(res).to.deep.equal([ { x: ['1', '2'], y: { x: 1, y: 'a', z: '1', o: { a: 1, b: '1' } } }, { x: ['5', '3', '4'], y: { x: 2, y: 'b', z: '2', o: { a: 2, b: '2' } } }, - { x: ['2'], y: { x: 3, y: 'c', z: '3', o: { a: 3, b: '3' } } } + { x: ['2'], y: { x: 3, y: 'c', z: '3', o: { a: 3, b: '3' } } }, ]) }) @@ -313,7 +350,7 @@ namespace JsonTests { expect(res).to.deep.equal([ { x: ['1', '2'], y: { x: 1, y: 'a', z: '1', o: { a: 1, b: '1' } } }, { x: ['5', '3', '4'], y: { x: 2, y: 'b', z: '2', o: { a: 2, b: '2' } } }, - { x: ['2'], y: { x: 3, y: 'c', z: '3', o: { a: 3, b: '3' } } } + { x: ['2'], y: { x: 3, y: 'c', z: '3', o: { a: 3, b: '3' } } }, ]) }) } From 9eb5cd1d2ff1364279f2a16e0a06102cd69daef8 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Wed, 1 Nov 2023 17:18:47 +0800 Subject: [PATCH 35/49] feat: add $.size, remove non-aggr $.count, fix sqlTypes --- .github/workflows/test.yaml | 5 ++++- packages/core/src/eval.ts | 6 +++++- packages/mongo/src/utils.ts | 9 ++++----- packages/mysql/src/index.ts | 1 - packages/sql-utils/src/index.ts | 33 ++++++++++++++++++++++++++++----- packages/sqlite/src/index.ts | 12 ++++-------- packages/tests/src/json.ts | 11 ++++++++++- packages/tests/src/selection.ts | 3 ++- 8 files changed, 57 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7d1ad795..0787be9d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -55,11 +55,14 @@ jobs: strategy: fail-fast: false matrix: + mysql-image: + - mongo:6.0 + - mongo:latest node-version: [16, 18, 20] services: mongo: - image: mongo + image: ${{ matrix.mongo-image }} ports: - 27017:27017 # https://stackoverflow.com/questions/66317184/github-actions-cannot-connect-to-mongodb-service diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index 894d08f3..6e1a5b11 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -93,13 +93,14 @@ export namespace Eval { max(value: Number): Expr min(value: Number): Expr count(value: Any): Expr + size(value: Any): Expr // json sum(value: (Number | Expr)[] | Expr): Expr avg(value: (Number | Expr)[] | Expr): Expr max(value: (Number | Expr)[] | Expr): Expr min(value: (Number | Expr)[] | Expr): Expr - count(value: (Any | Expr)[] | Expr): Expr + size(value: (Any | Expr)[] | Expr): Expr object>(fields: T): Expr object(row: Row.Cell): Expr @@ -195,6 +196,9 @@ Eval.min = unary('min', (expr, table) => Array.isArray(table) Eval.count = unary('count', (expr, table) => Array.isArray(table) ? new Set(table.map(data => executeAggr(expr, data))).size : new Set(Array.from(executeEval(table, expr))).size) +Eval.size = unary('size', (expr, table) => Array.isArray(table) + ? table.map(data => executeAggr(expr, data)).length + : Array.from(executeEval(table, expr)).length) operators.$object = (field, table) => valueMap(field, value => executeAggr(value, table)) Eval.object = (fields) => { diff --git a/packages/mongo/src/utils.ts b/packages/mongo/src/utils.ts index 0bc50aa7..f190c0b2 100644 --- a/packages/mongo/src/utils.ts +++ b/packages/mongo/src/utils.ts @@ -72,7 +72,7 @@ function transformFieldQuery(query: Query.FieldQuery, key: string, filters: Filt return result } -const aggrKeys = ['$sum', '$avg', '$min', '$max', '$count', '$array'] +const aggrKeys = ['$sum', '$avg', '$min', '$max', '$count', '$size', '$array'] export class Transformer { private counter = 0 @@ -118,10 +118,6 @@ export class Transformer { return this.transformEvalExpr(expr.$object || expr.$array) } - if (expr.$count) { - return { $size: this.transformEvalExpr(expr.$count) } - } - return valueMap(expr as any, (value) => { if (Array.isArray(value)) { return value.map(val => this.eval(val, group)) @@ -157,6 +153,9 @@ export class Transformer { if (type === '$count') { group![key] = { $addToSet: value } return { $size: '$' + key } + } else if (type === '$size') { + group![key] = { $push: value } + return { $size: '$' + key } } else if (type === '$array') { group![key] = { $push: value } return '$' + key diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index 069352b1..ec692c43 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -116,7 +116,6 @@ class MySQLBuilder extends Builder { this.evalOperators.$avg = (expr) => this.createAggr(expr, value => `avg(${value})`, value => `mj_avg(${value})`) this.evalOperators.$min = (expr) => this.createAggr(expr, value => `(0+min(${value}))`, value => `(0+mj_min(${value}))`) this.evalOperators.$max = (expr) => this.createAggr(expr, value => `(0+max(${value}))`, value => `(0+mj_max(${value}))`) - this.evalOperators.$count = (expr) => this.createAggr(expr, value => `count(distinct ${value})`, value => `mj_count(${value})`) this.define({ types: ['list'], diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 6ad9c9e6..e26b0d14 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -83,7 +83,7 @@ export class Builder { $size: (key, value) => { if (!value) return this.logicalNot(key) if (this.state.sqlTypes?.[this.unescapeId(key)] === 'json') { - return `json_length(${key}) = ${this.escape(value)}` + return `${this.jsonLength(key)} = ${this.escape(value)}` } else { return `${key} AND LENGTH(${key}) - LENGTH(REPLACE(${key}, ${this.escape(',')}, ${this.escape('')})) = ${this.escape(value)} - 1` } @@ -124,6 +124,24 @@ export class Builder { $min: (expr) => this.createAggr(expr, value => `(0+min(${value}))`), $max: (expr) => this.createAggr(expr, value => `(0+max(${value}))`), $count: (expr) => this.createAggr(expr, value => `count(distinct ${value})`), + $size: (expr) => { + if (this.state.group) { + this.state.group = false + const aggr = (this.parseAggr(expr)) + this.state.group = true + this.state.sqlType = 'raw' + return `count(distinct ${aggr})` + } else { + const aggr = this.parseAggr(expr) + if (this.state.sqlType === 'json') { + this.state.sqlType = 'raw' + return `${this.jsonLength(aggr)}` + } else { + this.state.sqlType = 'raw' + return `LENGTH(${aggr}) - LENGTH(REPLACE(${aggr}, ${this.escape(',')}, ${this.escape('')}))` + } + } + }, $object: (fields) => this.groupObject(fields), $array: (expr) => this.groupArray(this.parseAggr(expr)), @@ -183,6 +201,10 @@ export class Builder { return `NOT(${condition})` } + protected jsonLength(value: string) { + return `json_length(${value})` + } + protected unquoteJson(value: string) { const res = this.state.sqlType === 'json' ? `json_unquote(${value})` : value this.state.sqlType = 'raw' @@ -373,6 +395,9 @@ export class Builder { if (filter !== '1') prefix += ` ON ${filter}` } + const filter = this.parseQuery(query) + if (filter === '0') return + this.state.group = !!args[0].group const sqlTypes: Dict = {} const fields = args[0].fields ?? Object.fromEntries(Object @@ -384,13 +409,11 @@ export class Builder { sqlTypes[key] = this.state.sqlType! return escapeId(key) === value ? escapeId(key) : `${value} AS ${escapeId(key)}` }).join(', ') - this.state.sqlTypes = sqlTypes - - const filter = this.parseQuery(query) - if (filter === '0') return // get suffix let suffix = this.suffix(args[0]) + this.state.sqlTypes = sqlTypes + if (filter !== '1') { suffix = ` WHERE ${filter}` + suffix } diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index 7caefab3..7b4497a0 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -52,14 +52,6 @@ class SQLiteBuilder extends Builder { this.evalOperators.$if = (args) => `iif(${args.map(arg => this.parseEval(arg)).join(', ')})` this.evalOperators.$concat = (args) => `(${args.map(arg => this.parseEval(arg)).join('||')})` - this.queryOperators.$size = (key, value) => { - if (!value) return this.logicalNot(key) - if (this.state.sqlTypes?.[this.unescapeId(key)] === 'json') { - return `json_array_length(${key}) = ${this.escape(value)}` - } else { - return `${key} AND LENGTH(${key}) - LENGTH(REPLACE(${key}, ${this.escape(',')}, ${this.escape('')})) = ${this.escape(value)} - 1` - } - } this.define({ types: ['boolean'], @@ -99,6 +91,10 @@ class SQLiteBuilder extends Builder { } } + protected jsonLength(value: string) { + return `json_array_length(${value})` + } + protected unquoteJson(value: string) { return value } diff --git a/packages/tests/src/json.ts b/packages/tests/src/json.ts index ed5ac3b1..5216f7ed 100644 --- a/packages/tests/src/json.ts +++ b/packages/tests/src/json.ts @@ -89,6 +89,15 @@ namespace JsonTests { { id: 1, nums: [4, 5, 6] }, { id: 2, nums: [5, 6, 7] }, ]) + + await expect(database.select('baz', { + nums: { $size: 3 }, + }).project({ + size: row => $.size(row.nums), + }).execute()).to.eventually.deep.equal([ + { size: 3 }, + { size: 3 }, + ]) }) it('$el', async () => { @@ -291,7 +300,7 @@ namespace JsonTests { avg: row => $.avg(row.y), min: row => $.min(row.y), max: row => $.max(row.y), - count: row => $.count(row.y), + count: row => $.size(row.y), }) .orderBy(row => row.count) .execute() diff --git a/packages/tests/src/selection.ts b/packages/tests/src/selection.ts index 3a771eb7..cff4de48 100644 --- a/packages/tests/src/selection.ts +++ b/packages/tests/src/selection.ts @@ -144,13 +144,14 @@ namespace SelectionTests { await expect(database.select('foo') .groupBy({}, { count: row => $.count(row.id), + size: row => $.size(row.id), max: row => $.max(row.id), min: row => $.min(row.id), avg: row => $.avg(row.id), }) .execute() ).to.eventually.deep.equal([ - { avg: 2, count: 3, max: 3, min: 1 }, + { avg: 2, count: 3, max: 3, min: 1, size: 3 }, ]) }) } From 15ff1902c7bb2aab2985788ba016a23a5d8d606d Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Wed, 1 Nov 2023 17:21:34 +0800 Subject: [PATCH 36/49] ci: add more mongo verisions --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0787be9d..0a634c10 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -55,7 +55,7 @@ jobs: strategy: fail-fast: false matrix: - mysql-image: + mongo-image: - mongo:6.0 - mongo:latest node-version: [16, 18, 20] From 7189e6f8863bda8210f4f14792c5fe65ae10f1ee Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Wed, 1 Nov 2023 21:18:44 +0800 Subject: [PATCH 37/49] chore: remove parseAggr, refa createAggr --- packages/mysql/src/index.ts | 22 ++++++++---------- packages/sql-utils/src/index.ts | 40 ++++++++++++--------------------- packages/sqlite/src/index.ts | 22 +++++++++++------- packages/tests/src/query.ts | 13 +++++++++++ 4 files changed, 50 insertions(+), 47 deletions(-) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index ec692c43..d37a9730 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -112,10 +112,10 @@ class MySQLBuilder extends Builder { constructor(tables?: Dict, private compat: Compat = {}) { super(tables) - this.evalOperators.$sum = (expr) => this.createAggr(expr, value => `ifnull(sum(${value}), 0)`, value => `ifnull(mj_sum(${value}), 0)`) - this.evalOperators.$avg = (expr) => this.createAggr(expr, value => `avg(${value})`, value => `mj_avg(${value})`) - this.evalOperators.$min = (expr) => this.createAggr(expr, value => `(0+min(${value}))`, value => `(0+mj_min(${value}))`) - this.evalOperators.$max = (expr) => this.createAggr(expr, value => `(0+max(${value}))`, value => `(0+mj_max(${value}))`) + this.evalOperators.$sum = (expr) => this.createAggr(expr, value => `ifnull(sum(${value}), 0)`, undefined, value => `ifnull(mj_sum(${value}), 0)`) + this.evalOperators.$avg = (expr) => this.createAggr(expr, value => `avg(${value})`, undefined, value => `mj_avg(${value})`) + this.evalOperators.$min = (expr) => this.createAggr(expr, value => `(0+min(${value}))`, undefined, value => `(0+mj_min(${value}))`) + this.evalOperators.$max = (expr) => this.createAggr(expr, value => `(0+max(${value}))`, undefined, value => `(0+mj_max(${value}))`) this.define({ types: ['list'], @@ -139,13 +139,13 @@ class MySQLBuilder extends Builder { return super.escape(value, field) } - protected createAggr(expr: any, aggrfunc: (value: string) => string, compatfunc?: (value: string) => string) { - if (!this.state.group && compatfunc && (this.compat.mysql57 || this.compat.maria105)) { - const aggr = compatfunc(this.parseAggr(expr)) + protected createAggr(expr: any, aggr: (value: string) => string, nonaggr?: (value: string) => string, compat?: (value: string) => string) { + if (!this.state.group && compat && (this.compat.mysql57 || this.compat.maria105)) { + const value = compat(this.parseEval(expr, false)) this.state.sqlType = 'raw' - return aggr + return value } else { - return super.createAggr(expr, aggrfunc) + return super.createAggr(expr, aggr, nonaggr) } } @@ -420,10 +420,6 @@ INSERT INTO mtt VALUES(json_extract(j, concat('$[', i, ']'))); SET i=i+1; END WH await this.query(`CREATE FUNCTION mj_max (j JSON) RETURNS DOUBLE DETERMINISTIC BEGIN DECLARE n int; DECLARE i int; DECLARE r DOUBLE; DROP TEMPORARY TABLE IF EXISTS mtt; CREATE TEMPORARY TABLE mtt (value JSON); SELECT json_length(j) into n; set i = 0; WHILE i(sql: string, debug = true): Promise { diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index e26b0d14..65402302 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -124,27 +124,18 @@ export class Builder { $min: (expr) => this.createAggr(expr, value => `(0+min(${value}))`), $max: (expr) => this.createAggr(expr, value => `(0+max(${value}))`), $count: (expr) => this.createAggr(expr, value => `count(distinct ${value})`), - $size: (expr) => { - if (this.state.group) { - this.state.group = false - const aggr = (this.parseAggr(expr)) - this.state.group = true + $size: (expr) => this.createAggr(expr, value => `count(${value})`, value => { + if (this.state.sqlType === 'json') { this.state.sqlType = 'raw' - return `count(distinct ${aggr})` + return `${this.jsonLength(value)}` } else { - const aggr = this.parseAggr(expr) - if (this.state.sqlType === 'json') { - this.state.sqlType = 'raw' - return `${this.jsonLength(aggr)}` - } else { - this.state.sqlType = 'raw' - return `LENGTH(${aggr}) - LENGTH(REPLACE(${aggr}, ${this.escape(',')}, ${this.escape('')}))` - } + this.state.sqlType = 'raw' + return `if(${value}, LENGTH(${value}) - LENGTH(REPLACE(${value}, ${this.escape(',')}, ${this.escape('')})) + 1, 0)` } - }, + }), $object: (fields) => this.groupObject(fields), - $array: (expr) => this.groupArray(this.parseAggr(expr)), + $array: (expr) => this.groupArray(this.parseEval(expr, false)), } } @@ -211,17 +202,19 @@ export class Builder { return res } - protected createAggr(expr: any, aggrfunc: (value: string) => string) { + protected createAggr(expr: any, aggr: (value: string) => string, nonaggr?: (value: string) => string) { if (this.state.group) { this.state.group = false - const aggr = aggrfunc(this.parseAggr(expr)) + const value = aggr(this.parseEval(expr, false)) this.state.group = true this.state.sqlType = 'raw' - return aggr + return value } else { - const aggr = this.parseAggr(expr) + const value = this.parseEval(expr, false) + const res = nonaggr ? nonaggr(value) + : `(select ${aggr(`json_unquote(${escapeId('value')})`)} from json_table(${value}, '$[*]' columns (value json path '$')) ${randomId()})` this.state.sqlType = 'raw' - return `(select ${aggrfunc(`json_unquote(${escapeId('value')})`)} from json_table(${aggr}, '$[*]' columns (value json path '$')) ${randomId()})` + return res } } @@ -294,11 +287,6 @@ export class Builder { return this.escape(expr) } - protected parseAggr(expr: any) { - this.state.sqlType = 'raw' - return typeof expr === 'string' ? this.getRecursive(expr) : this.parseEvalExpr(expr) - } - protected transformJsonField(obj: string, path: string) { this.state.sqlType = 'json' return `json_extract(${obj}, '$${path}')` diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index 7b4497a0..d83fe621 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -52,6 +52,15 @@ class SQLiteBuilder extends Builder { this.evalOperators.$if = (args) => `iif(${args.map(arg => this.parseEval(arg)).join(', ')})` this.evalOperators.$concat = (args) => `(${args.map(arg => this.parseEval(arg)).join('||')})` + this.evalOperators.$size = (expr) => this.createAggr(expr, value => `count(${value})`, value => { + if (this.state.sqlType === 'json') { + this.state.sqlType = 'raw' + return `${this.jsonLength(value)}` + } else { + this.state.sqlType = 'raw' + return `iif(${value}, LENGTH(${value}) - LENGTH(REPLACE(${value}, ${this.escape(',')}, ${this.escape('')})) + 1, 0)` + } + }) this.define({ types: ['boolean'], @@ -99,15 +108,12 @@ class SQLiteBuilder extends Builder { return value } - protected createAggr(expr: any, aggrfunc: (value: string) => string) { - if (this.state.group) { - this.state.group = false - const ret = aggrfunc(this.parseAggr(expr)) - this.state.group = true - return ret + protected createAggr(expr: any, aggr: (value: string) => string, nonaggr?: (value: string) => string) { + if (!this.state.group && !nonaggr) { + const value = this.parseEval(expr, false) + return `(select ${aggr(escapeId('value'))} from json_each(${value}) ${randomId()})` } else { - const aggr = this.parseAggr(expr) - return `(select ${aggrfunc(escapeId('value'))} from json_each(${aggr}) ${randomId()})` + return super.createAggr(expr, aggr, nonaggr) } } diff --git a/packages/tests/src/query.ts b/packages/tests/src/query.ts index b695a918..eee5181a 100644 --- a/packages/tests/src/query.ts +++ b/packages/tests/src/query.ts @@ -271,6 +271,19 @@ namespace QueryOperators { })).eventually.to.have.length(2).with.shape([{ id: 2 }, { id: 3 }]) }) + size && it('$.size', async () => { + await expect(database.select('temp1') + .project({ x: row => $.size(row.list) }) + .orderBy(row => row.x) + .execute() + ).eventually.to.deep.equal([ + { x: 0 }, + { x: 1 }, + { x: 1 }, + { x: 2 }, + ]) + }) + element && it('$el shorthand', async () => { await expect(database.get('temp1', { list: { $el: 233 }, From 3c9cddf94ac431414acd01cbde48be64a4ef70a6 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Wed, 1 Nov 2023 21:26:45 +0800 Subject: [PATCH 38/49] ci: rename --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0a634c10..18467519 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -50,6 +50,7 @@ jobs: name: codecov mongo: + name: ${{ matrix.mongo-image }} (${{ matrix.node-version }}) runs-on: ubuntu-latest strategy: From 4cf938007858af58acee70f026d5c57b54db1a41 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 2 Nov 2023 01:11:33 +0800 Subject: [PATCH 39/49] stage: test $.in --- packages/mongo/src/utils.ts | 4 ++++ packages/sql-utils/src/index.ts | 26 ++++++++++++++++++++------ packages/sqlite/src/index.ts | 6 +++++- packages/tests/src/json.ts | 17 ++++++++++++++++- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/packages/mongo/src/utils.ts b/packages/mongo/src/utils.ts index f190c0b2..f9989ce8 100644 --- a/packages/mongo/src/utils.ts +++ b/packages/mongo/src/utils.ts @@ -118,6 +118,10 @@ export class Transformer { return this.transformEvalExpr(expr.$object || expr.$array) } + if (expr.$nin) { + return { $not: { $in: expr.$nin.map(val => this.eval(val, group)) } } + } + return valueMap(expr as any, (value) => { if (Array.isArray(value)) { return value.map(val => this.eval(val, group)) diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 65402302..c0e87987 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -118,6 +118,10 @@ export class Builder { $lt: this.binary('<'), $lte: this.binary('<='), + // membership + $in: ([key, value]) => this.createMemberQuery(this.parseEval(key), value, ''), + $nin: ([key, value]) => this.createMemberQuery(this.parseEval(key), value, ' NOT'), + // aggregation $sum: (expr) => this.createAggr(expr, value => `ifnull(sum(${value}), 0)`), $avg: (expr) => this.createAggr(expr, value => `avg(${value})`), @@ -147,9 +151,15 @@ export class Builder { return `${key} is ${value ? 'not ' : ''}null` } - protected createMemberQuery(key: string, value: any[], notStr = '') { - if (!value.length) return notStr ? '1' : '0' - return `${key}${notStr} in (${value.map(val => this.escape(val)).join(', ')})` + protected createMemberQuery(key: string, value: any, notStr = '') { + if (Array.isArray(value)) { + if (!value.length) return notStr ? '1' : '0' + return `${key}${notStr} in (${value.map(val => this.escape(val)).join(', ')})` + } else { + const res = this.jsonContains(this.parseEval(value, false), `json_extract(json_object('v', ${key}), '$.v')`) + this.state.sqlType = 'raw' + return notStr ? this.logicalNot(res) : res + } } protected createRegExpQuery(key: string, value: string | RegExp) { @@ -158,7 +168,7 @@ export class Builder { protected createElementQuery(key: string, value: any) { if (this.state.sqlTypes?.[this.unescapeId(key)] === 'json') { - return `json_contains(${key}, ${this.quote(JSON.stringify(value))})` + return this.jsonContains(key, this.quote(JSON.stringify(value))) } else { return `find_in_set(${this.escape(value)}, ${key})` } @@ -196,7 +206,11 @@ export class Builder { return `json_length(${value})` } - protected unquoteJson(value: string) { + protected jsonContains(obj: string, value: string) { + return `json_contains(${obj}, ${value})` + } + + protected jsonUnquote(value: string) { const res = this.state.sqlType === 'json' ? `json_unquote(${value})` : value this.state.sqlType = 'raw' return res @@ -331,7 +345,7 @@ export class Builder { if (typeof expr === 'string' || typeof expr === 'number' || typeof expr === 'boolean' || expr instanceof Date) { return this.escape(expr) } - return unquote ? this.unquoteJson(this.parseEvalExpr(expr)) : this.parseEvalExpr(expr) + return unquote ? this.jsonUnquote(this.parseEvalExpr(expr)) : this.parseEvalExpr(expr) } suffix(modifier: Modifier) { diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index d83fe621..bcaee0ff 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -104,7 +104,11 @@ class SQLiteBuilder extends Builder { return `json_array_length(${value})` } - protected unquoteJson(value: string) { + protected jsonContains(obj: string, value: string) { + return `json_array_contains(${obj}, ${value})` + } + + protected jsonUnquote(value: string) { return value } diff --git a/packages/tests/src/json.ts b/packages/tests/src/json.ts index 5216f7ed..8b13da9e 100644 --- a/packages/tests/src/json.ts +++ b/packages/tests/src/json.ts @@ -75,7 +75,7 @@ function JsonTests(database: Database) { await setup(database, 'baz', [ { id: 1, nums: [4, 5, 6] }, { id: 2, nums: [5, 6, 7] }, - { id: 3, nums: [6, 8] }, + { id: 3, nums: [7, 8] }, ]) }) } @@ -108,6 +108,21 @@ namespace JsonTests { { id: 2, nums: [5, 6, 7] }, ]) }) + + it('$in', async () => { + await expect(database.get('baz', row => $.in($.add(3, row.id), row.nums))) + .to.eventually.deep.equal([ + { id: 1, nums: [4, 5, 6] }, + { id: 2, nums: [5, 6, 7] }, + ]) + }) + + it('$nin', async () => { + await expect(database.get('baz', row => $.nin($.add(3, row.id), row.nums))) + .to.eventually.deep.equal([ + { id: 3, nums: [7, 8] }, + ]) + }) } export function selection(database: Database) { From 50c080618fe360892c151ee9ba45ba327df6fb47 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 2 Nov 2023 18:17:05 +0800 Subject: [PATCH 40/49] fix: compat aggr for all maria versions --- packages/mysql/src/index.ts | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index d37a9730..6c8924bb 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -69,6 +69,7 @@ function createIndex(keys: string | string[]) { } interface Compat { + maria?: boolean maria105?: boolean mysql57?: boolean } @@ -112,10 +113,10 @@ class MySQLBuilder extends Builder { constructor(tables?: Dict, private compat: Compat = {}) { super(tables) - this.evalOperators.$sum = (expr) => this.createAggr(expr, value => `ifnull(sum(${value}), 0)`, undefined, value => `ifnull(mj_sum(${value}), 0)`) - this.evalOperators.$avg = (expr) => this.createAggr(expr, value => `avg(${value})`, undefined, value => `mj_avg(${value})`) - this.evalOperators.$min = (expr) => this.createAggr(expr, value => `(0+min(${value}))`, undefined, value => `(0+mj_min(${value}))`) - this.evalOperators.$max = (expr) => this.createAggr(expr, value => `(0+max(${value}))`, undefined, value => `(0+mj_max(${value}))`) + this.evalOperators.$sum = (expr) => this.createAggr(expr, value => `ifnull(sum(${value}), 0)`, undefined, value => `ifnull(minato_cfunc_sum(${value}), 0)`) + this.evalOperators.$avg = (expr) => this.createAggr(expr, value => `avg(${value})`, undefined, value => `minato_cfunc_avg(${value})`) + this.evalOperators.$min = (expr) => this.createAggr(expr, value => `(0+min(${value}))`, undefined, value => `(0+minato_cfunc_min(${value}))`) + this.evalOperators.$max = (expr) => this.createAggr(expr, value => `(0+max(${value}))`, undefined, value => `(0+minato_cfunc_max(${value}))`) this.define({ types: ['list'], @@ -140,7 +141,7 @@ class MySQLBuilder extends Builder { } protected createAggr(expr: any, aggr: (value: string) => string, nonaggr?: (value: string) => string, compat?: (value: string) => string) { - if (!this.state.group && compat && (this.compat.mysql57 || this.compat.maria105)) { + if (!this.state.group && compat && (this.compat.mysql57 || this.compat.maria)) { const value = compat(this.parseEval(expr, false)) this.state.sqlType = 'raw' return value @@ -267,12 +268,14 @@ export class MySQLDriver extends Driver { /** synchronize table schema */ async prepare(name: string) { const version = Object.values((await this.query(`SELECT version()`))[0])[0] as string + // https://jira.mariadb.org/browse/MDEV-30623 + this._compat.maria = version.includes('MariaDB') // https://jira.mariadb.org/browse/MDEV-26506 this._compat.maria105 = !!version.match(/10.5.\d+-MariaDB/) // For json_table this._compat.mysql57 = !!version.match(/5.7.\d+/) - if (this._compat.mysql57 || this._compat.maria105) { + if (this._compat.mysql57 || this._compat.maria) { await this._setupCompatFunctions() } @@ -404,20 +407,20 @@ export class MySQLDriver extends Driver { async _setupCompatFunctions() { const log = () => logger.debug('Failed to setup compact functions') - await this.query(`DROP FUNCTION IF EXISTS mj_sum`).catch(log) - await this.query(`CREATE FUNCTION mj_sum (j JSON) RETURNS DOUBLE DETERMINISTIC BEGIN DECLARE n int; DECLARE i int; DECLARE r DOUBLE; + await this.query(`DROP FUNCTION IF EXISTS minato_cfunc_sum`).catch(log) + await this.query(`CREATE FUNCTION minato_cfunc_sum (j JSON) RETURNS DOUBLE DETERMINISTIC BEGIN DECLARE n int; DECLARE i int; DECLARE r DOUBLE; DROP TEMPORARY TABLE IF EXISTS mtt; CREATE TEMPORARY TABLE mtt (value JSON); SELECT json_length(j) into n; set i = 0; WHILE i Date: Fri, 3 Nov 2023 02:08:12 +0800 Subject: [PATCH 41/49] fix: check null for $.array --- packages/mysql/src/index.ts | 8 ++++---- packages/sql-utils/src/index.ts | 2 +- packages/sqlite/src/index.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index 6c8924bb..dae7edc8 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -151,11 +151,11 @@ class MySQLBuilder extends Builder { } protected groupArray(value: string) { - const res = this.compat.maria105 ? (this.state.sqlType === 'json' ? `concat('[', group_concat(${value}), ']')` - : `concat('[', group_concat(json_extract(json_object('v', ${value}), '$.v')), ']')`) - : super.groupArray(value) + if (!this.compat.maria105) return super.groupArray(value) + const res = this.state.sqlType === 'json' ? `concat('[', group_concat(${value}), ']')` + : `concat('[', group_concat(json_extract(json_object('v', ${value}), '$.v')), ']')` this.state.sqlType = 'json' - return res + return `ifnull(${res}, json_array())` } toUpdateExpr(item: any, key: string, field?: Field, upsert?: boolean) { diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index c0e87987..0a360cc7 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -244,7 +244,7 @@ export class Builder { protected groupArray(value: string) { this.state.sqlType = 'json' - return `json_arrayagg(${value})` + return `ifnull(json_arrayagg(${value}), json_array())` } protected parseFieldQuery(key: string, query: Query.FieldExpr) { diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index bcaee0ff..71a72176 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -124,7 +124,7 @@ class SQLiteBuilder extends Builder { protected groupArray(value: string) { const res = this.state.sqlType === 'json' ? `('[' || group_concat(${value}) || ']')` : `('[' || group_concat(json_quote(${value})) || ']')` this.state.sqlType = 'json' - return res + return `ifnull(${res}, json_array())` } protected transformJsonField(obj: string, path: string) { From d52fe16650d2d4201b0ba525079602738e7c62c9 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Fri, 3 Nov 2023 11:15:08 +0800 Subject: [PATCH 42/49] test: add empty --- packages/tests/src/json.ts | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/packages/tests/src/json.ts b/packages/tests/src/json.ts index 8b13da9e..e37b988b 100644 --- a/packages/tests/src/json.ts +++ b/packages/tests/src/json.ts @@ -100,6 +100,27 @@ namespace JsonTests { ]) }) + it('$.size on empty', async () => { + await expect(database.select('foo', { id: -1 }) + .groupBy({}, { + y: row => $.array(row.id), + }) + .execute() + ).eventually.to.deep.equal([ + { y: [] }, + ]) + + await expect(database.select('foo', { id: -1 }) + .groupBy({}, { + y: row => $.array(row.id), + }) + .orderBy(row => $.size(row.y)) + .execute() + ).eventually.to.deep.equal([ + { y: [] }, + ]) + }) + it('$el', async () => { await expect(database.get('baz', { nums: { $el: 5 }, @@ -114,14 +135,14 @@ namespace JsonTests { .to.eventually.deep.equal([ { id: 1, nums: [4, 5, 6] }, { id: 2, nums: [5, 6, 7] }, - ]) + ]) }) it('$nin', async () => { await expect(database.get('baz', row => $.nin($.add(3, row.id), row.nums))) .to.eventually.deep.equal([ { id: 3, nums: [7, 8] }, - ]) + ]) }) } @@ -171,7 +192,7 @@ namespace JsonTests { }) .execute(['x']) - expect(res).to.have.deep.members([ + expect(res).to.have.deep.members([ { x: [{ id: 1, value: 0 }] }, { x: [{ id: 1, value: 0 }] }, { x: [{ id: 2, value: 2 }] }, From 8c59e3c474c63cccaba08039aaf76cbcf68543f9 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Fri, 3 Nov 2023 12:28:46 +0800 Subject: [PATCH 43/49] revert: mongo related issue: kill --- packages/tests/src/json.ts | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/packages/tests/src/json.ts b/packages/tests/src/json.ts index e37b988b..bb0eb300 100644 --- a/packages/tests/src/json.ts +++ b/packages/tests/src/json.ts @@ -100,27 +100,6 @@ namespace JsonTests { ]) }) - it('$.size on empty', async () => { - await expect(database.select('foo', { id: -1 }) - .groupBy({}, { - y: row => $.array(row.id), - }) - .execute() - ).eventually.to.deep.equal([ - { y: [] }, - ]) - - await expect(database.select('foo', { id: -1 }) - .groupBy({}, { - y: row => $.array(row.id), - }) - .orderBy(row => $.size(row.y)) - .execute() - ).eventually.to.deep.equal([ - { y: [] }, - ]) - }) - it('$el', async () => { await expect(database.get('baz', { nums: { $el: 5 }, From eb65cd58818e967ad757fef6abe31c20f411bb82 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Fri, 3 Nov 2023 21:34:48 +0800 Subject: [PATCH 44/49] chore: sync --- packages/mysql/src/index.ts | 7 +++++++ packages/sql-utils/src/index.ts | 14 +++++++++++--- packages/sqlite/src/index.ts | 4 ++-- packages/tests/src/json.ts | 20 ++++++++++---------- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index dae7edc8..b22ee711 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -140,6 +140,13 @@ class MySQLBuilder extends Builder { return super.escape(value, field) } + protected jsonQuote(value: string, pure: boolean = false) { + if (pure) return this.compat.maria ? `json_extract(json_object('v', ${value}), '$.v')` : `cast(${value} as json)` + const res = this.state.sqlType === 'raw' ? (this.compat.maria ? `json_extract(json_object('v', ${value}), '$.v')` : `cast(${value} as json)`) : value + this.state.sqlType = 'json' + return res + } + protected createAggr(expr: any, aggr: (value: string) => string, nonaggr?: (value: string) => string, compat?: (value: string) => string) { if (!this.state.group && compat && (this.compat.mysql57 || this.compat.maria)) { const value = compat(this.parseEval(expr, false)) diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 0a360cc7..55f7a777 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -36,7 +36,7 @@ export class Builder { protected createEqualQuery = this.comparator('=') protected queryOperators: QueryOperators protected evalOperators: EvalOperators - state: State = {} + public state: State = {} constructor(public tables?: Dict) { this.queryOperators = { @@ -156,7 +156,7 @@ export class Builder { if (!value.length) return notStr ? '1' : '0' return `${key}${notStr} in (${value.map(val => this.escape(val)).join(', ')})` } else { - const res = this.jsonContains(this.parseEval(value, false), `json_extract(json_object('v', ${key}), '$.v')`) + const res = this.jsonContains(this.parseEval(value, false), this.jsonQuote(key, true)) this.state.sqlType = 'raw' return notStr ? this.logicalNot(res) : res } @@ -210,12 +210,20 @@ export class Builder { return `json_contains(${obj}, ${value})` } - protected jsonUnquote(value: string) { + protected jsonUnquote(value: string, pure: boolean = false) { + if (pure) return `json_unquote(${value})` const res = this.state.sqlType === 'json' ? `json_unquote(${value})` : value this.state.sqlType = 'raw' return res } + protected jsonQuote(value: string, pure: boolean = false) { + if (pure) return `cast(${value} as json)` + const res = this.state.sqlType === 'raw' ? `cast(${value} as json)` : value + this.state.sqlType = 'json' + return res + } + protected createAggr(expr: any, aggr: (value: string) => string, nonaggr?: (value: string) => string) { if (this.state.group) { this.state.group = false diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index 71a72176..c9e79d03 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -94,7 +94,7 @@ class SQLiteBuilder extends Builder { protected createElementQuery(key: string, value: any) { if (this.state.sqlTypes?.[this.unescapeId(key)] === 'json') { - return `json_array_contains(${key}, ${this.quote(JSON.stringify(value))})` + return this.jsonContains(key, this.quote(JSON.stringify(value))) } else { return `(',' || ${key} || ',') LIKE ${this.escape('%,' + value + ',%')}` } @@ -108,7 +108,7 @@ class SQLiteBuilder extends Builder { return `json_array_contains(${obj}, ${value})` } - protected jsonUnquote(value: string) { + protected jsonUnquote(value: string, pure: boolean = false) { return value } diff --git a/packages/tests/src/json.ts b/packages/tests/src/json.ts index bb0eb300..a824bbb1 100644 --- a/packages/tests/src/json.ts +++ b/packages/tests/src/json.ts @@ -140,7 +140,7 @@ namespace JsonTests { expect(res).to.deep.equal([ { obj: { id: 1, value: 0 } }, { obj: { id: 2, value: 2 } }, - { obj: { id: 3, value: 2 } } + { obj: { id: 3, value: 2 } }, ]) }) @@ -153,21 +153,21 @@ namespace JsonTests { str2: row.obj.z, obj: row.obj.o, a: row.obj.o.a, - }) + }), }) .execute() expect(res).to.deep.equal([ { obj: { a: 1, num: 1, obj: { a: 1, b: '1' }, str: 'a', str2: '1' } }, { obj: { a: 2, num: 2, obj: { a: 2, b: '2' }, str: 'b', str2: '2' } }, - { obj: { a: 3, num: 3, obj: { a: 3, b: '3' }, str: 'c', str2: '3' } } + { obj: { a: 3, num: 3, obj: { a: 3, b: '3' }, str: 'c', str2: '3' } }, ]) }) it('$.object on cell', async () => { const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) .groupBy('bar', { - x: row => $.array($.object(row.foo)) + x: row => $.array($.object(row.foo)), }) .execute(['x']) @@ -189,7 +189,7 @@ namespace JsonTests { expect(res).to.deep.equal([ { foo: { id: 1, value: 0 }, x: [1, 2], y: ['a', 'b'] }, - { foo: { id: 2, value: 2 }, x: [3], y: ['c'] } + { foo: { id: 2, value: 2 }, x: [3], y: ['c'] }, ]) }) @@ -208,8 +208,8 @@ namespace JsonTests { count2: ['1', '2', '3'], countnumber: [0, 1, 0], x: [1, 2, 3], - y: ['a', 'b', 'c'] - } + y: ['a', 'b', 'c'], + }, ]) }) @@ -253,7 +253,7 @@ namespace JsonTests { y: ['c'], z: ['3'], o: [{ a: 3, b: '3' }], - } + }, ]) }) @@ -282,7 +282,7 @@ namespace JsonTests { bars: [{ value: 0, value2: 2 }], x: [4], y: ['c'], - } + }, ]) }) @@ -341,7 +341,7 @@ namespace JsonTests { .execute() expect(res).to.deep.equal([ - { sum: 3, avg: 2.25, min: 1, max: 3 } + { sum: 3, avg: 2.25, min: 1, max: 3 }, ]) }) From 441a301cf412a79f6b676751149d59e8de240675 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Sun, 5 Nov 2023 01:07:12 +0800 Subject: [PATCH 45/49] feat: support driver.eval --- packages/mysql/src/index.ts | 10 +++++----- packages/sql-utils/src/index.ts | 13 ++++++++++--- packages/sqlite/src/index.ts | 6 +++--- packages/tests/src/json.ts | 26 +++++++++++++++++++++++--- 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index b22ee711..5e206fbf 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -513,11 +513,11 @@ INSERT INTO mtt VALUES(json_extract(j, concat('$[', i, ']'))); SET i=i+1; END WH async eval(sel: Selection.Immutable, expr: Eval.Expr) { const builder = new MySQLBuilder(sel.tables, this._compat) - builder.state.group = true - const output = builder.parseEval(expr) - const inner = builder.get(sel.table as Selection, true) - const [data] = await this.queue(`SELECT ${output} AS value FROM ${inner}`) - return data.value + const inner = builder.get(sel.table as Selection, true, true) + const output = builder.parseEval(expr, false) + const ref = inner.startsWith('(') && inner.endsWith(')') ? sel.ref : '' + const [data] = await this.queue(`SELECT ${output} AS value FROM ${inner} ${ref}`) + return builder.load(data.value) } async set(sel: Selection.Mutable, data: {}) { diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 55f7a777..051f3a5a 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -374,7 +374,7 @@ export class Builder { return sql } - get(sel: Selection.Immutable, inline = false) { + get(sel: Selection.Immutable, inline = false, group = false) { const { args, table, query, ref, model } = sel // get prefix @@ -408,7 +408,7 @@ export class Builder { const filter = this.parseQuery(query) if (filter === '0') return - this.state.group = !!args[0].group + this.state.group = group || !!args[0].group const sqlTypes: Dict = {} const fields = args[0].fields ?? Object.fromEntries(Object .entries(model.fields) @@ -453,7 +453,14 @@ export class Builder { return result } - load(model: Model, obj: any): any { + load(obj: any): any + load(model: Model, obj: any): any + load(model: any, obj?: any) { + if (!obj) { + const converter = this.types[this.state.sqlType!] + return converter ? converter.load(model) : model + } + const result = {} for (const key in obj) { if (!(key in model.fields)) continue diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index c9e79d03..8399f734 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -364,10 +364,10 @@ export class SQLiteDriver extends Driver { async eval(sel: Selection.Immutable, expr: Eval.Expr) { const builder = new SQLiteBuilder(sel.tables) builder.state.group = true - const output = builder.parseEval(expr) - const inner = builder.get(sel.table as Selection, true) + const inner = builder.get(sel.table as Selection, true, true) + const output = builder.parseEval(expr, false) const { value } = this.#get(`SELECT ${output} AS value FROM ${inner}`) - return value + return builder.load(value) } #update(sel: Selection.Mutable, indexFields: string[], updateFields: string[], update: {}, data: {}) { diff --git a/packages/tests/src/json.ts b/packages/tests/src/json.ts index a824bbb1..54bb99ac 100644 --- a/packages/tests/src/json.ts +++ b/packages/tests/src/json.ts @@ -179,18 +179,38 @@ namespace JsonTests { }) it('$.array groupBy', async () => { - const res = await database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) + await expect(database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) .groupBy(['foo'], { x: row => $.array(row.bar.obj.x), y: row => $.array(row.bar.obj.y), }) .orderBy(row => row.foo.id) .execute() - - expect(res).to.deep.equal([ + ).to.eventually.deep.equal([ { foo: { id: 1, value: 0 }, x: [1, 2], y: ['a', 'b'] }, { foo: { id: 2, value: 2 }, x: [3], y: ['c'] }, ]) + + await expect(database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) + .groupBy(['foo'], { + x: row => $.array(row.bar.obj.x), + y: row => $.array(row.bar.obj.y), + }) + .orderBy(row => row.foo.id) + .execute(row => $.array(row.y)) + ).to.eventually.deep.equal([ + ['a', 'b'], + ['c'], + ]) + + await expect(database.join(['foo', 'bar'] as const, (foo, bar) => $.eq(foo.id, bar.pid)) + .groupBy(['foo'], { + x: row => $.array(row.bar.obj.x), + y: row => $.array(row.bar.obj.y), + }) + .orderBy(row => row.foo.id) + .execute(row => $.count(row.y)) + ).to.eventually.deep.equal(2) }) it('$.array groupFull', async () => { From aac4efc829b7ba5a51e5bb71d788d2ce9038f9cc Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Sun, 5 Nov 2023 01:15:39 +0800 Subject: [PATCH 46/49] chore(sql-utils): protected state --- packages/sql-utils/src/index.ts | 2 +- packages/sqlite/src/index.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 051f3a5a..16025cd3 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -36,7 +36,7 @@ export class Builder { protected createEqualQuery = this.comparator('=') protected queryOperators: QueryOperators protected evalOperators: EvalOperators - public state: State = {} + protected state: State = {} constructor(public tables?: Dict) { this.queryOperators = { diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index 8399f734..670c2d5d 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -363,7 +363,6 @@ export class SQLiteDriver extends Driver { async eval(sel: Selection.Immutable, expr: Eval.Expr) { const builder = new SQLiteBuilder(sel.tables) - builder.state.group = true const inner = builder.get(sel.table as Selection, true, true) const output = builder.parseEval(expr, false) const { value } = this.#get(`SELECT ${output} AS value FROM ${inner}`) From c2f2ff951365b1ecf5710a0f41b079b771e7fcca Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Sun, 5 Nov 2023 14:04:21 +0800 Subject: [PATCH 47/49] refa: improve setup compat functions --- packages/mysql/src/index.ts | 43 ++++++++++++++++++--------------- packages/tests/src/selection.ts | 4 +++ 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/packages/mysql/src/index.ts b/packages/mysql/src/index.ts index 5e206fbf..129f9d62 100644 --- a/packages/mysql/src/index.ts +++ b/packages/mysql/src/index.ts @@ -266,14 +266,7 @@ export class MySQLDriver extends Driver { async start() { this.pool = createPool(this.config) - } - - async stop() { - this.pool.end() - } - /** synchronize table schema */ - async prepare(name: string) { const version = Object.values((await this.query(`SELECT version()`))[0])[0] as string // https://jira.mariadb.org/browse/MDEV-30623 this._compat.maria = version.includes('MariaDB') @@ -285,7 +278,14 @@ export class MySQLDriver extends Driver { if (this._compat.mysql57 || this._compat.maria) { await this._setupCompatFunctions() } + } + async stop() { + this.pool.end() + } + + /** synchronize table schema */ + async prepare(name: string) { const [columns, indexes] = await Promise.all([ this.queue([ `SELECT *`, @@ -413,23 +413,26 @@ export class MySQLDriver extends Driver { } async _setupCompatFunctions() { - const log = () => logger.debug('Failed to setup compact functions') - await this.query(`DROP FUNCTION IF EXISTS minato_cfunc_sum`).catch(log) - await this.query(`CREATE FUNCTION minato_cfunc_sum (j JSON) RETURNS DOUBLE DETERMINISTIC BEGIN DECLARE n int; DECLARE i int; DECLARE r DOUBLE; + try { + await this.query(`DROP FUNCTION IF EXISTS minato_cfunc_sum`) + await this.query(`CREATE FUNCTION minato_cfunc_sum (j JSON) RETURNS DOUBLE DETERMINISTIC BEGIN DECLARE n int; DECLARE i int; DECLARE r DOUBLE; DROP TEMPORARY TABLE IF EXISTS mtt; CREATE TEMPORARY TABLE mtt (value JSON); SELECT json_length(j) into n; set i = 0; WHILE i(sql: string, debug = true): Promise { diff --git a/packages/tests/src/selection.ts b/packages/tests/src/selection.ts index cff4de48..9ab2071f 100644 --- a/packages/tests/src/selection.ts +++ b/packages/tests/src/selection.ts @@ -107,6 +107,10 @@ namespace SelectionTests { { id: 2 }, { id: 3 }, ]) + + await expect(database.select('foo', row => $.eq(row.id, 1)).orderBy('id').execute(['id'])).to.eventually.deep.equal([ + { id: 1 }, + ]) }) it('callback', async () => { From bfa7e22c257d564b22b37bdbc4cb832b29d7fa0f Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 9 Nov 2023 16:50:04 +0800 Subject: [PATCH 48/49] refa: rename $.size to $.length --- packages/core/src/eval.ts | 12 +++++------- packages/mongo/src/utils.ts | 8 ++++++-- packages/sql-utils/src/index.ts | 2 +- packages/sqlite/src/index.ts | 2 +- packages/tests/src/json.ts | 4 ++-- packages/tests/src/query.ts | 4 ++-- packages/tests/src/selection.ts | 2 +- 7 files changed, 18 insertions(+), 16 deletions(-) diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index 6e1a5b11..f05d5c38 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -93,14 +93,14 @@ export namespace Eval { max(value: Number): Expr min(value: Number): Expr count(value: Any): Expr - size(value: Any): Expr + length(value: Any): Expr // json sum(value: (Number | Expr)[] | Expr): Expr avg(value: (Number | Expr)[] | Expr): Expr max(value: (Number | Expr)[] | Expr): Expr min(value: (Number | Expr)[] | Expr): Expr - size(value: (Any | Expr)[] | Expr): Expr + length(value: (Any | Expr)[] | Expr): Expr object>(fields: T): Expr object(row: Row.Cell): Expr @@ -193,12 +193,10 @@ Eval.max = unary('max', (expr, table) => Array.isArray(table) Eval.min = unary('min', (expr, table) => Array.isArray(table) ? Math.min(...table.map(data => executeAggr(expr, data))) : Math.min(...Array.from(executeEval(table, expr)))) -Eval.count = unary('count', (expr, table) => Array.isArray(table) - ? new Set(table.map(data => executeAggr(expr, data))).size - : new Set(Array.from(executeEval(table, expr))).size) -Eval.size = unary('size', (expr, table) => Array.isArray(table) +Eval.count = unary('count', (expr, table) => new Set(table.map(data => executeAggr(expr, data))).size) +defineProperty(Eval, 'length', unary('length', (expr, table) => Array.isArray(table) ? table.map(data => executeAggr(expr, data)).length - : Array.from(executeEval(table, expr)).length) + : Array.from(executeEval(table, expr)).length)) operators.$object = (field, table) => valueMap(field, value => executeAggr(value, table)) Eval.object = (fields) => { diff --git a/packages/mongo/src/utils.ts b/packages/mongo/src/utils.ts index f9989ce8..2822c4ed 100644 --- a/packages/mongo/src/utils.ts +++ b/packages/mongo/src/utils.ts @@ -72,7 +72,7 @@ function transformFieldQuery(query: Query.FieldQuery, key: string, filters: Filt return result } -const aggrKeys = ['$sum', '$avg', '$min', '$max', '$count', '$size', '$array'] +const aggrKeys = ['$sum', '$avg', '$min', '$max', '$count', '$length', '$array'] export class Transformer { private counter = 0 @@ -118,6 +118,10 @@ export class Transformer { return this.transformEvalExpr(expr.$object || expr.$array) } + if (expr.$length) { + return { $size: this.eval(expr.$length) } + } + if (expr.$nin) { return { $not: { $in: expr.$nin.map(val => this.eval(val, group)) } } } @@ -157,7 +161,7 @@ export class Transformer { if (type === '$count') { group![key] = { $addToSet: value } return { $size: '$' + key } - } else if (type === '$size') { + } else if (type === '$length') { group![key] = { $push: value } return { $size: '$' + key } } else if (type === '$array') { diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index 16025cd3..e2cfbd00 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -128,7 +128,7 @@ export class Builder { $min: (expr) => this.createAggr(expr, value => `(0+min(${value}))`), $max: (expr) => this.createAggr(expr, value => `(0+max(${value}))`), $count: (expr) => this.createAggr(expr, value => `count(distinct ${value})`), - $size: (expr) => this.createAggr(expr, value => `count(${value})`, value => { + $length: (expr) => this.createAggr(expr, value => `count(${value})`, value => { if (this.state.sqlType === 'json') { this.state.sqlType = 'raw' return `${this.jsonLength(value)}` diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index 670c2d5d..a6c81fb0 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -52,7 +52,7 @@ class SQLiteBuilder extends Builder { this.evalOperators.$if = (args) => `iif(${args.map(arg => this.parseEval(arg)).join(', ')})` this.evalOperators.$concat = (args) => `(${args.map(arg => this.parseEval(arg)).join('||')})` - this.evalOperators.$size = (expr) => this.createAggr(expr, value => `count(${value})`, value => { + this.evalOperators.$length = (expr) => this.createAggr(expr, value => `count(${value})`, value => { if (this.state.sqlType === 'json') { this.state.sqlType = 'raw' return `${this.jsonLength(value)}` diff --git a/packages/tests/src/json.ts b/packages/tests/src/json.ts index 54bb99ac..01e848f9 100644 --- a/packages/tests/src/json.ts +++ b/packages/tests/src/json.ts @@ -93,7 +93,7 @@ namespace JsonTests { await expect(database.select('baz', { nums: { $size: 3 }, }).project({ - size: row => $.size(row.nums), + size: row => $.length(row.nums), }).execute()).to.eventually.deep.equal([ { size: 3 }, { size: 3 }, @@ -335,7 +335,7 @@ namespace JsonTests { avg: row => $.avg(row.y), min: row => $.min(row.y), max: row => $.max(row.y), - count: row => $.size(row.y), + count: row => $.length(row.y), }) .orderBy(row => row.count) .execute() diff --git a/packages/tests/src/query.ts b/packages/tests/src/query.ts index eee5181a..b96e2b3f 100644 --- a/packages/tests/src/query.ts +++ b/packages/tests/src/query.ts @@ -271,9 +271,9 @@ namespace QueryOperators { })).eventually.to.have.length(2).with.shape([{ id: 2 }, { id: 3 }]) }) - size && it('$.size', async () => { + size && it('$.length', async () => { await expect(database.select('temp1') - .project({ x: row => $.size(row.list) }) + .project({ x: row => $.length(row.list) }) .orderBy(row => row.x) .execute() ).eventually.to.deep.equal([ diff --git a/packages/tests/src/selection.ts b/packages/tests/src/selection.ts index 9ab2071f..c4a33606 100644 --- a/packages/tests/src/selection.ts +++ b/packages/tests/src/selection.ts @@ -148,7 +148,7 @@ namespace SelectionTests { await expect(database.select('foo') .groupBy({}, { count: row => $.count(row.id), - size: row => $.size(row.id), + size: row => $.length(row.id), max: row => $.max(row.id), min: row => $.min(row.id), avg: row => $.avg(row.id), From 94a9512ab8d21cef4ed850183bb65d1a4cb609cf Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Thu, 9 Nov 2023 21:23:54 +0800 Subject: [PATCH 49/49] Apply suggestions from code review Co-authored-by: Shigma --- packages/core/src/eval.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/core/src/eval.ts b/packages/core/src/eval.ts index f05d5c38..55a40dee 100644 --- a/packages/core/src/eval.ts +++ b/packages/core/src/eval.ts @@ -96,11 +96,12 @@ export namespace Eval { length(value: Any): Expr // json - sum(value: (Number | Expr)[] | Expr): Expr - avg(value: (Number | Expr)[] | Expr): Expr - max(value: (Number | Expr)[] | Expr): Expr - min(value: (Number | Expr)[] | Expr): Expr - length(value: (Any | Expr)[] | Expr): Expr + sum(value: (number | Expr)[] | Expr): Expr + avg(value: (number | Expr)[] | Expr): Expr + max(value: (number | Expr)[] | Expr): Expr + min(value: (number | Expr)[] | Expr): Expr + size(value: (Any | Expr)[] | Expr): Expr + length(value: any[] | Expr): Expr object>(fields: T): Expr object(row: Row.Cell): Expr