feat(minato): first-class sql json type

.github/workflows/test.yaml

@@ -42,24 +42,28 @@ jobs:
       - name: Unit Test
         run: yarn test:json mysql
       - name: Report Coverage
-        if: ${{ matrix.node-version == 16 && matrix.mysql-image == 'mysql:5.7' }}
+        if: ${{ matrix.node-version == 16 }}
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage/coverage-final.json
           name: codecov
 
   mongo:
+    name: ${{ matrix.mongo-image }} (${{ matrix.node-version }})
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
       matrix:
+        mongo-image:
+          - mongo:6.0
+          - mongo:latest
         node-version: [16, 18, 20]
 
     services:
       mongo:
-        image: mongo
+        image: ${{ matrix.mongo-image }}
         ports:
           - 27017:27017
         # https://stackoverflow.com/questions/66317184/github-actions-cannot-connect-to-mongodb-service

packages/core/src/driver.ts

@@ -114,14 +114,14 @@ export class Database<S = any> {
   join<U extends Dict<TableLike<S>>>(tables: U, callback?: JoinCallback2<S, U>, optional?: Dict<boolean, Keys<U>>): Selection<TableMap2<S, U>>
   join(tables: any, query?: any, optional?: any) {
     if (Array.isArray(tables)) {
-      const sel = new Selection(this.getDriver(tables[0]), Object.fromEntries(tables.map((name) => [name, name])))
+      const sel = new Selection(this.getDriver(tables[0]), Object.fromEntries(tables.map((name) => [name, this.select(name)])))
       if (typeof query === 'function') {
         sel.args[0].having = Eval.and(query(...tables.map(name => sel.row[name])))
       }
       sel.args[0].optional = Object.fromEntries(tables.map((name, index) => [name, optional?.[index]]))
       return new Selection(this.getDriver(sel), sel)
     } else {
-      const sel = new Selection(this.getDriver(tables[0]), tables)
+      const sel = new Selection(this.getDriver(Object.values(tables)[0]), valueMap(tables, (t: TableLike<S>) => typeof t === 'string' ? this.select(t) : t))
       if (typeof query === 'function') {
         sel.args[0].having = Eval.and(query(sel.row))
       }

packages/core/src/eval.ts

@@ -1,5 +1,5 @@
-import { defineProperty, isNullable } from 'cosmokit'
-import { Comparable, Flatten, isComparable, makeRegExp } from './utils'
+import { defineProperty, Dict, isNullable, valueMap } from 'cosmokit'
+import { Comparable, Flatten, isComparable, makeRegExp, Row } from './utils'
 
 export function isEvalExpr(value: any): value is Eval.Expr {
   return value && Object.keys(value).some(key => key.startsWith('$'))
@@ -93,6 +93,19 @@ export namespace Eval {
     max(value: Number<false>): Expr<number, true>
     min(value: Number<false>): Expr<number, true>
     count(value: Any<false>): Expr<number, true>
+    length(value: Any<false>): Expr<number, true>
+
+    // json
+    sum<A extends boolean>(value: (number | Expr<number, A>)[] | Expr<number[], A>): Expr<number, A>
+    avg<A extends boolean>(value: (number | Expr<number, A>)[] | Expr<number[], A>): Expr<number, A>
+    max<A extends boolean>(value: (number | Expr<number, A>)[] | Expr<number[], A>): Expr<number, A>
+    min<A extends boolean>(value: (number | Expr<number, A>)[] | Expr<number[], A>): Expr<number, A>
+    size<A extends boolean>(value: (Any | Expr<Any, A>)[] | Expr<Any[], A>): Expr<number, A>
+    length<A extends boolean>(value: any[] | Expr<any[], A>): Expr<number, A>
+
+    object<T extends Dict<Expr>>(fields: T): Expr<T, false>
+    object<T extends any>(row: Row.Cell<T>): Expr<T, false>
+    array<T>(value: Expr<T, false>): Expr<T[], true>
   }
 }
 
@@ -165,12 +178,42 @@ Eval.or = multary('or', (args, data) => args.some(arg => executeEval(data, arg))
 Eval.not = unary('not', (value, data) => !executeEval(data, value))
 
 // aggregation
-Eval.sum = unary('sum', (expr, table) => table.reduce<number>((prev, curr) => prev + executeAggr(expr, curr), 0))
-Eval.avg = unary('avg', (expr, table) => table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length)
-Eval.max = unary('max', (expr, table) => Math.max(...table.map(data => executeAggr(expr, data))))
-Eval.min = unary('min', (expr, table) => Math.min(...table.map(data => executeAggr(expr, data))))
+Eval.sum = unary('sum', (expr, table) => Array.isArray(table)
+  ? table.reduce<number>((prev, curr) => prev + executeAggr(expr, curr), 0)
+  : Array.from<number>(executeEval(table, expr)).reduce((prev, curr) => prev + curr, 0))
+Eval.avg = unary('avg', (expr, table) => {
+  if (Array.isArray(table)) return table.reduce((prev, curr) => prev + executeAggr(expr, curr), 0) / table.length
+  else {
+    const array = Array.from<number>(executeEval(table, expr))
+    return array.reduce((prev, curr) => prev + curr, 0) / array.length
+  }
+})
+Eval.max = unary('max', (expr, table) => Array.isArray(table)
+  ? Math.max(...table.map(data => executeAggr(expr, data)))
+  : Math.max(...Array.from<number>(executeEval(table, expr))))
+Eval.min = unary('min', (expr, table) => Array.isArray(table)
+  ? Math.min(...table.map(data => executeAggr(expr, data)))
+  : Math.min(...Array.from<number>(executeEval(table, expr))))
 Eval.count = unary('count', (expr, table) => new Set(table.map(data => executeAggr(expr, data))).size)
-
+defineProperty(Eval, 'length', unary('length', (expr, table) => Array.isArray(table)
+  ? table.map(data => executeAggr(expr, data)).length
+  : Array.from(executeEval(table, expr)).length))
+
+operators.$object = (field, table) => valueMap(field, value => executeAggr(value, table))
+Eval.object = (fields) => {
+  if (fields.$model) {
+    const modelFields = Object.keys(fields.$model.fields)
+    const prefix: string = fields.$prefix
+    return Eval('object', Object.fromEntries(modelFields
+      .filter(path => path.startsWith(prefix))
+      .map(k => [k.slice(prefix.length), fields[k.slice(prefix.length)]]),
+    ))
+  }
+  return Eval('object', fields) as any
+}
+Eval.array = unary('array', (expr, table) => Array.isArray(table)
+  ? table.map(data => executeAggr(expr, data))
+  : Array.from(executeEval(table, expr)))
 export { Eval as $ }
 
 type MapUneval<S> = {

packages/core/src/model.ts

@@ -188,7 +188,7 @@ export class Model<S = any> {
         const field = fields.find(field => fullKey === field || fullKey.startsWith(field + '.'))
         if (field) {
           node[segments[0]] = this.resolveValue(key, value)
-        } else if (!value || typeof value !== 'object' || isEvalExpr(value) || Object.keys(value).length === 0) {
+        } else if (!value || typeof value !== 'object' || isEvalExpr(value) || Array.isArray(value) || Object.keys(value).length === 0) {
           if (strict) {
             throw new TypeError(`unknown field "${fullKey}" in model ${this.name}`)
           } else {

packages/core/src/selection.ts

@@ -11,7 +11,7 @@ export interface Modifier {
   limit: number
   offset: number
   sort: [Eval.Expr, Direction][]
-  group: string[]
+  group?: string[]
   having: Eval.Expr<boolean>
   fields?: Dict<Eval.Expr>
   optional: Dict<boolean>
@@ -22,17 +22,19 @@ namespace Executable {
 
   export interface Payload {
     type: Action
-    table: string | Selection | Dict<string | Selection.Immutable>
+    table: string | Selection | Dict<Selection.Immutable>
     ref: string
     query: Query.Expr
     args: any[]
   }
 }
 
-const createRow = (ref: string, expr = {}, prefix = '') => new Proxy(expr, {
+const createRow = (ref: string, expr = {}, prefix = '', model?: Model) => new Proxy(expr, {
   get(target, key) {
+    if (key === '$prefix') return prefix
+    if (key === '$model') return model
     if (typeof key === 'symbol' || key in target || key.startsWith('$')) return Reflect.get(target, key)
-    return createRow(ref, Eval('', [ref, `${prefix}${key}`]), `${prefix}${key}.`)
+    return createRow(ref, Eval('', [ref, `${prefix}${key}`]), `${prefix}${key}.`, model)
   },
 })
 
@@ -45,15 +47,15 @@ class Executable<S = any, T = any> {
 
   constructor(driver: Driver, payload: Executable.Payload) {
     Object.assign(this, payload)
-    const expr = {}
+    defineProperty(this, 'model', driver.model(this.table))
+    const expr = { $model: this.model }
     if (typeof payload.table !== 'string' && !(payload.table instanceof Selection)) {
       for (const key in payload.table) {
-        expr[key] = createRow(key)
+        expr[key] = createRow(key, {}, '', this.model)
       }
     }
     defineProperty(this, 'driver', driver)
-    defineProperty(this, 'row', createRow(this.ref, expr))
-    defineProperty(this, 'model', driver.model(this.table))
+    defineProperty(this, 'row', createRow(this.ref, expr, '', this.model))
   }
 
   protected resolveQuery(query?: Query<S>): Query.Expr<S>
@@ -130,13 +132,13 @@ export interface Selection extends Executable.Payload {
 export class Selection<S = any> extends Executable<S, S[]> {
   public tables: Dict<Model> = {}
 
-  constructor(driver: Driver, table: string | Selection | Dict<string | Selection.Immutable>, query?: Query) {
+  constructor(driver: Driver, table: string | Selection | Dict<Selection.Immutable>, query?: Query) {
     super(driver, {
       type: 'get',
       ref: randomId(),
       table,
       query: null as never,
-      args: [{ sort: [], limit: Infinity, offset: 0, group: [], having: Eval.and(), optional: {} }],
+      args: [{ sort: [], limit: Infinity, offset: 0, group: undefined, having: Eval.and(), optional: {} }],
     })
     this.tables[this.ref] = this.model
     this.query = this.resolveQuery(query)

packages/memory/src/index.ts

@@ -1,5 +1,5 @@
 import { clone, Dict, makeArray, noop, omit, pick, valueMap } from 'cosmokit'
-import { Database, Driver, Eval, executeEval, executeQuery, executeSort, executeUpdate, isEvalExpr, RuntimeError, Selection } from '@minatojs/core'
+import { Database, Driver, Eval, executeEval, executeQuery, executeSort, executeUpdate, RuntimeError, Selection } from '@minatojs/core'
 
 export namespace MemoryDriver {
   export interface Config {}
@@ -47,12 +47,9 @@ export class MemoryDriver extends Driver {
     const { ref, query, table, args, model } = sel
     const { fields, group, having } = sel.args[0]
     const data = this.table(table, having).filter(row => executeQuery(row, query, ref))
-    if (!group.length && fields && Object.values(args[0].fields ?? {}).some(x => isAggrExpr(x))) {
-      return [valueMap(fields!, (expr) => executeEval(data.map(row => ({ [ref]: row })), expr))]
-    }
 
     const branches: { index: Dict; table: any[] }[] = []
-    const groupFields = group.length ? pick(fields!, group) : fields
+    const groupFields = group ? pick(fields!, group) : fields
     for (let row of executeSort(data, args[0], ref)) {
       row = model.format(row, false)
       for (const key in model.fields) {
@@ -64,7 +61,7 @@ export class MemoryDriver extends Driver {
         index = valueMap(groupFields!, (expr) => executeEval({ [ref]: row }, expr))
       }
       let branch = branches.find((branch) => {
-        if (!groupFields) return false
+        if (!group || !groupFields) return false
         for (const key in groupFields) {
           if (branch.index[key] !== index[key]) return false
         }
@@ -77,7 +74,7 @@ export class MemoryDriver extends Driver {
       branch.table.push(row)
     }
     return branches.map(({ index, table }) => {
-      if (group.length) {
+      if (group) {
         if (having) {
           const value = executeEval(table.map(row => ({ [ref]: row, _: row })), having)
           if (!value) return
@@ -168,17 +165,4 @@ export class MemoryDriver extends Driver {
   }
 }
 
-const nonAggrKeys = ['$']
-const aggrKeys = ['$sum', '$avg', '$min', '$max', '$count']
-
-function isAggrExpr(value: any) {
-  if (!isEvalExpr(value)) return false
-  for (const [key, args] of Object.entries(value)) {
-    if (!key.startsWith('$')) continue
-    if (nonAggrKeys.includes(key)) return false
-    if (aggrKeys.includes(key) || ((Array.isArray(args) ? args : [args]).some(x => isAggrExpr(x)))) return true
-  }
-  return false
-}
-
 export default MemoryDriver

packages/mongo/src/utils.ts

@@ -72,7 +72,7 @@ function transformFieldQuery(query: Query.FieldQuery, key: string, filters: Filt
   return result
 }
 
-const aggrKeys = ['$sum', '$avg', '$min', '$max', '$count']
+const aggrKeys = ['$sum', '$avg', '$min', '$max', '$count', '$length', '$array']
 
 export class Transformer {
   private counter = 0
@@ -114,6 +114,18 @@ export class Transformer {
       return { $cond: expr.$if.map(val => this.eval(val, group)) }
     }
 
+    if (expr.$object || expr.$array) {
+      return this.transformEvalExpr(expr.$object || expr.$array)
+    }
+
+    if (expr.$length) {
+      return { $size: this.eval(expr.$length) }
+    }
+
+    if (expr.$nin) {
+      return { $not: { $in: expr.$nin.map(val => this.eval(val, group)) } }
+    }
+
     return valueMap(expr as any, (value) => {
       if (Array.isArray(value)) {
         return value.map(val => this.eval(val, group))
@@ -141,16 +153,24 @@ export class Transformer {
       return expr
     }
 
-    for (const type of aggrKeys) {
-      if (!expr[type]) continue
-      const key = this.createKey()
-      const value = this.transformAggr(expr[type])
-      if (type !== '$count') {
-        group![key] = { [type]: value }
-        return '$' + key
-      } else {
-        group![key] = { $addToSet: value }
-        return { $size: '$' + key }
+    if (group) {
+      for (const type of aggrKeys) {
+        if (!expr[type]) continue
+        const key = this.createKey()
+        const value = this.transformAggr(expr[type])
+        if (type === '$count') {
+          group![key] = { $addToSet: value }
+          return { $size: '$' + key }
+        } else if (type === '$length') {
+          group![key] = { $push: value }
+          return { $size: '$' + key }
+        } else if (type === '$array') {
+          group![key] = { $push: value }
+          return '$' + key
+        } else {
+          group![key] = { [type]: value }
+          return '$' + key
+        }
       }
     }
 
@@ -222,7 +242,7 @@ export class Transformer {
     }
 
     // groupBy, having, fields
-    if (group.length) {
+    if (group) {
       const $group: Dict = { _id: {} }
       const $project: Dict = { _id: 0 }
       stages.push({ $group })
@@ -242,10 +262,9 @@ export class Transformer {
       stages.push({ $project })
       $group['_id'] = model.parse($group['_id'], false)
     } else if (fields) {
-      const $group: Dict = { _id: null }
-      const $project = valueMap(fields, (expr) => this.eval(expr, $group))
+      const $project = valueMap(fields, (expr) => this.eval(expr))
       $project._id = 0
-      stages.push(...Object.keys($group).length === 1 ? [] : [{ $group }], { $project })
+      stages.push({ $project })
     } else {
       const $project: Dict = { _id: 0 }
       for (const key in model.fields) {