feat(minato): refactor to arrayBuffer, support full type def

packages/core/src/database.ts

@@ -1,4 +1,4 @@
-import { Dict, Intersect, makeArray, mapValues, MaybeArray, valueMap } from 'cosmokit'
+import { Dict, Intersect, makeArray, mapValues, MaybeArray, omit, valueMap } from 'cosmokit'
 import { Context, Service, Spread } from 'cordis'
 import { Flatten, Indexable, Keys, randomId, Row, unravel } from './utils.ts'
 import { Selection } from './selection.ts'
@@ -118,38 +118,30 @@ export class Database<S = any, N = any, C extends Context = Context> extends Ser
     ;(this.ctx as Context).emit('model', name)
   }
 
-  private parseField(field: any, transformers: Driver.Transformer[] = [], setInitial?: (value) => void, setField?: (value) => void): Type {
-    if (field === 'array') {
-      setInitial?.([])
-      setField?.({ type: 'json', initial: [] })
-      return Type.Array()
-    } else if (field === 'object') {
+  private _parseField(field: any, transformers: Driver.Transformer[] = [], setInitial?: (value) => void, setField?: (value) => void): Type {
+    if (field === 'object') {
       setInitial?.({})
       setField?.({ type: 'json', initial: {} })
       return Type.Object()
+    } else if (field === 'array') {
+      setInitial?.([])
+      setField?.({ type: 'json', initial: [] })
+      return Type.Array()
     } else if (typeof field === 'string' && this.types[field]) {
       transformers.push({
-        types: [this.types[field].type],
+        types: [field as any],
         load: this.types[field].load,
         dump: this.types[field].dump,
-      })
+      }, ...(this.types[field].transformers ?? []))
       setInitial?.(this.types[field].initial)
-      setField?.(this.types[field])
-      return Type.fromField(field as any)
-    } else if (typeof field === 'object' && field.load && field.dump) {
-      const name = this.define(field)
-      transformers.push({
-        types: [name as any],
-        load: field.load,
-        dump: field.dump,
-      })
-      // for transform type, intentionally assign a null initial on default
-      // setInitial?.(Field.getInitial(field.type, field.initial))
-      setInitial?.(field.initial)
-      setField?.({ ...field, deftype: field.type, type: name })
-      return Type.fromField(name as any)
+      setField?.({ ...this.types[field], type: field })
+      return Type.fromField(field)
+    } else if (typeof field === 'string') {
+      setInitial?.(Field.getInitial((field as any).split('(')[0]))
+      setField?.(field)
+      return Type.fromField(field.split('(')[0])
     } else if (typeof field === 'object' && field.type === 'object') {
-      const inner = unravel(field.inner, value => (value.type = 'object', value.inner ??= {}))
+      const inner = field.inner ? unravel(field.inner, value => (value.type = 'object', value.inner ??= {})) : Object.create(null)
       const initial = Object.create(null)
       const res = Type.Object(mapValues(inner, (x, k) => this.parseField(x, transformers, value => initial[k] = value)))
       setInitial?.(Field.getInitial('json', initial))
@@ -160,19 +152,45 @@ export class Database<S = any, N = any, C extends Context = Context> extends Ser
       setInitial?.([])
       setField?.({ initial: [], ...field, deftype: 'json', type: res })
       return res
-    } else if (typeof field === 'object') {
-      setInitial?.(Field.getInitial(field.type.split('(')[0], field.initial))
-      setField?.(field)
-      return Type.fromField(field.type.split('(')[0])
+    } else if (typeof field === 'object' && this.types[field.type]) {
+      transformers.push({
+        types: [field.type as any],
+        load: this.types[field.type].load,
+        dump: this.types[field.type].dump,
+      }, ...(this.types[field.type].transformers ?? []))
+      setInitial?.(field.initial === undefined ? this.types[field.type].initial : field.initial)
+      setField?.({ initial: this.types[field.type].initial, ...field })
+      return Type.fromField(field.type)
     } else {
-      setInitial?.(Field.getInitial(field.split('(')[0]))
+      setInitial?.(Field.getInitial(field.type, field.initial))
       setField?.(field)
-      return Type.fromField(field.split('(')[0])
+      return Type.fromField(field.type)
     }
   }
 
-  define<K extends Exclude<Keys<N>, Field.Type>>(name: K, field: Field.Transform<N[K]>): K
-  define<S>(field: Field.Transform<S>): Field.NewType<S>
+  private parseField(field: any, transformers: Driver.Transformer[] = [], setInitial?: (value) => void, setField?: (value: Field.Parsable) => void): Type {
+    let midfield
+    let type = this._parseField(field, transformers, setInitial, (value) => (midfield = value, setField?.(value)))
+    if (typeof field === 'object' && field.load && field.dump) {
+      if (type.inner) type = Type.fromField(this.define({ ...omit(midfield, ['load', 'dump']), type } as any))
+
+      const name = this.define({ ...field, deftype: midfield.deftype, type: type.type })
+      transformers.push({
+        types: [name as any],
+        load: field.load,
+        dump: field.dump,
+      })
+      // for transform type, intentionally assign a null initial on default
+      setInitial?.(field.initial)
+      setField?.({ ...field, deftype: midfield.deftype ?? this.types[type.type]?.deftype ?? type.type, initial: midfield.initial, type: name })
+      return Type.fromField(name as any)
+    }
+    if (typeof midfield === 'object') setField?.({ ...midfield, deftype: midfield.deftype ?? this.types[type.type]?.deftype ?? type?.type })
+    return type
+  }
+
+  define<K extends Exclude<Keys<N>, Field.Type | 'object' | 'array'>>(name: K, field: Field.Definition<N[K], N> | Field.Transform<N[K], any, N>): K
+  define<S>(field: Field.Definition<S, N> | Field.Transform<S, any, N>): Field.NewType<S>
   define(name: any, field?: any) {
     if (typeof name === 'object') {
       field = name
@@ -181,8 +199,14 @@ export class Database<S = any, N = any, C extends Context = Context> extends Ser
 
     if (name && this.types[name]) throw new Error(`type "${name}" already defined`)
     if (!name) while (this.types[name = '_define_' + randomId()]);
+
+    const transformers = []
+    const type = this._parseField(field, transformers, undefined, value => field = value)
+    field.transformers = transformers
+
     this[Context.current].effect(() => {
-      this.types[name] = { deftype: field.type, ...field, type: name }
+      this.types[name] = { ...field }
+      this.types[name].deftype ??= this.types[field.type]?.deftype ?? type.type as any
       return () => delete this.types[name]
     })
     return name as any

packages/core/src/driver.ts

@@ -35,8 +35,8 @@ export namespace Driver {
 
   export interface Transformer<S = any, T = any> {
     types: Field.Type<S>[]
-    dump: (value: S) => T | null
-    load: (value: T) => S | null
+    dump: (value: S | null) => T | null | void
+    load: (value: T | null) => S | null | void
   }
 }
 

packages/core/src/model.ts

@@ -1,7 +1,7 @@
-import { isNullable, makeArray, MaybeArray, valueMap } from 'cosmokit'
+import { Binary, clone, isNullable, makeArray, MaybeArray, valueMap } from 'cosmokit'
 import { Database } from './database.ts'
 import { Eval, isEvalExpr } from './eval.ts'
-import { clone, Flatten, isUint8Array, Keys } from './utils.ts'
+import { Flatten, Keys, unravel } from './utils.ts'
 import { Type } from './type.ts'
 import { Driver } from './driver.ts'
 
@@ -35,66 +35,69 @@ export namespace Field {
     : T extends string ? 'char' | 'string' | 'text'
     : T extends boolean ? 'boolean'
     : T extends Date ? 'timestamp' | 'date' | 'time'
-    : T extends Uint8Array ? 'binary'
-    : T extends unknown[] ? 'list' | 'json' | 'array'
-    : T extends object ? 'json' | 'object'
+    : T extends ArrayBuffer ? 'binary'
+    : T extends unknown[] ? 'list' | 'json'
+    : T extends object ? 'json'
     : 'expr'
 
   type Shorthand<S extends string> = S | `${S}(${any})`
 
   export type Object<T = any, N = any> = {
     type: 'object'
-    inner: Extension<T, N>
+    inner?: Extension<T, N>
   } & Omit<Field<T>, 'type'>
 
   export type Array<T = any, N = any> = {
     type: 'array'
-    inner?: Definition<T, N>
+    inner?: Literal<T, N> | Definition<T, N> | Transform<T, any, N>
   } & Omit<Field<T[]>, 'type'>
 
-  export type Transform<S = any, T = any> = {
-    type: Type<T>
-    dump: (value: S) => T | null
-    load: (value: T) => S | null
+  export type Transform<S = any, T = S, N = any> = {
+    type: Type<T> | Keys<N, T> | NewType<T> | 'object' | 'array'
+    dump: (value: S | null) => T | null | void
+    load: (value: T | null) => S | null | void
     initial?: S
-  } & Omit<Field<T>, 'type' | 'initial'>
-
-  type Parsed<T = any> = {
-    type: Type<T> | Field<T>['type']
-  } & Omit<Field<T>, 'type'>
+  } & Omit<Definition<T, N>, 'type' | 'initial'>
 
   export type Definition<T, N> =
-    | (Omit<Field<T>, 'type'> & { type: Type<T> })
-    | Object<T, N>
+    | (Omit<Field<T>, 'type'> & { type: Type<T> | Keys<N, T> | NewType<T> })
+    | (T extends object ? Object<T, N> : never)
     | (T extends (infer I)[] ? Array<I, N> : never)
+
+  export type Literal<T, N> =
     | Shorthand<Type<T>>
-    | Transform<T>
     | Keys<N, T>
     | NewType<T>
+    | (T extends object ? 'object' : never)
+    | (T extends unknown[] ? 'array' : never)
+
+  export type Parsable<T = any> = {
+    type: Type<T> | Field<T>['type']
+  } & Omit<Field<T>, 'type'>
 
   type MapField<O = any, N = any> = {
-    [K in keyof O]?: Definition<O[K], N>
+    [K in keyof O]?: Literal<O[K], N> | Definition<O[K], N> | Transform<O[K], any, N>
   }
 
   export type Extension<O = any, N = any> = MapField<Flatten<O>, N>
 
   const NewType = Symbol('newtype')
-  export type NewType<T> = { [NewType]?: T }
+  export type NewType<T> = string & { [NewType]: T }
 
   export type Config<O = any> = {
     [K in keyof O]?: Field<O[K]>
   }
 
   const regexp = /^(\w+)(?:\((.+)\))?$/
 
-  export function parse(source: string | Parsed): Field {
+  export function parse(source: string | Parsable): Field {
     if (typeof source === 'function') throw new TypeError('view field is not supported')
     if (typeof source !== 'string') {
       return {
         initial: null,
         deftype: source.type as any,
         ...source,
-        type: Type.isType(source.type) ? source.type : Type.fromField(source.type),
+        type: Type.fromField(source.type),
       }
     }
 
@@ -196,7 +199,7 @@ export class Model<S = any> {
   resolveValue(field: string | Field | Type, value: any) {
     if (isNullable(value)) return value
     if (typeof field === 'string') field = this.fields[field] as Field
-    if (field && !Type.isType(field)) field = Type.fromField(field)
+    if (field) field = Type.fromField(field)
     if (field?.type === 'time') {
       const date = new Date(0)
       date.setHours(value.getHours(), value.getMinutes(), value.getSeconds(), value.getMilliseconds())
@@ -259,6 +262,13 @@ export class Model<S = any> {
 
   parse(source: object, strict = true, prefix = '', result = {} as S) {
     const fields = Object.keys(this.fields)
+    if (strict && prefix === '') {
+      // initialize object layout
+      Object.assign(result as any, unravel(Object.fromEntries(fields
+        .filter(key => key.includes('.'))
+        .map(key => [key.slice(0, key.lastIndexOf('.')), {}])),
+      ))
+    }
     for (const key in source) {
       let node = result
       const segments = key.split('.').reverse()
@@ -271,7 +281,7 @@ export class Model<S = any> {
         const field = fields.find(field => fullKey === field || fullKey.startsWith(field + '.'))
         if (field) {
           node[segments[0]] = value
-        } else if (!value || typeof value !== 'object' || isEvalExpr(value) || Array.isArray(value) || isUint8Array(value) || Object.keys(value).length === 0) {
+        } else if (!value || typeof value !== 'object' || isEvalExpr(value) || Array.isArray(value) || Binary.is(value) || Object.keys(value).length === 0) {
           if (strict) {
             throw new TypeError(`unknown field "${fullKey}" in model ${this.name}`)
           } else {

packages/core/src/selection.ts

@@ -1,4 +1,4 @@
-import { defineProperty, Dict, valueMap } from 'cosmokit'
+import { defineProperty, Dict, filterKeys, valueMap } from 'cosmokit'
 import { Driver } from './driver.ts'
 import { Eval, executeEval } from './eval.ts'
 import { Model } from './model.ts'
@@ -59,7 +59,13 @@ const createRow = (ref: string, expr = {}, prefix = '', model?: Model) => new Pr
       // unknown field inside json
       type = Type.fromField('expr')
     }
-    return createRow(ref, Eval('', [ref, `${prefix}${key}`], type), `${prefix}${key}.`, model)
+
+    const row = createRow(ref, Eval('', [ref, `${prefix}${key}`], type), `${prefix}${key}.`, model)
+    if (Object.keys(model?.fields!).some(k => k.startsWith(`${prefix}${key}.`))) {
+      return createRow(ref, Eval.object(row), `${prefix}${key}.`, model)
+    } else {
+      return row
+    }
   },
 })
 
@@ -104,11 +110,14 @@ class Executable<S = any, T = any> {
     if (typeof fields === 'string') fields = [fields]
     if (Array.isArray(fields)) {
       const modelFields = Object.keys(this.model.fields)
-      const keys = fields.flatMap((key) => {
-        if (this.model.fields[key]) return key
-        return modelFields.filter(path => path.startsWith(key + '.'))
+      const entries = fields.flatMap((key) => {
+        if (this.model.fields[key]) return [[key, this.row[key]]]
+        else if (modelFields.some(path => path.startsWith(key + '.'))) {
+          return modelFields.filter(path => path.startsWith(key + '.')).map(path => [path, this.row[path]])
+        }
+        return [[key, key.split('.').reduce((row, k) => row[k], this.row)]]
       })
-      return Object.fromEntries(keys.map(key => [key, this.row[key]]))
+      return Object.fromEntries(entries)
     } else {
       return valueMap(fields, field => this.resolveField(field))
     }
@@ -260,6 +269,11 @@ export class Selection<S = any> extends Executable<S, S[]> {
         this.orderBy(field as any, cursor.sort[field])
       }
     }
+    if (cursor.fields) {
+      return super.execute().then(
+        rows => rows.map(row => filterKeys(row as any, key => (cursor.fields as string[]).some(k => k === key || k.startsWith(`${key}.`)))),
+      )
+    }
     return super.execute()
   }
 }

packages/core/src/type.ts

@@ -1,11 +1,12 @@
-import { defineProperty, isNullable, mapValues } from 'cosmokit'
+import { Binary, defineProperty, isNullable, mapValues } from 'cosmokit'
 import { Field } from './model.ts'
 import { Eval, isEvalExpr } from './eval.ts'
+import { Keys } from './utils.ts'
 
-export interface Type<T = any> {
+export interface Type<T = any, N = any> {
   [Type.kType]?: true
-  type: Field.Type<T>
-  inner?: T extends (infer I)[] ? Type<I> : Field.Type<T> extends 'json' ? { [key in keyof T]: Type<T[key]> } : never
+  type: Field.Type<T> | Keys<N, T> | Field.NewType<T>
+  inner?: T extends (infer I)[] ? Type<I, N> : Field.Type<T> extends 'json' ? { [key in keyof T]: Type<T[key], N> } : never
   array?: boolean
 }
 
@@ -42,14 +43,14 @@ export namespace Type {
     else if (typeof value === 'string') return String as any
     else if (typeof value === 'boolean') return Boolean as any
     else if (value instanceof Date) return fromField('timestamp' as any)
-    else if (ArrayBuffer.isView(value)) return fromField('binary' as any)
+    else if (Binary.is(value)) return fromField('binary' as any)
     else if (globalThis.Array.isArray(value)) return Array(value.length ? fromPrimitive(value[0]) : undefined) as any
     else if (typeof value === 'object') return fromField('json' as any)
     throw new TypeError(`invalid primitive: ${value}`)
   }
 
-  export function fromField<T>(field: Field<T> | Field.Type<T>): Type<T> {
-    if (isType(field)) throw new TypeError(`invalid field: ${JSON.stringify(field)}`)
+  export function fromField<T, N>(field: Type | Field<T> | Field.Type<T> | Keys<N, T> | Field.NewType<T>): Type<T, N> {
+    if (isType(field)) return field
     if (typeof field === 'string') return defineProperty({ type: field }, kType, true)
     else if (field.type) return field.type
     else if (field.expr?.[kType]) return field.expr[kType]
@@ -58,7 +59,7 @@ export namespace Type {
 
   export function fromTerm<T>(value: Eval.Term<T>): Type<T> {
     if (isEvalExpr(value)) return value[kType] ?? fromField('expr' as any)
-    else return fromPrimitive(value)
+    else return fromPrimitive(value as T)
   }
 
   export function isType(value: any): value is Type {
@@ -69,7 +70,7 @@ export namespace Type {
     return (type.type === 'json') && type.array
   }
 
-  export function getInner(type?: Type<any>, key?: string): Type | undefined {
+  export function getInner(type?: Type, key?: string): Type | undefined {
     if (!type?.inner) return
     if (isArray(type) && isNullable(key)) return type.inner
     if (isNullable(key)) return