From 1cd34926af969c08f4e284ddb22c28f8f5072191 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Fri, 19 Apr 2024 18:55:07 +0800 Subject: [PATCH 1/2] feat(minato): support left join in two-table joins --- packages/memory/src/index.ts | 12 ++++++++--- packages/mongo/src/builder.ts | 36 +++++++++++++++++++++++++------- packages/sql-utils/src/index.ts | 14 +++++++++---- packages/tests/src/selection.ts | 37 +++++++++++++++++++++++++++++++-- 4 files changed, 83 insertions(+), 16 deletions(-) diff --git a/packages/memory/src/index.ts b/packages/memory/src/index.ts index e3327e9d..b571862a 100644 --- a/packages/memory/src/index.ts +++ b/packages/memory/src/index.ts @@ -32,7 +32,7 @@ export class MemoryDriver extends Driver { } const { ref, query, table, args, model } = sel - const { fields, group, having } = sel.args[0] + const { fields, group, having, optional = {} } = sel.args[0] let data: any[] @@ -42,9 +42,15 @@ export class MemoryDriver extends Driver { if (!entries.length) return [] const [[name, rows], ...tail] = entries if (!tail.length) return rows.map(row => ({ [name]: row })) - return rows.flatMap(row => catesian(tail).map(tail => ({ ...tail, [name]: row }))) + return rows.flatMap(row => { + let res = catesian(tail).map(tail => ({ ...tail, [name]: row })) + if (Object.keys(table).length === tail.length + 1) { + res = res.map(row => ({ ...env, [ref]: row })).filter(data => executeEval(data, having)).map(x => x[ref]) + } + return !optional[tail[0]?.[0]] || res.length ? res : [{ [name]: row }] + }) } - data = catesian(entries).map(x => ({ ...env, [ref]: x })).filter(data => executeEval(data, having)).map(x => x[ref]) + data = catesian(entries) } else { data = this.table(table, env).filter(row => executeQuery(row, query, ref, env)) } diff --git a/packages/mongo/src/builder.ts b/packages/mongo/src/builder.ts index b73d5e02..47097c60 100644 --- a/packages/mongo/src/builder.ts +++ b/packages/mongo/src/builder.ts @@ -90,6 +90,7 @@ export class Builder { public evalKey?: string private refTables: string[] = [] private refVirtualKeys: Dict = {} + private joinTables: Dict = {} public aggrDefault: any private evalOperators: EvalOperators @@ -102,6 +103,12 @@ export class Builder { if (typeof arg === 'string') { this.walkedKeys.push(this.getActualKey(arg)) return this.recursivePrefix + this.getActualKey(arg) + } + const [joinRoot, ...rest] = (arg[1] as string).split('.') + if (this.tables.includes(`${arg[0]}.${joinRoot}`)) { + return this.recursivePrefix + rest.join('.') + } else if (`${arg[0]}.${joinRoot}` in this.joinTables) { + return `$$${this.joinTables[`${arg[0]}.${joinRoot}`]}.` + rest.join('.') } else if (this.tables.includes(arg[0])) { this.walkedKeys.push(this.getActualKey(arg[1])) return this.recursivePrefix + this.getActualKey(arg[1]) @@ -377,6 +384,7 @@ export class Builder { protected createSubquery(sel: Selection.Immutable) { const predecessor = new Builder(this.driver, Object.keys(sel.tables)) predecessor.refTables = [...this.refTables, ...this.tables] + predecessor.joinTables = { ...this.joinTables } predecessor.refVirtualKeys = this.refVirtualKeys return predecessor.select(sel) } @@ -392,7 +400,8 @@ export class Builder { this.table = predecessor.table this.pipeline.push(...predecessor.flushLookups(), ...predecessor.pipeline) } else { - for (const [name, subtable] of Object.entries(table)) { + const refs: Dict = {} + Object.entries(table).forEach(([name, subtable], i) => { const predecessor = this.createSubquery(subtable) if (!predecessor) return if (!this.table) { @@ -400,22 +409,35 @@ export class Builder { this.pipeline.push(...predecessor.flushLookups(), ...predecessor.pipeline, { $replaceRoot: { newRoot: { [name]: '$$ROOT' } }, }) - continue + refs[name] = subtable.ref + return + } + if (sel.args[0].having['$and'].length && i === Object.keys(table).length - 1) { + const thisTables = this.tables, thisJoinedTables = this.joinTables + this.tables = [...this.tables, `${sel.ref}.${name}`] + this.joinTables = { + ...this.joinTables, + [`${sel.ref}.${name}`]: sel.ref, + ...Object.fromEntries(Object.entries(refs).map(([name, ref]) => [`${sel.ref}.${name}`, ref])), + } + const $expr = this.eval(sel.args[0].having['$and'][0]) + predecessor.pipeline.push(...this.flushLookups(), { $match: { $expr } }) + this.tables = thisTables + this.joinTables = thisJoinedTables } const $lookup = { from: predecessor.table, as: name, + let: Object.fromEntries(Object.entries(refs).map(([name, ref]) => [ref, `$$ROOT.${name}`])), pipeline: predecessor.pipeline, } const $unwind = { path: `$${name}`, + preserveNullAndEmptyArrays: !!sel.args[0].optional?.[name], } this.pipeline.push({ $lookup }, { $unwind }) - } - if (sel.args[0].having['$and'].length) { - const $expr = this.eval(sel.args[0].having) - this.pipeline.push(...this.flushLookups(), { $match: { $expr } }) - } + refs[name] = subtable.ref + }) } // where diff --git a/packages/sql-utils/src/index.ts b/packages/sql-utils/src/index.ts index ac79eb3a..436cde5c 100644 --- a/packages/sql-utils/src/index.ts +++ b/packages/sql-utils/src/index.ts @@ -483,15 +483,21 @@ export class Builder { if (!prefix) return } else { this.state.innerTables = Object.fromEntries(Object.values(table).map(t => [t.ref, t.model])) - const joins: string[] = Object.entries(table).map(([key, table]) => { + const joins: [string, string][] = Object.entries(table).map(([key, table]) => { const restore = this.saveState({ tables: { ...table.tables } }) const t = `${this.get(table, true, false, false)} AS ${this.escapeId(table.ref)}` restore() - return t + return [key, t] }) - // the leading space is to prevent from being parsed as bracketed and added ref - prefix = ' ' + joins[0] + joins.slice(1, -1).map(join => ` JOIN ${join} ON ${this.$true}`).join(' ') + ` JOIN ` + joins.at(-1) + prefix = [ + // the leading space is to prevent from being parsed as bracketed and added ref + ' ', + joins[0][1], + ...joins.slice(1, -1).map(([key, join]) => `${args[0].optional?.[key] ? 'LEFT' : ''} JOIN ${join} ON ${this.$true}`), + `${args[0].optional?.[joins.at(-1)![0]] ? 'LEFT ' : ''}JOIN`, + joins.at(-1)![1], + ].join(' ') const filter = this.parseEval(args[0].having) prefix += ` ON ${filter}` } diff --git a/packages/tests/src/selection.ts b/packages/tests/src/selection.ts index 79ad64da..4e7bc4a8 100644 --- a/packages/tests/src/selection.ts +++ b/packages/tests/src/selection.ts @@ -287,6 +287,24 @@ namespace SelectionTests { ).to.eventually.have.length(2) }) + it('left join', async () => { + await expect(database + .join(['foo', 'bar'], (foo, bar) => $.eq(foo.value, bar.value), [false, true]) + .execute() + ).to.eventually.have.shape([ + { + foo: { value: 0, id: 1 }, + bar: { uid: 1, pid: 1, value: 0, id: 1 }, + }, + { + foo: { value: 0, id: 1 }, + bar: { uid: 1, pid: 2, value: 0, id: 3 }, + }, + { foo: { value: 2, id: 2 }, bar: {} }, + { foo: { value: 2, id: 3 }, bar: {} }, + ]) + }) + it('group', async () => { await expect(database.join(['foo', 'bar'], (foo, bar) => $.eq(foo.id, bar.pid)) .groupBy('foo', { count: row => $.sum(row.bar.uid) }) @@ -323,9 +341,9 @@ namespace SelectionTests { t1: database.select('bar').where(row => $.gt(row.pid, 1)), t2: database.select('bar').where(row => $.gt(row.uid, 1)), t3: database.select('bar').where(row => $.gt(row.id, 4)), - }, ({ t1, t2, t3 }) => $.gt($.add(t1.id, t2.id, t3.id), 0)) + }, ({ t1, t2, t3 }) => $.gt($.add(t1.id, t2.id, t3.id), 14)) .execute() - ).to.eventually.have.length(8) + ).to.eventually.have.length(4) }) it('aggregate', async () => { @@ -442,6 +460,21 @@ namespace SelectionTests { ).to.eventually.have.length(6) }) + it('selections', async () => { + const w = x => database.join(['bar', 'foo']).evaluate(row => $.add($.count(row.bar.id), -6, x)) + await expect(database + .join({ + t1: database.select('bar').where(row => $.gt(w(row.pid), 1)), + t2: database.select('bar').where(row => $.gt(row.uid, 1)), + t3: database.select('bar').where(row => $.gt(row.id, w(4))), + }, ({ t1, t2, t3 }) => $.gt($.add(t1.id, t2.id, w(t3.id)), 14)) + .project({ + val: row => $.add(row.t1.id, row.t2.id, w(row.t3.id)), + }) + .execute() + ).to.eventually.have.length(4) + }) + it('access from join', async () => { const w = x => database.select('bar').evaluate(row => $.add($.count(row.id), -6, x)) await expect(database From 60b73aa6b57ff4f15e0d05d1f4f2d9de86a646a5 Mon Sep 17 00:00:00 2001 From: Hieuzest Date: Mon, 22 Apr 2024 10:14:23 +0800 Subject: [PATCH 2/2] feat: allow left and right join in two table joins --- packages/core/src/database.ts | 6 +++++- packages/memory/src/index.ts | 2 +- packages/tests/src/selection.ts | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/core/src/database.ts b/packages/core/src/database.ts index efffd882..cfa69663 100644 --- a/packages/core/src/database.ts +++ b/packages/core/src/database.ts @@ -252,12 +252,16 @@ export class Database extends Servi if (Array.isArray(oldTables)) { tables = Object.fromEntries(oldTables.map((name) => [name, this.select(name)])) } - const sels = mapValues(tables, (t: TableLike) => { + let sels = mapValues(tables, (t: TableLike) => { return typeof t === 'string' ? this.select(t) : t }) if (Object.keys(sels).length === 0) throw new Error('no tables to join') const drivers = new Set(Object.values(sels).map(sel => sel.driver)) if (drivers.size !== 1) throw new Error('cannot join tables from different drivers') + if (Object.keys(sels).length === 2 && (optional?.[0] || optional?.[Object.keys(sels)[0]])) { + if (optional[1] || optional[Object.keys(sels)[1]]) throw new Error('full join is not supported') + sels = Object.fromEntries(Object.entries(sels).reverse()) + } const sel = new Selection([...drivers][0], sels) if (Array.isArray(oldTables)) { sel.args[0].having = Eval.and(query(...oldTables.map(name => sel.row[name]))) diff --git a/packages/memory/src/index.ts b/packages/memory/src/index.ts index 69f77fee..454f4e86 100644 --- a/packages/memory/src/index.ts +++ b/packages/memory/src/index.ts @@ -105,7 +105,7 @@ export class MemoryDriver extends Driver { } async stats() { - return { tables: valueMap(this._store, (rows, name) => ({ name, count: rows.length, size: 0 })), size: 0 } + return { tables: mapValues(this._store, (rows, name) => ({ name, count: rows.length, size: 0 })), size: 0 } } async get(sel: Selection.Immutable) { diff --git a/packages/tests/src/selection.ts b/packages/tests/src/selection.ts index 3941c0aa..03f6b35e 100644 --- a/packages/tests/src/selection.ts +++ b/packages/tests/src/selection.ts @@ -303,6 +303,24 @@ namespace SelectionTests { { foo: { value: 2, id: 2 }, bar: {} }, { foo: { value: 2, id: 3 }, bar: {} }, ]) + + await expect(database + .join(['foo', 'bar'], (foo, bar) => $.eq(foo.value, bar.value), [true, false]) + .execute() + ).to.eventually.have.shape([ + { + bar: { uid: 1, pid: 1, value: 0, id: 1 }, + foo: { value: 0, id: 1 }, + }, + { bar: { uid: 1, pid: 1, value: 1, id: 2 }, foo: {} }, + { + bar: { uid: 1, pid: 2, value: 0, id: 3 }, + foo: { value: 0, id: 1 }, + }, + { bar: { uid: 1, pid: 3, value: 1, id: 4 }, foo: {} }, + { bar: { uid: 2, pid: 1, value: 1, id: 5 }, foo: {} }, + { bar: { uid: 2, pid: 1, value: 1, id: 6 }, foo: {} }, + ]) }) it('group', async () => {