From 72edec4f34d1959773e6215812d33dbebac70092 Mon Sep 17 00:00:00 2001 From: Brayden Wilmoth Date: Wed, 28 Aug 2024 16:56:27 -0400 Subject: [PATCH 01/12] BigQuery Dialect --- src/query-builder/dialects/bigquery.ts | 68 ++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 src/query-builder/dialects/bigquery.ts diff --git a/src/query-builder/dialects/bigquery.ts b/src/query-builder/dialects/bigquery.ts new file mode 100644 index 0000000..8420e40 --- /dev/null +++ b/src/query-builder/dialects/bigquery.ts @@ -0,0 +1,68 @@ +import { QueryBuilder } from 'src/client'; +import { Query } from 'src/query'; +import { QueryType } from 'src/query-params'; +import { AbstractDialect } from '../index'; + +export class BigQueryDialect extends AbstractDialect { + escapeCharacter: string = '`' + + select(builder: QueryBuilder, type: QueryType, query: Query): Query { + let selectColumns = '' + let fromTable = '' + const joinClauses = builder.joins?.join(' ') || '' + + builder.columnsWithTable?.forEach((set, index) => { + if (index > 0) { + selectColumns += ', ' + } + + const schema = set.schema ? `${set.schema}.` : '' + let useTable = this.reservedKeywords.includes(set.table) + ? `"${set.table}"` + : set.table + + const columns = set.columns.map((column) => { + let useColumn = column + + if (this.reservedKeywords.includes(column)) { + useColumn = `"${column}"` + } + + return `\`${schema ?? ''}${useTable}\`.${useColumn}` + }) + + selectColumns += columns.join(', ') + + if (index === 0) { + fromTable = `\`${schema}${useTable}\`` + } + }) + + query.query = `SELECT ${selectColumns} FROM ${fromTable}` + + if (joinClauses) { + query.query += ` ${joinClauses}` + } + + if (builder.whereClauses?.length ?? 0 > 0) { + query.query += ` WHERE ${builder?.whereClauses?.join(' AND ')}` + } + + if (builder.orderBy !== undefined) { + query.query += ` ORDER BY ${builder.orderBy}` + } + + if (builder.limit !== undefined) { + query.query += ` LIMIT ${builder.limit}` + if (builder.offset) { + query.query += ` OFFSET ${builder.offset}` + } + } + + if (builder.groupBy !== undefined) { + query.query += ` GROUP BY ${builder.groupBy}` + } + + return query + } +} \ No newline at end of file From 1f9d6c111f5bdf5b0ad5953108b774d06777facd Mon Sep 17 00:00:00 2001 From: Brayden Wilmoth Date: Wed, 28 Aug 2024 17:09:12 -0400 Subject: [PATCH 02/12] Set BigQuery dialect for BigQuery connection --- package.json | 2 +- playground/index.js | 1 + src/connections/bigquery.ts | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 118704c..7d08291 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@outerbase/sdk", - "version": "1.0.15", + "version": "1.0.17", "description": "", "main": "dist/index.js", "module": "dist/index.js", diff --git a/playground/index.js b/playground/index.js index 964caef..1904a47 100644 --- a/playground/index.js +++ b/playground/index.js @@ -21,6 +21,7 @@ app.get('/test', async (req, res) => { // ]) // .query() + // let { data, query } = await db // .insert({ fname: 'John' }) // .into('test2') diff --git a/src/connections/bigquery.ts b/src/connections/bigquery.ts index e3ba3fb..a40ce37 100644 --- a/src/connections/bigquery.ts +++ b/src/connections/bigquery.ts @@ -2,7 +2,7 @@ import { QueryType } from '../query-params' import { Query, constructRawQuery } from '../query' import { Connection } from './index' import { Database, Table, TableColumn } from '../models/database' -import { DefaultDialect } from '../query-builder/dialects/default' +import { BigQueryDialect } from '../query-builder/dialects/bigquery' import { BigQuery } from '@google-cloud/bigquery' @@ -19,7 +19,7 @@ export class BigQueryConnection implements Connection { queryType = QueryType.positional // Default dialect for BigQuery - dialect = new DefaultDialect() + dialect = new BigQueryDialect() /** * Creates a new BigQuery object. From 685064d5cfc74e53e35bfb2e7baa85570cc6632f Mon Sep 17 00:00:00 2001 From: Brayden Wilmoth Date: Wed, 28 Aug 2024 19:13:31 -0400 Subject: [PATCH 03/12] Update comments on why select was reimplemented for BigQuery --- src/query-builder/dialects/bigquery.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/query-builder/dialects/bigquery.ts b/src/query-builder/dialects/bigquery.ts index 8420e40..2975596 100644 --- a/src/query-builder/dialects/bigquery.ts +++ b/src/query-builder/dialects/bigquery.ts @@ -4,8 +4,21 @@ import { QueryType } from 'src/query-params'; import { AbstractDialect } from '../index'; export class BigQueryDialect extends AbstractDialect { - escapeCharacter: string = '`' - + /** + * BigQuery currently entirely reimplements the `select` method due to the specialized + * nature of the BigQuery SQL dialect. When running a SELECT statement via BigQuery, + * both the schema and table name must be escaped with backticks. Since we currently do + * not have a better proper extraction method for the schema and table name, we manually + * re-implement what was already in the AbstractDialect class. + * + * In the future we should find a way where this `select` method can be reused from the + * AbstractDialect class in a better manner. + * + * @param builder + * @param type + * @param query + * @returns Query - The query object with the query string and parameters + */ select(builder: QueryBuilder, type: QueryType, query: Query): Query { let selectColumns = '' let fromTable = '' From a20048c2bee0fff5d01c174acc6112c8c303bb3e Mon Sep 17 00:00:00 2001 From: Brayden Wilmoth Date: Thu, 29 Aug 2024 13:07:14 -0400 Subject: [PATCH 04/12] New dialect options --- playground/index.js | 68 ------------------ src/query-builder/dialects/bigquery.ts | 97 ++++++++++++++------------ src/query-builder/index.ts | 21 +++++- 3 files changed, 71 insertions(+), 115 deletions(-) diff --git a/playground/index.js b/playground/index.js index 1904a47..e69de29 100644 --- a/playground/index.js +++ b/playground/index.js @@ -1,68 +0,0 @@ -import { CloudflareD1Connection, Outerbase } from '../dist/index.js'; -import express from 'express'; - -const app = express(); -const port = 4000; - -app.get('/test', async (req, res) => { - // Establish connection to your provider database - const d1 = new CloudflareD1Connection({ - apiKey: '', - accountId: '', - databaseId: '' - }); - - const db = Outerbase(d1); - // const dbSchema = await d1.fetchDatabaseSchema() - - // const { data, query } = await db - // .selectFrom([ - // { table: 'test2', columns: ['*'] } - // ]) - // .query() - - - // let { data, query } = await db - // .insert({ fname: 'John' }) - // .into('test2') - // .returning(['id']) - // .query(); - - // let { data, query } = await db - // .update({ fname: 'Johnny' }) - // .into('test2') - // .where(equals('id', '3', d1.dialect)) - // .query(); - - // let { data, query } = await db - // .deleteFrom('test2') - // .where(equals('id', '3')) - // .query(); - - // let data = {} - // let query = await db - // .createTable('test3') - // .schema('public') - // .columns([ - // { name: 'id', type: ColumnDataType.NUMBER, primaryKey: true }, - // { name: 'fname', type: ColumnDataType.STRING } - // ]) - // .toString(); - - // let data = {} - // let query = await db - // .renameTable('test3', 'test4') - // .toString(); - - // let data = {} - // let query = await db - // .dropTable('test4') - // .toString(); - - // let { data } = await db.queryRaw('SELECT * FROM playing_with_neon WHERE id = $1', ['1']); - res.json(data); -}); - -app.listen(port, () => { - console.log(`Server is running on http://localhost:${port}`); -}); diff --git a/src/query-builder/dialects/bigquery.ts b/src/query-builder/dialects/bigquery.ts index 2975596..9331dd3 100644 --- a/src/query-builder/dialects/bigquery.ts +++ b/src/query-builder/dialects/bigquery.ts @@ -4,6 +4,13 @@ import { QueryType } from 'src/query-params'; import { AbstractDialect } from '../index'; export class BigQueryDialect extends AbstractDialect { + formatSchemaAndTable(schema: string | undefined, table: string): string { + if (schema) { + return `\`${schema}.${table}\``; + } + return `\`${table}\``; + } + /** * BigQuery currently entirely reimplements the `select` method due to the specialized * nature of the BigQuery SQL dialect. When running a SELECT statement via BigQuery, @@ -19,63 +26,63 @@ export class BigQueryDialect extends AbstractDialect { * @param query * @returns Query - The query object with the query string and parameters */ - select(builder: QueryBuilder, type: QueryType, query: Query): Query { - let selectColumns = '' - let fromTable = '' - const joinClauses = builder.joins?.join(' ') || '' + // select(builder: QueryBuilder, type: QueryType, query: Query): Query { + // let selectColumns = '' + // let fromTable = '' + // const joinClauses = builder.joins?.join(' ') || '' - builder.columnsWithTable?.forEach((set, index) => { - if (index > 0) { - selectColumns += ', ' - } + // builder.columnsWithTable?.forEach((set, index) => { + // if (index > 0) { + // selectColumns += ', ' + // } - const schema = set.schema ? `${set.schema}.` : '' - let useTable = this.reservedKeywords.includes(set.table) - ? `"${set.table}"` - : set.table + // const schema = set.schema ? `${set.schema}.` : '' + // let useTable = this.reservedKeywords.includes(set.table) + // ? `"${set.table}"` + // : set.table - const columns = set.columns.map((column) => { - let useColumn = column + // const columns = set.columns.map((column) => { + // let useColumn = column - if (this.reservedKeywords.includes(column)) { - useColumn = `"${column}"` - } + // if (this.reservedKeywords.includes(column)) { + // useColumn = `"${column}"` + // } - return `\`${schema ?? ''}${useTable}\`.${useColumn}` - }) + // return `\`${schema ?? ''}${useTable}\`.${useColumn}` + // }) - selectColumns += columns.join(', ') + // selectColumns += columns.join(', ') - if (index === 0) { - fromTable = `\`${schema}${useTable}\`` - } - }) + // if (index === 0) { + // fromTable = `\`${schema}${useTable}\`` + // } + // }) - query.query = `SELECT ${selectColumns} FROM ${fromTable}` + // query.query = `SELECT ${selectColumns} FROM ${fromTable}` - if (joinClauses) { - query.query += ` ${joinClauses}` - } + // if (joinClauses) { + // query.query += ` ${joinClauses}` + // } - if (builder.whereClauses?.length ?? 0 > 0) { - query.query += ` WHERE ${builder?.whereClauses?.join(' AND ')}` - } + // if (builder.whereClauses?.length ?? 0 > 0) { + // query.query += ` WHERE ${builder?.whereClauses?.join(' AND ')}` + // } - if (builder.orderBy !== undefined) { - query.query += ` ORDER BY ${builder.orderBy}` - } + // if (builder.orderBy !== undefined) { + // query.query += ` ORDER BY ${builder.orderBy}` + // } - if (builder.limit !== undefined) { - query.query += ` LIMIT ${builder.limit}` - if (builder.offset) { - query.query += ` OFFSET ${builder.offset}` - } - } + // if (builder.limit !== undefined) { + // query.query += ` LIMIT ${builder.limit}` + // if (builder.offset) { + // query.query += ` OFFSET ${builder.offset}` + // } + // } - if (builder.groupBy !== undefined) { - query.query += ` GROUP BY ${builder.groupBy}` - } + // if (builder.groupBy !== undefined) { + // query.query += ` GROUP BY ${builder.groupBy}` + // } - return query - } + // return query + // } } \ No newline at end of file diff --git a/src/query-builder/index.ts b/src/query-builder/index.ts index 1146a3a..0b99d59 100644 --- a/src/query-builder/index.ts +++ b/src/query-builder/index.ts @@ -3,6 +3,8 @@ import { QueryBuilder } from "../client"; import { Query } from "../query"; interface Dialect { + formatSchemaAndTable(schema: string | undefined, table: string): string + select(builder: QueryBuilder, type: QueryType, query: Query): Query; insert(builder: QueryBuilder, type: QueryType, query: Query): Query; update(builder: QueryBuilder, type: QueryType, query: Query): Query; @@ -175,6 +177,20 @@ export abstract class AbstractDialect implements Dialect { 'WHERE', ]; + /** + * Formats the schema and table names according to the specific database dialect. + * @param schema The schema name (optional). + * @param table The table name. + * @returns The formatted schema and table combination. + */ + formatSchemaAndTable(schema: string | undefined, table: string): string { + // Default implementation (can be overridden by specific dialects) + if (schema) { + return `"${schema}".${table}`; + } + return table; + } + select(builder: QueryBuilder, type: QueryType, query: Query): Query { let selectColumns = '' let fromTable = '' @@ -190,6 +206,7 @@ export abstract class AbstractDialect implements Dialect { ? `"${set.table}"` : set.table + const formattedTable = this.formatSchemaAndTable(set.schema, set.table); const columns = set.columns.map((column) => { let useColumn = column @@ -197,13 +214,13 @@ export abstract class AbstractDialect implements Dialect { useColumn = `"${column}"` } - return `${schema ?? ''}${useTable}.${useColumn}` + return `${formattedTable}.${useColumn}` }) selectColumns += columns.join(', ') if (index === 0) { - fromTable = `${schema}${useTable}` + fromTable = formattedTable } }) From 94740ee4aa344c4e473e83e5edecd51a0640ff5f Mon Sep 17 00:00:00 2001 From: Brayden Wilmoth Date: Thu, 29 Aug 2024 14:04:17 -0400 Subject: [PATCH 05/12] Latest dialect changes --- src/query-builder/index.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/query-builder/index.ts b/src/query-builder/index.ts index 0b99d59..50cb924 100644 --- a/src/query-builder/index.ts +++ b/src/query-builder/index.ts @@ -191,6 +191,14 @@ export abstract class AbstractDialect implements Dialect { return table; } + formatFromSchemaAndTable(schema: string | undefined, table: string): string { + // Default implementation (can be overridden by specific dialects) + if (schema) { + return `"${schema}".${table}`; + } + return table; + } + select(builder: QueryBuilder, type: QueryType, query: Query): Query { let selectColumns = '' let fromTable = '' @@ -201,12 +209,8 @@ export abstract class AbstractDialect implements Dialect { selectColumns += ', ' } - const schema = set.schema ? `"${set.schema}".` : '' - let useTable = this.reservedKeywords.includes(set.table) - ? `"${set.table}"` - : set.table - const formattedTable = this.formatSchemaAndTable(set.schema, set.table); + const formattedFromTable = this.formatFromSchemaAndTable(set.schema, set.table); const columns = set.columns.map((column) => { let useColumn = column @@ -220,7 +224,7 @@ export abstract class AbstractDialect implements Dialect { selectColumns += columns.join(', ') if (index === 0) { - fromTable = formattedTable + fromTable = formattedFromTable } }) From 93223212f2c31fca1caa9e7a4dc602b3518a9779 Mon Sep 17 00:00:00 2001 From: Brayden Wilmoth Date: Thu, 29 Aug 2024 14:22:47 -0400 Subject: [PATCH 06/12] Formatted schema and table in CRUD operations --- src/query-builder/index.ts | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/query-builder/index.ts b/src/query-builder/index.ts index 50cb924..bf1d4fe 100644 --- a/src/query-builder/index.ts +++ b/src/query-builder/index.ts @@ -191,14 +191,6 @@ export abstract class AbstractDialect implements Dialect { return table; } - formatFromSchemaAndTable(schema: string | undefined, table: string): string { - // Default implementation (can be overridden by specific dialects) - if (schema) { - return `"${schema}".${table}`; - } - return table; - } - select(builder: QueryBuilder, type: QueryType, query: Query): Query { let selectColumns = '' let fromTable = '' @@ -210,7 +202,6 @@ export abstract class AbstractDialect implements Dialect { } const formattedTable = this.formatSchemaAndTable(set.schema, set.table); - const formattedFromTable = this.formatFromSchemaAndTable(set.schema, set.table); const columns = set.columns.map((column) => { let useColumn = column @@ -224,7 +215,7 @@ export abstract class AbstractDialect implements Dialect { selectColumns += columns.join(', ') if (index === 0) { - fromTable = formattedFromTable + fromTable = formattedTable } }) @@ -263,7 +254,8 @@ export abstract class AbstractDialect implements Dialect { ? columns.map((column) => `:${column}`).join(', ') : columns.map(() => '?').join(', ') - query.query = `INSERT INTO ${builder.schema ? `"${builder.schema}".` : ''}${builder.table || ''} (${columns.join( + const formattedTable = this.formatSchemaAndTable(builder.schema, builder.table || ''); + query.query = `INSERT INTO ${formattedTable} (${columns.join( ', ' )}) VALUES (${placeholders})` @@ -281,6 +273,7 @@ export abstract class AbstractDialect implements Dialect { return query } + const formattedTable = this.formatSchemaAndTable(builder.schema, builder.table || ''); const columnsToUpdate = Object.keys(builder.data || {}) const setClauses = type === QueryType.named @@ -291,7 +284,7 @@ export abstract class AbstractDialect implements Dialect { .map((column) => `${column} = ?`) .join(', ') - query.query = `UPDATE ${builder.schema ? `"${builder.schema}".` : ''}${builder.table || ''} SET ${setClauses}` + query.query = `UPDATE ${formattedTable} SET ${setClauses}` if (builder.whereClauses?.length > 0) { query.query += ` WHERE ${builder.whereClauses.join(' AND ')}` } @@ -313,7 +306,8 @@ export abstract class AbstractDialect implements Dialect { return query } - query.query = `DELETE FROM ${builder.schema ? `"${builder.schema}".` : ''}${builder.table || ''}` + const formattedTable = this.formatSchemaAndTable(builder.schema, builder.table || ''); + query.query = `DELETE FROM ${formattedTable}` if (builder.whereClauses?.length > 0) { query.query += ` WHERE ${builder.whereClauses.join(' AND ')}` } @@ -326,6 +320,7 @@ export abstract class AbstractDialect implements Dialect { // return query // } + const formattedTable = this.formatSchemaAndTable(builder.schema, builder.table || ''); const columns = builder?.columns?.map((column) => { const dataType = this.mapDataType(column.type) @@ -340,7 +335,7 @@ export abstract class AbstractDialect implements Dialect { query.query = ` CREATE TABLE IF NOT EXISTS - ${builder.schema ? `"${builder.schema}".` : ''}${builder.table} + ${formattedTable} (${columns.join(', ')}) ` From 5cf13a180153740eb9bff8068cefcd8ba3856bf5 Mon Sep 17 00:00:00 2001 From: Brayden Wilmoth Date: Thu, 29 Aug 2024 15:10:24 -0400 Subject: [PATCH 07/12] Support DuckDB dialect --- src/connections/motherduck.ts | 4 ++-- src/query-builder/dialects/duckdb.ts | 10 ++++++++++ src/query-builder/index.ts | 11 +++++++++-- 3 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 src/query-builder/dialects/duckdb.ts diff --git a/src/connections/motherduck.ts b/src/connections/motherduck.ts index 3f45c99..286b4e8 100644 --- a/src/connections/motherduck.ts +++ b/src/connections/motherduck.ts @@ -8,7 +8,7 @@ import { TableIndex, TableIndexType, } from '../models/database' -import { DefaultDialect } from '../query-builder/dialects/default' +import { DuckDbDialect } from '../query-builder/dialects/duckdb' import duckDB from 'duckdb' type DuckDBParameters = { @@ -24,7 +24,7 @@ export class DuckDBConnection implements Connection { queryType = QueryType.positional // Default dialect for MotherDuck - dialect = new DefaultDialect() + dialect = new DuckDbDialect() constructor(private _: DuckDBParameters) { this.duckDB = new duckDB.Database(_.path, { diff --git a/src/query-builder/dialects/duckdb.ts b/src/query-builder/dialects/duckdb.ts new file mode 100644 index 0000000..4f7346c --- /dev/null +++ b/src/query-builder/dialects/duckdb.ts @@ -0,0 +1,10 @@ +import { QueryBuilder } from 'src/client'; +import { Query } from 'src/query'; +import { QueryType } from 'src/query-params'; +import { AbstractDialect } from '../index'; + +export class DuckDbDialect extends AbstractDialect { + formatSchemaAndTable(schema: string | undefined, table: string): string { + return table; + } +} \ No newline at end of file diff --git a/src/query-builder/index.ts b/src/query-builder/index.ts index bf1d4fe..2c8e351 100644 --- a/src/query-builder/index.ts +++ b/src/query-builder/index.ts @@ -184,7 +184,13 @@ export abstract class AbstractDialect implements Dialect { * @returns The formatted schema and table combination. */ formatSchemaAndTable(schema: string | undefined, table: string): string { - // Default implementation (can be overridden by specific dialects) + if (schema) { + return `"${schema}".${table}`; + } + return table; + } + + formatFromSchemaAndTable(schema: string | undefined, table: string): string { if (schema) { return `"${schema}".${table}`; } @@ -202,6 +208,7 @@ export abstract class AbstractDialect implements Dialect { } const formattedTable = this.formatSchemaAndTable(set.schema, set.table); + const formattedFromTable = this.formatFromSchemaAndTable(set.schema, set.table); const columns = set.columns.map((column) => { let useColumn = column @@ -215,7 +222,7 @@ export abstract class AbstractDialect implements Dialect { selectColumns += columns.join(', ') if (index === 0) { - fromTable = formattedTable + fromTable = formattedFromTable } }) From 1ad333d2de918cfda51fcb4a1cd22ed4c0b70d4d Mon Sep 17 00:00:00 2001 From: Caleb Mabry <36182383+caleb-mabry@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:11:23 -0400 Subject: [PATCH 08/12] Motherduck database check --- src/connections/motherduck.ts | 65 +++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/src/connections/motherduck.ts b/src/connections/motherduck.ts index 3f45c99..22f8eb7 100644 --- a/src/connections/motherduck.ts +++ b/src/connections/motherduck.ts @@ -28,7 +28,7 @@ export class DuckDBConnection implements Connection { constructor(private _: DuckDBParameters) { this.duckDB = new duckDB.Database(_.path, { - motherduck_token: _.token + motherduck_token: _.token, }) this.connection = this.duckDB.connect() } @@ -82,10 +82,7 @@ export class DuckDBConnection implements Connection { ) result = res } else { - const { res } = await this.runQuery( - connection, - query.query - ) + const { res } = await this.runQuery(connection, query.query) result = res } } catch (e) { @@ -103,7 +100,7 @@ export class DuckDBConnection implements Connection { public async fetchDatabaseSchema(): Promise { let database: Database = [] - + type DuckDBTables = { database: string schema: string @@ -112,16 +109,32 @@ export class DuckDBConnection implements Connection { column_types: string[] temporary: boolean } - + type DuckDBSettingsRow = { + name: string + value: string + description: string + input_type: string + scope: string + } + const { data: currentDatabaseResponse, error } = await this.query({ + query: `SELECT * FROM duckdb_settings();`, + }) + + const currentDatabase = (currentDatabaseResponse as DuckDBSettingsRow[]) + .find((row) => row.name === 'temp_directory') + ?.value.split(':')[1] + .split('.')[0] const result = await this.query({ query: `PRAGMA show_tables_expanded;`, }) - + const tables = result.data as DuckDBTables[] - + const currentTables = tables.filter( + (table) => table.database === currentDatabase + ) const schemaMap: { [key: string]: Table[] } = {} - - for (const table of tables) { + + for (const table of currentTables) { type DuckDBTableInfo = { cid: number name: string @@ -133,9 +146,9 @@ export class DuckDBConnection implements Connection { const tableInfoResult = await this.query({ query: `PRAGMA table_info('${table.database}.${table.schema}.${table.name}')`, }) - + const tableInfo = tableInfoResult.data as DuckDBTableInfo[] - + const constraints: TableIndex[] = [] const columns = tableInfo.map((column) => { if (column.pk) { @@ -145,7 +158,7 @@ export class DuckDBConnection implements Connection { columns: [column.name], }) } - + const currentColumn: TableColumn = { name: column.name, type: column.type, @@ -156,50 +169,50 @@ export class DuckDBConnection implements Connection { unique: column.pk, references: [], // DuckDB currently doesn't have a pragma for foreign keys } - + return currentColumn }) - + const currentTable: Table = { name: table.name, - schema: table.schema, // Assign schema name to the table + schema: table.schema, // Assign schema name to the table columns: columns, indexes: constraints, } - + if (!schemaMap[table.schema]) { schemaMap[table.schema] = [] } - + schemaMap[table.schema].push(currentTable) } - + database = Object.entries(schemaMap).map(([schemaName, tables]) => { return { - [schemaName]: tables + [schemaName]: tables, } }) - + return database - } + } runQuery = async ( connection: duckDB.Connection, query: string, ...params: any[] - ): Promise<{ stmt: duckDB.Statement; res: any[]; }> => { + ): Promise<{ stmt: duckDB.Statement; res: any[] }> => { return new Promise((resolve, reject) => { connection.prepare(query, (err, stmt) => { if (err) { return reject(err) } - + stmt.all(...params, (err, res) => { if (err) { stmt.finalize() return reject(err) } - + resolve({ stmt, res }) stmt.finalize() }) From f49eecfb666a5073dd04de324c94c96f8171cf87 Mon Sep 17 00:00:00 2001 From: Brayden Wilmoth Date: Thu, 29 Aug 2024 15:26:57 -0400 Subject: [PATCH 09/12] Fix BigQuery FROM statements --- src/query-builder/dialects/bigquery.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/query-builder/dialects/bigquery.ts b/src/query-builder/dialects/bigquery.ts index 9331dd3..5cae8dd 100644 --- a/src/query-builder/dialects/bigquery.ts +++ b/src/query-builder/dialects/bigquery.ts @@ -11,6 +11,13 @@ export class BigQueryDialect extends AbstractDialect { return `\`${table}\``; } + formatFromSchemaAndTable(schema: string | undefined, table: string): string { + if (schema) { + return `\`${schema}.${table}\``; + } + return `\`${table}\``; + } + /** * BigQuery currently entirely reimplements the `select` method due to the specialized * nature of the BigQuery SQL dialect. When running a SELECT statement via BigQuery, From 2f9242844d4a725d9a411cbf1e6791aceac49a7a Mon Sep 17 00:00:00 2001 From: Brayden Wilmoth Date: Thu, 29 Aug 2024 16:02:01 -0400 Subject: [PATCH 10/12] Fix dialect for deleting data in dialect --- src/query-builder/dialects/bigquery.ts | 75 -------------------------- src/query-builder/index.ts | 2 +- 2 files changed, 1 insertion(+), 76 deletions(-) diff --git a/src/query-builder/dialects/bigquery.ts b/src/query-builder/dialects/bigquery.ts index 5cae8dd..1bd298c 100644 --- a/src/query-builder/dialects/bigquery.ts +++ b/src/query-builder/dialects/bigquery.ts @@ -17,79 +17,4 @@ export class BigQueryDialect extends AbstractDialect { } return `\`${table}\``; } - - /** - * BigQuery currently entirely reimplements the `select` method due to the specialized - * nature of the BigQuery SQL dialect. When running a SELECT statement via BigQuery, - * both the schema and table name must be escaped with backticks. Since we currently do - * not have a better proper extraction method for the schema and table name, we manually - * re-implement what was already in the AbstractDialect class. - * - * In the future we should find a way where this `select` method can be reused from the - * AbstractDialect class in a better manner. - * - * @param builder - * @param type - * @param query - * @returns Query - The query object with the query string and parameters - */ - // select(builder: QueryBuilder, type: QueryType, query: Query): Query { - // let selectColumns = '' - // let fromTable = '' - // const joinClauses = builder.joins?.join(' ') || '' - - // builder.columnsWithTable?.forEach((set, index) => { - // if (index > 0) { - // selectColumns += ', ' - // } - - // const schema = set.schema ? `${set.schema}.` : '' - // let useTable = this.reservedKeywords.includes(set.table) - // ? `"${set.table}"` - // : set.table - - // const columns = set.columns.map((column) => { - // let useColumn = column - - // if (this.reservedKeywords.includes(column)) { - // useColumn = `"${column}"` - // } - - // return `\`${schema ?? ''}${useTable}\`.${useColumn}` - // }) - - // selectColumns += columns.join(', ') - - // if (index === 0) { - // fromTable = `\`${schema}${useTable}\`` - // } - // }) - - // query.query = `SELECT ${selectColumns} FROM ${fromTable}` - - // if (joinClauses) { - // query.query += ` ${joinClauses}` - // } - - // if (builder.whereClauses?.length ?? 0 > 0) { - // query.query += ` WHERE ${builder?.whereClauses?.join(' AND ')}` - // } - - // if (builder.orderBy !== undefined) { - // query.query += ` ORDER BY ${builder.orderBy}` - // } - - // if (builder.limit !== undefined) { - // query.query += ` LIMIT ${builder.limit}` - // if (builder.offset) { - // query.query += ` OFFSET ${builder.offset}` - // } - // } - - // if (builder.groupBy !== undefined) { - // query.query += ` GROUP BY ${builder.groupBy}` - // } - - // return query - // } } \ No newline at end of file diff --git a/src/query-builder/index.ts b/src/query-builder/index.ts index 2c8e351..c4dfa90 100644 --- a/src/query-builder/index.ts +++ b/src/query-builder/index.ts @@ -313,7 +313,7 @@ export abstract class AbstractDialect implements Dialect { return query } - const formattedTable = this.formatSchemaAndTable(builder.schema, builder.table || ''); + const formattedTable = this.formatFromSchemaAndTable(builder.schema, builder.table || ''); query.query = `DELETE FROM ${formattedTable}` if (builder.whereClauses?.length > 0) { query.query += ` WHERE ${builder.whereClauses.join(' AND ')}` From f5d482143b6b6c9fb51ea8d6060029f6765759ca Mon Sep 17 00:00:00 2001 From: Brayden Wilmoth Date: Wed, 4 Sep 2024 12:39:25 -0400 Subject: [PATCH 11/12] Select raw support --- src/ai/prompts/ob1/3.ts | 39 +++++++++++ src/ai/providers/cloudflare.ts | 5 +- src/client.ts | 10 +++ src/query-builder/dialects/bigquery.ts | 3 - src/query-builder/dialects/duckdb.ts | 3 - src/query-builder/dialects/mysql.ts | 10 +-- src/query-builder/index.ts | 92 ++++++++++++++++++++------ 7 files changed, 127 insertions(+), 35 deletions(-) diff --git a/src/ai/prompts/ob1/3.ts b/src/ai/prompts/ob1/3.ts index 4c810c7..aa6912e 100644 --- a/src/ai/prompts/ob1/3.ts +++ b/src/ai/prompts/ob1/3.ts @@ -1,3 +1,42 @@ export const PROMPT = ` + Your job is to ONLY return a block of Javascript code for a Cloudflare worker. + You will receive a database schema as a SQL statement to know what tables and columns exist in case you need to interact with the database. + + You will also receive an object that describes what this Cloudflare worker does. + Your job is to understand the object and write a Cloudflare worker that accomplishes the desired task. + + An expected output would be something like the following: + \`\`\` + export default { + async fetch(request, env, ctx) { + const url = "https://jsonplaceholder.typicode.com/todos/1"; + + // gatherResponse returns both content-type & response body as a string + async function gatherResponse(response) { + const { headers } = response; + const contentType = headers.get("content-type") || ""; + if (contentType.includes("application/json")) { + return { contentType, result: JSON.stringify(await response.json()) }; + } + return { contentType, result: response.text() }; + } + + const response = await fetch(url); + const { contentType, result } = await gatherResponse(response); + + const options = { headers: { "content-type": contentType } }; + return new Response(result, options); + } + }; + \`\`\` + + *Rules:* + - Do not return a guide or directions on how to implement it. + - Only return a block of Javascript code! + - The code block should be able to be deployed to Cloudflare Workers. + - Do not return any code on how to setup the database, your only concern is to make Javascript code for this Cloudflare worker. + - If you need to interact with the database you can do so by doing "sdk.query()" and passing in the SQL query you need to run. + + Important: Only respond with a single block of Javascript code, nothing else. ` \ No newline at end of file diff --git a/src/ai/providers/cloudflare.ts b/src/ai/providers/cloudflare.ts index 3824771..1f968ca 100644 --- a/src/ai/providers/cloudflare.ts +++ b/src/ai/providers/cloudflare.ts @@ -7,8 +7,9 @@ export class CloudflareAi implements AiProvider { constructor(_?: { model: string, apiKey: string }) { if (!_) return; - this.model = _.model; - this.apiKey = _.apiKey; + + if (_.model) this.model = _.model; + if (_.apiKey) this.apiKey = _.apiKey; } async startConversation(systemPrompt: string, message: string): Promise { diff --git a/src/client.ts b/src/client.ts index 47abf43..d877f5e 100644 --- a/src/client.ts +++ b/src/client.ts @@ -39,6 +39,8 @@ export interface QueryBuilder { asClass?: any groupBy?: string + selectRawValue?: string + // General operation values, such as when renaming tables referencing the old and new name originalValue?: string newValue?: string @@ -49,6 +51,8 @@ export interface OuterbaseType { selectFrom: ( columnsArray: { schema?: string; table: string; columns: string[] }[] ) => OuterbaseType + selectRaw: (statement: string) => OuterbaseType + insert: (data: { [key: string]: any }) => OuterbaseType update: (data: { [key: string]: any }) => OuterbaseType deleteFrom: (table: string) => OuterbaseType @@ -112,6 +116,12 @@ export function Outerbase(connection: Connection): OuterbaseType { return this }, + selectRaw(statement) { + this.queryBuilder.action = QueryBuilderAction.SELECT + this.queryBuilder.selectRawValue = statement + + return this + }, where(condition) { // Check if `condition` is an array of conditions if (Array.isArray(condition)) { diff --git a/src/query-builder/dialects/bigquery.ts b/src/query-builder/dialects/bigquery.ts index 1bd298c..caad71c 100644 --- a/src/query-builder/dialects/bigquery.ts +++ b/src/query-builder/dialects/bigquery.ts @@ -1,6 +1,3 @@ -import { QueryBuilder } from 'src/client'; -import { Query } from 'src/query'; -import { QueryType } from 'src/query-params'; import { AbstractDialect } from '../index'; export class BigQueryDialect extends AbstractDialect { diff --git a/src/query-builder/dialects/duckdb.ts b/src/query-builder/dialects/duckdb.ts index 4f7346c..faa34f1 100644 --- a/src/query-builder/dialects/duckdb.ts +++ b/src/query-builder/dialects/duckdb.ts @@ -1,6 +1,3 @@ -import { QueryBuilder } from 'src/client'; -import { Query } from 'src/query'; -import { QueryType } from 'src/query-params'; import { AbstractDialect } from '../index'; export class DuckDbDialect extends AbstractDialect { diff --git a/src/query-builder/dialects/mysql.ts b/src/query-builder/dialects/mysql.ts index 591d916..ea1c01c 100644 --- a/src/query-builder/dialects/mysql.ts +++ b/src/query-builder/dialects/mysql.ts @@ -1,20 +1,14 @@ import { AbstractDialect, ColumnDataType } from '../index'; -import { QueryBuilder } from '../../client'; -import { QueryType } from '../../query-params'; export class MySQLDialect extends AbstractDialect { mapDataType(dataType: ColumnDataType): string { switch (dataType.toLowerCase()) { case ColumnDataType.STRING: - return 'VARCHAR(255)'; // MySQL specific VARCHAR length + return 'VARCHAR(255)'; case ColumnDataType.BOOLEAN: - return 'TINYINT(1)'; // MySQL uses TINYINT for boolean + return 'TINYINT(1)'; default: return super.mapDataType(dataType); } } - - // select(builder: QueryBuilder, type: QueryType): string { - // return `SELECT MYSQL` - // } } \ No newline at end of file diff --git a/src/query-builder/index.ts b/src/query-builder/index.ts index c4dfa90..ef1ac9c 100644 --- a/src/query-builder/index.ts +++ b/src/query-builder/index.ts @@ -15,6 +15,21 @@ interface Dialect { dropTable(builder: QueryBuilder, type: QueryType, query: Query): Query; renameTable(builder: QueryBuilder, type: QueryType, query: Query): Query; + // Column operations + // addColumn?: (name: string, table: string, schema?: string) => Promise + // dropColumn?: (name: string, table: string, schema?: string) => Promise + // renameColumn?: (name: string, original: string, table: string, schema?: string) => Promise + // updateColumn?: (name: string, column: TableColumn, table: string, schema?: string) => Promise + + // // // Index operations + // createIndex?: (index: TableIndex, table: string, schema?: string) => Promise + // dropIndex?: (name: string, table: string, schema?: string) => Promise + // renameIndex?: (name: string, original: string, table: string, schema?: string) => Promise + + // // // Schema operations + // createSchema?: (name: string) => Promise + // dropSchema?: (name: string) => Promise + equals(a: any, b: string): string; equalsNumber(a: any, b: any): string; equalsColumn(a: any, b: any): string; @@ -49,21 +64,6 @@ interface Dialect { notBetweenNumbers(a: any, b: any, c: any): string; ascending(a: any): string; descending(a: any): string; - - // Column operations - // addColumn?: (name: string, table: string, schema?: string) => Promise - // dropColumn?: (name: string, table: string, schema?: string) => Promise - // renameColumn?: (name: string, original: string, table: string, schema?: string) => Promise - // updateColumn?: (name: string, column: TableColumn, table: string, schema?: string) => Promise - - // // Index operations - // createIndex?: (index: TableIndex, table: string, schema?: string) => Promise - // dropIndex?: (name: string, table: string, schema?: string) => Promise - // renameIndex?: (name: string, original: string, table: string, schema?: string) => Promise - - // // Schema operations - // createSchema?: (name: string) => Promise - // dropSchema?: (name: string) => Promise } export enum ColumnDataType { @@ -79,6 +79,16 @@ export enum ColumnDataType { } export abstract class AbstractDialect implements Dialect { + /** + * WORK IN PROGRESS! + * This code is not used anywhere in the SDK at the moment. It is a work in progress to add support for SQL functions + * in the query builder. The idea is to allow users to use SQL functions in their queries, and the query builder will + * automatically format the query to use the correct SQL function for the specific database dialect. + * + * The `sqlFunctions` object is a map of SQL function names to their implementations. The `getFunction` method is used + * to get the implementation of a specific SQL function. The `addFunction` method is used to add a new SQL function + * to the `sqlFunctions` object. + */ protected sqlFunctions: { [key: string]: (...args: string[]) => string } = { now: () => 'NOW()', concat: (...args: string[]) => `CONCAT(${args.join(', ')})`, @@ -86,6 +96,12 @@ export abstract class AbstractDialect implements Dialect { abs: (value: string) => `ABS(${value})`, }; + /** + * Retrieves the implementation of the SQL function with the given name. + * + * @param funcName + * @returns Returns the implementation of the SQL function with the given name. + */ getFunction(funcName: string): (...args: string[]) => string { if (this.sqlFunctions[funcName]) { return this.sqlFunctions[funcName]; @@ -93,12 +109,23 @@ export abstract class AbstractDialect implements Dialect { throw new Error(`SQL function '${funcName}' not supported in this dialect.`); } - // Allow specific dialects to add or override functions + /** + * Adds a new SQL function to the `sqlFunctions` object. If a function with the same name already exists, it will be + * overwritten with the new implementation. + * + * @param funcName + * @param implementation + */ protected addFunction(funcName: string, implementation: (...args: string[]) => string) { this.sqlFunctions[funcName] = implementation; } - // Maps generic data types to database-specific data types + /** + * Maps the data type from the SDK to the equivalent data type for the specific database dialect. + * + * @param dataType + * @returns Returns the equivalent data type for the specific database dialect. + */ mapDataType(dataType: ColumnDataType | string): string { switch (dataType) { case ColumnDataType.STRING: @@ -114,6 +141,11 @@ export abstract class AbstractDialect implements Dialect { } } + /** + * Words that are reserved in the query language and may require special character wrapping to prevent + * collisions with the database engine executing the query. Each dialect may need to override this property + * with the reserved keywords for the specific database engine. + */ reservedKeywords: string[] = [ 'ADD', 'ALL', @@ -178,7 +210,13 @@ export abstract class AbstractDialect implements Dialect { ]; /** - * Formats the schema and table names according to the specific database dialect. + * Formats how the schema and table name should be used in the SELECT statement. + * + * @why When implementing support for BigQuery, the SELECT statement takes only a table name, where the FROM + * statement takes the schema and table name. It also requires both the schema and name to be wrapped in + * backticks together, and not separately. This method allows for formatting the schema and table name in a way + * that is compatible with the specific database dialect. + * See also - `formatFromSchemaAndTable` * @param schema The schema name (optional). * @param table The table name. * @returns The formatted schema and table combination. @@ -190,6 +228,18 @@ export abstract class AbstractDialect implements Dialect { return table; } + /** + * Formats how the schema and table name should be used in the FROM statement. + * + * @why When implementing support for BigQuery, the FROM statement takes a fully qualified schema and table name, + * where the SELECT statement only takes the table name. It also requires both the schema and name to be wrapped + * in backticks together, and not separately. This method allows for formatting the schema and table name in a way + * that is compatible with the specific database dialect. + * See also - `formatSchemaAndTable` + * @param schema + * @param table + * @returns The formatted schema and table combination. + */ formatFromSchemaAndTable(schema: string | undefined, table: string): string { if (schema) { return `"${schema}".${table}`; @@ -226,7 +276,11 @@ export abstract class AbstractDialect implements Dialect { } }) - query.query = `SELECT ${selectColumns} FROM ${fromTable}` + query.query = `SELECT + ${selectColumns} + ${builder.selectRawValue ? builder.selectRawValue : ''} + FROM ${fromTable} + ` if (joinClauses) { query.query += ` ${joinClauses}` From 2f90823b26b7edd2b92ba54a4b4d0206d0c1d793 Mon Sep 17 00:00:00 2001 From: Brayden Wilmoth Date: Wed, 4 Sep 2024 12:49:26 -0400 Subject: [PATCH 12/12] Format SQL better --- src/query-builder/index.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/query-builder/index.ts b/src/query-builder/index.ts index ef1ac9c..04404a2 100644 --- a/src/query-builder/index.ts +++ b/src/query-builder/index.ts @@ -276,11 +276,7 @@ export abstract class AbstractDialect implements Dialect { } }) - query.query = `SELECT - ${selectColumns} - ${builder.selectRawValue ? builder.selectRawValue : ''} - FROM ${fromTable} - ` + query.query = `SELECT ${selectColumns} ${builder.selectRawValue ? builder.selectRawValue : ''} FROM ${fromTable}` if (joinClauses) { query.query += ` ${joinClauses}`