diff --git a/playground/index.js b/playground/index.js index 5cce2f5..964caef 100644 --- a/playground/index.js +++ b/playground/index.js @@ -1,11 +1,10 @@ -import { CloudflareD1Connection, Outerbase, NeonHttpConnection, OuterbaseConnection, equalsNumber, equals } from '../dist/index.js'; +import { CloudflareD1Connection, Outerbase } from '../dist/index.js'; import express from 'express'; -import { ColumnDataType } from '../dist/query-builder/index.js'; const app = express(); const port = 4000; -app.get('/test/cloudflare', async (req, res) => { +app.get('/test', async (req, res) => { // Establish connection to your provider database const d1 = new CloudflareD1Connection({ apiKey: '', @@ -16,9 +15,11 @@ app.get('/test/cloudflare', async (req, res) => { const db = Outerbase(d1); // const dbSchema = await d1.fetchDatabaseSchema() - // const { data, query } = await db.selectFrom([ - // { table: 'test2', columns: ['*'] } - // ]).query() + // const { data, query } = await db + // .selectFrom([ + // { table: 'test2', columns: ['*'] } + // ]) + // .query() // let { data, query } = await db // .insert({ fname: 'John' }) @@ -37,15 +38,15 @@ app.get('/test/cloudflare', async (req, res) => { // .where(equals('id', '3')) // .query(); - let data = {} - let query = await db - .createTable('test3') - .schema('public') - .columns([ - { name: 'id', type: ColumnDataType.NUMBER, primaryKey: true }, - { name: 'fname', type: ColumnDataType.STRING } - ]) - .toString(); + // let data = {} + // let query = await db + // .createTable('test3') + // .schema('public') + // .columns([ + // { name: 'id', type: ColumnDataType.NUMBER, primaryKey: true }, + // { name: 'fname', type: ColumnDataType.STRING } + // ]) + // .toString(); // let data = {} // let query = await db @@ -57,14 +58,6 @@ app.get('/test/cloudflare', async (req, res) => { // .dropTable('test4') // .toString(); - console.log('Running Query: ', query) - - // db. - // - ACTION - // - CONDITIONS - // - RETURNING - // - query() / toString() - // let { data } = await db.queryRaw('SELECT * FROM playing_with_neon WHERE id = $1', ['1']); res.json(data); }); diff --git a/playground/package-lock.json b/playground/package-lock.json new file mode 100644 index 0000000..00fb945 --- /dev/null +++ b/playground/package-lock.json @@ -0,0 +1,630 @@ +{ + "name": "playground", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "playground", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "express": "^4.19.2" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "1.20.2", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind": { + "version": "1.0.7", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.6.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "license": "MIT" + }, + "node_modules/debug": { + "version": "2.6.9", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "1.0.2", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.19.2", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.2", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.6.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.2.0", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/finalhandler": { + "version": "1.2.0", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.2.4", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gopd": { + "version": "1.0.1", + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-proto": { + "version": "1.0.3", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.0", + "license": "MIT", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "license": "MIT" + }, + "node_modules/methods": { + "version": "1.1.2", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.1", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.7", + "license": "MIT" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.11.0", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.2", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "license": "MIT" + }, + "node_modules/send": { + "version": "0.18.0", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.15.0", + "license": "MIT", + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.18.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/set-function-length": { + "version": "1.2.2", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.0.6", + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.1", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + } + } +} diff --git a/src/connections/cloudflare.ts b/src/connections/cloudflare.ts index 8fd4008..0d2d6bd 100644 --- a/src/connections/cloudflare.ts +++ b/src/connections/cloudflare.ts @@ -116,28 +116,25 @@ export class CloudflareD1Connection implements Connection { public async fetchDatabaseSchema(): Promise { const exclude_tables = ['_cf_kv'] - + let database: Database = [] - let schema: Schema = { - tables: [] - } - + const schemaMap: { [key: string]: Table[] } = {} + const { data } = await this.query({ query: `SELECT * FROM sqlite_master WHERE type='table' AND name NOT LIKE '_lite%' AND name NOT LIKE 'sqlite_%'` }) - + for (const table of data) { // Skip excluded tables if (exclude_tables.includes(table.name?.toLowerCase())) continue; - - // Add tables to the schema + if (table.type === 'table') { - const { data: tableData } = await this.query({ query: `SELECT * FROM PRAGMA_TABLE_INFO('${table.name}')`}) - + const { data: tableData } = await this.query({ query: `PRAGMA table_info('${table.name}')` }) + // TODO: This is not returning any data. Need to investigate why. - const { data: indexData } = await this.query({ query: `PRAGMA index_list('${table.name}')`}) - + const { data: indexData } = await this.query({ query: `PRAGMA index_list('${table.name}')` }) + console.log('Table: ', tableData) console.log('Index: ', indexData) - + let constraints: TableIndex[] = [] let columns = tableData.map((column: any) => { if (column.pk === 1) { @@ -147,7 +144,7 @@ export class CloudflareD1Connection implements Connection { columns: [column.name] }) } - + const currentColumn: TableColumn = { name: column.name, type: column.type, @@ -159,23 +156,34 @@ export class CloudflareD1Connection implements Connection { // TODO: Need to support foreign key references references: [] } - + return currentColumn }) - + let current: Table = { name: table.name, columns: columns, indexes: constraints } - - schema.tables.push(current) + + // Use the table's schema name (assuming all tables belong to the same schema, e.g., "main") + const schemaName = 'main' // Replace with the actual schema name if needed + + if (!schemaMap[schemaName]) { + schemaMap[schemaName] = [] + } + + schemaMap[schemaName].push(current) } } - - // Add schema to database - database.push(schema) - + + // Transform schemaMap into the final Database structure + database = Object.entries(schemaMap).map(([schemaName, tables]) => { + return { + [schemaName]: tables + } + }) + return database - } + } } diff --git a/src/connections/motherduck.ts b/src/connections/motherduck.ts index b7b0f72..3f45c99 100644 --- a/src/connections/motherduck.ts +++ b/src/connections/motherduck.ts @@ -1,70 +1,35 @@ import { QueryType } from '../query-params' -import { Query } from '../query' +import { Query, constructRawQuery } from '../query' import { Connection } from './index' import { Database, - Schema, Table, TableColumn, TableIndex, TableIndexType, } from '../models/database' import { DefaultDialect } from '../query-builder/dialects/default' -import duckDB, { Callback, DuckDbError } from 'duckdb' +import duckDB from 'duckdb' -const reconstructQuery = (query: string, params: any[]): string => { - let i = 0 - return query.replace(/\?/g, () => { - const param = params[i++] - if (typeof param === 'string') { - return `'${param.replace(/'/g, "''")}'` // Properly escape single quotes in strings - } - if (param === null) { - return 'NULL' - } - return param - }) -} - -const runQuery = async ( - connection: duckDB.Connection, - query: string, - ...params: any[] -): Promise<{ stmt: duckDB.Statement; res: any[]; fullQuery: string }> => { - return new Promise((resolve, reject) => { - connection.prepare(query, (err, stmt) => { - if (err) { - return reject(err) - } - - const fullQuery = reconstructQuery(query, params) - - stmt.all(...params, (err, res) => { - if (err) { - stmt.finalize() - return reject(err) - } - - resolve({ stmt, res, fullQuery }) - stmt.finalize() - }) - }) - }) -} type DuckDBParameters = { path: string - options?: Record - callback?: duckDB.Callback + token: string } + export class DuckDBConnection implements Connection { + duckDB: duckDB.Database | undefined + connection: duckDB.Connection | undefined + + // Default query type to positional for MotherDuck queryType = QueryType.positional + // Default dialect for MotherDuck dialect = new DefaultDialect() - duckDB: duckDB.Database | undefined - connection: duckDB.Connection | undefined constructor(private _: DuckDBParameters) { - this.duckDB = new duckDB.Database(_.path, _.options, _.callback) + this.duckDB = new duckDB.Database(_.path, { + motherduck_token: _.token + }) this.connection = this.duckDB.connect() } @@ -103,49 +68,42 @@ export class DuckDBConnection implements Connection { query: Query ): Promise<{ data: any; error: Error | null; query: string }> { const connection = this.connection - if (!connection) throw new Error('Please create a connection.') + if (!connection) throw new Error('No DuckDB connection was found.') - let result - let statement = reconstructQuery(query.query, query.parameters as any[]) + let result = null + let error = null try { if (Array.isArray(query.parameters)) { - const { res, fullQuery } = await runQuery( + const { res } = await this.runQuery( connection, query.query, ...query.parameters ) result = res - statement = fullQuery } else { - const { res, fullQuery } = await runQuery( + const { res } = await this.runQuery( connection, query.query ) result = res - statement = fullQuery - } - - return { - data: result, - error: null, - query: statement, } } catch (e) { - const error = e instanceof Error ? e : new Error(String(e)) - return { - data: null, - error: error, - query: statement, - } + error = e instanceof Error ? e : new Error(String(e)) + } + + const rawSQL = constructRawQuery(query) + + return { + data: result, + error: error, + query: rawSQL, } } + public async fetchDatabaseSchema(): Promise { let database: Database = [] - let schema: Schema = { - tables: [], - } - + type DuckDBTables = { database: string schema: string @@ -154,13 +112,15 @@ export class DuckDBConnection implements Connection { column_types: string[] temporary: boolean } - + const result = await this.query({ query: `PRAGMA show_tables_expanded;`, }) - + const tables = result.data as DuckDBTables[] - + + const schemaMap: { [key: string]: Table[] } = {} + for (const table of tables) { type DuckDBTableInfo = { cid: number @@ -173,9 +133,9 @@ export class DuckDBConnection implements Connection { const tableInfoResult = await this.query({ query: `PRAGMA table_info('${table.database}.${table.schema}.${table.name}')`, }) - + const tableInfo = tableInfoResult.data as DuckDBTableInfo[] - + const constraints: TableIndex[] = [] const columns = tableInfo.map((column) => { if (column.pk) { @@ -185,7 +145,7 @@ export class DuckDBConnection implements Connection { columns: [column.name], }) } - + const currentColumn: TableColumn = { name: column.name, type: column.type, @@ -196,21 +156,54 @@ export class DuckDBConnection implements Connection { unique: column.pk, references: [], // DuckDB currently doesn't have a pragma for foreign keys } - + return currentColumn }) - + const currentTable: Table = { name: table.name, + schema: table.schema, // Assign schema name to the table columns: columns, indexes: constraints, } - - schema.tables.push(currentTable) + + if (!schemaMap[table.schema]) { + schemaMap[table.schema] = [] + } + + schemaMap[table.schema].push(currentTable) } - - database.push(schema) - + + database = Object.entries(schemaMap).map(([schemaName, tables]) => { + return { + [schemaName]: tables + } + }) + return database + } + + runQuery = async ( + connection: duckDB.Connection, + query: string, + ...params: any[] + ): Promise<{ stmt: duckDB.Statement; res: any[]; }> => { + return new Promise((resolve, reject) => { + connection.prepare(query, (err, stmt) => { + if (err) { + return reject(err) + } + + stmt.all(...params, (err, res) => { + if (err) { + stmt.finalize() + return reject(err) + } + + resolve({ stmt, res }) + stmt.finalize() + }) + }) + }) } } diff --git a/src/models/database.ts b/src/models/database.ts index de2c7ab..cc614dc 100644 --- a/src/models/database.ts +++ b/src/models/database.ts @@ -2,12 +2,7 @@ export type Database = Schema[] // Schema consists of a name and an array of tables -export type Schema = { - // Unique name of the schema, some databases may not support this - name?: string - // Tables that are part of the schema - tables: Table[] -} +export type Schema = Record // Table consists of a name, schema, an array of columns, and an array of indexes export type Table = {