Skip to content

Commit

Permalink
Merge pull request #49 from outerbase/bwilmoth/bigquery-dialect
Browse files Browse the repository at this point in the history
BigQuery Dialect
  • Loading branch information
Brayden committed Sep 6, 2024
2 parents 7237d8f + 2f90823 commit d68decb
Show file tree
Hide file tree
Showing 11 changed files with 224 additions and 137 deletions.
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@outerbase/sdk",
"version": "1.0.15",
"version": "1.0.17",
"description": "",
"main": "dist/index.js",
"module": "dist/index.js",
Expand Down
67 changes: 0 additions & 67 deletions playground/index.js
Original file line number Diff line number Diff line change
@@ -1,67 +0,0 @@
import { CloudflareD1Connection, Outerbase } from '../dist/index.js';
import express from 'express';

const app = express();
const port = 4000;

app.get('/test', async (req, res) => {
// Establish connection to your provider database
const d1 = new CloudflareD1Connection({
apiKey: '',
accountId: '',
databaseId: ''
});

const db = Outerbase(d1);
// const dbSchema = await d1.fetchDatabaseSchema()

// const { data, query } = await db
// .selectFrom([
// { table: 'test2', columns: ['*'] }
// ])
// .query()

// let { data, query } = await db
// .insert({ fname: 'John' })
// .into('test2')
// .returning(['id'])
// .query();

// let { data, query } = await db
// .update({ fname: 'Johnny' })
// .into('test2')
// .where(equals('id', '3', d1.dialect))
// .query();

// let { data, query } = await db
// .deleteFrom('test2')
// .where(equals('id', '3'))
// .query();

// let data = {}
// let query = await db
// .createTable('test3')
// .schema('public')
// .columns([
// { name: 'id', type: ColumnDataType.NUMBER, primaryKey: true },
// { name: 'fname', type: ColumnDataType.STRING }
// ])
// .toString();

// let data = {}
// let query = await db
// .renameTable('test3', 'test4')
// .toString();

// let data = {}
// let query = await db
// .dropTable('test4')
// .toString();

// let { data } = await db.queryRaw('SELECT * FROM playing_with_neon WHERE id = $1', ['1']);
res.json(data);
});

app.listen(port, () => {
console.log(`Server is running on http://localhost:${port}`);
});
39 changes: 39 additions & 0 deletions src/ai/prompts/ob1/3.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,42 @@
export const PROMPT = `
Your job is to ONLY return a block of Javascript code for a Cloudflare worker.
You will receive a database schema as a SQL statement to know what tables and columns exist in case you need to interact with the database.
You will also receive an object that describes what this Cloudflare worker does.
Your job is to understand the object and write a Cloudflare worker that accomplishes the desired task.
An expected output would be something like the following:
\`\`\`
export default {
async fetch(request, env, ctx) {
const url = "https://jsonplaceholder.typicode.com/todos/1";
// gatherResponse returns both content-type & response body as a string
async function gatherResponse(response) {
const { headers } = response;
const contentType = headers.get("content-type") || "";
if (contentType.includes("application/json")) {
return { contentType, result: JSON.stringify(await response.json()) };
}
return { contentType, result: response.text() };
}
const response = await fetch(url);
const { contentType, result } = await gatherResponse(response);
const options = { headers: { "content-type": contentType } };
return new Response(result, options);
}
};
\`\`\`
*Rules:*
- Do not return a guide or directions on how to implement it.
- Only return a block of Javascript code!
- The code block should be able to be deployed to Cloudflare Workers.
- Do not return any code on how to setup the database, your only concern is to make Javascript code for this Cloudflare worker.
- If you need to interact with the database you can do so by doing "sdk.query()" and passing in the SQL query you need to run.
Important: Only respond with a single block of Javascript code, nothing else.
`
5 changes: 3 additions & 2 deletions src/ai/providers/cloudflare.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ export class CloudflareAi implements AiProvider {

constructor(_?: { model: string, apiKey: string }) {
if (!_) return;
this.model = _.model;
this.apiKey = _.apiKey;

if (_.model) this.model = _.model;
if (_.apiKey) this.apiKey = _.apiKey;
}

async startConversation(systemPrompt: string, message: string): Promise<OperationResponse> {
Expand Down
10 changes: 10 additions & 0 deletions src/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ export interface QueryBuilder {
asClass?: any
groupBy?: string

selectRawValue?: string

// General operation values, such as when renaming tables referencing the old and new name
originalValue?: string
newValue?: string
Expand All @@ -49,6 +51,8 @@ export interface OuterbaseType {
selectFrom: (
columnsArray: { schema?: string; table: string; columns: string[] }[]
) => OuterbaseType
selectRaw: (statement: string) => OuterbaseType

insert: (data: { [key: string]: any }) => OuterbaseType
update: (data: { [key: string]: any }) => OuterbaseType
deleteFrom: (table: string) => OuterbaseType
Expand Down Expand Up @@ -112,6 +116,12 @@ export function Outerbase(connection: Connection): OuterbaseType {

return this
},
selectRaw(statement) {
this.queryBuilder.action = QueryBuilderAction.SELECT
this.queryBuilder.selectRawValue = statement

return this
},
where(condition) {
// Check if `condition` is an array of conditions
if (Array.isArray(condition)) {
Expand Down
4 changes: 2 additions & 2 deletions src/connections/bigquery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { QueryType } from '../query-params'
import { Query, constructRawQuery } from '../query'
import { Connection } from './index'
import { Database, Table, TableColumn } from '../models/database'
import { DefaultDialect } from '../query-builder/dialects/default'
import { BigQueryDialect } from '../query-builder/dialects/bigquery'

import { BigQuery } from '@google-cloud/bigquery'

Expand All @@ -19,7 +19,7 @@ export class BigQueryConnection implements Connection {
queryType = QueryType.positional

// Default dialect for BigQuery
dialect = new DefaultDialect()
dialect = new BigQueryDialect()

/**
* Creates a new BigQuery object.
Expand Down
69 changes: 41 additions & 28 deletions src/connections/motherduck.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {
TableIndex,
TableIndexType,
} from '../models/database'
import { DefaultDialect } from '../query-builder/dialects/default'
import { DuckDbDialect } from '../query-builder/dialects/duckdb'
import duckDB from 'duckdb'

type DuckDBParameters = {
Expand All @@ -24,11 +24,11 @@ export class DuckDBConnection implements Connection {
queryType = QueryType.positional

// Default dialect for MotherDuck
dialect = new DefaultDialect()
dialect = new DuckDbDialect()

constructor(private _: DuckDBParameters) {
this.duckDB = new duckDB.Database(_.path, {
motherduck_token: _.token
motherduck_token: _.token,
})
this.connection = this.duckDB.connect()
}
Expand Down Expand Up @@ -82,10 +82,7 @@ export class DuckDBConnection implements Connection {
)
result = res
} else {
const { res } = await this.runQuery(
connection,
query.query
)
const { res } = await this.runQuery(connection, query.query)
result = res
}
} catch (e) {
Expand All @@ -103,7 +100,7 @@ export class DuckDBConnection implements Connection {

public async fetchDatabaseSchema(): Promise<Database> {
let database: Database = []

type DuckDBTables = {
database: string
schema: string
Expand All @@ -112,16 +109,32 @@ export class DuckDBConnection implements Connection {
column_types: string[]
temporary: boolean
}

type DuckDBSettingsRow = {
name: string
value: string
description: string
input_type: string
scope: string
}
const { data: currentDatabaseResponse, error } = await this.query({
query: `SELECT * FROM duckdb_settings();`,
})

const currentDatabase = (currentDatabaseResponse as DuckDBSettingsRow[])
.find((row) => row.name === 'temp_directory')
?.value.split(':')[1]
.split('.')[0]
const result = await this.query({
query: `PRAGMA show_tables_expanded;`,
})

const tables = result.data as DuckDBTables[]

const currentTables = tables.filter(
(table) => table.database === currentDatabase
)
const schemaMap: { [key: string]: Table[] } = {}
for (const table of tables) {

for (const table of currentTables) {
type DuckDBTableInfo = {
cid: number
name: string
Expand All @@ -133,9 +146,9 @@ export class DuckDBConnection implements Connection {
const tableInfoResult = await this.query({
query: `PRAGMA table_info('${table.database}.${table.schema}.${table.name}')`,
})

const tableInfo = tableInfoResult.data as DuckDBTableInfo[]

const constraints: TableIndex[] = []
const columns = tableInfo.map((column) => {
if (column.pk) {
Expand All @@ -145,7 +158,7 @@ export class DuckDBConnection implements Connection {
columns: [column.name],
})
}

const currentColumn: TableColumn = {
name: column.name,
type: column.type,
Expand All @@ -156,50 +169,50 @@ export class DuckDBConnection implements Connection {
unique: column.pk,
references: [], // DuckDB currently doesn't have a pragma for foreign keys
}

return currentColumn
})

const currentTable: Table = {
name: table.name,
schema: table.schema, // Assign schema name to the table
schema: table.schema, // Assign schema name to the table
columns: columns,
indexes: constraints,
}

if (!schemaMap[table.schema]) {
schemaMap[table.schema] = []
}

schemaMap[table.schema].push(currentTable)
}

database = Object.entries(schemaMap).map(([schemaName, tables]) => {
return {
[schemaName]: tables
[schemaName]: tables,
}
})

return database
}
}

runQuery = async (
connection: duckDB.Connection,
query: string,
...params: any[]
): Promise<{ stmt: duckDB.Statement; res: any[]; }> => {
): Promise<{ stmt: duckDB.Statement; res: any[] }> => {
return new Promise((resolve, reject) => {
connection.prepare(query, (err, stmt) => {
if (err) {
return reject(err)
}

stmt.all(...params, (err, res) => {
if (err) {
stmt.finalize()
return reject(err)
}

resolve({ stmt, res })
stmt.finalize()
})
Expand Down
17 changes: 17 additions & 0 deletions src/query-builder/dialects/bigquery.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import { AbstractDialect } from '../index';

export class BigQueryDialect extends AbstractDialect {
formatSchemaAndTable(schema: string | undefined, table: string): string {
if (schema) {
return `\`${schema}.${table}\``;
}
return `\`${table}\``;
}

formatFromSchemaAndTable(schema: string | undefined, table: string): string {
if (schema) {
return `\`${schema}.${table}\``;
}
return `\`${table}\``;
}
}
7 changes: 7 additions & 0 deletions src/query-builder/dialects/duckdb.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
import { AbstractDialect } from '../index';

export class DuckDbDialect extends AbstractDialect {
formatSchemaAndTable(schema: string | undefined, table: string): string {
return table;
}
}
10 changes: 2 additions & 8 deletions src/query-builder/dialects/mysql.ts
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
import { AbstractDialect, ColumnDataType } from '../index';
import { QueryBuilder } from '../../client';
import { QueryType } from '../../query-params';

export class MySQLDialect extends AbstractDialect {
mapDataType(dataType: ColumnDataType): string {
switch (dataType.toLowerCase()) {
case ColumnDataType.STRING:
return 'VARCHAR(255)'; // MySQL specific VARCHAR length
return 'VARCHAR(255)';
case ColumnDataType.BOOLEAN:
return 'TINYINT(1)'; // MySQL uses TINYINT for boolean
return 'TINYINT(1)';
default:
return super.mapDataType(dataType);
}
}

// select(builder: QueryBuilder, type: QueryType): string {
// return `SELECT MYSQL`
// }
}
Loading

0 comments on commit d68decb

Please sign in to comment.