From 4a2757ce080491f0c57d02945eb183f23d08377a Mon Sep 17 00:00:00 2001 From: Leon Alvarez Del Canto Date: Tue, 30 May 2023 22:56:37 +0200 Subject: [PATCH 01/12] Fix jsonb field escaping data before insert fix issue with escaping data on jsonb pg --- drizzle-orm/src/pg-core/columns/jsonb.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drizzle-orm/src/pg-core/columns/jsonb.ts b/drizzle-orm/src/pg-core/columns/jsonb.ts index e4bbc1bdf..a5a3c2f5f 100644 --- a/drizzle-orm/src/pg-core/columns/jsonb.ts +++ b/drizzle-orm/src/pg-core/columns/jsonb.ts @@ -41,8 +41,8 @@ export class PgJsonb extends PgColumn return 'jsonb'; } - override mapToDriverValue(value: T['data']): string { - return JSON.stringify(value); + override mapToDriverValue(value: T['data']): T['data'] { + return value; } override mapFromDriverValue(value: T['data'] | string): T['data'] { From 2335b3edc9a82dd0adbbd4ccd23ef90125c07d40 Mon Sep 17 00:00:00 2001 From: LeonAlvarez Date: Sun, 4 Jun 2023 09:35:56 +0200 Subject: [PATCH 02/12] fix postgres.js.tests.ts --- integration-tests/tests/postgres.js.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-tests/tests/postgres.js.test.ts b/integration-tests/tests/postgres.js.test.ts index 124b741e4..4223e0278 100644 --- a/integration-tests/tests/postgres.js.test.ts +++ b/integration-tests/tests/postgres.js.test.ts @@ -771,7 +771,7 @@ test.serial('build query insert with onConflict do update', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do update set "name" = $3', - params: ['John', '["foo","bar"]', 'John1'], + params: ['John', ["foo","bar"], 'John1'], }); }); @@ -785,7 +785,7 @@ test.serial('build query insert with onConflict do update / multiple columns', a t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id","name") do update set "name" = $3', - params: ['John', '["foo","bar"]', 'John1'], + params: ['John', ["foo","bar"], 'John1'], }); }); @@ -799,7 +799,7 @@ test.serial('build query insert with onConflict do nothing', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict do nothing', - params: ['John', '["foo","bar"]'], + params: ['John', ["foo","bar"]], }); }); @@ -813,7 +813,7 @@ test.serial('build query insert with onConflict do nothing + target', async (t) t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do nothing', - params: ['John', '["foo","bar"]'], + params: ['John', ["foo","bar"]], }); }); From b014fd86d9faacbfa1e81ee785ca241eb02fdb8e Mon Sep 17 00:00:00 2001 From: LeonAlvarez Date: Sat, 10 Jun 2023 12:52:09 +0200 Subject: [PATCH 03/12] postgres.js jsonb test improvements --- integration-tests/tests/postgres.js.test.ts | 41 ++++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/integration-tests/tests/postgres.js.test.ts b/integration-tests/tests/postgres.js.test.ts index 4223e0278..1deff11d5 100644 --- a/integration-tests/tests/postgres.js.test.ts +++ b/integration-tests/tests/postgres.js.test.ts @@ -92,6 +92,16 @@ const usersMigratorTable = pgTable('users12', { email: text('email').notNull(), }); +type MetaData = { + foo: string; + bar: number; +} +const metaDataTable = pgTable('meta_data', { + id: serial('id').primaryKey(), + data: jsonb('data').$type(), +}); + + interface Context { docker: Docker; pgContainer: Docker.Container; @@ -226,6 +236,14 @@ test.beforeEach(async (t) => { ) `, ); + await ctx.db.execute( + sql` + create table meta_data ( + id serial primary key, + data jsonb + ) + `, + ); }); test.serial('select all fields', async (t) => { @@ -360,6 +378,20 @@ test.serial('insert + select', async (t) => { ]); }); +test.serial('json object insert', async (t) => { + const { db } = t.context; + + await db.insert(metaDataTable).values({ data: {foo: 'bar', bar: 33} }); + const result = await db.select({ + id: metaDataTable.id, + data: metaDataTable.data, + }).from(metaDataTable).where( + sql.raw("data->>`foo`=`bar`") + ).toSQL() + + t.deepEqual(result, [{ id: 1, data: {foo: 'bar', bar: 33}}]); +}); + test.serial('json insert', async (t) => { const { db } = t.context; @@ -368,8 +400,13 @@ test.serial('json insert', async (t) => { id: usersTable.id, name: usersTable.name, jsonb: usersTable.jsonb, - }).from(usersTable); - + }).from(usersTable).where( + and( + eq(usersTable.name, 'John'), + sql.raw(`jsonb->>0 = 'foo'`), + ) + ); + t.deepEqual(result, [{ id: 1, name: 'John', jsonb: ['foo', 'bar'] }]); }); From 38d63fa56aedf25b0bbcec6c1f9b70726ce92d80 Mon Sep 17 00:00:00 2001 From: LeonAlvarez Date: Sat, 10 Jun 2023 13:04:07 +0200 Subject: [PATCH 04/12] fix pg tests array jsonb for pg tests --- integration-tests/tests/awsdatapi.test.ts | 8 ++++---- integration-tests/tests/pg-schema.test.ts | 8 ++++---- integration-tests/tests/pg.custom.test.ts | 8 ++++---- integration-tests/tests/pg.test.ts | 8 ++++---- integration-tests/tests/vercel-pg.test.ts | 8 ++++---- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/integration-tests/tests/awsdatapi.test.ts b/integration-tests/tests/awsdatapi.test.ts index 9cba8c1db..c2d2eae9d 100644 --- a/integration-tests/tests/awsdatapi.test.ts +++ b/integration-tests/tests/awsdatapi.test.ts @@ -569,7 +569,7 @@ test.serial('build query insert with onConflict do update', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values (:1, :2) on conflict ("id") do update set "name" = :3', - params: ['John', '["foo","bar"]', 'John1'], + params: ['John', ["foo","bar"], 'John1'], // typings: ['none', 'json', 'none'] }); }); @@ -584,7 +584,7 @@ test.serial('build query insert with onConflict do update / multiple columns', a t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values (:1, :2) on conflict ("id","name") do update set "name" = :3', - params: ['John', '["foo","bar"]', 'John1'], + params: ['John', ["foo","bar"], 'John1'], // typings: ['none', 'json', 'none'] }); }); @@ -599,7 +599,7 @@ test.serial('build query insert with onConflict do nothing', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values (:1, :2) on conflict do nothing', - params: ['John', '["foo","bar"]'], + params: ['John', ["foo","bar"]], // typings: ['none', 'json'] }); }); @@ -614,7 +614,7 @@ test.serial('build query insert with onConflict do nothing + target', async (t) t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values (:1, :2) on conflict ("id") do nothing', - params: ['John', '["foo","bar"]'], + params: ['John', ["foo","bar"]], // typings: ['none', 'json'] }); }); diff --git a/integration-tests/tests/pg-schema.test.ts b/integration-tests/tests/pg-schema.test.ts index f126dccd5..92fc28641 100644 --- a/integration-tests/tests/pg-schema.test.ts +++ b/integration-tests/tests/pg-schema.test.ts @@ -653,7 +653,7 @@ test.serial('build query insert with onConflict do update', async (t) => { t.deepEqual(query, { sql: 'insert into "mySchema"."users" ("name", "jsonb") values ($1, $2) on conflict ("id") do update set "name" = $3', - params: ['John', '["foo","bar"]', 'John1'], + params: ['John', ["foo","bar"], 'John1'], }); }); @@ -668,7 +668,7 @@ test.serial('build query insert with onConflict do update / multiple columns', a t.deepEqual(query, { sql: 'insert into "mySchema"."users" ("name", "jsonb") values ($1, $2) on conflict ("id","name") do update set "name" = $3', - params: ['John', '["foo","bar"]', 'John1'], + params: ['John', ["foo","bar"], 'John1'], }); }); @@ -682,7 +682,7 @@ test.serial('build query insert with onConflict do nothing', async (t) => { t.deepEqual(query, { sql: 'insert into "mySchema"."users" ("name", "jsonb") values ($1, $2) on conflict do nothing', - params: ['John', '["foo","bar"]'], + params: ['John', ["foo","bar"]], }); }); @@ -696,7 +696,7 @@ test.serial('build query insert with onConflict do nothing + target', async (t) t.deepEqual(query, { sql: 'insert into "mySchema"."users" ("name", "jsonb") values ($1, $2) on conflict ("id") do nothing', - params: ['John', '["foo","bar"]'], + params: ['John', ["foo","bar"]], }); }); diff --git a/integration-tests/tests/pg.custom.test.ts b/integration-tests/tests/pg.custom.test.ts index e2a1f67d1..231c16b81 100644 --- a/integration-tests/tests/pg.custom.test.ts +++ b/integration-tests/tests/pg.custom.test.ts @@ -661,7 +661,7 @@ test.serial('build query insert with onConflict do update', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do update set "name" = $3', - params: ['John', '["foo","bar"]', 'John1'], + params: ['John', ["foo","bar"], 'John1'], }); }); @@ -675,7 +675,7 @@ test.serial('build query insert with onConflict do update / multiple columns', a t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id","name") do update set "name" = $3', - params: ['John', '["foo","bar"]', 'John1'], + params: ['John', ["foo","bar"], 'John1'], }); }); @@ -689,7 +689,7 @@ test.serial('build query insert with onConflict do nothing', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict do nothing', - params: ['John', '["foo","bar"]'], + params: ['John', ["foo","bar"]], }); }); @@ -703,7 +703,7 @@ test.serial('build query insert with onConflict do nothing + target', async (t) t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do nothing', - params: ['John', '["foo","bar"]'], + params: ['John', ["foo","bar"]], }); }); diff --git a/integration-tests/tests/pg.test.ts b/integration-tests/tests/pg.test.ts index b81bc9981..1f87d3185 100644 --- a/integration-tests/tests/pg.test.ts +++ b/integration-tests/tests/pg.test.ts @@ -927,7 +927,7 @@ test.serial('build query insert with onConflict do update', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do update set "name" = $3', - params: ['John', '["foo","bar"]', 'John1'], + params: ['John', ["foo","bar"], 'John1'], }); }); @@ -942,7 +942,7 @@ test.serial('build query insert with onConflict do update / multiple columns', a t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id","name") do update set "name" = $3', - params: ['John', '["foo","bar"]', 'John1'], + params: ['John', ["foo","bar"], 'John1'], }); }); @@ -957,7 +957,7 @@ test.serial('build query insert with onConflict do nothing', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict do nothing', - params: ['John', '["foo","bar"]'], + params: ['John', ["foo","bar"]], }); }); @@ -972,7 +972,7 @@ test.serial('build query insert with onConflict do nothing + target', async (t) t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do nothing', - params: ['John', '["foo","bar"]'], + params: ['John', ["foo","bar"]], }); }); diff --git a/integration-tests/tests/vercel-pg.test.ts b/integration-tests/tests/vercel-pg.test.ts index fa97a529c..b468102d8 100644 --- a/integration-tests/tests/vercel-pg.test.ts +++ b/integration-tests/tests/vercel-pg.test.ts @@ -927,7 +927,7 @@ test.serial('build query insert with onConflict do update', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do update set "name" = $3', - params: ['John', '["foo","bar"]', 'John1'], + params: ['John', ["foo","bar"], 'John1'], }); }); @@ -942,7 +942,7 @@ test.serial('build query insert with onConflict do update / multiple columns', a t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id","name") do update set "name" = $3', - params: ['John', '["foo","bar"]', 'John1'], + params: ['John', ["foo","bar"], 'John1'], }); }); @@ -957,7 +957,7 @@ test.serial('build query insert with onConflict do nothing', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict do nothing', - params: ['John', '["foo","bar"]'], + params: ['John', ["foo","bar"]], }); }); @@ -972,7 +972,7 @@ test.serial('build query insert with onConflict do nothing + target', async (t) t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do nothing', - params: ['John', '["foo","bar"]'], + params: ['John', ["foo","bar"]], }); }); From ab9171ef3ac96d9951f3e8e3a75df19cc6f019a1 Mon Sep 17 00:00:00 2001 From: LeonAlvarez Date: Sat, 10 Jun 2023 13:52:09 +0200 Subject: [PATCH 05/12] WIP - custom dialect for postgresjs - revert pg-core changes --- drizzle-orm/src/pg-core/columns/jsonb.ts | 4 +- drizzle-orm/src/postgres-js/columns/jsonb.ts | 58 ++++++++++++++++++++ drizzle-orm/src/postgres-js/dialect.ts | 38 +++++++++++++ drizzle-orm/src/postgres-js/driver.ts | 4 +- 4 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 drizzle-orm/src/postgres-js/columns/jsonb.ts create mode 100644 drizzle-orm/src/postgres-js/dialect.ts diff --git a/drizzle-orm/src/pg-core/columns/jsonb.ts b/drizzle-orm/src/pg-core/columns/jsonb.ts index a5a3c2f5f..e4bbc1bdf 100644 --- a/drizzle-orm/src/pg-core/columns/jsonb.ts +++ b/drizzle-orm/src/pg-core/columns/jsonb.ts @@ -41,8 +41,8 @@ export class PgJsonb extends PgColumn return 'jsonb'; } - override mapToDriverValue(value: T['data']): T['data'] { - return value; + override mapToDriverValue(value: T['data']): string { + return JSON.stringify(value); } override mapFromDriverValue(value: T['data'] | string): T['data'] { diff --git a/drizzle-orm/src/postgres-js/columns/jsonb.ts b/drizzle-orm/src/postgres-js/columns/jsonb.ts new file mode 100644 index 000000000..e160bba6d --- /dev/null +++ b/drizzle-orm/src/postgres-js/columns/jsonb.ts @@ -0,0 +1,58 @@ +import type { ColumnBaseConfig, ColumnHKTBase } from '~/column'; +import type { ColumnBuilderBaseConfig, ColumnBuilderHKTBase, MakeColumnConfig } from '~/column-builder'; +import type { AnyPgTable } from '~/pg-core/table'; +import type { Assume } from '~/utils'; +import { PgColumn, PgColumnBuilder } from '~/pg-core/columns'; + +export interface PgJsonbBuilderHKT extends ColumnBuilderHKTBase { + _type: PgJsonbBuilder>; + _columnHKT: PgJsonbHKT; +} + +export interface PgJsonbHKT extends ColumnHKTBase { + _type: PgJsonb>; +} + +export type PgJsonbBuilderInitial = PgJsonbBuilder<{ + name: TName; + data: unknown; + driverParam: unknown; + notNull: false; + hasDefault: false; +}>; + +export class PgJsonbBuilder extends PgColumnBuilder { + /** @internal */ + override build( + table: AnyPgTable<{ name: TTableName }>, + ): PgJsonb> { + return new PgJsonb>(table, this.config); + } +} + +export class PgJsonb extends PgColumn { + declare protected $pgColumnBrand: 'PgJsonb'; + + constructor(table: AnyPgTable<{ name: T['tableName'] }>, config: PgJsonbBuilder['config']) { + super(table, config); + } + + getSQLType(): string { + return 'jsonb'; + } + + override mapFromDriverValue(value: T['data'] | string): T['data'] { + if (typeof value === 'string') { + try { + return JSON.parse(value); + } catch { + return value as T['data']; + } + } + return value; + } +} + +export function jsonb(name: TName): PgJsonbBuilderInitial { + return new PgJsonbBuilder(name); +} diff --git a/drizzle-orm/src/postgres-js/dialect.ts b/drizzle-orm/src/postgres-js/dialect.ts new file mode 100644 index 000000000..b1a44f1c1 --- /dev/null +++ b/drizzle-orm/src/postgres-js/dialect.ts @@ -0,0 +1,38 @@ +import { + PgDate, + PgJson, + PgNumeric, + PgTime, + PgTimestamp, + PgUUID, +} from "~/pg-core/columns"; + +import { type DriverValueEncoder, type QueryTypingsValue } from "~/sql"; +import { PgJsonb } from "./columns/jsonb"; +import { PgDialect } from "~/pg-core/dialect"; + +export class PgJSDialect extends PgDialect { + public constructor() { + super(); + } + + override prepareTyping( + encoder: DriverValueEncoder + ): QueryTypingsValue { + if (encoder instanceof PgJsonb || encoder instanceof PgJson) { + return "json"; + } else if (encoder instanceof PgNumeric) { + return "decimal"; + } else if (encoder instanceof PgTime) { + return "time"; + } else if (encoder instanceof PgTimestamp) { + return "timestamp"; + } else if (encoder instanceof PgDate) { + return "date"; + } else if (encoder instanceof PgUUID) { + return "uuid"; + } else { + return "none"; + } + } +} diff --git a/drizzle-orm/src/postgres-js/driver.ts b/drizzle-orm/src/postgres-js/driver.ts index 5747b4108..bcfe5a49b 100644 --- a/drizzle-orm/src/postgres-js/driver.ts +++ b/drizzle-orm/src/postgres-js/driver.ts @@ -1,7 +1,6 @@ import type { Sql } from 'postgres'; import { DefaultLogger } from '~/logger'; import { PgDatabase } from '~/pg-core/db'; -import { PgDialect } from '~/pg-core/dialect'; import { createTableRelationsHelpers, extractTablesRelationalConfig, @@ -11,6 +10,7 @@ import { import { type DrizzleConfig } from '~/utils'; import type { PostgresJsQueryResultHKT } from './session'; import { PostgresJsSession } from './session'; +import { PgJSDialect } from './dialect'; export type PostgresJsDatabase< TSchema extends Record = Record, @@ -20,7 +20,7 @@ export function drizzle = Record = {}, ): PostgresJsDatabase { - const dialect = new PgDialect(); + const dialect = new PgJSDialect(); let logger; if (config.logger === true) { logger = new DefaultLogger(); From eec997f8c6340506a634725315c43dcc2c2615fc Mon Sep 17 00:00:00 2001 From: LeonAlvarez Date: Sat, 10 Jun 2023 14:30:05 +0200 Subject: [PATCH 06/12] revert changes on pg tests --- integration-tests/tests/awsdatapi.test.ts | 8 ++++---- integration-tests/tests/pg-schema.test.ts | 8 ++++---- integration-tests/tests/pg.custom.test.ts | 8 ++++---- integration-tests/tests/pg.test.ts | 8 ++++---- integration-tests/tests/vercel-pg.test.ts | 8 ++++---- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/integration-tests/tests/awsdatapi.test.ts b/integration-tests/tests/awsdatapi.test.ts index c2d2eae9d..9cba8c1db 100644 --- a/integration-tests/tests/awsdatapi.test.ts +++ b/integration-tests/tests/awsdatapi.test.ts @@ -569,7 +569,7 @@ test.serial('build query insert with onConflict do update', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values (:1, :2) on conflict ("id") do update set "name" = :3', - params: ['John', ["foo","bar"], 'John1'], + params: ['John', '["foo","bar"]', 'John1'], // typings: ['none', 'json', 'none'] }); }); @@ -584,7 +584,7 @@ test.serial('build query insert with onConflict do update / multiple columns', a t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values (:1, :2) on conflict ("id","name") do update set "name" = :3', - params: ['John', ["foo","bar"], 'John1'], + params: ['John', '["foo","bar"]', 'John1'], // typings: ['none', 'json', 'none'] }); }); @@ -599,7 +599,7 @@ test.serial('build query insert with onConflict do nothing', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values (:1, :2) on conflict do nothing', - params: ['John', ["foo","bar"]], + params: ['John', '["foo","bar"]'], // typings: ['none', 'json'] }); }); @@ -614,7 +614,7 @@ test.serial('build query insert with onConflict do nothing + target', async (t) t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values (:1, :2) on conflict ("id") do nothing', - params: ['John', ["foo","bar"]], + params: ['John', '["foo","bar"]'], // typings: ['none', 'json'] }); }); diff --git a/integration-tests/tests/pg-schema.test.ts b/integration-tests/tests/pg-schema.test.ts index 92fc28641..f126dccd5 100644 --- a/integration-tests/tests/pg-schema.test.ts +++ b/integration-tests/tests/pg-schema.test.ts @@ -653,7 +653,7 @@ test.serial('build query insert with onConflict do update', async (t) => { t.deepEqual(query, { sql: 'insert into "mySchema"."users" ("name", "jsonb") values ($1, $2) on conflict ("id") do update set "name" = $3', - params: ['John', ["foo","bar"], 'John1'], + params: ['John', '["foo","bar"]', 'John1'], }); }); @@ -668,7 +668,7 @@ test.serial('build query insert with onConflict do update / multiple columns', a t.deepEqual(query, { sql: 'insert into "mySchema"."users" ("name", "jsonb") values ($1, $2) on conflict ("id","name") do update set "name" = $3', - params: ['John', ["foo","bar"], 'John1'], + params: ['John', '["foo","bar"]', 'John1'], }); }); @@ -682,7 +682,7 @@ test.serial('build query insert with onConflict do nothing', async (t) => { t.deepEqual(query, { sql: 'insert into "mySchema"."users" ("name", "jsonb") values ($1, $2) on conflict do nothing', - params: ['John', ["foo","bar"]], + params: ['John', '["foo","bar"]'], }); }); @@ -696,7 +696,7 @@ test.serial('build query insert with onConflict do nothing + target', async (t) t.deepEqual(query, { sql: 'insert into "mySchema"."users" ("name", "jsonb") values ($1, $2) on conflict ("id") do nothing', - params: ['John', ["foo","bar"]], + params: ['John', '["foo","bar"]'], }); }); diff --git a/integration-tests/tests/pg.custom.test.ts b/integration-tests/tests/pg.custom.test.ts index 231c16b81..e2a1f67d1 100644 --- a/integration-tests/tests/pg.custom.test.ts +++ b/integration-tests/tests/pg.custom.test.ts @@ -661,7 +661,7 @@ test.serial('build query insert with onConflict do update', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do update set "name" = $3', - params: ['John', ["foo","bar"], 'John1'], + params: ['John', '["foo","bar"]', 'John1'], }); }); @@ -675,7 +675,7 @@ test.serial('build query insert with onConflict do update / multiple columns', a t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id","name") do update set "name" = $3', - params: ['John', ["foo","bar"], 'John1'], + params: ['John', '["foo","bar"]', 'John1'], }); }); @@ -689,7 +689,7 @@ test.serial('build query insert with onConflict do nothing', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict do nothing', - params: ['John', ["foo","bar"]], + params: ['John', '["foo","bar"]'], }); }); @@ -703,7 +703,7 @@ test.serial('build query insert with onConflict do nothing + target', async (t) t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do nothing', - params: ['John', ["foo","bar"]], + params: ['John', '["foo","bar"]'], }); }); diff --git a/integration-tests/tests/pg.test.ts b/integration-tests/tests/pg.test.ts index 1f87d3185..b81bc9981 100644 --- a/integration-tests/tests/pg.test.ts +++ b/integration-tests/tests/pg.test.ts @@ -927,7 +927,7 @@ test.serial('build query insert with onConflict do update', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do update set "name" = $3', - params: ['John', ["foo","bar"], 'John1'], + params: ['John', '["foo","bar"]', 'John1'], }); }); @@ -942,7 +942,7 @@ test.serial('build query insert with onConflict do update / multiple columns', a t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id","name") do update set "name" = $3', - params: ['John', ["foo","bar"], 'John1'], + params: ['John', '["foo","bar"]', 'John1'], }); }); @@ -957,7 +957,7 @@ test.serial('build query insert with onConflict do nothing', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict do nothing', - params: ['John', ["foo","bar"]], + params: ['John', '["foo","bar"]'], }); }); @@ -972,7 +972,7 @@ test.serial('build query insert with onConflict do nothing + target', async (t) t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do nothing', - params: ['John', ["foo","bar"]], + params: ['John', '["foo","bar"]'], }); }); diff --git a/integration-tests/tests/vercel-pg.test.ts b/integration-tests/tests/vercel-pg.test.ts index b468102d8..fa97a529c 100644 --- a/integration-tests/tests/vercel-pg.test.ts +++ b/integration-tests/tests/vercel-pg.test.ts @@ -927,7 +927,7 @@ test.serial('build query insert with onConflict do update', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do update set "name" = $3', - params: ['John', ["foo","bar"], 'John1'], + params: ['John', '["foo","bar"]', 'John1'], }); }); @@ -942,7 +942,7 @@ test.serial('build query insert with onConflict do update / multiple columns', a t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id","name") do update set "name" = $3', - params: ['John', ["foo","bar"], 'John1'], + params: ['John', '["foo","bar"]', 'John1'], }); }); @@ -957,7 +957,7 @@ test.serial('build query insert with onConflict do nothing', async (t) => { t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict do nothing', - params: ['John', ["foo","bar"]], + params: ['John', '["foo","bar"]'], }); }); @@ -972,7 +972,7 @@ test.serial('build query insert with onConflict do nothing + target', async (t) t.deepEqual(query, { sql: 'insert into "users" ("name", "jsonb") values ($1, $2) on conflict ("id") do nothing', - params: ['John', ["foo","bar"]], + params: ['John', '["foo","bar"]'], }); }); From 821916d321ba0038a8b060156679c82232854df4 Mon Sep 17 00:00:00 2001 From: LeonAlvarez Date: Sat, 10 Jun 2023 21:37:33 +0200 Subject: [PATCH 07/12] postgresjs dialect --- drizzle-orm/src/postgres-js/columns/jsonb.ts | 4 ++ drizzle-orm/src/postgres-js/dialect.ts | 57 ++++++++------------ 2 files changed, 26 insertions(+), 35 deletions(-) diff --git a/drizzle-orm/src/postgres-js/columns/jsonb.ts b/drizzle-orm/src/postgres-js/columns/jsonb.ts index e160bba6d..aad592dee 100644 --- a/drizzle-orm/src/postgres-js/columns/jsonb.ts +++ b/drizzle-orm/src/postgres-js/columns/jsonb.ts @@ -41,6 +41,10 @@ export class PgJsonb extends PgColumn return 'jsonb'; } + override mapToDriverValue(value: T['data']): T['driverParam'] { + return value as any; + } + override mapFromDriverValue(value: T['data'] | string): T['data'] { if (typeof value === 'string') { try { diff --git a/drizzle-orm/src/postgres-js/dialect.ts b/drizzle-orm/src/postgres-js/dialect.ts index b1a44f1c1..01934a01b 100644 --- a/drizzle-orm/src/postgres-js/dialect.ts +++ b/drizzle-orm/src/postgres-js/dialect.ts @@ -1,38 +1,25 @@ -import { - PgDate, - PgJson, - PgNumeric, - PgTime, - PgTimestamp, - PgUUID, -} from "~/pg-core/columns"; - -import { type DriverValueEncoder, type QueryTypingsValue } from "~/sql"; +import { PgDialect, PgJson, PgNumeric, PgTime, PgTimestamp, PgDate, PgUUID } from "~/pg-core"; import { PgJsonb } from "./columns/jsonb"; -import { PgDialect } from "~/pg-core/dialect"; +import { type DriverValueEncoder, type QueryTypingsValue } from "~/sql"; export class PgJSDialect extends PgDialect { - public constructor() { - super(); - } - - override prepareTyping( - encoder: DriverValueEncoder - ): QueryTypingsValue { - if (encoder instanceof PgJsonb || encoder instanceof PgJson) { - return "json"; - } else if (encoder instanceof PgNumeric) { - return "decimal"; - } else if (encoder instanceof PgTime) { - return "time"; - } else if (encoder instanceof PgTimestamp) { - return "timestamp"; - } else if (encoder instanceof PgDate) { - return "date"; - } else if (encoder instanceof PgUUID) { - return "uuid"; - } else { - return "none"; - } - } -} + override prepareTyping(encoder: DriverValueEncoder): QueryTypingsValue { + if ( + encoder instanceof PgJsonb || encoder instanceof PgJson + ) { + return 'json'; + } else if (encoder instanceof PgNumeric) { + return 'decimal'; + } else if (encoder instanceof PgTime) { + return 'time'; + } else if (encoder instanceof PgTimestamp) { + return 'timestamp'; + } else if (encoder instanceof PgDate) { + return 'date'; + } else if (encoder instanceof PgUUID) { + return 'uuid'; + } else { + return 'none'; + } + } +} \ No newline at end of file From f39c46b4e6e94c7ada95ff531155dee2ce0ffd2f Mon Sep 17 00:00:00 2001 From: LeonAlvarez Date: Sat, 10 Jun 2023 23:18:32 +0200 Subject: [PATCH 08/12] improve pg-schema json test --- integration-tests/tests/pg-schema.test.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/integration-tests/tests/pg-schema.test.ts b/integration-tests/tests/pg-schema.test.ts index f126dccd5..d8a3865a6 100644 --- a/integration-tests/tests/pg-schema.test.ts +++ b/integration-tests/tests/pg-schema.test.ts @@ -3,7 +3,7 @@ import 'dotenv/config'; import type { TestFn } from 'ava'; import anyTest from 'ava'; import Docker from 'dockerode'; -import { asc, eq, Name, placeholder, sql } from 'drizzle-orm'; +import { asc, eq, Name, placeholder, sql, and } from 'drizzle-orm'; import type { NodePgDatabase } from 'drizzle-orm/node-postgres'; import { drizzle } from 'drizzle-orm/node-postgres'; import { @@ -305,8 +305,13 @@ test.serial('json insert', async (t) => { id: usersTable.id, name: usersTable.name, jsonb: usersTable.jsonb, - }).from(usersTable); - + }).from(usersTable).where( + and( + eq(usersTable.name, 'John'), + sql.raw(`jsonb->>0 = 'foo'`), + ) + ); + t.deepEqual(result, [{ id: 1, name: 'John', jsonb: ['foo', 'bar'] }]); }); From ed57b652b9b3cabd5f7f9c79a1ecd4bcf48f1070 Mon Sep 17 00:00:00 2001 From: LeonAlvarez Date: Sun, 11 Jun 2023 03:39:20 +0200 Subject: [PATCH 09/12] JSON object test --- integration-tests/tests/pg-schema.test.ts | 30 +++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/integration-tests/tests/pg-schema.test.ts b/integration-tests/tests/pg-schema.test.ts index d8a3865a6..7ca264d1d 100644 --- a/integration-tests/tests/pg-schema.test.ts +++ b/integration-tests/tests/pg-schema.test.ts @@ -55,6 +55,15 @@ const publicUsersTable = pgTable('users', { createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), }); +type MetaData = { + foo: string; + bar: number; +} +const metaDataTable = pgTable('meta_data', { + id: serial('id').primaryKey(), + data: jsonb('data').$type(), +}); + interface Context { docker: Docker; pgContainer: Docker.Container; @@ -122,8 +131,8 @@ test.before(async (t) => { test.after.always(async (t) => { const ctx = t.context; - await ctx.client?.end().catch(console.error); - await ctx.pgContainer?.stop().catch(console.error); + // await ctx.client?.end().catch(console.error); + // await ctx.pgContainer?.stop().catch(console.error); }); test.beforeEach(async (t) => { @@ -163,6 +172,14 @@ test.beforeEach(async (t) => { ) `, ); + await ctx.db.execute( + sql` + create table meta_data ( + id serial primary key, + data jsonb + ) + `, + ); }); test.serial('select all fields', async (t) => { @@ -297,6 +314,15 @@ test.serial('insert + select', async (t) => { ]); }); +test.serial('json object insert', async (t) => { + const { db } = t.context; + + await db.insert(metaDataTable).values({ data: {foo: 'bar', bar: 33} }); + const {rows: result} = await db.execute(sql`select "id", "data" from "meta_data" where data->>'foo'='bar'`); + + t.deepEqual(result, [{ id: 1, data: {foo: 'bar', bar: 33}}]); +}); + test.serial('json insert', async (t) => { const { db } = t.context; From 422b888918e23dea4905cd1068cafbb4a2673132 Mon Sep 17 00:00:00 2001 From: LeonAlvarez Date: Sun, 11 Jun 2023 03:44:25 +0200 Subject: [PATCH 10/12] revert dialect --- drizzle-orm/src/postgres-js/columns/jsonb.ts | 62 -------------------- drizzle-orm/src/postgres-js/dialect.ts | 25 -------- drizzle-orm/src/postgres-js/driver.ts | 4 +- integration-tests/tests/postgres.js.test.ts | 11 +--- 4 files changed, 5 insertions(+), 97 deletions(-) delete mode 100644 drizzle-orm/src/postgres-js/columns/jsonb.ts delete mode 100644 drizzle-orm/src/postgres-js/dialect.ts diff --git a/drizzle-orm/src/postgres-js/columns/jsonb.ts b/drizzle-orm/src/postgres-js/columns/jsonb.ts deleted file mode 100644 index aad592dee..000000000 --- a/drizzle-orm/src/postgres-js/columns/jsonb.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { ColumnBaseConfig, ColumnHKTBase } from '~/column'; -import type { ColumnBuilderBaseConfig, ColumnBuilderHKTBase, MakeColumnConfig } from '~/column-builder'; -import type { AnyPgTable } from '~/pg-core/table'; -import type { Assume } from '~/utils'; -import { PgColumn, PgColumnBuilder } from '~/pg-core/columns'; - -export interface PgJsonbBuilderHKT extends ColumnBuilderHKTBase { - _type: PgJsonbBuilder>; - _columnHKT: PgJsonbHKT; -} - -export interface PgJsonbHKT extends ColumnHKTBase { - _type: PgJsonb>; -} - -export type PgJsonbBuilderInitial = PgJsonbBuilder<{ - name: TName; - data: unknown; - driverParam: unknown; - notNull: false; - hasDefault: false; -}>; - -export class PgJsonbBuilder extends PgColumnBuilder { - /** @internal */ - override build( - table: AnyPgTable<{ name: TTableName }>, - ): PgJsonb> { - return new PgJsonb>(table, this.config); - } -} - -export class PgJsonb extends PgColumn { - declare protected $pgColumnBrand: 'PgJsonb'; - - constructor(table: AnyPgTable<{ name: T['tableName'] }>, config: PgJsonbBuilder['config']) { - super(table, config); - } - - getSQLType(): string { - return 'jsonb'; - } - - override mapToDriverValue(value: T['data']): T['driverParam'] { - return value as any; - } - - override mapFromDriverValue(value: T['data'] | string): T['data'] { - if (typeof value === 'string') { - try { - return JSON.parse(value); - } catch { - return value as T['data']; - } - } - return value; - } -} - -export function jsonb(name: TName): PgJsonbBuilderInitial { - return new PgJsonbBuilder(name); -} diff --git a/drizzle-orm/src/postgres-js/dialect.ts b/drizzle-orm/src/postgres-js/dialect.ts deleted file mode 100644 index 01934a01b..000000000 --- a/drizzle-orm/src/postgres-js/dialect.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { PgDialect, PgJson, PgNumeric, PgTime, PgTimestamp, PgDate, PgUUID } from "~/pg-core"; -import { PgJsonb } from "./columns/jsonb"; -import { type DriverValueEncoder, type QueryTypingsValue } from "~/sql"; - -export class PgJSDialect extends PgDialect { - override prepareTyping(encoder: DriverValueEncoder): QueryTypingsValue { - if ( - encoder instanceof PgJsonb || encoder instanceof PgJson - ) { - return 'json'; - } else if (encoder instanceof PgNumeric) { - return 'decimal'; - } else if (encoder instanceof PgTime) { - return 'time'; - } else if (encoder instanceof PgTimestamp) { - return 'timestamp'; - } else if (encoder instanceof PgDate) { - return 'date'; - } else if (encoder instanceof PgUUID) { - return 'uuid'; - } else { - return 'none'; - } - } -} \ No newline at end of file diff --git a/drizzle-orm/src/postgres-js/driver.ts b/drizzle-orm/src/postgres-js/driver.ts index bcfe5a49b..7bbabd29b 100644 --- a/drizzle-orm/src/postgres-js/driver.ts +++ b/drizzle-orm/src/postgres-js/driver.ts @@ -10,7 +10,7 @@ import { import { type DrizzleConfig } from '~/utils'; import type { PostgresJsQueryResultHKT } from './session'; import { PostgresJsSession } from './session'; -import { PgJSDialect } from './dialect'; +import { PgDialect } from '~/pg-core/dialect'; export type PostgresJsDatabase< TSchema extends Record = Record, @@ -20,7 +20,7 @@ export function drizzle = Record = {}, ): PostgresJsDatabase { - const dialect = new PgJSDialect(); + const dialect = new PgDialect(); let logger; if (config.logger === true) { logger = new DefaultLogger(); diff --git a/integration-tests/tests/postgres.js.test.ts b/integration-tests/tests/postgres.js.test.ts index 1deff11d5..88f2afc55 100644 --- a/integration-tests/tests/postgres.js.test.ts +++ b/integration-tests/tests/postgres.js.test.ts @@ -101,7 +101,6 @@ const metaDataTable = pgTable('meta_data', { data: jsonb('data').$type(), }); - interface Context { docker: Docker; pgContainer: Docker.Container; @@ -382,16 +381,12 @@ test.serial('json object insert', async (t) => { const { db } = t.context; await db.insert(metaDataTable).values({ data: {foo: 'bar', bar: 33} }); - const result = await db.select({ - id: metaDataTable.id, - data: metaDataTable.data, - }).from(metaDataTable).where( - sql.raw("data->>`foo`=`bar`") - ).toSQL() + // const {rows: result} = await db.execute(sql`select "id", "data" from "meta_data" where data->>'foo'='bar'`); - t.deepEqual(result, [{ id: 1, data: {foo: 'bar', bar: 33}}]); + // t.deepEqual(result, [{ id: 1, data: {foo: 'bar', bar: 33}}]); }); + test.serial('json insert', async (t) => { const { db } = t.context; From c1c5f757c3b6b8c948618b1f4539d870acb1a77d Mon Sep 17 00:00:00 2001 From: LeonAlvarez Date: Sun, 11 Jun 2023 04:02:15 +0200 Subject: [PATCH 11/12] fix tests --- integration-tests/tests/pg-schema.test.ts | 4 ++-- integration-tests/tests/postgres.js.test.ts | 25 +++++++++++++++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/integration-tests/tests/pg-schema.test.ts b/integration-tests/tests/pg-schema.test.ts index 7ca264d1d..2e09742e3 100644 --- a/integration-tests/tests/pg-schema.test.ts +++ b/integration-tests/tests/pg-schema.test.ts @@ -131,8 +131,8 @@ test.before(async (t) => { test.after.always(async (t) => { const ctx = t.context; - // await ctx.client?.end().catch(console.error); - // await ctx.pgContainer?.stop().catch(console.error); + await ctx.client?.end().catch(console.error); + await ctx.pgContainer?.stop().catch(console.error); }); test.beforeEach(async (t) => { diff --git a/integration-tests/tests/postgres.js.test.ts b/integration-tests/tests/postgres.js.test.ts index 88f2afc55..2e791199d 100644 --- a/integration-tests/tests/postgres.js.test.ts +++ b/integration-tests/tests/postgres.js.test.ts @@ -37,6 +37,7 @@ import { timestamp, uuid as pgUuid, varchar, + //customType, } from 'drizzle-orm/pg-core'; import type { PostgresJsDatabase } from 'drizzle-orm/postgres-js'; import { drizzle } from 'drizzle-orm/postgres-js'; @@ -48,6 +49,25 @@ import { type Equal, Expect } from './utils'; const QUERY_LOGGING = false; +// const customJsonb = customType<{ data: any }>({ +// dataType() { +// return 'jsonb'; +// }, +// toDriver(val) { +// val as any; +// }, +// fromDriver(value) { +// if (typeof value === 'string') { +// try { +// return JSON.parse(value) as any; +// } catch { +// return value as any; +// } +// } +// return value as any; +// }, +// }); + const usersTable = pgTable('users', { id: serial('id').primaryKey(), name: text('name').notNull(), @@ -96,6 +116,7 @@ type MetaData = { foo: string; bar: number; } + const metaDataTable = pgTable('meta_data', { id: serial('id').primaryKey(), data: jsonb('data').$type(), @@ -381,9 +402,9 @@ test.serial('json object insert', async (t) => { const { db } = t.context; await db.insert(metaDataTable).values({ data: {foo: 'bar', bar: 33} }); - // const {rows: result} = await db.execute(sql`select "id", "data" from "meta_data" where data->>'foo'='bar'`); + const result = await db.execute(sql`select "id", "data" from "meta_data" where data->>'foo'='bar'`); - // t.deepEqual(result, [{ id: 1, data: {foo: 'bar', bar: 33}}]); + t.deepEqual([result.at(0)], [{ id: 1, data: {foo: 'bar', bar: 33}}]); }); From 2732cdf977ee59c99d02fdaec26b562b602eec93 Mon Sep 17 00:00:00 2001 From: LeonAlvarez Date: Sun, 11 Jun 2023 11:16:50 +0200 Subject: [PATCH 12/12] jsonb workaround for postgres.js --- drizzle-orm/src/pg-core/columns/jsonb.ts | 5 +++-- drizzle-orm/src/postgres-js/driver.ts | 2 +- integration-tests/tests/pg-schema.test.ts | 10 ++++++++-- integration-tests/tests/postgres.js.test.ts | 10 ++++++++-- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/drizzle-orm/src/pg-core/columns/jsonb.ts b/drizzle-orm/src/pg-core/columns/jsonb.ts index e4bbc1bdf..0529f00d5 100644 --- a/drizzle-orm/src/pg-core/columns/jsonb.ts +++ b/drizzle-orm/src/pg-core/columns/jsonb.ts @@ -41,8 +41,9 @@ export class PgJsonb extends PgColumn return 'jsonb'; } - override mapToDriverValue(value: T['data']): string { - return JSON.stringify(value); + override mapToDriverValue(value: T['data']): T['data'] { + return value; + //return JSON.stringify(value); } override mapFromDriverValue(value: T['data'] | string): T['data'] { diff --git a/drizzle-orm/src/postgres-js/driver.ts b/drizzle-orm/src/postgres-js/driver.ts index 7bbabd29b..5747b4108 100644 --- a/drizzle-orm/src/postgres-js/driver.ts +++ b/drizzle-orm/src/postgres-js/driver.ts @@ -1,6 +1,7 @@ import type { Sql } from 'postgres'; import { DefaultLogger } from '~/logger'; import { PgDatabase } from '~/pg-core/db'; +import { PgDialect } from '~/pg-core/dialect'; import { createTableRelationsHelpers, extractTablesRelationalConfig, @@ -10,7 +11,6 @@ import { import { type DrizzleConfig } from '~/utils'; import type { PostgresJsQueryResultHKT } from './session'; import { PostgresJsSession } from './session'; -import { PgDialect } from '~/pg-core/dialect'; export type PostgresJsDatabase< TSchema extends Record = Record, diff --git a/integration-tests/tests/pg-schema.test.ts b/integration-tests/tests/pg-schema.test.ts index 2e09742e3..c8b55a53b 100644 --- a/integration-tests/tests/pg-schema.test.ts +++ b/integration-tests/tests/pg-schema.test.ts @@ -317,8 +317,14 @@ test.serial('insert + select', async (t) => { test.serial('json object insert', async (t) => { const { db } = t.context; - await db.insert(metaDataTable).values({ data: {foo: 'bar', bar: 33} }); - const {rows: result} = await db.execute(sql`select "id", "data" from "meta_data" where data->>'foo'='bar'`); + const bar = 33 + const foo = 'bar'; + + await db.insert(metaDataTable).values({ data: {foo, bar} }); + const result = await db.select({ + id: metaDataTable.id, + data: metaDataTable.data, + }).from(metaDataTable).where(and(sql`data->'foo' = ${foo}`, sql`data->'bar' = ${bar}`)); t.deepEqual(result, [{ id: 1, data: {foo: 'bar', bar: 33}}]); }); diff --git a/integration-tests/tests/postgres.js.test.ts b/integration-tests/tests/postgres.js.test.ts index 2e791199d..d54b92bd7 100644 --- a/integration-tests/tests/postgres.js.test.ts +++ b/integration-tests/tests/postgres.js.test.ts @@ -401,8 +401,14 @@ test.serial('insert + select', async (t) => { test.serial('json object insert', async (t) => { const { db } = t.context; - await db.insert(metaDataTable).values({ data: {foo: 'bar', bar: 33} }); - const result = await db.execute(sql`select "id", "data" from "meta_data" where data->>'foo'='bar'`); + const bar = 33 + const foo = 'bar'; + + await db.insert(metaDataTable).values({ data: {foo, bar} }); + const result = await db.select({ + id: metaDataTable.id, + data: metaDataTable.data, + }).from(metaDataTable).where(and(sql`data->'foo' = ${foo}`, sql`data->'bar' = ${bar}`)); t.deepEqual([result.at(0)], [{ id: 1, data: {foo: 'bar', bar: 33}}]); });