From 9f60e04d09c665f70b87b98394648fc4c4db321e Mon Sep 17 00:00:00 2001 From: Michael Kret Date: Fri, 1 Sep 2023 14:07:44 +0300 Subject: [PATCH 1/4] :zap: fix, optimizations --- .../v2/actions/database/insert.operation.ts | 24 +++++++++++++++---- .../v2/actions/database/update.operation.ts | 24 +++++++++++++++---- .../v2/actions/database/upsert.operation.ts | 24 +++++++++++++++---- .../nodes/Postgres/v2/actions/router.ts | 3 ++- .../nodes/Postgres/v2/methods/listSearch.ts | 8 +++---- .../nodes/Postgres/v2/methods/loadOptions.ts | 4 ++-- 6 files changed, 68 insertions(+), 19 deletions(-) diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts b/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts index 29bfd90495b40..d8f0fd39fa6e8 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts @@ -161,15 +161,28 @@ export async function execute( db: PgpDatabase, ): Promise { items = replaceEmptyStringsByNulls(items, nodeOptions.replaceEmptyStrings as boolean); + const nodeVersion = nodeOptions.typeVersion as number; + + let schema = this.getNodeParameter('schema', 0, undefined, { + extractValue: true, + }) as string; + + let table = this.getNodeParameter('table', 0, undefined, { + extractValue: true, + }) as string; + + let tableSchema = await getTableSchema(db, schema, table); + let currentSchema = schema; + let currentTable = table; const queries: QueryWithValues[] = []; for (let i = 0; i < items.length; i++) { - const schema = this.getNodeParameter('schema', i, undefined, { + schema = this.getNodeParameter('schema', i, undefined, { extractValue: true, }) as string; - const table = this.getNodeParameter('table', i, undefined, { + table = this.getNodeParameter('table', i, undefined, { extractValue: true, }) as string; @@ -183,7 +196,6 @@ export async function execute( let query = `INSERT INTO $1:name.$2:name($3:name) VALUES($3:csv)${onConflict}`; let values: QueryValues = [schema, table]; - const nodeVersion = this.getNode().typeVersion; const dataMode = nodeVersion < 2.2 ? (this.getNodeParameter('dataMode', i) as string) @@ -209,7 +221,11 @@ export async function execute( } } - const tableSchema = await getTableSchema(db, schema, table); + if (currentSchema !== schema || currentTable !== table) { + currentSchema = schema; + currentTable = table; + tableSchema = await getTableSchema(db, schema, table); + } values.push(checkItemAgainstSchema(this.getNode(), item, tableSchema, i)); diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts b/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts index a45f2bbb378f7..54f6c3688c510 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts @@ -198,19 +198,31 @@ export async function execute( db: PgpDatabase, ): Promise { items = replaceEmptyStringsByNulls(items, nodeOptions.replaceEmptyStrings as boolean); + const nodeVersion = nodeOptions.typeVersion as number; + + let schema = this.getNodeParameter('schema', 0, undefined, { + extractValue: true, + }) as string; + + let table = this.getNodeParameter('table', 0, undefined, { + extractValue: true, + }) as string; + + let tableSchema = await getTableSchema(db, schema, table); + let currentSchema = schema; + let currentTable = table; const queries: QueryWithValues[] = []; for (let i = 0; i < items.length; i++) { - const schema = this.getNodeParameter('schema', i, undefined, { + schema = this.getNodeParameter('schema', i, undefined, { extractValue: true, }) as string; - const table = this.getNodeParameter('table', i, undefined, { + table = this.getNodeParameter('table', i, undefined, { extractValue: true, }) as string; - const nodeVersion = this.getNode().typeVersion; const columnsToMatchOn: string[] = nodeVersion < 2.2 ? [this.getNodeParameter('columnToMatchOn', i) as string] @@ -286,7 +298,11 @@ export async function execute( } } - const tableSchema = await getTableSchema(db, schema, table); + if (currentSchema !== schema || currentTable !== table) { + currentSchema = schema; + currentTable = table; + tableSchema = await getTableSchema(db, schema, table); + } item = checkItemAgainstSchema(this.getNode(), item, tableSchema, i); diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts b/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts index c2893b91ae66e..d7a851adf60b5 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts @@ -197,19 +197,31 @@ export async function execute( db: PgpDatabase, ): Promise { items = replaceEmptyStringsByNulls(items, nodeOptions.replaceEmptyStrings as boolean); + const nodeVersion = nodeOptions.typeVersion as number; + + let schema = this.getNodeParameter('schema', 0, undefined, { + extractValue: true, + }) as string; + + let table = this.getNodeParameter('table', 0, undefined, { + extractValue: true, + }) as string; + + let tableSchema = await getTableSchema(db, schema, table); + let currentSchema = schema; + let currentTable = table; const queries: QueryWithValues[] = []; for (let i = 0; i < items.length; i++) { - const schema = this.getNodeParameter('schema', i, undefined, { + schema = this.getNodeParameter('schema', i, undefined, { extractValue: true, }) as string; - const table = this.getNodeParameter('table', i, undefined, { + table = this.getNodeParameter('table', i, undefined, { extractValue: true, }) as string; - const nodeVersion = this.getNode().typeVersion; const columnsToMatchOn: string[] = nodeVersion < 2.2 ? [this.getNodeParameter('columnToMatchOn', i) as string] @@ -255,7 +267,11 @@ export async function execute( ); } - const tableSchema = await getTableSchema(db, schema, table); + if (currentSchema !== schema || currentTable !== table) { + currentSchema = schema; + currentTable = table; + tableSchema = await getTableSchema(db, schema, table); + } item = checkItemAgainstSchema(this.getNode(), item, tableSchema, i); diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/router.ts b/packages/nodes-base/nodes/Postgres/v2/actions/router.ts index ca061b752428e..632a0da04b59a 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/router.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/router.ts @@ -57,7 +57,8 @@ export async function router(this: IExecuteFunctions): Promise { const credentials = await this.getCredentials('postgres'); const options = { nodeVersion: this.getNode().typeVersion }; - const { db, pgp, sshClient } = await configurePostgres(credentials, options); + const { db, sshClient } = await configurePostgres(credentials, options); const schema = this.getNodeParameter('schema', 0, { extractValue: true, @@ -54,6 +54,6 @@ export async function tableSearch(this: ILoadOptionsFunctions): Promise Date: Fri, 1 Sep 2023 14:58:35 +0300 Subject: [PATCH 2/4] :zap: removed duplicated code --- .../v2/actions/database/insert.operation.ts | 11 ++++------- .../v2/actions/database/update.operation.ts | 11 ++++------- .../v2/actions/database/upsert.operation.ts | 11 ++++------- .../nodes-base/nodes/Postgres/v2/helpers/utils.ts | 13 +++++++++++++ 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts b/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts index d8f0fd39fa6e8..ec18b2ff550b1 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts @@ -17,6 +17,7 @@ import type { import { addReturning, checkItemAgainstSchema, + configureTableSchemaUpdater, getTableSchema, prepareItem, replaceEmptyStringsByNulls, @@ -171,9 +172,9 @@ export async function execute( extractValue: true, }) as string; + const updateTableSchema = configureTableSchemaUpdater(schema, table); + let tableSchema = await getTableSchema(db, schema, table); - let currentSchema = schema; - let currentTable = table; const queries: QueryWithValues[] = []; @@ -221,11 +222,7 @@ export async function execute( } } - if (currentSchema !== schema || currentTable !== table) { - currentSchema = schema; - currentTable = table; - tableSchema = await getTableSchema(db, schema, table); - } + tableSchema = await updateTableSchema(db, tableSchema, schema, table); values.push(checkItemAgainstSchema(this.getNode(), item, tableSchema, i)); diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts b/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts index 54f6c3688c510..72d93ff0eadd6 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts @@ -18,6 +18,7 @@ import type { import { addReturning, checkItemAgainstSchema, + configureTableSchemaUpdater, doesRowExist, getTableSchema, prepareItem, @@ -208,9 +209,9 @@ export async function execute( extractValue: true, }) as string; + const updateTableSchema = configureTableSchemaUpdater(schema, table); + let tableSchema = await getTableSchema(db, schema, table); - let currentSchema = schema; - let currentTable = table; const queries: QueryWithValues[] = []; @@ -298,11 +299,7 @@ export async function execute( } } - if (currentSchema !== schema || currentTable !== table) { - currentSchema = schema; - currentTable = table; - tableSchema = await getTableSchema(db, schema, table); - } + tableSchema = await updateTableSchema(db, tableSchema, schema, table); item = checkItemAgainstSchema(this.getNode(), item, tableSchema, i); diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts b/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts index d7a851adf60b5..dce4609bd4f28 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts @@ -21,6 +21,7 @@ import { getTableSchema, prepareItem, replaceEmptyStringsByNulls, + configureTableSchemaUpdater, } from '../../helpers/utils'; import { optionsCollection } from '../common.descriptions'; @@ -207,9 +208,9 @@ export async function execute( extractValue: true, }) as string; + const updateTableSchema = configureTableSchemaUpdater(schema, table); + let tableSchema = await getTableSchema(db, schema, table); - let currentSchema = schema; - let currentTable = table; const queries: QueryWithValues[] = []; @@ -267,11 +268,7 @@ export async function execute( ); } - if (currentSchema !== schema || currentTable !== table) { - currentSchema = schema; - currentTable = table; - tableSchema = await getTableSchema(db, schema, table); - } + tableSchema = await updateTableSchema(db, tableSchema, schema, table); item = checkItemAgainstSchema(this.getNode(), item, tableSchema, i); diff --git a/packages/nodes-base/nodes/Postgres/v2/helpers/utils.ts b/packages/nodes-base/nodes/Postgres/v2/helpers/utils.ts index a800ee044b60e..8569f8ab75a91 100644 --- a/packages/nodes-base/nodes/Postgres/v2/helpers/utils.ts +++ b/packages/nodes-base/nodes/Postgres/v2/helpers/utils.ts @@ -458,3 +458,16 @@ export function checkItemAgainstSchema( return item; } + +export const configureTableSchemaUpdater = (initialSchema: string, initialTable: string) => { + let currentSchema = initialSchema; + let currentTable = initialTable; + return async (db: PgpDatabase, tableSchema: ColumnInfo[], schema: string, table: string) => { + if (currentSchema !== schema || currentTable !== table) { + currentSchema = schema; + currentTable = table; + tableSchema = await getTableSchema(db, schema, table); + } + return tableSchema; + }; +}; From 56ef5134a598a36f0f24bb1aa178bb8cc06c8495 Mon Sep 17 00:00:00 2001 From: Michael Kret Date: Fri, 1 Sep 2023 21:22:24 +0300 Subject: [PATCH 3/4] :zap: tests fix --- .../nodes/Postgres/test/v2/operations.test.ts | 14 ++++++++------ .../nodes/Postgres/v2/helpers/interfaces.ts | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/nodes-base/nodes/Postgres/test/v2/operations.test.ts b/packages/nodes-base/nodes/Postgres/test/v2/operations.test.ts index c2f5d7e57d85d..6c7976d181067 100644 --- a/packages/nodes-base/nodes/Postgres/test/v2/operations.test.ts +++ b/packages/nodes-base/nodes/Postgres/test/v2/operations.test.ts @@ -89,7 +89,7 @@ describe('Test PostgresV2, deleteTable operation', () => { }, ], }, - options: {}, + options: { typeVersion: 2.1 }, }; const nodeOptions = nodeParameters.options as IDataObject; @@ -168,7 +168,7 @@ describe('Test PostgresV2, deleteTable operation', () => { cachedResultName: 'my_table', }, deleteCommand: 'drop', - options: {}, + options: { typeVersion: 2.1 }, }; const nodeOptions = nodeParameters.options as IDataObject; @@ -256,7 +256,7 @@ describe('Test PostgresV2, insert operation', () => { }, ], }, - options: {}, + options: { typeVersion: 2.1 }, }; const columnsInfo: ColumnInfo[] = [ { column_name: 'id', data_type: 'integer', is_nullable: 'NO', udt_name: '' }, @@ -299,7 +299,7 @@ describe('Test PostgresV2, insert operation', () => { mode: 'list', }, dataMode: 'autoMapInputData', - options: {}, + options: { typeVersion: 2.1 }, }; const columnsInfo: ColumnInfo[] = [ { column_name: 'id', data_type: 'integer', is_nullable: 'NO', udt_name: '' }, @@ -509,6 +509,7 @@ describe('Test PostgresV2, update operation', () => { }, options: { outputColumns: ['json', 'foo'], + typeVersion: 2.1, }, }; const columnsInfo: ColumnInfo[] = [ @@ -565,7 +566,7 @@ describe('Test PostgresV2, update operation', () => { }, dataMode: 'autoMapInputData', columnToMatchOn: 'id', - options: {}, + options: { typeVersion: 2.1 }, }; const columnsInfo: ColumnInfo[] = [ { column_name: 'id', data_type: 'integer', is_nullable: 'NO', udt_name: '' }, @@ -668,6 +669,7 @@ describe('Test PostgresV2, upsert operation', () => { }, options: { outputColumns: ['json'], + typeVersion: 2.1, }, }; const columnsInfo: ColumnInfo[] = [ @@ -724,7 +726,7 @@ describe('Test PostgresV2, upsert operation', () => { }, dataMode: 'autoMapInputData', columnToMatchOn: 'id', - options: {}, + options: { typeVersion: 2.1 }, }; const columnsInfo: ColumnInfo[] = [ { column_name: 'id', data_type: 'integer', is_nullable: 'NO', udt_name: '' }, diff --git a/packages/nodes-base/nodes/Postgres/v2/helpers/interfaces.ts b/packages/nodes-base/nodes/Postgres/v2/helpers/interfaces.ts index 93b0291158ca2..13f731a02af0c 100644 --- a/packages/nodes-base/nodes/Postgres/v2/helpers/interfaces.ts +++ b/packages/nodes-base/nodes/Postgres/v2/helpers/interfaces.ts @@ -15,7 +15,7 @@ export type ColumnInfo = { column_name: string; data_type: string; is_nullable: string; - udt_name: string; + udt_name?: string; column_default?: string; }; export type EnumInfo = { From 0d0545f7de60002ef046089eed8cd813640ecf79 Mon Sep 17 00:00:00 2001 From: Michael Kret Date: Fri, 1 Sep 2023 21:40:52 +0300 Subject: [PATCH 4/4] :zap: type fix --- .../nodes-base/nodes/Postgres/v2/methods/resourceMapping.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/nodes-base/nodes/Postgres/v2/methods/resourceMapping.ts b/packages/nodes-base/nodes/Postgres/v2/methods/resourceMapping.ts index d100228133609..bf2b80bbc9de8 100644 --- a/packages/nodes-base/nodes/Postgres/v2/methods/resourceMapping.ts +++ b/packages/nodes-base/nodes/Postgres/v2/methods/resourceMapping.ts @@ -70,7 +70,8 @@ export async function getMappingColumns( const canBeUsedToMatch = operation === 'upsert' ? unique.some((u) => u.attname === col.column_name) : true; const type = mapPostgresType(col.data_type); - const options = type === 'options' ? getEnumValues(enumInfo, col.udt_name) : undefined; + const options = + type === 'options' ? getEnumValues(enumInfo, col.udt_name as string) : undefined; const isAutoIncrement = col.column_default?.startsWith('nextval'); return { id: col.column_name,