From e00a2ee9cd71804c6192b813aec40d02e0979b53 Mon Sep 17 00:00:00 2001 From: Michael Kret Date: Fri, 26 Jul 2024 12:20:10 +0300 Subject: [PATCH 1/3] pagination and timeout fix --- .../test/v2/node/executeQuery.test.ts | 6 +- .../test/v2/node/insert.autoMapMode.test.ts | 6 +- .../test/v2/node/insert.manualMode.test.ts | 6 +- .../database/executeQuery.operation.ts | 61 ++++++++++++++----- .../v2/actions/database/insert.operation.ts | 6 +- .../Google/BigQuery/v2/methods/listSearch.ts | 8 +-- .../Google/BigQuery/v2/methods/loadOptions.ts | 6 +- .../Google/BigQuery/v2/transport/index.ts | 17 +++--- 8 files changed, 74 insertions(+), 42 deletions(-) diff --git a/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQuery.test.ts b/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQuery.test.ts index 39cefa7319002..f52b412f84151 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQuery.test.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQuery.test.ts @@ -47,8 +47,8 @@ describe('Test Google BigQuery V2, executeQuery', () => { const testNode = async (testData: WorkflowTestData, types: INodeTypes) => { const { result } = await executeWorkflow(testData, types); - expect(transport.googleApiRequest).toHaveBeenCalledTimes(2); - expect(transport.googleApiRequest).toHaveBeenCalledWith( + expect(transport.googleBigQueryApiRequest).toHaveBeenCalledTimes(2); + expect(transport.googleBigQueryApiRequest).toHaveBeenCalledWith( 'POST', '/v2/projects/test-project/jobs', { @@ -60,7 +60,7 @@ describe('Test Google BigQuery V2, executeQuery', () => { }, }, ); - expect(transport.googleApiRequest).toHaveBeenCalledWith( + expect(transport.googleBigQueryApiRequest).toHaveBeenCalledWith( 'GET', '/v2/projects/test-project/queries/job_123', undefined, diff --git a/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.autoMapMode.test.ts b/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.autoMapMode.test.ts index 001a06315f202..052963cff5b11 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.autoMapMode.test.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.autoMapMode.test.ts @@ -55,13 +55,13 @@ describe('Test Google BigQuery V2, insert auto map', () => { const testNode = async (testData: WorkflowTestData, types: INodeTypes) => { const { result } = await executeWorkflow(testData, types); - expect(transport.googleApiRequest).toHaveBeenCalledTimes(2); - expect(transport.googleApiRequest).toHaveBeenCalledWith( + expect(transport.googleBigQueryApiRequest).toHaveBeenCalledTimes(2); + expect(transport.googleBigQueryApiRequest).toHaveBeenCalledWith( 'GET', '/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text', {}, ); - expect(transport.googleApiRequest).toHaveBeenCalledWith( + expect(transport.googleBigQueryApiRequest).toHaveBeenCalledWith( 'POST', '/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text/insertAll', { diff --git a/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.manualMode.test.ts b/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.manualMode.test.ts index 0d0215d5cd594..87cc6b9cf7290 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.manualMode.test.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.manualMode.test.ts @@ -56,13 +56,13 @@ describe('Test Google BigQuery V2, insert define manually', () => { const testNode = async (testData: WorkflowTestData, types: INodeTypes) => { const { result } = await executeWorkflow(testData, types); - expect(transport.googleApiRequest).toHaveBeenCalledTimes(2); - expect(transport.googleApiRequest).toHaveBeenCalledWith( + expect(transport.googleBigQueryApiRequest).toHaveBeenCalledTimes(2); + expect(transport.googleBigQueryApiRequest).toHaveBeenCalledWith( 'GET', '/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json', {}, ); - expect(transport.googleApiRequest).toHaveBeenCalledWith( + expect(transport.googleBigQueryApiRequest).toHaveBeenCalledWith( 'POST', '/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json/insertAll', { diff --git a/packages/nodes-base/nodes/Google/BigQuery/v2/actions/database/executeQuery.operation.ts b/packages/nodes-base/nodes/Google/BigQuery/v2/actions/database/executeQuery.operation.ts index 7278e502b380c..6556e34bccfe8 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/v2/actions/database/executeQuery.operation.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/v2/actions/database/executeQuery.operation.ts @@ -9,7 +9,7 @@ import { ApplicationError, NodeOperationError, sleep } from 'n8n-workflow'; import type { ResponseWithJobReference } from '../../helpers/interfaces'; import { prepareOutput } from '../../helpers/utils'; -import { googleApiRequest } from '../../transport'; +import { googleBigQeryApiRequestAllItems, googleBigQueryApiRequest } from '../../transport'; import { getResolvables, updateDisplayOptions } from '@utils/utilities'; const properties: INodeProperties[] = [ @@ -108,18 +108,21 @@ const properties: INodeProperties[] = [ 'Limits the bytes billed for this query. Queries with bytes billed above this limit will fail (without incurring a charge). String in Int64Value format', }, { - displayName: 'Max Results', + displayName: 'Max Results Per Page', name: 'maxResults', type: 'number', default: 1000, - description: 'The maximum number of rows of data to return', + description: + 'Maximum number of results to return per page of results. This is particularly useful when dealing with large datasets. It will not affect the total number of results returned, e.g. rows in a table. You can use LIMIT in your SQL query to limit the number of rows returned.', }, { displayName: 'Timeout', name: 'timeoutMs', type: 'number', default: 10000, - description: 'How long to wait for the query to complete, in milliseconds', + hint: 'How long to wait for the query to complete, in milliseconds', + description: + 'Specifies the maximum amount of time, in milliseconds, that the client is willing to wait for the query to complete. Be aware that the call is not guaranteed to wait for the specified timeout; it typically returns after around 200 seconds (200,000 milliseconds), even if the query is not complete.', }, { displayName: 'Raw Output', @@ -154,19 +157,31 @@ const displayOptions = { export const description = updateDisplayOptions(displayOptions, properties); export async function execute(this: IExecuteFunctions): Promise { - // https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query - const items = this.getInputData(); const length = items.length; const returnData: INodeExecutionData[] = []; let jobs = []; + let maxResults = 1000; + let timeoutMs = 10000; for (let i = 0; i < length; i++) { try { let sqlQuery = this.getNodeParameter('sqlQuery', i) as string; - const options = this.getNodeParameter('options', i); + + const options = this.getNodeParameter('options', i) as { + defaultDataset?: string; + dryRun?: boolean; + includeSchema?: boolean; + location?: string; + maximumBytesBilled?: string; + maxResults?: number; + timeoutMs?: number; + rawOutput?: boolean; + useLegacySql?: boolean; + }; + const projectId = this.getNodeParameter('projectId', i, undefined, { extractValue: true, }); @@ -179,15 +194,25 @@ export async function execute(this: IExecuteFunctions): Promise { const projectId = this.getNodeParameter('projectId', undefined, { extractValue: true, }); const returnData: INodePropertyOptions[] = []; - const { datasets } = await googleApiRequest.call( + const { datasets } = await googleBigQueryApiRequest.call( this, 'GET', `/v2/projects/${projectId}/datasets`, @@ -33,7 +33,7 @@ export async function getSchema(this: ILoadOptionsFunctions): Promise Date: Sat, 27 Jul 2024 08:09:10 +0300 Subject: [PATCH 2/3] typo fix --- .../BigQuery/v2/actions/database/executeQuery.operation.ts | 6 +++--- .../nodes-base/nodes/Google/BigQuery/v2/transport/index.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/nodes-base/nodes/Google/BigQuery/v2/actions/database/executeQuery.operation.ts b/packages/nodes-base/nodes/Google/BigQuery/v2/actions/database/executeQuery.operation.ts index 6556e34bccfe8..e3633b0fbc877 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/v2/actions/database/executeQuery.operation.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/v2/actions/database/executeQuery.operation.ts @@ -9,7 +9,7 @@ import { ApplicationError, NodeOperationError, sleep } from 'n8n-workflow'; import type { ResponseWithJobReference } from '../../helpers/interfaces'; import { prepareOutput } from '../../helpers/utils'; -import { googleBigQeryApiRequestAllItems, googleBigQueryApiRequest } from '../../transport'; +import { googleBigQueryApiRequestAllItems, googleBigQueryApiRequest } from '../../transport'; import { getResolvables, updateDisplayOptions } from '@utils/utilities'; const properties: INodeProperties[] = [ @@ -255,7 +255,7 @@ export async function execute(this: IExecuteFunctions): Promise Date: Sat, 27 Jul 2024 08:40:58 +0300 Subject: [PATCH 3/3] tests fix --- .../Google/BigQuery/test/v2/node/executeQuery.test.ts | 11 ++++++----- .../BigQuery/test/v2/node/insert.autoMapMode.test.ts | 2 +- .../BigQuery/test/v2/node/insert.manualMode.test.ts | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQuery.test.ts b/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQuery.test.ts index f52b412f84151..f7a0965d3d2ac 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQuery.test.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/executeQuery.test.ts @@ -10,7 +10,7 @@ jest.mock('../../../v2/transport', () => { const originalModule = jest.requireActual('../../../v2/transport'); return { ...originalModule, - googleApiRequest: jest.fn(async (method: IHttpRequestMethods, resource: string) => { + googleBigQueryApiRequest: jest.fn(async (method: IHttpRequestMethods, resource: string) => { if (resource === '/v2/projects/test-project/jobs' && method === 'POST') { return { jobReference: { @@ -25,7 +25,7 @@ jest.mock('../../../v2/transport', () => { return {}; } }), - // googleApiRequestAllItems: jest.fn(async () => {}), + googleBigQueryApiRequestAllItems: jest.fn(async () => ({ rows: [], schema: {} })), }; }); @@ -47,7 +47,8 @@ describe('Test Google BigQuery V2, executeQuery', () => { const testNode = async (testData: WorkflowTestData, types: INodeTypes) => { const { result } = await executeWorkflow(testData, types); - expect(transport.googleBigQueryApiRequest).toHaveBeenCalledTimes(2); + expect(transport.googleBigQueryApiRequest).toHaveBeenCalledTimes(1); + expect(transport.googleBigQueryApiRequestAllItems).toHaveBeenCalledTimes(1); expect(transport.googleBigQueryApiRequest).toHaveBeenCalledWith( 'POST', '/v2/projects/test-project/jobs', @@ -60,11 +61,11 @@ describe('Test Google BigQuery V2, executeQuery', () => { }, }, ); - expect(transport.googleBigQueryApiRequest).toHaveBeenCalledWith( + expect(transport.googleBigQueryApiRequestAllItems).toHaveBeenCalledWith( 'GET', '/v2/projects/test-project/queries/job_123', undefined, - {}, + { location: undefined, maxResults: 1000, timeoutMs: 10000 }, ); expect(result.finished).toEqual(true); diff --git a/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.autoMapMode.test.ts b/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.autoMapMode.test.ts index 052963cff5b11..4ee4627838cce 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.autoMapMode.test.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.autoMapMode.test.ts @@ -10,7 +10,7 @@ jest.mock('../../../v2/transport', () => { const originalModule = jest.requireActual('../../../v2/transport'); return { ...originalModule, - googleApiRequest: jest.fn(async (method: IHttpRequestMethods, resource: string) => { + googleBigQueryApiRequest: jest.fn(async (method: IHttpRequestMethods, resource: string) => { if ( resource === '/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text' && diff --git a/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.manualMode.test.ts b/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.manualMode.test.ts index 87cc6b9cf7290..f4d94dda8b963 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.manualMode.test.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/test/v2/node/insert.manualMode.test.ts @@ -10,7 +10,7 @@ jest.mock('../../../v2/transport', () => { const originalModule = jest.requireActual('../../../v2/transport'); return { ...originalModule, - googleApiRequest: jest.fn(async (method: IHttpRequestMethods, resource: string) => { + googleBigQueryApiRequest: jest.fn(async (method: IHttpRequestMethods, resource: string) => { if ( resource === '/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json' &&