From 0772a4efd65cc43004caad1fcd754fbcfaba777f Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Wed, 24 Apr 2024 14:19:16 -0400 Subject: [PATCH 01/14] CUMULUS-3692:Granule list endpoint for basic postgres query --- packages/api/endpoints/granules.js | 12 ++- packages/db/src/index.ts | 3 + packages/db/src/search/BaseSearch.ts | 119 ++++++++++++++++++++++++++ packages/db/src/translate/granules.ts | 78 +++++++++++------ 4 files changed, 184 insertions(+), 28 deletions(-) create mode 100644 packages/db/src/search/BaseSearch.ts diff --git a/packages/api/endpoints/granules.js b/packages/api/endpoints/granules.js index fd634ce0555..d0ded6c4d03 100644 --- a/packages/api/endpoints/granules.js +++ b/packages/api/endpoints/granules.js @@ -12,6 +12,7 @@ const { v4: uuidv4 } = require('uuid'); const Logger = require('@cumulus/logger'); const { deconstructCollectionId } = require('@cumulus/message/Collections'); const { RecordDoesNotExist } = require('@cumulus/errors'); +const { BaseSearch } = require('@cumulus/db'); const { CollectionPgModel, @@ -101,6 +102,7 @@ function _createNewGranuleDateValue() { * @returns {Promise} the promise of express response object */ async function list(req, res) { + log.trace(`list query ${JSON.stringify(req.query)}`); const { getRecoveryStatus, ...queryStringParameters } = req.query; let es; @@ -113,7 +115,15 @@ async function list(req, res) { } else { es = new Search({ queryStringParameters }, 'granule', process.env.ES_INDEX); } - const result = await es.query(); + let result; + // TODO the condition should be removed after we support all the query parameters + if (Object.keys(queryStringParameters).filter((item) => !['limit', 'page', 'sort_key'].includes(item)).length === 0) { + log.info('list perform db search'); + const dbClient = new BaseSearch({ queryStringParameters }, 'granule'); + result = await dbClient.query(); + } else { + result = await es.query(); + } if (getRecoveryStatus === 'true') { return res.send(await addOrcaRecoveryStatus(result)); } diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 1f4a747dcbf..299952a2e59 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -136,6 +136,9 @@ export { export { QuerySearchClient, } from './lib/QuerySearchClient'; +export { + BaseSearch, +} from './search/BaseSearch'; export { AsyncOperationPgModel } from './models/async_operation'; export { BasePgModel } from './models/base'; diff --git a/packages/db/src/search/BaseSearch.ts b/packages/db/src/search/BaseSearch.ts new file mode 100644 index 00000000000..a67122b6274 --- /dev/null +++ b/packages/db/src/search/BaseSearch.ts @@ -0,0 +1,119 @@ +import { Knex } from 'knex'; + +import Logger from '@cumulus/logger'; +import { getKnexClient } from '../connection'; +import { TableNames } from '../tables'; +import { translatePostgresGranuleToApiGranuleWithoutDbQuery } from '../translate/granules'; + +const log = new Logger({ sender: '@cumulus/db/DbClient' }); +/** + * Class to handle fetching results for an arbitrary PostgreSQL query and + * paging through them. + */ +class BaseSearch { + //readonly query: Knex.QueryBuilder; + readonly limit: number; + readonly page: number; + offset: number; + params: object; + type: string | null; + + constructor(event: any, type = null) { + let params: any = {}; + const logLimit = 10; + //this.query = query; + this.type = type; + + // this will allow us to receive payload + // from GET and POST requests + if (event.queryStringParameters) { + params = event.queryStringParameters; + } + + // get page number + this.page = Number.parseInt((params.page) ? params.page : 1, 10); + this.params = params; + //log.debug('Generated params:', params, logDetails); + + this.limit = Number.parseInt((params.limit) ? params.limit : logLimit, 10); + + this.offset = (this.page - 1) * this.limit; + } + + _buildSearch(knex: Knex) + : { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + } { + const { + granules: granulesTable, + collections: collectionsTable, + providers: providersTable, + } = TableNames; + const countQuery = knex(granulesTable) + .count(`${granulesTable}.cumulus_id`) + .innerJoin(collectionsTable, `${granulesTable}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`) + .leftJoin(providersTable, `${granulesTable}.provider_cumulus_id`, `${providersTable}.cumulus_id`); + + const searchQuery = knex(granulesTable) + .select(`${granulesTable}.*`) + .select({ + providerName: `${providersTable}.name`, + collectionName: `${collectionsTable}.name`, + collectionVersion: `${collectionsTable}.version`, + }) + .innerJoin(collectionsTable, `${granulesTable}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`) + .leftJoin(providersTable, `${granulesTable}.provider_cumulus_id`, `${providersTable}.cumulus_id`) + .limit(this.limit) + .offset(this.offset); + return { countQuery, searchQuery }; + } + + _metaTemplate(): any { + return { + name: 'cumulus-api', + stack: process.env.stackName, + table: this.type, + }; + } + + async query() { + const knex = await getKnexClient(); + const { countQuery, searchQuery } = this._buildSearch(knex); + try { + const meta = this._metaTemplate(); + meta.limit = this.limit; + meta.page = this.page; + const countResult = await countQuery; + log.trace(`Count response: ${JSON.stringify(countResult)}`); + meta.count = Number(countResult[0]?.count ?? 0); + + const searchResult = await searchQuery; + log.trace(`Search response: ${JSON.stringify(searchResult)}`); + const convertedResult = searchResult.map((item: any) => { + log.trace(`About to translate item: ${JSON.stringify(item)}`); + const granulePgRecord = item; + const collectionPgRecord = { + cumulus_id: item.collection_cumulus_id, + name: item.collectionName, + version: item.collectionVersion, + }; + const providerPgRecord = item.provider_cumulus_id + ?? { cumulus_id: item.provider_cumulus_id, name: item.providerName }; + log.trace(JSON.stringify(item)); + return translatePostgresGranuleToApiGranuleWithoutDbQuery({ + granulePgRecord, collectionPgRecord, providerPgRecord, + }); + }); + + return { + meta, + results: convertedResult, + }; + } catch (error) { + return error; + } + } +} + +export { BaseSearch }; diff --git a/packages/db/src/translate/granules.ts b/packages/db/src/translate/granules.ts index 45b22ca14b5..cb2cded3b15 100644 --- a/packages/db/src/translate/granules.ts +++ b/packages/db/src/translate/granules.ts @@ -14,12 +14,56 @@ import { FilePgModel } from '../models/file'; import { getExecutionInfoByGranuleCumulusId } from '../lib/execution'; import { PostgresCollectionRecord } from '../types/collection'; +import { PostgresExecutionRecord } from '../types/execution'; import { PostgresGranule, PostgresGranuleRecord } from '../types/granule'; +import { PostgresFileRecord } from '../types/file'; +import { PostgresPdrRecord } from '../types/pdr'; import { GranuleWithProviderAndCollectionInfo } from '../types/query'; import { PostgresProviderRecord } from '../types/provider'; import { translatePostgresFileToApiFile } from './file'; +export const translatePostgresGranuleToApiGranuleWithoutDbQuery = ({ + granulePgRecord, + collectionPgRecord, + executionUrls = [], + files = [], + pdr, + providerPgRecord, +}: { + granulePgRecord: PostgresGranuleRecord, + collectionPgRecord: Pick, + executionUrls?: Partial[], + files?: PostgresFileRecord[], + pdr?: PostgresPdrRecord, + providerPgRecord?: Pick, +}): ApiGranuleRecord => removeNilProperties({ + beginningDateTime: granulePgRecord.beginning_date_time?.toISOString(), + cmrLink: granulePgRecord.cmr_link, + collectionId: constructCollectionId(collectionPgRecord.name, collectionPgRecord.version), + createdAt: granulePgRecord.created_at?.getTime(), + duration: granulePgRecord.duration, + endingDateTime: granulePgRecord.ending_date_time?.toISOString(), + error: granulePgRecord.error, + execution: executionUrls[0] ? executionUrls[0].url : undefined, + files: files.length > 0 ? files.map((file) => translatePostgresFileToApiFile(file)) : [], + granuleId: granulePgRecord.granule_id, + lastUpdateDateTime: granulePgRecord.last_update_date_time?.toISOString(), + pdrName: pdr ? pdr.name : undefined, + processingEndDateTime: granulePgRecord.processing_end_date_time?.toISOString(), + processingStartDateTime: granulePgRecord.processing_start_date_time?.toISOString(), + productionDateTime: granulePgRecord.production_date_time?.toISOString(), + productVolume: granulePgRecord.product_volume, + provider: providerPgRecord ? providerPgRecord.name : undefined, + published: granulePgRecord.published, + queryFields: granulePgRecord.query_fields, + status: granulePgRecord.status as GranuleStatus, + timestamp: granulePgRecord.timestamp?.getTime(), + timeToArchive: granulePgRecord.time_to_archive, + timeToPreprocess: granulePgRecord.time_to_process, + updatedAt: granulePgRecord.updated_at?.getTime(), +}); + /** * Generate an API Granule object from a Postgres Granule with associated Files. * @@ -88,34 +132,14 @@ export const translatePostgresGranuleToApiGranule = async ({ ); } - const apiGranule: ApiGranuleRecord = removeNilProperties({ - beginningDateTime: granulePgRecord.beginning_date_time?.toISOString(), - cmrLink: granulePgRecord.cmr_link, - collectionId: constructCollectionId(collection.name, collection.version), - createdAt: granulePgRecord.created_at?.getTime(), - duration: granulePgRecord.duration, - endingDateTime: granulePgRecord.ending_date_time?.toISOString(), - error: granulePgRecord.error, - execution: executionUrls[0] ? executionUrls[0].url : undefined, - files: files.length > 0 ? files.map((file) => translatePostgresFileToApiFile(file)) : [], - granuleId: granulePgRecord.granule_id, - lastUpdateDateTime: granulePgRecord.last_update_date_time?.toISOString(), - pdrName: pdr ? pdr.name : undefined, - processingEndDateTime: granulePgRecord.processing_end_date_time?.toISOString(), - processingStartDateTime: granulePgRecord.processing_start_date_time?.toISOString(), - productionDateTime: granulePgRecord.production_date_time?.toISOString(), - productVolume: granulePgRecord.product_volume, - provider: provider ? provider.name : undefined, - published: granulePgRecord.published, - queryFields: granulePgRecord.query_fields, - status: granulePgRecord.status as GranuleStatus, - timestamp: granulePgRecord.timestamp?.getTime(), - timeToArchive: granulePgRecord.time_to_archive, - timeToPreprocess: granulePgRecord.time_to_process, - updatedAt: granulePgRecord.updated_at?.getTime(), + return translatePostgresGranuleToApiGranuleWithoutDbQuery({ + granulePgRecord, + collectionPgRecord: collection, + executionUrls, + files, + pdr, + providerPgRecord: provider, }); - - return apiGranule; }; /** From dfdf68f2fbd10e7e24aaaca1ceec788a33e407cd Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Thu, 25 Apr 2024 22:30:46 -0400 Subject: [PATCH 02/14] refactor --- packages/api/endpoints/granules.js | 6 +- packages/db/src/index.ts | 3 + packages/db/src/search/BaseSearch.ts | 98 ++++++++----------------- packages/db/src/search/GranuleSearch.ts | 67 +++++++++++++++++ 4 files changed, 105 insertions(+), 69 deletions(-) create mode 100644 packages/db/src/search/GranuleSearch.ts diff --git a/packages/api/endpoints/granules.js b/packages/api/endpoints/granules.js index d0ded6c4d03..7ceb6fa4e83 100644 --- a/packages/api/endpoints/granules.js +++ b/packages/api/endpoints/granules.js @@ -12,7 +12,7 @@ const { v4: uuidv4 } = require('uuid'); const Logger = require('@cumulus/logger'); const { deconstructCollectionId } = require('@cumulus/message/Collections'); const { RecordDoesNotExist } = require('@cumulus/errors'); -const { BaseSearch } = require('@cumulus/db'); +const { GranuleSearch } = require('@cumulus/db'); const { CollectionPgModel, @@ -119,8 +119,8 @@ async function list(req, res) { // TODO the condition should be removed after we support all the query parameters if (Object.keys(queryStringParameters).filter((item) => !['limit', 'page', 'sort_key'].includes(item)).length === 0) { log.info('list perform db search'); - const dbClient = new BaseSearch({ queryStringParameters }, 'granule'); - result = await dbClient.query(); + const dbSearch = new GranuleSearch({ queryStringParameters }, 'granule'); + result = await dbSearch.query(); } else { result = await es.query(); } diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 299952a2e59..c761e630c90 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -139,6 +139,9 @@ export { export { BaseSearch, } from './search/BaseSearch'; +export { + GranuleSearch, +} from './search/GranuleSearch'; export { AsyncOperationPgModel } from './models/async_operation'; export { BasePgModel } from './models/base'; diff --git a/packages/db/src/search/BaseSearch.ts b/packages/db/src/search/BaseSearch.ts index a67122b6274..948319c4345 100644 --- a/packages/db/src/search/BaseSearch.ts +++ b/packages/db/src/search/BaseSearch.ts @@ -1,75 +1,50 @@ import { Knex } from 'knex'; - import Logger from '@cumulus/logger'; import { getKnexClient } from '../connection'; -import { TableNames } from '../tables'; -import { translatePostgresGranuleToApiGranuleWithoutDbQuery } from '../translate/granules'; -const log = new Logger({ sender: '@cumulus/db/DbClient' }); +const log = new Logger({ sender: '@cumulus/db/BaseSearch' }); + /** - * Class to handle fetching results for an arbitrary PostgreSQL query and - * paging through them. + * Class to build db search query and return result */ class BaseSearch { - //readonly query: Knex.QueryBuilder; readonly limit: number; readonly page: number; offset: number; - params: object; + params: any; type: string | null; constructor(event: any, type = null) { - let params: any = {}; const logLimit = 10; - //this.query = query; this.type = type; - - // this will allow us to receive payload - // from GET and POST requests - if (event.queryStringParameters) { - params = event.queryStringParameters; - } - - // get page number - this.page = Number.parseInt((params.page) ? params.page : 1, 10); - this.params = params; - //log.debug('Generated params:', params, logDetails); - - this.limit = Number.parseInt((params.limit) ? params.limit : logLimit, 10); - + this.params = event.queryStringParameters ?? {}; + this.page = Number.parseInt((this.params.page) ? this.params.page : 1, 10); + this.limit = Number.parseInt((this.params.limit) ? this.params.limit : logLimit, 10); this.offset = (this.page - 1) * this.limit; } - _buildSearch(knex: Knex) + protected buildBasicQuery(knex: Knex): { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + } { + log.trace(`buildBasicQuery is not implemented ${knex.constructor.name}`); + throw new Error('buildBasicQuery is not implemented'); + } + + private _buildSearch(knex: Knex) : { countQuery: Knex.QueryBuilder, searchQuery: Knex.QueryBuilder, } { - const { - granules: granulesTable, - collections: collectionsTable, - providers: providersTable, - } = TableNames; - const countQuery = knex(granulesTable) - .count(`${granulesTable}.cumulus_id`) - .innerJoin(collectionsTable, `${granulesTable}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`) - .leftJoin(providersTable, `${granulesTable}.provider_cumulus_id`, `${providersTable}.cumulus_id`); - - const searchQuery = knex(granulesTable) - .select(`${granulesTable}.*`) - .select({ - providerName: `${providersTable}.name`, - collectionName: `${collectionsTable}.name`, - collectionVersion: `${collectionsTable}.version`, - }) - .innerJoin(collectionsTable, `${granulesTable}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`) - .leftJoin(providersTable, `${granulesTable}.provider_cumulus_id`, `${providersTable}.cumulus_id`) - .limit(this.limit) - .offset(this.offset); - return { countQuery, searchQuery }; + const { countQuery, searchQuery } = this.buildBasicQuery(knex); + const updatedQuery = searchQuery.modify((queryBuilder) => { + if (this.limit) queryBuilder.limit(this.limit); + if (this.offset) queryBuilder.offset(this.offset); + }); + return { countQuery, searchQuery: updatedQuery }; } - _metaTemplate(): any { + private _metaTemplate(): any { return { name: 'cumulus-api', stack: process.env.stackName, @@ -77,6 +52,11 @@ class BaseSearch { }; } + protected translatePostgresRecordsToApiRecords(pgRecords: any) { + log.error(`translatePostgresRecordsToApiRecords is not implemented ${pgRecords[0]}`); + throw new Error('translatePostgresRecordsToApiRecords is not implemented'); + } + async query() { const knex = await getKnexClient(); const { countQuery, searchQuery } = this._buildSearch(knex); @@ -88,27 +68,13 @@ class BaseSearch { log.trace(`Count response: ${JSON.stringify(countResult)}`); meta.count = Number(countResult[0]?.count ?? 0); - const searchResult = await searchQuery; - log.trace(`Search response: ${JSON.stringify(searchResult)}`); - const convertedResult = searchResult.map((item: any) => { - log.trace(`About to translate item: ${JSON.stringify(item)}`); - const granulePgRecord = item; - const collectionPgRecord = { - cumulus_id: item.collection_cumulus_id, - name: item.collectionName, - version: item.collectionVersion, - }; - const providerPgRecord = item.provider_cumulus_id - ?? { cumulus_id: item.provider_cumulus_id, name: item.providerName }; - log.trace(JSON.stringify(item)); - return translatePostgresGranuleToApiGranuleWithoutDbQuery({ - granulePgRecord, collectionPgRecord, providerPgRecord, - }); - }); + const pgRecords = await searchQuery; + log.trace(`Search response: ${JSON.stringify(pgRecords)}`); + const apiRecords = this.translatePostgresRecordsToApiRecords(pgRecords); return { meta, - results: convertedResult, + results: apiRecords, }; } catch (error) { return error; diff --git a/packages/db/src/search/GranuleSearch.ts b/packages/db/src/search/GranuleSearch.ts new file mode 100644 index 00000000000..aa616a05936 --- /dev/null +++ b/packages/db/src/search/GranuleSearch.ts @@ -0,0 +1,67 @@ +import { Knex } from 'knex'; + +import Logger from '@cumulus/logger'; +import { translatePostgresGranuleToApiGranuleWithoutDbQuery } from '../translate/granules'; + +import { BaseSearch } from './BaseSearch'; +import { TableNames } from '../tables'; + +const log = new Logger({ sender: '@cumulus/db/GranuleSearch' }); + +export class GranuleSearch extends BaseSearch { + constructor(event: any, type = null) { + super(event, type); + } + + protected buildBasicQuery(knex: Knex) + : { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + } { + const { + granules: granulesTable, + collections: collectionsTable, + providers: providersTable, + pdrs: pdrsTable, + } = TableNames; + const countQuery = knex(granulesTable) + .count(`${granulesTable}.cumulus_id`) + .innerJoin(collectionsTable, `${granulesTable}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`) + .leftJoin(providersTable, `${granulesTable}.provider_cumulus_id`, `${providersTable}.cumulus_id`) + .leftJoin(pdrsTable, `${granulesTable}.pdr_cumulus_id`, `${pdrsTable}.cumulus_id`); + + const searchQuery = knex(granulesTable) + .select(`${granulesTable}.*`) + .select({ + providerName: `${providersTable}.name`, + collectionName: `${collectionsTable}.name`, + collectionVersion: `${collectionsTable}.version`, + pdrName: `${pdrsTable}.name`, + }) + .innerJoin(collectionsTable, `${granulesTable}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`) + .leftJoin(providersTable, `${granulesTable}.provider_cumulus_id`, `${providersTable}.cumulus_id`) + .leftJoin(pdrsTable, `${granulesTable}.pdr_cumulus_id`, `${pdrsTable}.cumulus_id`) + .limit(this.limit) + .offset(this.offset); + return { countQuery, searchQuery }; + } + + protected translatePostgresRecordsToApiRecords(pgRecords: any[]) { + const apiRecords = pgRecords.map((item: any) => { + log.trace(`About to translate item: ${JSON.stringify(item)}`); + const granulePgRecord = item; + const collectionPgRecord = { + cumulus_id: item.collection_cumulus_id, + name: item.collectionName, + version: item.collectionVersion, + }; + const providerPgRecord = item.provider_cumulus_id + ?? { cumulus_id: item.provider_cumulus_id, name: item.providerName }; + log.trace(JSON.stringify(item)); + return translatePostgresGranuleToApiGranuleWithoutDbQuery({ + granulePgRecord, collectionPgRecord, providerPgRecord, + }); + }); + return apiRecords; + } +} From 4efe9908d308af56ec0794086b57e7b93890161c Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Fri, 26 Apr 2024 10:14:35 -0400 Subject: [PATCH 03/14] refactor --- packages/api/endpoints/granules.js | 2 +- packages/db/src/search/BaseSearch.ts | 4 ++-- packages/db/src/search/GranuleSearch.ts | 8 +++----- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/api/endpoints/granules.js b/packages/api/endpoints/granules.js index 7ceb6fa4e83..c3efe2d414a 100644 --- a/packages/api/endpoints/granules.js +++ b/packages/api/endpoints/granules.js @@ -119,7 +119,7 @@ async function list(req, res) { // TODO the condition should be removed after we support all the query parameters if (Object.keys(queryStringParameters).filter((item) => !['limit', 'page', 'sort_key'].includes(item)).length === 0) { log.info('list perform db search'); - const dbSearch = new GranuleSearch({ queryStringParameters }, 'granule'); + const dbSearch = new GranuleSearch({ queryStringParameters }); result = await dbSearch.query(); } else { result = await es.query(); diff --git a/packages/db/src/search/BaseSearch.ts b/packages/db/src/search/BaseSearch.ts index 948319c4345..1443a1d6661 100644 --- a/packages/db/src/search/BaseSearch.ts +++ b/packages/db/src/search/BaseSearch.ts @@ -12,9 +12,9 @@ class BaseSearch { readonly page: number; offset: number; params: any; - type: string | null; + type?: string; - constructor(event: any, type = null) { + constructor(event: any, type?: string) { const logLimit = 10; this.type = type; this.params = event.queryStringParameters ?? {}; diff --git a/packages/db/src/search/GranuleSearch.ts b/packages/db/src/search/GranuleSearch.ts index aa616a05936..04fb0ff62e6 100644 --- a/packages/db/src/search/GranuleSearch.ts +++ b/packages/db/src/search/GranuleSearch.ts @@ -9,8 +9,8 @@ import { TableNames } from '../tables'; const log = new Logger({ sender: '@cumulus/db/GranuleSearch' }); export class GranuleSearch extends BaseSearch { - constructor(event: any, type = null) { - super(event, type); + constructor(event: any) { + super(event, 'granule'); } protected buildBasicQuery(knex: Knex) @@ -40,9 +40,7 @@ export class GranuleSearch extends BaseSearch { }) .innerJoin(collectionsTable, `${granulesTable}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`) .leftJoin(providersTable, `${granulesTable}.provider_cumulus_id`, `${providersTable}.cumulus_id`) - .leftJoin(pdrsTable, `${granulesTable}.pdr_cumulus_id`, `${pdrsTable}.cumulus_id`) - .limit(this.limit) - .offset(this.offset); + .leftJoin(pdrsTable, `${granulesTable}.pdr_cumulus_id`, `${pdrsTable}.cumulus_id`); return { countQuery, searchQuery }; } From 5cc833cb38b0316231f6a066a91367ffae07f41e Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Mon, 29 Apr 2024 14:59:10 -0400 Subject: [PATCH 04/14] typing --- packages/db/src/search/BaseSearch.ts | 91 ++++++++--- packages/db/src/search/GranuleSearch.ts | 56 +++++-- packages/db/src/translate/granules.ts | 2 +- packages/db/src/types/search.ts | 15 ++ .../db/tests/search/test-GranuleSearch.js | 148 ++++++++++++++++++ 5 files changed, 273 insertions(+), 39 deletions(-) create mode 100644 packages/db/src/types/search.ts create mode 100644 packages/db/tests/search/test-GranuleSearch.js diff --git a/packages/db/src/search/BaseSearch.ts b/packages/db/src/search/BaseSearch.ts index 1443a1d6661..ebc79b3fc7b 100644 --- a/packages/db/src/search/BaseSearch.ts +++ b/packages/db/src/search/BaseSearch.ts @@ -1,36 +1,64 @@ import { Knex } from 'knex'; import Logger from '@cumulus/logger'; import { getKnexClient } from '../connection'; +import { BaseRecord } from '../types/base'; +import { ParsedQueryParameters, QueryEvent, QueryStringParameters } from '../types/search'; const log = new Logger({ sender: '@cumulus/db/BaseSearch' }); +export interface Meta { + name: string, + stack?: string, + table?: string, + limit?: number, + page?: number, + count?: number, +} + /** - * Class to build db search query and return result + * Class to build and execute db search query */ class BaseSearch { - readonly limit: number; - readonly page: number; - offset: number; - params: any; - type?: string; + readonly type?: string; + readonly queryStringParameters: QueryStringParameters; + // parsed from queryStringParameters + parsedQueryParameters: ParsedQueryParameters = {}; - constructor(event: any, type?: string) { - const logLimit = 10; + constructor(event: QueryEvent, type?: string) { this.type = type; - this.params = event.queryStringParameters ?? {}; - this.page = Number.parseInt((this.params.page) ? this.params.page : 1, 10); - this.limit = Number.parseInt((this.params.limit) ? this.params.limit : logLimit, 10); - this.offset = (this.page - 1) * this.limit; + this.queryStringParameters = event?.queryStringParameters ?? {}; + this.parsedQueryParameters.page = Number.parseInt( + (this.queryStringParameters.page) ?? '1', + 10 + ); + this.parsedQueryParameters.limit = Number.parseInt( + (this.queryStringParameters.limit) ?? '10', + 10 + ); + this.parsedQueryParameters.offset = (this.parsedQueryParameters.page - 1) + * this.parsedQueryParameters.limit; } + /** + * build basic query + * + * @param knex - DB client + * @throws - function is not implemented + */ protected buildBasicQuery(knex: Knex): { countQuery: Knex.QueryBuilder, searchQuery: Knex.QueryBuilder, } { - log.trace(`buildBasicQuery is not implemented ${knex.constructor.name}`); + log.debug(`buildBasicQuery is not implemented ${knex.constructor.name}`); throw new Error('buildBasicQuery is not implemented'); } + /** + * build the search query + * + * @param knex - DB client + * @returns queries for getting count and search result + */ private _buildSearch(knex: Knex) : { countQuery: Knex.QueryBuilder, @@ -38,13 +66,18 @@ class BaseSearch { } { const { countQuery, searchQuery } = this.buildBasicQuery(knex); const updatedQuery = searchQuery.modify((queryBuilder) => { - if (this.limit) queryBuilder.limit(this.limit); - if (this.offset) queryBuilder.offset(this.offset); + if (this.parsedQueryParameters.limit) queryBuilder.limit(this.parsedQueryParameters.limit); + if (this.parsedQueryParameters.offset) queryBuilder.offset(this.parsedQueryParameters.offset); }); return { countQuery, searchQuery: updatedQuery }; } - private _metaTemplate(): any { + /** + * metadata template for query result + * + * @returns metadata template + */ + private _metaTemplate(): Meta { return { name: 'cumulus-api', stack: process.env.stackName, @@ -52,24 +85,34 @@ class BaseSearch { }; } - protected translatePostgresRecordsToApiRecords(pgRecords: any) { + /** + * Translate postgres records to api records + * + * @param pgRecords - postgres records returned from query + * @throws - function is not implemented + */ + protected translatePostgresRecordsToApiRecords(pgRecords: BaseRecord[]) { log.error(`translatePostgresRecordsToApiRecords is not implemented ${pgRecords[0]}`); throw new Error('translatePostgresRecordsToApiRecords is not implemented'); } - async query() { - const knex = await getKnexClient(); + /** + * build and execute search query + * + * @param testKnex - knex for testing + * @returns search result + */ + async query(testKnex: Knex | undefined) { + const knex = testKnex ?? await getKnexClient(); const { countQuery, searchQuery } = this._buildSearch(knex); try { - const meta = this._metaTemplate(); - meta.limit = this.limit; - meta.page = this.page; const countResult = await countQuery; - log.trace(`Count response: ${JSON.stringify(countResult)}`); + const meta = this._metaTemplate(); + meta.limit = this.parsedQueryParameters.limit; + meta.page = this.parsedQueryParameters.page; meta.count = Number(countResult[0]?.count ?? 0); const pgRecords = await searchQuery; - log.trace(`Search response: ${JSON.stringify(pgRecords)}`); const apiRecords = this.translatePostgresRecordsToApiRecords(pgRecords); return { diff --git a/packages/db/src/search/GranuleSearch.ts b/packages/db/src/search/GranuleSearch.ts index 04fb0ff62e6..8ff2ec6eb74 100644 --- a/packages/db/src/search/GranuleSearch.ts +++ b/packages/db/src/search/GranuleSearch.ts @@ -1,18 +1,44 @@ import { Knex } from 'knex'; +import { ApiGranuleRecord } from '@cumulus/types/api/granules'; import Logger from '@cumulus/logger'; -import { translatePostgresGranuleToApiGranuleWithoutDbQuery } from '../translate/granules'; +import { BaseRecord } from '../types/base'; import { BaseSearch } from './BaseSearch'; +import { PostgresGranuleRecord } from '../types/granule'; +import { QueryEvent } from '../types/search'; + import { TableNames } from '../tables'; +import { translatePostgresGranuleToApiGranuleWithoutDbQuery } from '../translate/granules'; + +const log = new Logger({ sender: '@cumulus/db/BaseSearch' }); -const log = new Logger({ sender: '@cumulus/db/GranuleSearch' }); +export interface GranuleRecord extends BaseRecord, PostgresGranuleRecord { + cumulus_id: number, + updated_at: Date, + collection_cumulus_id: number, + collectionName: string, + collectionVersion: string, + pdr_cumulus_id: number, + pdrName?: string, + provider_cumulus_id?: number, + providerName?: string, +} +/** + * Class to build and execute db search query for granules + */ export class GranuleSearch extends BaseSearch { - constructor(event: any) { + constructor(event: QueryEvent) { super(event, 'granule'); } + /** + * build basic query + * + * @param knex - DB client + * @returns queries for getting count and search result + */ protected buildBasicQuery(knex: Knex) : { countQuery: Knex.QueryBuilder, @@ -25,10 +51,7 @@ export class GranuleSearch extends BaseSearch { pdrs: pdrsTable, } = TableNames; const countQuery = knex(granulesTable) - .count(`${granulesTable}.cumulus_id`) - .innerJoin(collectionsTable, `${granulesTable}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`) - .leftJoin(providersTable, `${granulesTable}.provider_cumulus_id`, `${providersTable}.cumulus_id`) - .leftJoin(pdrsTable, `${granulesTable}.pdr_cumulus_id`, `${pdrsTable}.cumulus_id`); + .count(`${granulesTable}.cumulus_id`); const searchQuery = knex(granulesTable) .select(`${granulesTable}.*`) @@ -44,20 +67,25 @@ export class GranuleSearch extends BaseSearch { return { countQuery, searchQuery }; } - protected translatePostgresRecordsToApiRecords(pgRecords: any[]) { - const apiRecords = pgRecords.map((item: any) => { - log.trace(`About to translate item: ${JSON.stringify(item)}`); + /** + * Translate postgres records to api records + * + * @param pgRecords - postgres records returned from query + * @returns translated api records + */ + protected translatePostgresRecordsToApiRecords(pgRecords: GranuleRecord[]) : ApiGranuleRecord[] { + log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); + const apiRecords = pgRecords.map((item: GranuleRecord) => { const granulePgRecord = item; const collectionPgRecord = { cumulus_id: item.collection_cumulus_id, name: item.collectionName, version: item.collectionVersion, }; - const providerPgRecord = item.provider_cumulus_id - ?? { cumulus_id: item.provider_cumulus_id, name: item.providerName }; - log.trace(JSON.stringify(item)); + const pdr = item.pdrName ? { name: item.pdrName } : undefined; + const providerPgRecord = item.providerName ? { name: item.providerName } : undefined; return translatePostgresGranuleToApiGranuleWithoutDbQuery({ - granulePgRecord, collectionPgRecord, providerPgRecord, + granulePgRecord, collectionPgRecord, pdr, providerPgRecord, }); }); return apiRecords; diff --git a/packages/db/src/translate/granules.ts b/packages/db/src/translate/granules.ts index cb2cded3b15..57a04d96409 100644 --- a/packages/db/src/translate/granules.ts +++ b/packages/db/src/translate/granules.ts @@ -35,7 +35,7 @@ export const translatePostgresGranuleToApiGranuleWithoutDbQuery = ({ collectionPgRecord: Pick, executionUrls?: Partial[], files?: PostgresFileRecord[], - pdr?: PostgresPdrRecord, + pdr?: Pick, providerPgRecord?: Pick, }): ApiGranuleRecord => removeNilProperties({ beginningDateTime: granulePgRecord.beginning_date_time?.toISOString(), diff --git a/packages/db/src/types/search.ts b/packages/db/src/types/search.ts new file mode 100644 index 00000000000..bc1920d8081 --- /dev/null +++ b/packages/db/src/types/search.ts @@ -0,0 +1,15 @@ +export interface QueryStringParameters { + limit?: string, + page?: string, + [key: string]: string | string[] | undefined, +} + +export interface QueryEvent { + queryStringParameters?: QueryStringParameters, +} + +export interface ParsedQueryParameters { + limit?: number, + offset?: number, + page?: number, +} diff --git a/packages/db/tests/search/test-GranuleSearch.js b/packages/db/tests/search/test-GranuleSearch.js new file mode 100644 index 00000000000..a18690d70b0 --- /dev/null +++ b/packages/db/tests/search/test-GranuleSearch.js @@ -0,0 +1,148 @@ +const test = require('ava'); +const cryptoRandomString = require('crypto-random-string'); +const range = require('lodash/range'); + +const { constructCollectionId } = require('@cumulus/message/Collections'); + +const { + CollectionPgModel, + fakeCollectionRecordFactory, + fakeGranuleRecordFactory, + fakePdrRecordFactory, + fakeProviderRecordFactory, + generateLocalTestDb, + GranulePgModel, + GranuleSearch, + PdrPgModel, + ProviderPgModel, + migrationDir, +} = require('../../dist'); + +const testDbName = `granule_${cryptoRandomString({ length: 10 })}`; + +test.before(async (t) => { + const { knexAdmin, knex } = await generateLocalTestDb( + testDbName, + migrationDir + ); + t.context.knexAdmin = knexAdmin; + t.context.knex = knex; + + // Create collection + t.context.collectionPgModel = new CollectionPgModel(); + t.context.collectionName = 'fakeCollection'; + t.context.collectionVersion = 'v1'; + + const collectionName2 = 'fakeCollection2'; + const collectionVersion2 = 'v2'; + + t.context.collectionId = constructCollectionId( + t.context.collectionName, + t.context.collectionVersion + ); + + t.context.collectionId2 = constructCollectionId( + collectionName2, + collectionVersion2 + ); + + t.context.testPgCollection = fakeCollectionRecordFactory({ + name: t.context.collectionName, + version: t.context.collectionVersion, + }); + t.context.testPgCollection2 = fakeCollectionRecordFactory({ + name: collectionName2, + version: collectionVersion2, + }); + + const [pgCollection] = await t.context.collectionPgModel.create( + t.context.knex, + t.context.testPgCollection + ); + const [pgCollection2] = await t.context.collectionPgModel.create( + t.context.knex, + t.context.testPgCollection2 + ); + t.context.collectionCumulusId = pgCollection.cumulus_id; + t.context.collectionCumulusId2 = pgCollection2.cumulus_id; + + // Create provider + t.context.providerPgModel = new ProviderPgModel(); + t.context.provider = fakeProviderRecordFactory(); + + const [pgProvider] = await t.context.providerPgModel.create( + t.context.knex, + t.context.provider + ); + t.context.providerCumulusId = pgProvider.cumulus_id; + + // Create PDR + t.context.pdrPgModel = new PdrPgModel(); + t.context.pdr = fakePdrRecordFactory({ + collection_cumulus_id: pgCollection.cumulus_id, + provider_cumulus_id: t.context.providerCumulusId, + }); + const [pgPdr] = await t.context.pdrPgModel.create( + t.context.knex, + t.context.pdr + ); + t.context.pdrCumulusId = pgPdr.cumulus_id; + + // Create Granule + t.context.granulePgModel = new GranulePgModel(); + t.context.pgGranules = await t.context.granulePgModel.insert( + knex, + range(100).map((num) => fakeGranuleRecordFactory({ + collection_cumulus_id: (num % 2) + ? t.context.collectionCumulusId : t.context.collectionCumulusId2, + pdr_cumulus_id: t.context.pdrCumulusId, + provider_cumulus_id: t.context.providerCumulusId, + })) + ); +}); + +test('Granule search returns 10 granule records by default', async (t) => { + const { knex } = t.context; + const dbSearch = new GranuleSearch(); + const response = await dbSearch.query(knex); + + t.is(response.meta.count, 100); + + const apiGranules = response.results || {}; + t.is(apiGranules.length, 10); + const validatedRecords = apiGranules.filter((granule) => ( + [t.context.collectionId, t.context.collectionId2].includes(granule.collectionId) + && granule.provider === t.context.provider.name + && granule.pdrName === t.context.pdr.name)); + t.is(validatedRecords.length, apiGranules.length); +}); + +test('Granule search supports page and limit params', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 20, + page: 2, + }; + let dbSearch = new GranuleSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 20); + + queryStringParameters = { + limit: 11, + page: 10, + }; + dbSearch = new GranuleSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 1); + + queryStringParameters = { + limit: 10, + page: 11, + }; + dbSearch = new GranuleSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 0); +}); From 883e65fd5f0cd80f99f57a6d6b64d6be2d3e5358 Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Mon, 29 Apr 2024 15:22:35 -0400 Subject: [PATCH 05/14] add changelog entry --- CHANGELOG.md | 6 ++++++ packages/api/endpoints/granules.js | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 044ea2bf226..b7f74e86b2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## Unreleased +### Replace ElasticSearch Phase 1 + +- **CUMULUS-3692** + - Update granules List endpoints to query postgres for basic queries + + ### Migration Notes #### CUMULUS-3449 Please follow instructions before upgrading Cumulus. diff --git a/packages/api/endpoints/granules.js b/packages/api/endpoints/granules.js index c3efe2d414a..0f4b2cc1f55 100644 --- a/packages/api/endpoints/granules.js +++ b/packages/api/endpoints/granules.js @@ -118,7 +118,7 @@ async function list(req, res) { let result; // TODO the condition should be removed after we support all the query parameters if (Object.keys(queryStringParameters).filter((item) => !['limit', 'page', 'sort_key'].includes(item)).length === 0) { - log.info('list perform db search'); + log.debug('list perform db search'); const dbSearch = new GranuleSearch({ queryStringParameters }); result = await dbSearch.query(); } else { From cd20bd452f32c915ccfee4d5f3141bbd1b876c2b Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Mon, 29 Apr 2024 15:52:39 -0400 Subject: [PATCH 06/14] skip search_after --- packages/api/tests/endpoints/test-granules.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/api/tests/endpoints/test-granules.js b/packages/api/tests/endpoints/test-granules.js index 63fb708a6a5..9ed8345f8bc 100644 --- a/packages/api/tests/endpoints/test-granules.js +++ b/packages/api/tests/endpoints/test-granules.js @@ -401,7 +401,8 @@ test.after.always(async (t) => { await cleanupTestIndex(t.context); }); -test.serial('default lists and paginates correctly with search_after', async (t) => { +// TODO postgres query doesn't return searchContext +test.serial.skip('default lists and paginates correctly with search_after', async (t) => { const granuleIds = t.context.fakePGGranules.map((i) => i.granule_id); const response = await request(app) .get('/granules') @@ -3846,7 +3847,8 @@ test.serial('PUT returns 404 if collection is not part of URI', async (t) => { t.is(response.statusCode, 404); }); -test.serial('default paginates correctly with search_after', async (t) => { +// TODO postgres query doesn't return searchContext +test.serial.skip('default paginates correctly with search_after', async (t) => { const response = await request(app) .get('/granules?limit=1') .set('Accept', 'application/json') From 08d635a6542b46a008dbf87e5ad9e475e52fd444 Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Mon, 29 Apr 2024 16:59:52 -0400 Subject: [PATCH 07/14] skip searchafter unit tests --- packages/api/tests/endpoints/granules/test-searchafter-10k.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/api/tests/endpoints/granules/test-searchafter-10k.js b/packages/api/tests/endpoints/granules/test-searchafter-10k.js index 61f7f740cd2..ccc927c01ee 100644 --- a/packages/api/tests/endpoints/granules/test-searchafter-10k.js +++ b/packages/api/tests/endpoints/granules/test-searchafter-10k.js @@ -36,7 +36,8 @@ test.after.always(async (t) => { await t.context.esClient.client.indices.delete({ index: t.context.esIndex }); }); -test.serial('CUMULUS-2930 /GET granules allows searching past 10K results windows with searchContext', async (t) => { +// TODO postgres query doesn't return searchContext +test.serial.skip('CUMULUS-2930 /GET granules allows searching past 10K results windows with searchContext', async (t) => { const numGranules = 12 * 1000; // create granules in batches of 1000 From e3926c72b7bc1c1bb711a3a089b92423b567cff2 Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Tue, 30 Apr 2024 15:35:51 -0400 Subject: [PATCH 08/14] add granule list test --- packages/api/tests/endpoints/test-granules.js | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/packages/api/tests/endpoints/test-granules.js b/packages/api/tests/endpoints/test-granules.js index 9ed8345f8bc..b00d41da951 100644 --- a/packages/api/tests/endpoints/test-granules.js +++ b/packages/api/tests/endpoints/test-granules.js @@ -447,6 +447,48 @@ test.serial.skip('default lists and paginates correctly with search_after', asyn t.not(meta.searchContext === newMeta.searchContext); }); +test.serial('default lists and paginates correctly', async (t) => { + const granuleIds = t.context.fakePGGranules.map((i) => i.granule_id); + const response = await request(app) + .get('/granules') + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .expect(200); + + const { meta, results } = response.body; + t.is(results.length, 4); + t.is(meta.stack, process.env.stackName); + t.is(meta.table, 'granule'); + t.is(meta.count, 4); + results.forEach((r) => { + t.true(granuleIds.includes(r.granuleId)); + }); + // default paginates correctly with search_after + const firstResponse = await request(app) + .get('/granules?limit=1') + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .expect(200); + + const { meta: firstMeta, results: firstResults } = firstResponse.body; + t.is(firstResults.length, 1); + t.is(firstMeta.page, 1); + + const newResponse = await request(app) + .get('/granules?limit=1&page=2') + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .expect(200); + + const { meta: newMeta, results: newResults } = newResponse.body; + t.is(newResults.length, 1); + t.is(newMeta.page, 2); + + t.true(granuleIds.includes(results[0].granuleId)); + t.true(granuleIds.includes(newResults[0].granuleId)); + t.not(results[0].granuleId, newResults[0].granuleId); +}); + test.serial('CUMULUS-911 GET without pathParameters and without an Authorization header returns an Authorization Missing response', async (t) => { const response = await request(app) .get('/granules') From 935641dd47f710830f50d0eda852636176a8d0f4 Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Wed, 1 May 2024 10:45:28 -0400 Subject: [PATCH 09/14] rename --- packages/db/src/search/BaseSearch.ts | 22 +++++++++++----------- packages/db/src/types/search.ts | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/db/src/search/BaseSearch.ts b/packages/db/src/search/BaseSearch.ts index ebc79b3fc7b..78b4752b1dd 100644 --- a/packages/db/src/search/BaseSearch.ts +++ b/packages/db/src/search/BaseSearch.ts @@ -2,7 +2,7 @@ import { Knex } from 'knex'; import Logger from '@cumulus/logger'; import { getKnexClient } from '../connection'; import { BaseRecord } from '../types/base'; -import { ParsedQueryParameters, QueryEvent, QueryStringParameters } from '../types/search'; +import { DbQueryParameters, QueryEvent, QueryStringParameters } from '../types/search'; const log = new Logger({ sender: '@cumulus/db/BaseSearch' }); @@ -21,22 +21,22 @@ export interface Meta { class BaseSearch { readonly type?: string; readonly queryStringParameters: QueryStringParameters; - // parsed from queryStringParameters - parsedQueryParameters: ParsedQueryParameters = {}; + // parsed from queryStringParameters for query build + dbQueryParameters: DbQueryParameters = {}; constructor(event: QueryEvent, type?: string) { this.type = type; this.queryStringParameters = event?.queryStringParameters ?? {}; - this.parsedQueryParameters.page = Number.parseInt( + this.dbQueryParameters.page = Number.parseInt( (this.queryStringParameters.page) ?? '1', 10 ); - this.parsedQueryParameters.limit = Number.parseInt( + this.dbQueryParameters.limit = Number.parseInt( (this.queryStringParameters.limit) ?? '10', 10 ); - this.parsedQueryParameters.offset = (this.parsedQueryParameters.page - 1) - * this.parsedQueryParameters.limit; + this.dbQueryParameters.offset = (this.dbQueryParameters.page - 1) + * this.dbQueryParameters.limit; } /** @@ -66,8 +66,8 @@ class BaseSearch { } { const { countQuery, searchQuery } = this.buildBasicQuery(knex); const updatedQuery = searchQuery.modify((queryBuilder) => { - if (this.parsedQueryParameters.limit) queryBuilder.limit(this.parsedQueryParameters.limit); - if (this.parsedQueryParameters.offset) queryBuilder.offset(this.parsedQueryParameters.offset); + if (this.dbQueryParameters.limit) queryBuilder.limit(this.dbQueryParameters.limit); + if (this.dbQueryParameters.offset) queryBuilder.offset(this.dbQueryParameters.offset); }); return { countQuery, searchQuery: updatedQuery }; } @@ -108,8 +108,8 @@ class BaseSearch { try { const countResult = await countQuery; const meta = this._metaTemplate(); - meta.limit = this.parsedQueryParameters.limit; - meta.page = this.parsedQueryParameters.page; + meta.limit = this.dbQueryParameters.limit; + meta.page = this.dbQueryParameters.page; meta.count = Number(countResult[0]?.count ?? 0); const pgRecords = await searchQuery; diff --git a/packages/db/src/types/search.ts b/packages/db/src/types/search.ts index bc1920d8081..e19771de79e 100644 --- a/packages/db/src/types/search.ts +++ b/packages/db/src/types/search.ts @@ -8,7 +8,7 @@ export interface QueryEvent { queryStringParameters?: QueryStringParameters, } -export interface ParsedQueryParameters { +export interface DbQueryParameters { limit?: number, offset?: number, page?: number, From c1e724a76d3eaecb1294243ca594e6132ec2e284 Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Thu, 2 May 2024 20:48:54 -0400 Subject: [PATCH 10/14] refactor --- packages/db/src/search/BaseSearch.ts | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/db/src/search/BaseSearch.ts b/packages/db/src/search/BaseSearch.ts index 78b4752b1dd..cf212a54324 100644 --- a/packages/db/src/search/BaseSearch.ts +++ b/packages/db/src/search/BaseSearch.ts @@ -39,20 +39,6 @@ class BaseSearch { * this.dbQueryParameters.limit; } - /** - * build basic query - * - * @param knex - DB client - * @throws - function is not implemented - */ - protected buildBasicQuery(knex: Knex): { - countQuery: Knex.QueryBuilder, - searchQuery: Knex.QueryBuilder, - } { - log.debug(`buildBasicQuery is not implemented ${knex.constructor.name}`); - throw new Error('buildBasicQuery is not implemented'); - } - /** * build the search query * @@ -85,6 +71,20 @@ class BaseSearch { }; } + /** + * build basic query + * + * @param knex - DB client + * @throws - function is not implemented + */ + protected buildBasicQuery(knex: Knex): { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + } { + log.debug(`buildBasicQuery is not implemented ${knex.constructor.name}`); + throw new Error('buildBasicQuery is not implemented'); + } + /** * Translate postgres records to api records * From 421a2dda6027256c3cabb041db30fa8f71b4de42 Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Mon, 6 May 2024 15:05:25 -0400 Subject: [PATCH 11/14] update comment --- packages/api/tests/endpoints/test-granules.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/tests/endpoints/test-granules.js b/packages/api/tests/endpoints/test-granules.js index b00d41da951..33694d842bf 100644 --- a/packages/api/tests/endpoints/test-granules.js +++ b/packages/api/tests/endpoints/test-granules.js @@ -463,7 +463,7 @@ test.serial('default lists and paginates correctly', async (t) => { results.forEach((r) => { t.true(granuleIds.includes(r.granuleId)); }); - // default paginates correctly with search_after + // default paginates correctly const firstResponse = await request(app) .get('/granules?limit=1') .set('Accept', 'application/json') From 9942fac1ad316c2700fb0064b264818380d39f98 Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Mon, 6 May 2024 17:26:44 -0400 Subject: [PATCH 12/14] update jsdoc --- packages/db/src/translate/granules.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/db/src/translate/granules.ts b/packages/db/src/translate/granules.ts index 57a04d96409..11bfdbfc778 100644 --- a/packages/db/src/translate/granules.ts +++ b/packages/db/src/translate/granules.ts @@ -23,6 +23,19 @@ import { PostgresProviderRecord } from '../types/provider'; import { translatePostgresFileToApiFile } from './file'; +/** + * Generate an API Granule object from the granule and associated Postgres objects without + * querying the database + * + * @param params - params + * @param params.granulePgRecord - Granule from Postgres + * @param params.collectionPgRecord - Collection from Postgres + * @param [params.executionUrls] - executionUrls from Postgres + * @param [params.files] - granule files from Postgres + * @param [params.pdr] - pdr from Postgres + * @param [params.providerPgRecord] - provider from Postgres + * @returns An API Granule with associated Files + */ export const translatePostgresGranuleToApiGranuleWithoutDbQuery = ({ granulePgRecord, collectionPgRecord, From 39922be4f4b0762d14689e67a1687da7f2491dc1 Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Tue, 7 May 2024 11:18:00 -0400 Subject: [PATCH 13/14] use type over interface,add log --- packages/db/src/search/BaseSearch.ts | 14 +++++++------- packages/db/src/types/search.ts | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/db/src/search/BaseSearch.ts b/packages/db/src/search/BaseSearch.ts index cf212a54324..00b703e9897 100644 --- a/packages/db/src/search/BaseSearch.ts +++ b/packages/db/src/search/BaseSearch.ts @@ -6,14 +6,14 @@ import { DbQueryParameters, QueryEvent, QueryStringParameters } from '../types/s const log = new Logger({ sender: '@cumulus/db/BaseSearch' }); -export interface Meta { +export type Meta = { name: string, stack?: string, table?: string, limit?: number, page?: number, count?: number, -} +}; /** * Class to build and execute db search query @@ -51,11 +51,10 @@ class BaseSearch { searchQuery: Knex.QueryBuilder, } { const { countQuery, searchQuery } = this.buildBasicQuery(knex); - const updatedQuery = searchQuery.modify((queryBuilder) => { - if (this.dbQueryParameters.limit) queryBuilder.limit(this.dbQueryParameters.limit); - if (this.dbQueryParameters.offset) queryBuilder.offset(this.dbQueryParameters.offset); - }); - return { countQuery, searchQuery: updatedQuery }; + if (this.dbQueryParameters.limit) searchQuery.limit(this.dbQueryParameters.limit); + if (this.dbQueryParameters.offset) searchQuery.offset(this.dbQueryParameters.offset); + + return { countQuery, searchQuery }; } /** @@ -120,6 +119,7 @@ class BaseSearch { results: apiRecords, }; } catch (error) { + log.error(`Error caught in search query for ${JSON.stringify(this.queryStringParameters)}`, error); return error; } } diff --git a/packages/db/src/types/search.ts b/packages/db/src/types/search.ts index e19771de79e..50a3664ef48 100644 --- a/packages/db/src/types/search.ts +++ b/packages/db/src/types/search.ts @@ -1,15 +1,15 @@ -export interface QueryStringParameters { +export type QueryStringParameters = { limit?: string, page?: string, [key: string]: string | string[] | undefined, -} +}; -export interface QueryEvent { +export type QueryEvent = { queryStringParameters?: QueryStringParameters, -} +}; -export interface DbQueryParameters { +export type DbQueryParameters = { limit?: number, offset?: number, page?: number, -} +}; From fb9c51fb10e758a24842063065680980fc93b298 Mon Sep 17 00:00:00 2001 From: jennyhliu Date: Tue, 7 May 2024 18:17:01 -0400 Subject: [PATCH 14/14] update test description --- packages/api/tests/endpoints/test-granules.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/tests/endpoints/test-granules.js b/packages/api/tests/endpoints/test-granules.js index 33694d842bf..5ec292f04e7 100644 --- a/packages/api/tests/endpoints/test-granules.js +++ b/packages/api/tests/endpoints/test-granules.js @@ -447,7 +447,7 @@ test.serial.skip('default lists and paginates correctly with search_after', asyn t.not(meta.searchContext === newMeta.searchContext); }); -test.serial('default lists and paginates correctly', async (t) => { +test.serial('default lists and paginates correctly from querying database', async (t) => { const granuleIds = t.context.fakePGGranules.map((i) => i.granule_id); const response = await request(app) .get('/granules')