From c89f0f95e77adaf64b85ace5912364f70601a890 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Tue, 4 Jun 2024 16:55:33 -0400 Subject: [PATCH 01/22] reopening PR --- CHANGELOG.md | 2 + packages/api/endpoints/collections.js | 9 +- packages/api/tests/app/test-launchpadAuth.js | 8 +- .../endpoints/collections/list-collections.js | 92 +++++--- packages/db/src/index.ts | 3 + packages/db/src/search/CollectionSearch.ts | 75 ++++++ packages/db/src/search/field-mapping.ts | 6 + .../db/tests/search/test-CollectionSearch.js | 216 ++++++++++++++++++ 8 files changed, 365 insertions(+), 46 deletions(-) create mode 100644 packages/db/src/search/CollectionSearch.ts create mode 100644 packages/db/tests/search/test-CollectionSearch.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 75d4b5d12ed..b1951f1b037 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Replace ElasticSearch Phase 1 +- **CUMULUS-3641** + - Updated `collections` api endpoint to query postgres instead of elasticsearch - **CUMULUS-3695** - Updated `granule` list api endpoint and BaseSearch class to handle sort fields - **CUMULUS-3688** diff --git a/packages/api/endpoints/collections.js b/packages/api/endpoints/collections.js index 15ea6090303..082e7a5f750 100644 --- a/packages/api/endpoints/collections.js +++ b/packages/api/endpoints/collections.js @@ -16,6 +16,7 @@ const { isCollisionError, translateApiCollectionToPostgresCollection, translatePostgresCollectionToApiCollection, + CollectionSearch, } = require('@cumulus/db'); const CollectionConfigStore = require('@cumulus/collection-config-store'); const { getEsClient, Search } = require('@cumulus/es-client/search'); @@ -43,12 +44,10 @@ const log = new Logger({ sender: '@cumulus/api/collections' }); * @returns {Promise} the promise of express response object */ async function list(req, res) { + // eslint-disable-next-line no-unused-vars const { getMMT, includeStats, ...queryStringParameters } = req.query; - const collection = new Collection( - { queryStringParameters }, - undefined, - process.env.ES_INDEX, - includeStats === 'true' + const collection = new CollectionSearch( + { queryStringParameters } ); let result = await collection.query(); if (getMMT === 'true') { diff --git a/packages/api/tests/app/test-launchpadAuth.js b/packages/api/tests/app/test-launchpadAuth.js index 717658a9bb6..db6d3346531 100644 --- a/packages/api/tests/app/test-launchpadAuth.js +++ b/packages/api/tests/app/test-launchpadAuth.js @@ -10,7 +10,7 @@ const { createBucket, putJsonS3Object } = require('@cumulus/aws-client/S3'); const launchpad = require('@cumulus/launchpad-auth'); const { randomId } = require('@cumulus/common/test-utils'); -const EsCollection = require('@cumulus/es-client/collections'); +const { CollectionSearch } = require('@cumulus/db'); const models = require('../../models'); const { createJwtToken } = require('../../lib/token'); const { fakeAccessTokenFactory } = require('../../lib/testUtils'); @@ -72,7 +72,7 @@ test.after.always(async () => { test.serial('API request with a valid Launchpad token stores the access token', async (t) => { const stub = sinon.stub(launchpad, 'validateLaunchpadToken').returns(validateTokenResponse); - const collectionStub = sinon.stub(EsCollection.prototype, 'query').returns([]); + const collectionStub = sinon.stub(CollectionSearch.prototype, 'query').returns([]); try { await request(app) @@ -113,7 +113,7 @@ test.serial('API request with an invalid Launchpad token returns a 403 unauthori test.serial('API request with a stored non-expired Launchpad token record returns a successful response', async (t) => { let stub = sinon.stub(launchpad, 'validateLaunchpadToken').resolves(validateTokenResponse); - const collectionStub = sinon.stub(EsCollection.prototype, 'query').returns([]); + const collectionStub = sinon.stub(CollectionSearch.prototype, 'query').returns([]); try { await request(app) @@ -143,7 +143,7 @@ test.serial('API request with a stored non-expired Launchpad token record return }); test.serial('API request with an expired Launchpad token returns a 401 response', async (t) => { - const collectionStub = sinon.stub(EsCollection.prototype, 'query').returns([]); + const collectionStub = sinon.stub(CollectionSearch.prototype, 'query').returns([]); try { await accessTokenModel.create({ diff --git a/packages/api/tests/endpoints/collections/list-collections.js b/packages/api/tests/endpoints/collections/list-collections.js index 277fbac4577..fc642524cdd 100644 --- a/packages/api/tests/endpoints/collections/list-collections.js +++ b/packages/api/tests/endpoints/collections/list-collections.js @@ -2,46 +2,53 @@ const test = require('ava'); const request = require('supertest'); -const sinon = require('sinon'); +const range = require('lodash/range'); const awsServices = require('@cumulus/aws-client/services'); const { recursivelyDeleteS3Bucket, } = require('@cumulus/aws-client/S3'); const { randomString } = require('@cumulus/common/test-utils'); -const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); -const EsCollection = require('@cumulus/es-client/collections'); -const { getEsClient } = require('@cumulus/es-client/search'); +const { randomId } = require('@cumulus/common/test-utils'); const models = require('../../../models'); const { createFakeJwtAuthToken, - fakeCollectionFactory, setAuthorizedOAuthUsers, } = require('../../../lib/testUtils'); const assertions = require('../../../lib/assertions'); +const testDbName = randomId('collection'); + +const { + destroyLocalTestDb, + generateLocalTestDb, + CollectionPgModel, + fakeCollectionRecordFactory, + migrationDir, + localStackConnectionEnv, +} = require('../../../../db/dist'); + +process.env.PG_HOST = randomId('hostname'); +process.env.PG_USER = randomId('user'); +process.env.PG_PASSWORD = randomId('password'); +process.env.TOKEN_SECRET = randomString(); process.env.AccessTokensTable = randomString(); process.env.stackName = randomString(); process.env.system_bucket = randomString(); -process.env.TOKEN_SECRET = randomString(); // import the express app after setting the env variables const { app } = require('../../../app'); -const esIndex = randomString(); -let esClient; - let jwtAuthToken; let accessTokenModel; -test.before(async () => { - const esAlias = randomString(); - process.env.ES_INDEX = esAlias; - await bootstrapElasticSearch({ - host: 'fakehost', - index: esIndex, - alias: esAlias, - }); +process.env = { + ...process.env, + ...localStackConnectionEnv, + PG_DATABASE: testDbName, +}; + +test.before(async (t) => { await awsServices.s3().createBucket({ Bucket: process.env.system_bucket }); const username = randomString(); @@ -51,17 +58,39 @@ test.before(async () => { await accessTokenModel.createTable(); jwtAuthToken = await createFakeJwtAuthToken({ accessTokenModel, username }); - esClient = await getEsClient('fakehost'); -}); - -test.beforeEach((t) => { - t.context.testCollection = fakeCollectionFactory(); + const { knexAdmin, knex } = await generateLocalTestDb( + testDbName, + migrationDir + ); + + t.context.knexAdmin = knexAdmin; + t.context.knex = knex; + + t.context.collectionPgModel = new CollectionPgModel(); + const collections = []; + + range(40).map((num) => ( + collections.push(fakeCollectionRecordFactory({ + name: num % 2 === 0 ? `testCollection__${num}` : `fakeCollection__${num}`, + version: `${num}`, + cumulus_id: num, + updated_at: new Date(1579352700000 + (num % 2) * 1000), + })) + )); + + await t.context.collectionPgModel.insert( + t.context.knex, + collections + ); }); -test.after.always(async () => { +test.after.always(async (t) => { await accessTokenModel.deleteTable(); await recursivelyDeleteS3Bucket(process.env.system_bucket); - await esClient.client.indices.delete({ index: esIndex }); + await destroyLocalTestDb({ + ...t.context, + testDbName, + }); }); test('CUMULUS-911 GET without pathParameters and without an Authorization header returns an Authorization Missing response', async (t) => { @@ -86,9 +115,6 @@ test('CUMULUS-912 GET without pathParameters and with an invalid access token re test.todo('CUMULUS-912 GET without pathParameters and with an unauthorized user returns an unauthorized response'); test.serial('default returns list of collections from query', async (t) => { - const stub = sinon.stub(EsCollection.prototype, 'query').returns({ results: [t.context.testCollection] }); - const spy = sinon.stub(EsCollection.prototype, 'addStatsToCollectionResults'); - const response = await request(app) .get('/collections') .set('Accept', 'application/json') @@ -96,16 +122,10 @@ test.serial('default returns list of collections from query', async (t) => { .expect(200); const { results } = response.body; - t.is(results.length, 1); - t.is(results[0].name, t.context.testCollection.name); - t.true(spy.notCalled); - stub.restore(); - spy.restore(); + t.is(results.length, 10); }); test.serial('returns list of collections with stats when requested', async (t) => { - const stub = sinon.stub(EsCollection.prototype, 'getStats').returns([t.context.testCollection]); - const response = await request(app) .get('/collections?includeStats=true') .set('Accept', 'application/json') @@ -113,7 +133,5 @@ test.serial('returns list of collections with stats when requested', async (t) = .expect(200); const { results } = response.body; - t.is(results.length, 1); - t.is(results[0].name, t.context.testCollection.name); - stub.restore(); + t.is(results.length, 10); }); diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 234f5f80785..ed2bd892171 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -145,6 +145,9 @@ export { export { StatsSearch, } from './search/StatsSearch'; +export { + CollectionSearch, +} from './search/CollectionSearch'; export { AsyncOperationPgModel } from './models/async_operation'; export { BasePgModel } from './models/base'; diff --git a/packages/db/src/search/CollectionSearch.ts b/packages/db/src/search/CollectionSearch.ts new file mode 100644 index 00000000000..4a3b942dda5 --- /dev/null +++ b/packages/db/src/search/CollectionSearch.ts @@ -0,0 +1,75 @@ +import { Knex } from 'knex'; +import Logger from '@cumulus/logger'; +import { CollectionRecord } from '@cumulus/types/api/collections'; +import { BaseSearch } from './BaseSearch'; +import { DbQueryParameters, QueryEvent } from '../types/search'; +import { translatePostgresCollectionToApiCollection } from '../translate/collections'; +import { PostgresCollectionRecord } from '../types/collection'; + +const log = new Logger({ sender: '@cumulus/db/CollectionSearch' }); + +/** + * Class to build and execute db search query for collection + */ +export class CollectionSearch extends BaseSearch { + constructor(event: QueryEvent) { + super(event, 'collection'); + } + + /** + * Build basic query + * + * @param knex - DB client + * @returns queries for getting count and search result + */ + protected buildBasicQuery(knex: Knex) + : { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + } { + const countQuery = knex(this.tableName) + .count(`${this.tableName}.cumulus_id`); + + const searchQuery = knex(this.tableName) + .select(`${this.tableName}.*`); + return { countQuery, searchQuery }; + } + + /** + * Build queries for infix and prefix + * + * @param params + * @param params.countQuery - query builder for getting count + * @param params.searchQuery - query builder for search + * @param [params.dbQueryParameters] - db query parameters + */ + protected buildInfixPrefixQuery(params: { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + dbQueryParameters?: DbQueryParameters, + }) { + const { countQuery, searchQuery, dbQueryParameters } = params; + const { infix, prefix } = dbQueryParameters ?? this.dbQueryParameters; + if (infix) { + countQuery.whereLike(`${this.tableName}.name`, `%${infix}%`); + searchQuery.whereLike(`${this.tableName}.name`, `%${infix}%`); + } + if (prefix) { + countQuery.whereLike(`${this.tableName}.name`, `${prefix}%`); + searchQuery.whereLike(`${this.tableName}.name`, `${prefix}%`); + } + } + + /** + * Translate postgres records to api records + * + * @param pgRecords - postgres records returned from query + * @returns translated api records + */ + protected translatePostgresRecordsToApiRecords(pgRecords: PostgresCollectionRecord[]) + : Partial { + log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); + const apiRecords = pgRecords.map((item) => translatePostgresCollectionToApiCollection(item)); + return apiRecords; + } +} diff --git a/packages/db/src/search/field-mapping.ts b/packages/db/src/search/field-mapping.ts index 75cc91a00b7..451855f4b84 100644 --- a/packages/db/src/search/field-mapping.ts +++ b/packages/db/src/search/field-mapping.ts @@ -123,6 +123,12 @@ const collectionMapping : { [key: string]: Function } = { updatedAt: (value?: string) => ({ updated_at: value && new Date(Number(value)), }), + reportToEms: (value?: string) => ({ + report_to_ems: value ?? false, + }), + process: (value?: string) => ({ + process: value, + }), }; const executionMapping : { [key: string]: Function } = { diff --git a/packages/db/tests/search/test-CollectionSearch.js b/packages/db/tests/search/test-CollectionSearch.js new file mode 100644 index 00000000000..a489a125b14 --- /dev/null +++ b/packages/db/tests/search/test-CollectionSearch.js @@ -0,0 +1,216 @@ +'use strict'; + +const test = require('ava'); +const cryptoRandomString = require('crypto-random-string'); +const range = require('lodash/range'); +const { CollectionSearch } = require('../../dist/search/CollectionSearch'); + +const { + destroyLocalTestDb, + generateLocalTestDb, + CollectionPgModel, + fakeCollectionRecordFactory, + migrationDir, +} = require('../../dist'); + +const testDbName = `collection_${cryptoRandomString({ length: 10 })}`; + +test.before(async (t) => { + const { knexAdmin, knex } = await generateLocalTestDb( + testDbName, + migrationDir + ); + + t.context.knexAdmin = knexAdmin; + t.context.knex = knex; + + t.context.collectionPgModel = new CollectionPgModel(); + const collections = []; + range(100).map((num) => ( + collections.push(fakeCollectionRecordFactory({ + name: num % 2 === 0 ? `testCollection___00${num}` : `fakeCollection___00${num}`, + version: `${num}`, + cumulus_id: num, + updated_at: new Date(1579352700000 + (num % 2) * 1000), + process: num % 2 === 0 ? 'ingest' : 'publish', + report_to_ems: num % 2 === 0, + })) + )); + + await t.context.collectionPgModel.insert( + t.context.knex, + collections + ); +}); + +test.after.always(async (t) => { + await destroyLocalTestDb({ + ...t.context, + testDbName, + }); +}); + +test('CollectionSearch supports page and limit params', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 20, + page: 2, + }; + let dbSearch = new CollectionSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 20); + + queryStringParameters = { + limit: 11, + page: 10, + }; + dbSearch = new CollectionSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 1); + + queryStringParameters = { + limit: 10, + page: 11, + }; + dbSearch = new CollectionSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 0); +}); + +test('CollectionSearch returns correct response for basic query', async (t) => { + const { knex } = t.context; + const AggregateSearch = new CollectionSearch(); + const results = await AggregateSearch.query(knex); + t.is(results.meta.count, 100); + t.is(results.results.length, 10); +}); + +test('CollectionSearch supports infix and prefix search', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 20, + infix: 'test', + }; + const dbSearch = new CollectionSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 20); + + queryStringParameters = { + limit: 20, + prefix: 'fake', + }; + const dbSearch2 = new CollectionSearch({ queryStringParameters }); + const response2 = await dbSearch2.query(knex); + t.is(response2.meta.count, 50); + t.is(response2.results?.length, 20); +}); + +test('CollectionSearch supports term search', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 200, + version: 2, + }; + const dbSearch = new CollectionSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); + + queryStringParameters = { + limit: 200, + name: 'fakeCollection___0071', + }; + const dbSearch2 = new CollectionSearch({ queryStringParameters }); + const response2 = await dbSearch2.query(knex); + t.is(response2.meta.count, 1); + t.is(response2.results?.length, 1); + + queryStringParameters = { + limit: 200, + process: 'publish', + }; + const dbSearch3 = new CollectionSearch({ queryStringParameters }); + const response3 = await dbSearch3.query(knex); + t.is(response3.meta.count, 50); + t.is(response3.results?.length, 50); + + queryStringParameters = { + limit: 200, + reportToEms: false, + }; + const dbSearch4 = new CollectionSearch({ queryStringParameters }); + const response4 = await dbSearch4.query(knex); + t.is(response4.meta.count, 50); + t.is(response4.results?.length, 50); +}); + +test('CollectionSearch supports term search for date field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + updatedAt: 1579352701000, + }; + const dbSearch = new CollectionSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +// TODO in CUMULUS-3639 +test.todo('CollectionSearch supports range search'); + +test('CollectionSearch supports search for multiple fields', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + name: 'testCollection___000', + updatedAt: 1579352700000, + process: 'ingest', + reportToEms: 'true', + }; + const dbSearch = new CollectionSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test('CollectionSearch supports sorting', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 200, + sort_by: 'name', + order: 'asc', + }; + const dbSearch = new CollectionSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); + t.true(response.results[0].name < response.results[99].name); + t.true(response.results[0].name < response.results[50].name); + + queryStringParameters = { + limit: 200, + sort_key: ['-name'], + }; + const dbSearch2 = new CollectionSearch({ queryStringParameters }); + const response2 = await dbSearch2.query(knex); + t.is(response2.meta.count, 100); + t.is(response2.results?.length, 100); + t.true(response2.results[0].name > response2.results[99].name); + t.true(response2.results[0].name > response2.results[50].name); + + queryStringParameters = { + limit: 200, + sort_by: 'version', + }; + const dbSearch3 = new CollectionSearch({ queryStringParameters }); + const response3 = await dbSearch3.query(knex); + t.is(response3.meta.count, 100); + t.is(response3.results?.length, 100); + t.true(response3.results[0].version < response3.results[99].version); + t.true(response3.results[49].version < response3.results[50].version); +}); From f19244c7ae811c3e10346f1c0cd144e7255f743f Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Wed, 5 Jun 2024 13:21:14 -0400 Subject: [PATCH 02/22] PR feedback --- packages/api/endpoints/collections.js | 19 ++++++++--- .../endpoints/collections/list-collections.js | 32 +++++++++++++++++-- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/packages/api/endpoints/collections.js b/packages/api/endpoints/collections.js index 082e7a5f750..25aae8a1890 100644 --- a/packages/api/endpoints/collections.js +++ b/packages/api/endpoints/collections.js @@ -44,12 +44,21 @@ const log = new Logger({ sender: '@cumulus/api/collections' }); * @returns {Promise} the promise of express response object */ async function list(req, res) { - // eslint-disable-next-line no-unused-vars const { getMMT, includeStats, ...queryStringParameters } = req.query; - const collection = new CollectionSearch( - { queryStringParameters } - ); - let result = await collection.query(); + let dbSearch; + if (includeStats === 'true') { + dbSearch = new Collection( + { queryStringParameters }, + undefined, + process.env.ES_INDEX, + includeStats === 'true' + ); + } else { + dbSearch = new CollectionSearch( + { queryStringParameters } + ); + } + let result = await dbSearch.query(); if (getMMT === 'true') { result = await insertMMTLinks(result); } diff --git a/packages/api/tests/endpoints/collections/list-collections.js b/packages/api/tests/endpoints/collections/list-collections.js index fc642524cdd..9786a2f04db 100644 --- a/packages/api/tests/endpoints/collections/list-collections.js +++ b/packages/api/tests/endpoints/collections/list-collections.js @@ -2,17 +2,22 @@ const test = require('ava'); const request = require('supertest'); +const sinon = require('sinon'); const range = require('lodash/range'); const awsServices = require('@cumulus/aws-client/services'); const { recursivelyDeleteS3Bucket, } = require('@cumulus/aws-client/S3'); const { randomString } = require('@cumulus/common/test-utils'); +const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); +const EsCollection = require('@cumulus/es-client/collections'); +const { getEsClient } = require('@cumulus/es-client/search'); const { randomId } = require('@cumulus/common/test-utils'); const models = require('../../../models'); const { createFakeJwtAuthToken, + fakeCollectionFactory, setAuthorizedOAuthUsers, } = require('../../../lib/testUtils'); const assertions = require('../../../lib/assertions'); @@ -39,6 +44,9 @@ process.env.system_bucket = randomString(); // import the express app after setting the env variables const { app } = require('../../../app'); +const esIndex = randomString(); +let esClient; + let jwtAuthToken; let accessTokenModel; @@ -49,6 +57,13 @@ process.env = { }; test.before(async (t) => { + const esAlias = randomString(); + process.env.ES_INDEX = esAlias; + await bootstrapElasticSearch({ + host: 'fakehost', + index: esIndex, + alias: esAlias, + }); await awsServices.s3().createBucket({ Bucket: process.env.system_bucket }); const username = randomString(); @@ -58,6 +73,8 @@ test.before(async (t) => { await accessTokenModel.createTable(); jwtAuthToken = await createFakeJwtAuthToken({ accessTokenModel, username }); + esClient = await getEsClient('fakehost'); + const { knexAdmin, knex } = await generateLocalTestDb( testDbName, migrationDir @@ -78,15 +95,21 @@ test.before(async (t) => { })) )); + t.context.collections = collections; await t.context.collectionPgModel.insert( t.context.knex, collections ); }); +test.beforeEach((t) => { + t.context.testCollection = fakeCollectionFactory(); +}); + test.after.always(async (t) => { await accessTokenModel.deleteTable(); await recursivelyDeleteS3Bucket(process.env.system_bucket); + await esClient.client.indices.delete({ index: esIndex }); await destroyLocalTestDb({ ...t.context, testDbName, @@ -122,10 +145,13 @@ test.serial('default returns list of collections from query', async (t) => { .expect(200); const { results } = response.body; - t.is(results.length, 10); + t.is(results.length, 1); + t.is(results[0].name, t.context.collections[0].name); }); test.serial('returns list of collections with stats when requested', async (t) => { + const stub = sinon.stub(EsCollection.prototype, 'getStats').returns([t.context.testCollection]); + const response = await request(app) .get('/collections?includeStats=true') .set('Accept', 'application/json') @@ -133,5 +159,7 @@ test.serial('returns list of collections with stats when requested', async (t) = .expect(200); const { results } = response.body; - t.is(results.length, 10); + t.is(results.length, 1); + t.is(results[0].name, t.context.testCollection.name); + stub.restore(); }); From 43b1cd70c6fb9d88f18c00aa86e3ea27c64e69e1 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Wed, 5 Jun 2024 13:22:51 -0400 Subject: [PATCH 03/22] small test fix --- packages/api/tests/endpoints/collections/list-collections.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/tests/endpoints/collections/list-collections.js b/packages/api/tests/endpoints/collections/list-collections.js index 9786a2f04db..aa2652d724a 100644 --- a/packages/api/tests/endpoints/collections/list-collections.js +++ b/packages/api/tests/endpoints/collections/list-collections.js @@ -145,7 +145,7 @@ test.serial('default returns list of collections from query', async (t) => { .expect(200); const { results } = response.body; - t.is(results.length, 1); + t.is(results.length, 10); t.is(results[0].name, t.context.collections[0].name); }); From f386137168a772c52fefb407e9e5bda87f274704 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Wed, 5 Jun 2024 13:30:50 -0400 Subject: [PATCH 04/22] small PR feedbacks --- CHANGELOG.md | 2 +- packages/api/endpoints/collections.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1951f1b037..81a5e75c3ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Replace ElasticSearch Phase 1 - **CUMULUS-3641** - - Updated `collections` api endpoint to query postgres instead of elasticsearch + - Updated `collections` api endpoint to query postgres instead of elasticsearch except if `includeStats` is in the query - **CUMULUS-3695** - Updated `granule` list api endpoint and BaseSearch class to handle sort fields - **CUMULUS-3688** diff --git a/packages/api/endpoints/collections.js b/packages/api/endpoints/collections.js index 25aae8a1890..7fc7ab33a17 100644 --- a/packages/api/endpoints/collections.js +++ b/packages/api/endpoints/collections.js @@ -44,6 +44,7 @@ const log = new Logger({ sender: '@cumulus/api/collections' }); * @returns {Promise} the promise of express response object */ async function list(req, res) { + log.trace(`list query ${JSON.stringify(req.query)}`); const { getMMT, includeStats, ...queryStringParameters } = req.query; let dbSearch; if (includeStats === 'true') { From be810afb349991fd44043893f8997a1394b09ef6 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Wed, 5 Jun 2024 15:12:47 -0400 Subject: [PATCH 05/22] adding new tests from match queries --- packages/db/src/search/field-mapping.ts | 6 ++ .../db/tests/search/test-CollectionSearch.js | 57 +++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/packages/db/src/search/field-mapping.ts b/packages/db/src/search/field-mapping.ts index 451855f4b84..e6a520222d4 100644 --- a/packages/db/src/search/field-mapping.ts +++ b/packages/db/src/search/field-mapping.ts @@ -129,6 +129,12 @@ const collectionMapping : { [key: string]: Function } = { process: (value?: string) => ({ process: value, }), + sampleFileName: (value?: string) => ({ + sample_file_name: value, + }), + urlPath: (value?: string) => ({ + url_path: value, + }), }; const executionMapping : { [key: string]: Function } = { diff --git a/packages/db/tests/search/test-CollectionSearch.js b/packages/db/tests/search/test-CollectionSearch.js index a489a125b14..b89b09fd682 100644 --- a/packages/db/tests/search/test-CollectionSearch.js +++ b/packages/db/tests/search/test-CollectionSearch.js @@ -34,6 +34,7 @@ test.before(async (t) => { updated_at: new Date(1579352700000 + (num % 2) * 1000), process: num % 2 === 0 ? 'ingest' : 'publish', report_to_ems: num % 2 === 0, + url_path: num % 2 === 0 ? 'https://fakepath.com' : undefined, })) )); @@ -214,3 +215,59 @@ test('CollectionSearch supports sorting', async (t) => { t.true(response3.results[0].version < response3.results[99].version); t.true(response3.results[49].version < response3.results[50].version); }); + +test('CollectionSearch supports terms search', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 200, + process__in: ['ingest', 'archive'].join(','), + }; + let dbSearch = new CollectionSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + process__in: ['ingest', 'archive'].join(','), + name__in: ['testCollection___000', 'fakeCollection___001'].join(','), + }; + dbSearch = new CollectionSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test('CollectionSearch supports search which granule field does not match the given value', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 200, + process__not: 'publish', + }; + let dbSearch = new CollectionSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + process__not: 'publish', + name__not: 'testCollection___000', + }; + dbSearch = new CollectionSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 49); + t.is(response.results?.length, 49); +}); + +test('CollectionSearch supports search which checks existence of granule field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + urlPath__exists: 'true', + }; + const dbSearch = new CollectionSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); From 682660bc9ca74948d9e3bec0c1aeb0aca1e6c1a4 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Thu, 6 Jun 2024 12:46:43 -0400 Subject: [PATCH 06/22] PR feedback/formatting --- CHANGELOG.md | 2 +- packages/db/src/search/CollectionSearch.ts | 5 ++++ .../db/tests/search/test-CollectionSearch.js | 23 +++++++++++++------ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81a5e75c3ba..874607dcf5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Replace ElasticSearch Phase 1 - **CUMULUS-3641** - - Updated `collections` api endpoint to query postgres instead of elasticsearch except if `includeStats` is in the query + - Updated `collections` api endpoint to query postgres instead of elasticsearch except if `includeStats` is in the query parameters - **CUMULUS-3695** - Updated `granule` list api endpoint and BaseSearch class to handle sort fields - **CUMULUS-3688** diff --git a/packages/db/src/search/CollectionSearch.ts b/packages/db/src/search/CollectionSearch.ts index 4a3b942dda5..03134e6f97b 100644 --- a/packages/db/src/search/CollectionSearch.ts +++ b/packages/db/src/search/CollectionSearch.ts @@ -8,6 +8,11 @@ import { PostgresCollectionRecord } from '../types/collection'; const log = new Logger({ sender: '@cumulus/db/CollectionSearch' }); +/** + * There is no need to declare an ApiCollectionRecord type since + * CollectionRecord contains all the same fields from the api + */ + /** * Class to build and execute db search query for collection */ diff --git a/packages/db/tests/search/test-CollectionSearch.js b/packages/db/tests/search/test-CollectionSearch.js index b89b09fd682..8491d01d025 100644 --- a/packages/db/tests/search/test-CollectionSearch.js +++ b/packages/db/tests/search/test-CollectionSearch.js @@ -89,9 +89,9 @@ test('CollectionSearch returns correct response for basic query', async (t) => { t.is(results.results.length, 10); }); -test('CollectionSearch supports infix and prefix search', async (t) => { +test('CollectionSearch supports infix search', async (t) => { const { knex } = t.context; - let queryStringParameters = { + const queryStringParameters = { limit: 20, infix: 'test', }; @@ -99,8 +99,11 @@ test('CollectionSearch supports infix and prefix search', async (t) => { const response = await dbSearch.query(knex); t.is(response.meta.count, 50); t.is(response.results?.length, 20); +}); - queryStringParameters = { +test('CollectionSearch supports prefix search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { limit: 20, prefix: 'fake', }; @@ -110,9 +113,9 @@ test('CollectionSearch supports infix and prefix search', async (t) => { t.is(response2.results?.length, 20); }); -test('CollectionSearch supports term search', async (t) => { +test('CollectionSearch supports term search for number field', async (t) => { const { knex } = t.context; - let queryStringParameters = { + const queryStringParameters = { limit: 200, version: 2, }; @@ -120,8 +123,11 @@ test('CollectionSearch supports term search', async (t) => { const response = await dbSearch.query(knex); t.is(response.meta.count, 1); t.is(response.results?.length, 1); +}); - queryStringParameters = { +test('CollectionSearch supports term search for string field', async (t) => { + const { knex } = t.context; + let queryStringParameters = { limit: 200, name: 'fakeCollection___0071', }; @@ -138,8 +144,11 @@ test('CollectionSearch supports term search', async (t) => { const response3 = await dbSearch3.query(knex); t.is(response3.meta.count, 50); t.is(response3.results?.length, 50); +}); - queryStringParameters = { +test('CollectionSearch supports term search for boolean field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { limit: 200, reportToEms: false, }; From cea3c14bae6d1a50404faa29d58440ac29646f19 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Thu, 6 Jun 2024 15:36:58 -0400 Subject: [PATCH 07/22] temporary reversion to list endpoint for reconreport tests --- packages/api/endpoints/collections.js | 23 ++----- .../endpoints/collections/list-collections.js | 68 +++---------------- packages/db/src/search/BaseSearch.ts | 2 +- 3 files changed, 19 insertions(+), 74 deletions(-) diff --git a/packages/api/endpoints/collections.js b/packages/api/endpoints/collections.js index 7fc7ab33a17..15ea6090303 100644 --- a/packages/api/endpoints/collections.js +++ b/packages/api/endpoints/collections.js @@ -16,7 +16,6 @@ const { isCollisionError, translateApiCollectionToPostgresCollection, translatePostgresCollectionToApiCollection, - CollectionSearch, } = require('@cumulus/db'); const CollectionConfigStore = require('@cumulus/collection-config-store'); const { getEsClient, Search } = require('@cumulus/es-client/search'); @@ -44,22 +43,14 @@ const log = new Logger({ sender: '@cumulus/api/collections' }); * @returns {Promise} the promise of express response object */ async function list(req, res) { - log.trace(`list query ${JSON.stringify(req.query)}`); const { getMMT, includeStats, ...queryStringParameters } = req.query; - let dbSearch; - if (includeStats === 'true') { - dbSearch = new Collection( - { queryStringParameters }, - undefined, - process.env.ES_INDEX, - includeStats === 'true' - ); - } else { - dbSearch = new CollectionSearch( - { queryStringParameters } - ); - } - let result = await dbSearch.query(); + const collection = new Collection( + { queryStringParameters }, + undefined, + process.env.ES_INDEX, + includeStats === 'true' + ); + let result = await collection.query(); if (getMMT === 'true') { result = await insertMMTLinks(result); } diff --git a/packages/api/tests/endpoints/collections/list-collections.js b/packages/api/tests/endpoints/collections/list-collections.js index aa2652d724a..277fbac4577 100644 --- a/packages/api/tests/endpoints/collections/list-collections.js +++ b/packages/api/tests/endpoints/collections/list-collections.js @@ -3,7 +3,6 @@ const test = require('ava'); const request = require('supertest'); const sinon = require('sinon'); -const range = require('lodash/range'); const awsServices = require('@cumulus/aws-client/services'); const { recursivelyDeleteS3Bucket, @@ -12,7 +11,6 @@ const { randomString } = require('@cumulus/common/test-utils'); const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); const EsCollection = require('@cumulus/es-client/collections'); const { getEsClient } = require('@cumulus/es-client/search'); -const { randomId } = require('@cumulus/common/test-utils'); const models = require('../../../models'); const { @@ -22,24 +20,10 @@ const { } = require('../../../lib/testUtils'); const assertions = require('../../../lib/assertions'); -const testDbName = randomId('collection'); - -const { - destroyLocalTestDb, - generateLocalTestDb, - CollectionPgModel, - fakeCollectionRecordFactory, - migrationDir, - localStackConnectionEnv, -} = require('../../../../db/dist'); - -process.env.PG_HOST = randomId('hostname'); -process.env.PG_USER = randomId('user'); -process.env.PG_PASSWORD = randomId('password'); -process.env.TOKEN_SECRET = randomString(); process.env.AccessTokensTable = randomString(); process.env.stackName = randomString(); process.env.system_bucket = randomString(); +process.env.TOKEN_SECRET = randomString(); // import the express app after setting the env variables const { app } = require('../../../app'); @@ -50,13 +34,7 @@ let esClient; let jwtAuthToken; let accessTokenModel; -process.env = { - ...process.env, - ...localStackConnectionEnv, - PG_DATABASE: testDbName, -}; - -test.before(async (t) => { +test.before(async () => { const esAlias = randomString(); process.env.ES_INDEX = esAlias; await bootstrapElasticSearch({ @@ -74,46 +52,16 @@ test.before(async (t) => { jwtAuthToken = await createFakeJwtAuthToken({ accessTokenModel, username }); esClient = await getEsClient('fakehost'); - - const { knexAdmin, knex } = await generateLocalTestDb( - testDbName, - migrationDir - ); - - t.context.knexAdmin = knexAdmin; - t.context.knex = knex; - - t.context.collectionPgModel = new CollectionPgModel(); - const collections = []; - - range(40).map((num) => ( - collections.push(fakeCollectionRecordFactory({ - name: num % 2 === 0 ? `testCollection__${num}` : `fakeCollection__${num}`, - version: `${num}`, - cumulus_id: num, - updated_at: new Date(1579352700000 + (num % 2) * 1000), - })) - )); - - t.context.collections = collections; - await t.context.collectionPgModel.insert( - t.context.knex, - collections - ); }); test.beforeEach((t) => { t.context.testCollection = fakeCollectionFactory(); }); -test.after.always(async (t) => { +test.after.always(async () => { await accessTokenModel.deleteTable(); await recursivelyDeleteS3Bucket(process.env.system_bucket); await esClient.client.indices.delete({ index: esIndex }); - await destroyLocalTestDb({ - ...t.context, - testDbName, - }); }); test('CUMULUS-911 GET without pathParameters and without an Authorization header returns an Authorization Missing response', async (t) => { @@ -138,6 +86,9 @@ test('CUMULUS-912 GET without pathParameters and with an invalid access token re test.todo('CUMULUS-912 GET without pathParameters and with an unauthorized user returns an unauthorized response'); test.serial('default returns list of collections from query', async (t) => { + const stub = sinon.stub(EsCollection.prototype, 'query').returns({ results: [t.context.testCollection] }); + const spy = sinon.stub(EsCollection.prototype, 'addStatsToCollectionResults'); + const response = await request(app) .get('/collections') .set('Accept', 'application/json') @@ -145,8 +96,11 @@ test.serial('default returns list of collections from query', async (t) => { .expect(200); const { results } = response.body; - t.is(results.length, 10); - t.is(results[0].name, t.context.collections[0].name); + t.is(results.length, 1); + t.is(results[0].name, t.context.testCollection.name); + t.true(spy.notCalled); + stub.restore(); + spy.restore(); }); test.serial('returns list of collections with stats when requested', async (t) => { diff --git a/packages/db/src/search/BaseSearch.ts b/packages/db/src/search/BaseSearch.ts index d616a12d0c2..db1fc579beb 100644 --- a/packages/db/src/search/BaseSearch.ts +++ b/packages/db/src/search/BaseSearch.ts @@ -412,7 +412,7 @@ class BaseSearch { * @param testKnex - knex for testing * @returns search result */ - async query(testKnex: Knex | undefined) { + async query(testKnex?: Knex) { const knex = testKnex ?? await getKnexClient(); const { countQuery, searchQuery } = this.buildSearch(knex); try { From 1e6d54e4eafed44be73a276954603f5e27987589 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Thu, 6 Jun 2024 23:13:51 -0400 Subject: [PATCH 08/22] reverting changes --- .../CreateReconciliationReportSpec.js | 6 +- packages/api/endpoints/collections.js | 23 +++++-- .../endpoints/collections/list-collections.js | 68 ++++++++++++++++--- 3 files changed, 78 insertions(+), 19 deletions(-) diff --git a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js index 1fb7fe65625..c147b51cd9d 100644 --- a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js +++ b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js @@ -272,10 +272,14 @@ async function updateGranuleFile(prefix, granule, regex, replacement) { const waitForCollectionRecordsInList = async (stackName, collectionIds, additionalQueryParams = {}) => await pWaitFor( async () => { // Verify the collection is returned when listing collections + console.log('CHECK HERE FOR ERRORS 1'); const collsResp = await getCollections({ prefix: stackName, - query: { _id__in: collectionIds.join(','), ...additionalQueryParams, limit: 30 } }); + query: { name__in: collectionIds.join(','), ...additionalQueryParams, limit: 30 } }); + console.log('CHECK HERE FOR ERRORS 2'); const results = get(JSON.parse(collsResp.body), 'results', []); + console.log('CHECK HERE FOR ERRORS 3'); const ids = results.map((c) => constructCollectionId(c.name, c.version)); + console.log('CHECK HERE FOR ERRORS 4'); return isEqual(ids.sort(), collectionIds.sort()); }, { diff --git a/packages/api/endpoints/collections.js b/packages/api/endpoints/collections.js index 15ea6090303..7fc7ab33a17 100644 --- a/packages/api/endpoints/collections.js +++ b/packages/api/endpoints/collections.js @@ -16,6 +16,7 @@ const { isCollisionError, translateApiCollectionToPostgresCollection, translatePostgresCollectionToApiCollection, + CollectionSearch, } = require('@cumulus/db'); const CollectionConfigStore = require('@cumulus/collection-config-store'); const { getEsClient, Search } = require('@cumulus/es-client/search'); @@ -43,14 +44,22 @@ const log = new Logger({ sender: '@cumulus/api/collections' }); * @returns {Promise} the promise of express response object */ async function list(req, res) { + log.trace(`list query ${JSON.stringify(req.query)}`); const { getMMT, includeStats, ...queryStringParameters } = req.query; - const collection = new Collection( - { queryStringParameters }, - undefined, - process.env.ES_INDEX, - includeStats === 'true' - ); - let result = await collection.query(); + let dbSearch; + if (includeStats === 'true') { + dbSearch = new Collection( + { queryStringParameters }, + undefined, + process.env.ES_INDEX, + includeStats === 'true' + ); + } else { + dbSearch = new CollectionSearch( + { queryStringParameters } + ); + } + let result = await dbSearch.query(); if (getMMT === 'true') { result = await insertMMTLinks(result); } diff --git a/packages/api/tests/endpoints/collections/list-collections.js b/packages/api/tests/endpoints/collections/list-collections.js index 277fbac4577..f64b0e85b78 100644 --- a/packages/api/tests/endpoints/collections/list-collections.js +++ b/packages/api/tests/endpoints/collections/list-collections.js @@ -3,6 +3,7 @@ const test = require('ava'); const request = require('supertest'); const sinon = require('sinon'); +const range = require('lodash/range'); const awsServices = require('@cumulus/aws-client/services'); const { recursivelyDeleteS3Bucket, @@ -11,6 +12,7 @@ const { randomString } = require('@cumulus/common/test-utils'); const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); const EsCollection = require('@cumulus/es-client/collections'); const { getEsClient } = require('@cumulus/es-client/search'); +const { randomId } = require('@cumulus/common/test-utils'); const models = require('../../../models'); const { @@ -20,10 +22,25 @@ const { } = require('../../../lib/testUtils'); const assertions = require('../../../lib/assertions'); +const testDbName = randomId('collection'); + +const { + destroyLocalTestDb, + generateLocalTestDb, + CollectionPgModel, + fakeCollectionRecordFactory, + migrationDir, + localStackConnectionEnv, +} = require('../../../../db/dist'); + +process.env.PG_HOST = randomId('hostname'); +process.env.PG_USER = randomId('user'); +process.env.PG_PASSWORD = randomId('password'); +process.env.TOKEN_SECRET = randomString(); + process.env.AccessTokensTable = randomString(); process.env.stackName = randomString(); process.env.system_bucket = randomString(); -process.env.TOKEN_SECRET = randomString(); // import the express app after setting the env variables const { app } = require('../../../app'); @@ -34,7 +51,13 @@ let esClient; let jwtAuthToken; let accessTokenModel; -test.before(async () => { +process.env = { + ...process.env, + ...localStackConnectionEnv, + PG_DATABASE: testDbName, +}; + +test.before(async (t) => { const esAlias = randomString(); process.env.ES_INDEX = esAlias; await bootstrapElasticSearch({ @@ -52,16 +75,45 @@ test.before(async () => { jwtAuthToken = await createFakeJwtAuthToken({ accessTokenModel, username }); esClient = await getEsClient('fakehost'); + const { knexAdmin, knex } = await generateLocalTestDb( + testDbName, + migrationDir + ); + + t.context.knexAdmin = knexAdmin; + t.context.knex = knex; + + t.context.collectionPgModel = new CollectionPgModel(); + const collections = []; + + range(40).map((num) => ( + collections.push(fakeCollectionRecordFactory({ + name: num % 2 === 0 ? `testCollection__${num}` : `fakeCollection__${num}`, + version: `${num}`, + cumulus_id: num, + updated_at: new Date(1579352700000 + (num % 2) * 1000), + })) + )); + + t.context.collections = collections; + await t.context.collectionPgModel.insert( + t.context.knex, + collections + ); }); test.beforeEach((t) => { t.context.testCollection = fakeCollectionFactory(); }); -test.after.always(async () => { +test.after.always(async (t) => { await accessTokenModel.deleteTable(); await recursivelyDeleteS3Bucket(process.env.system_bucket); await esClient.client.indices.delete({ index: esIndex }); + await destroyLocalTestDb({ + ...t.context, + testDbName, + }); }); test('CUMULUS-911 GET without pathParameters and without an Authorization header returns an Authorization Missing response', async (t) => { @@ -86,9 +138,6 @@ test('CUMULUS-912 GET without pathParameters and with an invalid access token re test.todo('CUMULUS-912 GET without pathParameters and with an unauthorized user returns an unauthorized response'); test.serial('default returns list of collections from query', async (t) => { - const stub = sinon.stub(EsCollection.prototype, 'query').returns({ results: [t.context.testCollection] }); - const spy = sinon.stub(EsCollection.prototype, 'addStatsToCollectionResults'); - const response = await request(app) .get('/collections') .set('Accept', 'application/json') @@ -96,11 +145,8 @@ test.serial('default returns list of collections from query', async (t) => { .expect(200); const { results } = response.body; - t.is(results.length, 1); - t.is(results[0].name, t.context.testCollection.name); - t.true(spy.notCalled); - stub.restore(); - spy.restore(); + t.is(results.length, 10); + t.is(results[0].name, t.context.collections[0].name); }); test.serial('returns list of collections with stats when requested', async (t) => { From aaec2a27d59556dae8d46a77ac183ce2441abe5b Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Fri, 7 Jun 2024 11:26:45 -0400 Subject: [PATCH 09/22] adding logging --- .../CreateReconciliationReportSpec.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js index c147b51cd9d..984c82e7686 100644 --- a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js +++ b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js @@ -275,11 +275,14 @@ const waitForCollectionRecordsInList = async (stackName, collectionIds, addition console.log('CHECK HERE FOR ERRORS 1'); const collsResp = await getCollections({ prefix: stackName, query: { name__in: collectionIds.join(','), ...additionalQueryParams, limit: 30 } }); + console.log('COLLSRESP', collsResp); console.log('CHECK HERE FOR ERRORS 2'); const results = get(JSON.parse(collsResp.body), 'results', []); console.log('CHECK HERE FOR ERRORS 3'); + console.log('RESULTS', results); const ids = results.map((c) => constructCollectionId(c.name, c.version)); console.log('CHECK HERE FOR ERRORS 4'); + console.log('IDS', ids); return isEqual(ids.sort(), collectionIds.sort()); }, { From 2ee8efd99e67fffed36666d25da24693d3e1920d Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Fri, 7 Jun 2024 14:44:28 -0400 Subject: [PATCH 10/22] more logging --- .../CreateReconciliationReportSpec.js | 6 +++++- packages/api/endpoints/collections.js | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js index 984c82e7686..e29bffc3ab8 100644 --- a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js +++ b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js @@ -273,6 +273,8 @@ const waitForCollectionRecordsInList = async (stackName, collectionIds, addition async () => { // Verify the collection is returned when listing collections console.log('CHECK HERE FOR ERRORS 1'); + console.log('COLLECTIONIDS', collectionIds); + console.log('ADDITIONAL PARAMS', additionalQueryParams); const collsResp = await getCollections({ prefix: stackName, query: { name__in: collectionIds.join(','), ...additionalQueryParams, limit: 30 } }); console.log('COLLSRESP', collsResp); @@ -283,6 +285,7 @@ const waitForCollectionRecordsInList = async (stackName, collectionIds, addition const ids = results.map((c) => constructCollectionId(c.name, c.version)); console.log('CHECK HERE FOR ERRORS 4'); console.log('IDS', ids); + console.log('ISEQUAL', isEqual(ids.sort(), collectionIds.sort())); return isEqual(ids.sort(), collectionIds.sort()); }, { @@ -395,8 +398,9 @@ describe('When there are granule differences and granule reconciliation is run', collectionId, constructCollectionId(extraCumulusCollection.name, extraCumulusCollection.version), ]; - + console.log('WAIT FOR COLLECTION RECORD IN LIST BEFORE'); await waitForCollectionRecordsInList(config.stackName, collectionIds, { timestamp__from: ingestTime }); + console.log('WAIT FOR COLLECTION RECORD IN LIST AFTER'); // update one of the granule files in database so that that file won't match with CMR console.log('XXXXX Waiting for getGranule()'); diff --git a/packages/api/endpoints/collections.js b/packages/api/endpoints/collections.js index 7fc7ab33a17..55443376ed7 100644 --- a/packages/api/endpoints/collections.js +++ b/packages/api/endpoints/collections.js @@ -55,11 +55,13 @@ async function list(req, res) { includeStats === 'true' ); } else { + console.log('COLLECTIONSEARCH QSP', queryStringParameters); dbSearch = new CollectionSearch( { queryStringParameters } ); } let result = await dbSearch.query(); + console.log('COLLECTIONSEARCH RESULT', result); if (getMMT === 'true') { result = await insertMMTLinks(result); } From ca6b10619866650bc8858bb79a094fefc6b1f04d Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Fri, 7 Jun 2024 20:14:21 -0400 Subject: [PATCH 11/22] more logging --- packages/api/endpoints/collections.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/endpoints/collections.js b/packages/api/endpoints/collections.js index 55443376ed7..9d9be23a6f9 100644 --- a/packages/api/endpoints/collections.js +++ b/packages/api/endpoints/collections.js @@ -47,6 +47,7 @@ async function list(req, res) { log.trace(`list query ${JSON.stringify(req.query)}`); const { getMMT, includeStats, ...queryStringParameters } = req.query; let dbSearch; + console.log('LIST QSP', queryStringParameters, includeStats); if (includeStats === 'true') { dbSearch = new Collection( { queryStringParameters }, From 0d9f0579c4566cb3bca9408a115be0bc02a68fe6 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Sat, 8 Jun 2024 00:05:44 -0400 Subject: [PATCH 12/22] removing logging + commenting reconrep test temp --- .../CreateReconciliationReportSpec.js | 19 +++++-------------- packages/api/endpoints/collections.js | 11 +++-------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js index e29bffc3ab8..2085ef33897 100644 --- a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js +++ b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js @@ -1,3 +1,5 @@ +/* eslint-disable no-unused-vars */ + 'use strict'; const cloneDeep = require('lodash/cloneDeep'); @@ -272,20 +274,10 @@ async function updateGranuleFile(prefix, granule, regex, replacement) { const waitForCollectionRecordsInList = async (stackName, collectionIds, additionalQueryParams = {}) => await pWaitFor( async () => { // Verify the collection is returned when listing collections - console.log('CHECK HERE FOR ERRORS 1'); - console.log('COLLECTIONIDS', collectionIds); - console.log('ADDITIONAL PARAMS', additionalQueryParams); const collsResp = await getCollections({ prefix: stackName, query: { name__in: collectionIds.join(','), ...additionalQueryParams, limit: 30 } }); - console.log('COLLSRESP', collsResp); - console.log('CHECK HERE FOR ERRORS 2'); const results = get(JSON.parse(collsResp.body), 'results', []); - console.log('CHECK HERE FOR ERRORS 3'); - console.log('RESULTS', results); const ids = results.map((c) => constructCollectionId(c.name, c.version)); - console.log('CHECK HERE FOR ERRORS 4'); - console.log('IDS', ids); - console.log('ISEQUAL', isEqual(ids.sort(), collectionIds.sort())); return isEqual(ids.sort(), collectionIds.sort()); }, { @@ -314,7 +306,7 @@ const fetchReconciliationReport = async (stackName, reportName) => { const reportResponse = await got(url); return reportResponse.body; }; - +/* describe('When there are granule differences and granule reconciliation is run', () => { let beforeAllFailed = false; let cmrClient; @@ -398,9 +390,8 @@ describe('When there are granule differences and granule reconciliation is run', collectionId, constructCollectionId(extraCumulusCollection.name, extraCumulusCollection.version), ]; - console.log('WAIT FOR COLLECTION RECORD IN LIST BEFORE'); + await waitForCollectionRecordsInList(config.stackName, collectionIds, { timestamp__from: ingestTime }); - console.log('WAIT FOR COLLECTION RECORD IN LIST AFTER'); // update one of the granule files in database so that that file won't match with CMR console.log('XXXXX Waiting for getGranule()'); @@ -1001,4 +992,4 @@ describe('When there are granule differences and granule reconciliation is run', }); await cleanupProviders(config.stackName, config.bucket, providersDir, testSuffix); }); -}); +});*/ diff --git a/packages/api/endpoints/collections.js b/packages/api/endpoints/collections.js index 9d9be23a6f9..c66ffe9cd36 100644 --- a/packages/api/endpoints/collections.js +++ b/packages/api/endpoints/collections.js @@ -46,8 +46,9 @@ const log = new Logger({ sender: '@cumulus/api/collections' }); async function list(req, res) { log.trace(`list query ${JSON.stringify(req.query)}`); const { getMMT, includeStats, ...queryStringParameters } = req.query; - let dbSearch; - console.log('LIST QSP', queryStringParameters, includeStats); + let dbSearch = new CollectionSearch( + { queryStringParameters } + ); if (includeStats === 'true') { dbSearch = new Collection( { queryStringParameters }, @@ -55,14 +56,8 @@ async function list(req, res) { process.env.ES_INDEX, includeStats === 'true' ); - } else { - console.log('COLLECTIONSEARCH QSP', queryStringParameters); - dbSearch = new CollectionSearch( - { queryStringParameters } - ); } let result = await dbSearch.query(); - console.log('COLLECTIONSEARCH RESULT', result); if (getMMT === 'true') { result = await insertMMTLinks(result); } From d0ae5794492a3e5511cba3bbb2ee62d06eac9af8 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Mon, 10 Jun 2024 10:54:10 -0400 Subject: [PATCH 13/22] commenting out failing createReconReport spec --- .../CreateReconciliationReportSpec.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js index 2085ef33897..716e74b1369 100644 --- a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js +++ b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js @@ -1,5 +1,5 @@ /* eslint-disable no-unused-vars */ - +/* 'use strict'; const cloneDeep = require('lodash/cloneDeep'); @@ -108,6 +108,7 @@ const ingestAndPublishGranuleExecutionArns = []; * @param {string} sourceBucket - testing source bucket * @returns {Promise} The collection created */ +/* const createActiveCollection = async (prefix, sourceBucket) => { // The S3 path where granules will be ingested from const sourcePath = `${prefix}/tmp/${randomId('test-')}`; @@ -306,7 +307,7 @@ const fetchReconciliationReport = async (stackName, reportName) => { const reportResponse = await got(url); return reportResponse.body; }; -/* + describe('When there are granule differences and granule reconciliation is run', () => { let beforeAllFailed = false; let cmrClient; From b2826421776fd97d992fb91fa7e3073073b713e1 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Mon, 10 Jun 2024 12:01:29 -0400 Subject: [PATCH 14/22] removing comment --- .../CreateReconciliationReportSpec.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js index 716e74b1369..80f1911838e 100644 --- a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js +++ b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js @@ -1,5 +1,3 @@ -/* eslint-disable no-unused-vars */ -/* 'use strict'; const cloneDeep = require('lodash/cloneDeep'); @@ -108,7 +106,7 @@ const ingestAndPublishGranuleExecutionArns = []; * @param {string} sourceBucket - testing source bucket * @returns {Promise} The collection created */ -/* + const createActiveCollection = async (prefix, sourceBucket) => { // The S3 path where granules will be ingested from const sourcePath = `${prefix}/tmp/${randomId('test-')}`; @@ -276,7 +274,7 @@ const waitForCollectionRecordsInList = async (stackName, collectionIds, addition async () => { // Verify the collection is returned when listing collections const collsResp = await getCollections({ prefix: stackName, - query: { name__in: collectionIds.join(','), ...additionalQueryParams, limit: 30 } }); + query: { _id__in: collectionIds.join(','), ...additionalQueryParams, includeStats: true, limit: 30 } }); const results = get(JSON.parse(collsResp.body), 'results', []); const ids = results.map((c) => constructCollectionId(c.name, c.version)); return isEqual(ids.sort(), collectionIds.sort()); @@ -993,4 +991,4 @@ describe('When there are granule differences and granule reconciliation is run', }); await cleanupProviders(config.stackName, config.bucket, providersDir, testSuffix); }); -});*/ +}); From 3ca493dd85cab956d4b33bf510a71a5093ee9e74 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Mon, 10 Jun 2024 13:05:12 -0400 Subject: [PATCH 15/22] reverting changes to reconReport test --- .../CreateReconciliationReportSpec.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js index 80f1911838e..1fb7fe65625 100644 --- a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js +++ b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js @@ -106,7 +106,6 @@ const ingestAndPublishGranuleExecutionArns = []; * @param {string} sourceBucket - testing source bucket * @returns {Promise} The collection created */ - const createActiveCollection = async (prefix, sourceBucket) => { // The S3 path where granules will be ingested from const sourcePath = `${prefix}/tmp/${randomId('test-')}`; @@ -274,7 +273,7 @@ const waitForCollectionRecordsInList = async (stackName, collectionIds, addition async () => { // Verify the collection is returned when listing collections const collsResp = await getCollections({ prefix: stackName, - query: { _id__in: collectionIds.join(','), ...additionalQueryParams, includeStats: true, limit: 30 } }); + query: { _id__in: collectionIds.join(','), ...additionalQueryParams, limit: 30 } }); const results = get(JSON.parse(collsResp.body), 'results', []); const ids = results.map((c) => constructCollectionId(c.name, c.version)); return isEqual(ids.sort(), collectionIds.sort()); From 52281e8e8042034928821c1c765aa99b3ba8f86e Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Mon, 10 Jun 2024 14:07:26 -0400 Subject: [PATCH 16/22] reverting previous change --- .../CreateReconciliationReportSpec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js index 1fb7fe65625..5462f04c5f9 100644 --- a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js +++ b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js @@ -273,7 +273,7 @@ const waitForCollectionRecordsInList = async (stackName, collectionIds, addition async () => { // Verify the collection is returned when listing collections const collsResp = await getCollections({ prefix: stackName, - query: { _id__in: collectionIds.join(','), ...additionalQueryParams, limit: 30 } }); + query: { _id__in: collectionIds.join(','), ...additionalQueryParams, includeStats: true, limit: 30 } }); const results = get(JSON.parse(collsResp.body), 'results', []); const ids = results.map((c) => constructCollectionId(c.name, c.version)); return isEqual(ids.sort(), collectionIds.sort()); From db45ecd3bbd7f40086a6013100d4637c6d12916c Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Tue, 11 Jun 2024 16:49:25 -0400 Subject: [PATCH 17/22] adding ts-check --- packages/api/endpoints/collections.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/api/endpoints/collections.js b/packages/api/endpoints/collections.js index c66ffe9cd36..1e2f6b3518a 100644 --- a/packages/api/endpoints/collections.js +++ b/packages/api/endpoints/collections.js @@ -1,3 +1,5 @@ +//@ts-check + 'use strict'; const router = require('express-promise-router')(); @@ -46,9 +48,7 @@ const log = new Logger({ sender: '@cumulus/api/collections' }); async function list(req, res) { log.trace(`list query ${JSON.stringify(req.query)}`); const { getMMT, includeStats, ...queryStringParameters } = req.query; - let dbSearch = new CollectionSearch( - { queryStringParameters } - ); + let dbSearch; if (includeStats === 'true') { dbSearch = new Collection( { queryStringParameters }, @@ -56,6 +56,10 @@ async function list(req, res) { process.env.ES_INDEX, includeStats === 'true' ); + } else { + dbSearch = new CollectionSearch( + { queryStringParameters } + ); } let result = await dbSearch.query(); if (getMMT === 'true') { From 9f613bab05e8fcccba769135a4addda489f41189 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Wed, 12 Jun 2024 15:03:35 -0400 Subject: [PATCH 18/22] PR feedback --- .../db/tests/search/test-CollectionSearch.js | 81 +++++++++++-------- .../db/tests/search/test-GranuleSearch.js | 2 +- 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/packages/db/tests/search/test-CollectionSearch.js b/packages/db/tests/search/test-CollectionSearch.js index 8491d01d025..613efd88e2b 100644 --- a/packages/db/tests/search/test-CollectionSearch.js +++ b/packages/db/tests/search/test-CollectionSearch.js @@ -51,6 +51,14 @@ test.after.always(async (t) => { }); }); +test('CollectionSearch returns 10 collections by default', async (t) => { + const { knex } = t.context; + const AggregateSearch = new CollectionSearch(); + const results = await AggregateSearch.query(knex); + t.is(results.meta.count, 100); + t.is(results.results.length, 10); +}); + test('CollectionSearch supports page and limit params', async (t) => { const { knex } = t.context; let queryStringParameters = { @@ -81,14 +89,6 @@ test('CollectionSearch supports page and limit params', async (t) => { t.is(response.results?.length, 0); }); -test('CollectionSearch returns correct response for basic query', async (t) => { - const { knex } = t.context; - const AggregateSearch = new CollectionSearch(); - const results = await AggregateSearch.query(knex); - t.is(results.meta.count, 100); - t.is(results.results.length, 10); -}); - test('CollectionSearch supports infix search', async (t) => { const { knex } = t.context; const queryStringParameters = { @@ -113,6 +113,30 @@ test('CollectionSearch supports prefix search', async (t) => { t.is(response2.results?.length, 20); }); +test('CollectionSearch supports term search for boolean field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + reportToEms: false, + }; + const dbSearch4 = new CollectionSearch({ queryStringParameters }); + const response4 = await dbSearch4.query(knex); + t.is(response4.meta.count, 50); + t.is(response4.results?.length, 50); +}); + +test('CollectionSearch supports term search for date field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + updatedAt: 1579352701000, + }; + const dbSearch = new CollectionSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + test('CollectionSearch supports term search for number field', async (t) => { const { knex } = t.context; const queryStringParameters = { @@ -146,30 +170,6 @@ test('CollectionSearch supports term search for string field', async (t) => { t.is(response3.results?.length, 50); }); -test('CollectionSearch supports term search for boolean field', async (t) => { - const { knex } = t.context; - const queryStringParameters = { - limit: 200, - reportToEms: false, - }; - const dbSearch4 = new CollectionSearch({ queryStringParameters }); - const response4 = await dbSearch4.query(knex); - t.is(response4.meta.count, 50); - t.is(response4.results?.length, 50); -}); - -test('CollectionSearch supports term search for date field', async (t) => { - const { knex } = t.context; - const queryStringParameters = { - limit: 200, - updatedAt: 1579352701000, - }; - const dbSearch = new CollectionSearch({ queryStringParameters }); - const response = await dbSearch.query(knex); - t.is(response.meta.count, 50); - t.is(response.results?.length, 50); -}); - // TODO in CUMULUS-3639 test.todo('CollectionSearch supports range search'); @@ -188,6 +188,19 @@ test('CollectionSearch supports search for multiple fields', async (t) => { t.is(response.results?.length, 1); }); +test('CollectionSearch non-existing fields are ignored', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new CollectionSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); +}); + test('CollectionSearch supports sorting', async (t) => { const { knex } = t.context; let queryStringParameters = { @@ -247,7 +260,7 @@ test('CollectionSearch supports terms search', async (t) => { t.is(response.results?.length, 1); }); -test('CollectionSearch supports search which granule field does not match the given value', async (t) => { +test('CollectionSearch supports search when collection field does not match the given value', async (t) => { const { knex } = t.context; let queryStringParameters = { limit: 200, @@ -269,7 +282,7 @@ test('CollectionSearch supports search which granule field does not match the gi t.is(response.results?.length, 49); }); -test('CollectionSearch supports search which checks existence of granule field', async (t) => { +test('CollectionSearch supports search which checks existence of collection field', async (t) => { const { knex } = t.context; const queryStringParameters = { limit: 200, diff --git a/packages/db/tests/search/test-GranuleSearch.js b/packages/db/tests/search/test-GranuleSearch.js index 370330d2128..50a70c0c06e 100644 --- a/packages/db/tests/search/test-GranuleSearch.js +++ b/packages/db/tests/search/test-GranuleSearch.js @@ -643,7 +643,7 @@ test('GranuleSearch supports error.Error terms search', async (t) => { t.is(response.results?.length, 0); }); -test('GranuleSearch supports search which granule field does not match the given value', async (t) => { +test('GranuleSearch supports search when granule field does not match the given value', async (t) => { const { knex } = t.context; let queryStringParameters = { limit: 200, From bc40d5719720805edf217fab4165198f4513cc24 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Wed, 12 Jun 2024 15:15:34 -0400 Subject: [PATCH 19/22] PR feedback --- packages/db/src/search/CollectionSearch.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/db/src/search/CollectionSearch.ts b/packages/db/src/search/CollectionSearch.ts index 03134e6f97b..bc32488961f 100644 --- a/packages/db/src/search/CollectionSearch.ts +++ b/packages/db/src/search/CollectionSearch.ts @@ -56,12 +56,10 @@ export class CollectionSearch extends BaseSearch { const { countQuery, searchQuery, dbQueryParameters } = params; const { infix, prefix } = dbQueryParameters ?? this.dbQueryParameters; if (infix) { - countQuery.whereLike(`${this.tableName}.name`, `%${infix}%`); - searchQuery.whereLike(`${this.tableName}.name`, `%${infix}%`); + [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `%${infix}%`)); } if (prefix) { - countQuery.whereLike(`${this.tableName}.name`, `${prefix}%`); - searchQuery.whereLike(`${this.tableName}.name`, `${prefix}%`); + [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `%${prefix}%`)); } } From 6aeb39f944f473f815f7199f2462545a35bd51a9 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Wed, 12 Jun 2024 17:13:29 -0400 Subject: [PATCH 20/22] adding in test --- packages/db/src/search/CollectionSearch.ts | 12 ++++++++++-- packages/db/tests/search/test-CollectionSearch.js | 13 +++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/packages/db/src/search/CollectionSearch.ts b/packages/db/src/search/CollectionSearch.ts index bc32488961f..d8b1b805432 100644 --- a/packages/db/src/search/CollectionSearch.ts +++ b/packages/db/src/search/CollectionSearch.ts @@ -1,4 +1,6 @@ import { Knex } from 'knex'; +import pick from 'lodash/pick'; + import Logger from '@cumulus/logger'; import { CollectionRecord } from '@cumulus/types/api/collections'; import { BaseSearch } from './BaseSearch'; @@ -70,9 +72,15 @@ export class CollectionSearch extends BaseSearch { * @returns translated api records */ protected translatePostgresRecordsToApiRecords(pgRecords: PostgresCollectionRecord[]) - : Partial { + : Partial[] { log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); - const apiRecords = pgRecords.map((item) => translatePostgresCollectionToApiCollection(item)); + const apiRecords = pgRecords.map((item) => { + const apiRecord = translatePostgresCollectionToApiCollection(item); + + return this.dbQueryParameters.fields + ? pick(apiRecord, this.dbQueryParameters.fields) + : apiRecord; + }); return apiRecords; } } diff --git a/packages/db/tests/search/test-CollectionSearch.js b/packages/db/tests/search/test-CollectionSearch.js index 613efd88e2b..5147075fae9 100644 --- a/packages/db/tests/search/test-CollectionSearch.js +++ b/packages/db/tests/search/test-CollectionSearch.js @@ -201,6 +201,19 @@ test('CollectionSearch non-existing fields are ignored', async (t) => { t.is(response.results?.length, 100); }); +test('CollectionSearch returns fields specified', async (t) => { + const { knex } = t.context; + const fields = 'name,version,reportToEms,process'; + const queryStringParameters = { + fields, + }; + const dbSearch = new CollectionSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 10); + response.results.forEach((granule) => t.deepEqual(Object.keys(granule), fields.split(','))); +}); + test('CollectionSearch supports sorting', async (t) => { const { knex } = t.context; let queryStringParameters = { From e208d5e0df12ac4343b822b18f1175341ca5966a Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Wed, 12 Jun 2024 18:28:01 -0400 Subject: [PATCH 21/22] PR feedback fix --- packages/db/tests/search/test-CollectionSearch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/tests/search/test-CollectionSearch.js b/packages/db/tests/search/test-CollectionSearch.js index 5147075fae9..3598cda5edd 100644 --- a/packages/db/tests/search/test-CollectionSearch.js +++ b/packages/db/tests/search/test-CollectionSearch.js @@ -211,7 +211,7 @@ test('CollectionSearch returns fields specified', async (t) => { const response = await dbSearch.query(knex); t.is(response.meta.count, 100); t.is(response.results?.length, 10); - response.results.forEach((granule) => t.deepEqual(Object.keys(granule), fields.split(','))); + response.results.forEach((collection) => t.deepEqual(Object.keys(collection), fields.split(','))); }); test('CollectionSearch supports sorting', async (t) => { From 54330231336327d912fb12c570142c9024ffd486 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Wed, 12 Jun 2024 22:26:57 -0400 Subject: [PATCH 22/22] PR feedback --- packages/db/src/search/field-mapping.ts | 2 +- packages/db/tests/search/test-field-mapping.js | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/db/src/search/field-mapping.ts b/packages/db/src/search/field-mapping.ts index e6a520222d4..9a196243d11 100644 --- a/packages/db/src/search/field-mapping.ts +++ b/packages/db/src/search/field-mapping.ts @@ -124,7 +124,7 @@ const collectionMapping : { [key: string]: Function } = { updated_at: value && new Date(Number(value)), }), reportToEms: (value?: string) => ({ - report_to_ems: value ?? false, + report_to_ems: (value === 'true'), }), process: (value?: string) => ({ process: value, diff --git a/packages/db/tests/search/test-field-mapping.js b/packages/db/tests/search/test-field-mapping.js index b1d18befd30..4fca79ec82f 100644 --- a/packages/db/tests/search/test-field-mapping.js +++ b/packages/db/tests/search/test-field-mapping.js @@ -105,6 +105,9 @@ test('mapQueryStringFieldToDbField correctly converts all collection api fields const queryStringParameters = { createdAt: '1591312763823', name: 'MOD11A1', + reportToEms: 'true', + urlPath: 'http://fakepath.com', + sampleFileName: 'hello.txt', version: '006', updatedAt: 1591384094512, }; @@ -113,6 +116,9 @@ test('mapQueryStringFieldToDbField correctly converts all collection api fields created_at: new Date(1591312763823), name: 'MOD11A1', version: '006', + report_to_ems: true, + url_path: 'http://fakepath.com', + sample_file_name: 'hello.txt', updated_at: new Date(1591384094512), };