CUMULUS-3641 - Update Collections LIST endpoint to query Postgres basic

CHANGELOG.md

@@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ### Replace ElasticSearch Phase 1
 
+- **CUMULUS-3641**
+ - Updated `collections` api endpoint to query postgres instead of elasticsearch except if `includeStats` is in the query parameters
 - **CUMULUS-3695**
  - Updated `granule` list api endpoint and BaseSearch class to handle sort fields
 - **CUMULUS-3688**

example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js

@@ -273,7 +273,7 @@ const waitForCollectionRecordsInList = async (stackName, collectionIds, addition
   async () => {
     // Verify the collection is returned when listing collections
     const collsResp = await getCollections({ prefix: stackName,
-      query: { _id__in: collectionIds.join(','), ...additionalQueryParams, limit: 30 } });
+      query: { _id__in: collectionIds.join(','), ...additionalQueryParams, includeStats: true, limit: 30 } });
     const results = get(JSON.parse(collsResp.body), 'results', []);
     const ids = results.map((c) => constructCollectionId(c.name, c.version));
     return isEqual(ids.sort(), collectionIds.sort());

packages/api/endpoints/collections.js

@@ -1,3 +1,5 @@
+//@ts-check
+
 'use strict';
 
 const router = require('express-promise-router')();
@@ -16,6 +18,7 @@ const {
   isCollisionError,
   translateApiCollectionToPostgresCollection,
   translatePostgresCollectionToApiCollection,
+  CollectionSearch,
 } = require('@cumulus/db');
 const CollectionConfigStore = require('@cumulus/collection-config-store');
 const { getEsClient, Search } = require('@cumulus/es-client/search');
@@ -43,14 +46,22 @@ const log = new Logger({ sender: '@cumulus/api/collections' });
  * @returns {Promise<Object>} the promise of express response object
  */
 async function list(req, res) {
+  log.trace(`list query ${JSON.stringify(req.query)}`);
   const { getMMT, includeStats, ...queryStringParameters } = req.query;
-  const collection = new Collection(
-    { queryStringParameters },
-    undefined,
-    process.env.ES_INDEX,
-    includeStats === 'true'
-  );
-  let result = await collection.query();
+  let dbSearch;
+  if (includeStats === 'true') {
+    dbSearch = new Collection(
+      { queryStringParameters },
+      undefined,
+      process.env.ES_INDEX,
+      includeStats === 'true'
+    );
+  } else {
+    dbSearch = new CollectionSearch(
+      { queryStringParameters }
+    );
+  }
+  let result = await dbSearch.query();
   if (getMMT === 'true') {
     result = await insertMMTLinks(result);
   }

packages/api/tests/app/test-launchpadAuth.js

@@ -10,7 +10,7 @@ const { createBucket, putJsonS3Object } = require('@cumulus/aws-client/S3');
 const launchpad = require('@cumulus/launchpad-auth');
 const { randomId } = require('@cumulus/common/test-utils');
 
-const EsCollection = require('@cumulus/es-client/collections');
+const { CollectionSearch } = require('@cumulus/db');
 const models = require('../../models');
 const { createJwtToken } = require('../../lib/token');
 const { fakeAccessTokenFactory } = require('../../lib/testUtils');
@@ -72,7 +72,7 @@ test.after.always(async () => {
 
 test.serial('API request with a valid Launchpad token stores the access token', async (t) => {
   const stub = sinon.stub(launchpad, 'validateLaunchpadToken').returns(validateTokenResponse);
-  const collectionStub = sinon.stub(EsCollection.prototype, 'query').returns([]);
+  const collectionStub = sinon.stub(CollectionSearch.prototype, 'query').returns([]);
 
   try {
     await request(app)
@@ -113,7 +113,7 @@ test.serial('API request with an invalid Launchpad token returns a 403 unauthori
 
 test.serial('API request with a stored non-expired Launchpad token record returns a successful response', async (t) => {
   let stub = sinon.stub(launchpad, 'validateLaunchpadToken').resolves(validateTokenResponse);
-  const collectionStub = sinon.stub(EsCollection.prototype, 'query').returns([]);
+  const collectionStub = sinon.stub(CollectionSearch.prototype, 'query').returns([]);
 
   try {
     await request(app)
@@ -143,7 +143,7 @@ test.serial('API request with a stored non-expired Launchpad token record return
 });
 
 test.serial('API request with an expired Launchpad token returns a 401 response', async (t) => {
-  const collectionStub = sinon.stub(EsCollection.prototype, 'query').returns([]);
+  const collectionStub = sinon.stub(CollectionSearch.prototype, 'query').returns([]);
 
   try {
     await accessTokenModel.create({

packages/api/tests/endpoints/collections/list-collections.js

@@ -3,6 +3,7 @@
 const test = require('ava');
 const request = require('supertest');
 const sinon = require('sinon');
+const range = require('lodash/range');
 const awsServices = require('@cumulus/aws-client/services');
 const {
   recursivelyDeleteS3Bucket,
@@ -11,6 +12,7 @@ const { randomString } = require('@cumulus/common/test-utils');
 const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap');
 const EsCollection = require('@cumulus/es-client/collections');
 const { getEsClient } = require('@cumulus/es-client/search');
+const { randomId } = require('@cumulus/common/test-utils');
 
 const models = require('../../../models');
 const {
@@ -20,10 +22,25 @@ const {
 } = require('../../../lib/testUtils');
 const assertions = require('../../../lib/assertions');
 
+const testDbName = randomId('collection');
+
+const {
+  destroyLocalTestDb,
+  generateLocalTestDb,
+  CollectionPgModel,
+  fakeCollectionRecordFactory,
+  migrationDir,
+  localStackConnectionEnv,
+} = require('../../../../db/dist');
+
+process.env.PG_HOST = randomId('hostname');
+process.env.PG_USER = randomId('user');
+process.env.PG_PASSWORD = randomId('password');
+process.env.TOKEN_SECRET = randomString();
+
 process.env.AccessTokensTable = randomString();
 process.env.stackName = randomString();
 process.env.system_bucket = randomString();
-process.env.TOKEN_SECRET = randomString();
 
 // import the express app after setting the env variables
 const { app } = require('../../../app');
@@ -34,7 +51,13 @@ let esClient;
 let jwtAuthToken;
 let accessTokenModel;
 
-test.before(async () => {
+process.env = {
+  ...process.env,
+  ...localStackConnectionEnv,
+  PG_DATABASE: testDbName,
+};
+
+test.before(async (t) => {
   const esAlias = randomString();
   process.env.ES_INDEX = esAlias;
   await bootstrapElasticSearch({
@@ -52,16 +75,45 @@ test.before(async () => {
 
   jwtAuthToken = await createFakeJwtAuthToken({ accessTokenModel, username });
   esClient = await getEsClient('fakehost');
+  const { knexAdmin, knex } = await generateLocalTestDb(
+    testDbName,
+    migrationDir
+  );
+
+  t.context.knexAdmin = knexAdmin;
+  t.context.knex = knex;
+
+  t.context.collectionPgModel = new CollectionPgModel();
+  const collections = [];
+
+  range(40).map((num) => (
+    collections.push(fakeCollectionRecordFactory({
+      name: num % 2 === 0 ? `testCollection__${num}` : `fakeCollection__${num}`,
+      version: `${num}`,
+      cumulus_id: num,
+      updated_at: new Date(1579352700000 + (num % 2) * 1000),
+    }))
+  ));
+
+  t.context.collections = collections;
+  await t.context.collectionPgModel.insert(
+    t.context.knex,
+    collections
+  );
 });
 
 test.beforeEach((t) => {
   t.context.testCollection = fakeCollectionFactory();
 });
 
-test.after.always(async () => {
+test.after.always(async (t) => {
   await accessTokenModel.deleteTable();
   await recursivelyDeleteS3Bucket(process.env.system_bucket);
   await esClient.client.indices.delete({ index: esIndex });
+  await destroyLocalTestDb({
+    ...t.context,
+    testDbName,
+  });
 });
 
 test('CUMULUS-911 GET without pathParameters and without an Authorization header returns an Authorization Missing response', async (t) => {
@@ -86,21 +138,15 @@ test('CUMULUS-912 GET without pathParameters and with an invalid access token re
 test.todo('CUMULUS-912 GET without pathParameters and with an unauthorized user returns an unauthorized response');
 
 test.serial('default returns list of collections from query', async (t) => {
-  const stub = sinon.stub(EsCollection.prototype, 'query').returns({ results: [t.context.testCollection] });
-  const spy = sinon.stub(EsCollection.prototype, 'addStatsToCollectionResults');
-
   const response = await request(app)
     .get('/collections')
     .set('Accept', 'application/json')
     .set('Authorization', `Bearer ${jwtAuthToken}`)
     .expect(200);
 
   const { results } = response.body;
-  t.is(results.length, 1);
-  t.is(results[0].name, t.context.testCollection.name);
-  t.true(spy.notCalled);
-  stub.restore();
-  spy.restore();
+  t.is(results.length, 10);
+  t.is(results[0].name, t.context.collections[0].name);
 });
 
 test.serial('returns list of collections with stats when requested', async (t) => {

packages/db/src/index.ts

@@ -145,6 +145,9 @@ export {
 export {
   StatsSearch,
 } from './search/StatsSearch';
+export {
+  CollectionSearch,
+} from './search/CollectionSearch';
 
 export { AsyncOperationPgModel } from './models/async_operation';
 export { BasePgModel } from './models/base';

packages/db/src/search/BaseSearch.ts

@@ -412,7 +412,7 @@ class BaseSearch {
    * @param testKnex - knex for testing
    * @returns search result
    */
-  async query(testKnex: Knex | undefined) {
+  async query(testKnex?: Knex) {
     const knex = testKnex ?? await getKnexClient();
     const { countQuery, searchQuery } = this.buildSearch(knex);
     try {

packages/db/src/search/CollectionSearch.ts

@@ -0,0 +1,86 @@
+import { Knex } from 'knex';
+import pick from 'lodash/pick';
+
+import Logger from '@cumulus/logger';
+import { CollectionRecord } from '@cumulus/types/api/collections';
+import { BaseSearch } from './BaseSearch';
+import { DbQueryParameters, QueryEvent } from '../types/search';
+import { translatePostgresCollectionToApiCollection } from '../translate/collections';
+import { PostgresCollectionRecord } from '../types/collection';
+
+const log = new Logger({ sender: '@cumulus/db/CollectionSearch' });
+
+/**
+ * There is no need to declare an ApiCollectionRecord type since
+ * CollectionRecord contains all the same fields from the api
+ */
+
+/**
+ * Class to build and execute db search query for collection
+ */
+export class CollectionSearch extends BaseSearch {
+  constructor(event: QueryEvent) {
+    super(event, 'collection');
+  }
+
+  /**
+   * Build basic query
+   *
+   * @param knex - DB client
+   * @returns queries for getting count and search result
+   */
+  protected buildBasicQuery(knex: Knex)
+    : {
+      countQuery: Knex.QueryBuilder,
+      searchQuery: Knex.QueryBuilder,
+    } {
+    const countQuery = knex(this.tableName)
+      .count(`${this.tableName}.cumulus_id`);
+
+    const searchQuery = knex(this.tableName)
+      .select(`${this.tableName}.*`);
+    return { countQuery, searchQuery };
+  }
+
+  /**
+   * Build queries for infix and prefix
+   *
+   * @param params
+   * @param params.countQuery - query builder for getting count
+   * @param params.searchQuery - query builder for search
+   * @param [params.dbQueryParameters] - db query parameters
+   */
+  protected buildInfixPrefixQuery(params: {
+    countQuery: Knex.QueryBuilder,
+    searchQuery: Knex.QueryBuilder,
+    dbQueryParameters?: DbQueryParameters,
+  }) {
+    const { countQuery, searchQuery, dbQueryParameters } = params;
+    const { infix, prefix } = dbQueryParameters ?? this.dbQueryParameters;
+    if (infix) {
+      [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `%${infix}%`));
+    }
+    if (prefix) {
+      [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `%${prefix}%`));
+    }
+  }
+
+  /**
+   * Translate postgres records to api records
+   *
+   * @param pgRecords - postgres records returned from query
+   * @returns translated api records
+   */
+  protected translatePostgresRecordsToApiRecords(pgRecords: PostgresCollectionRecord[])
+    : Partial<CollectionRecord>[] {
+    log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `);
+    const apiRecords = pgRecords.map((item) => {
+      const apiRecord = translatePostgresCollectionToApiCollection(item);
+
+      return this.dbQueryParameters.fields
+        ? pick(apiRecord, this.dbQueryParameters.fields)
+        : apiRecord;
+    });
+    return apiRecords;
+  }
+}

packages/db/src/search/field-mapping.ts

@@ -123,6 +123,18 @@ const collectionMapping : { [key: string]: Function } = {
   updatedAt: (value?: string) => ({
     updated_at: value && new Date(Number(value)),
   }),
+  reportToEms: (value?: string) => ({
+    report_to_ems: (value === 'true'),
+  }),
+  process: (value?: string) => ({
+    process: value,
+  }),
+  sampleFileName: (value?: string) => ({
+    sample_file_name: value,
+  }),
+  urlPath: (value?: string) => ({
+    url_path: value,
+  }),
 };
 
 const executionMapping : { [key: string]: Function } = {