diff --git a/serverless-configs/env.yml b/serverless-configs/env.yml index 56519ea2e..8681b8726 100644 --- a/serverless-configs/env.yml +++ b/serverless-configs/env.yml @@ -26,6 +26,10 @@ default_env: &default_env # Timeout for lambda lambdaTimeout: ${env:LAMBDA_TIMEOUT, '30'} + # Refetch variables + maxRetries: ${env:MAX_RETRIES, '1'} + retryDelay: ${env:RETRY_DELAY, '1000'} + # Pinned UMM versions ummCollectionVersion: '1.18.1' ummGranuleVersion: '1.6.5' diff --git a/src/cmr/concepts/__tests__/concept.test.js b/src/cmr/concepts/__tests__/concept.test.js index 3494e7cb9..12e6e835f 100644 --- a/src/cmr/concepts/__tests__/concept.test.js +++ b/src/cmr/concepts/__tests__/concept.test.js @@ -30,22 +30,6 @@ describe('collection', () => { expect(concept.getItemCount()).toEqual(84) }) }) - - describe('both json and umm keys but count is different', () => { - test('throws an error', () => { - const concept = new Concept('concept', {}, { - jsonKeys: ['jsonKey'], - ummKeys: ['ummKey'] - }) - - concept.setJsonItemCount(84) - concept.setUmmItemCount(32617) - - expect(() => { - concept.getItemCount() - }).toThrow('Inconsistent data prevented GraphQL from correctly parsing results (JSON Hits: 84, UMM Hits: 32617)') - }) - }) }) }) }) diff --git a/src/cmr/concepts/concept.js b/src/cmr/concepts/concept.js index e0eac60ee..23036074f 100644 --- a/src/cmr/concepts/concept.js +++ b/src/cmr/concepts/concept.js @@ -180,15 +180,8 @@ export default class Concept { */ getItemCount() { const { jsonKeys = [], ummKeys = [] } = this.requestInfo - if (jsonKeys.length) { if (ummKeys.length) { - // If both json and umm keys are being requested ensure that each endpoint - // returned the same number of results - if (this.jsonItemCount !== this.ummItemCount) { - throw new Error(`Inconsistent data prevented GraphQL from correctly parsing results (JSON Hits: ${this.jsonItemCount}, UMM Hits: ${this.ummItemCount})`) - } - // Both endpoints returned the same value, return either value here return this.ummItemCount } @@ -205,6 +198,131 @@ export default class Concept { return 0 } + /** + * Fetches missing items for a umm or json endpoint with retry logic + * @param {Array} missingIds - Array of concept IDs for missing items + * @param {Array} keys - Array of requested keys to fetch for each item (ummKeys or jsonKeys) + * @param {Function} fetchFunction - Function to fetch data from CMR (fetchUmm or fetchJson) + * @param {Function} parseFunction - Function to parse the fetched data + * @param {Integer} retryCount - Current retry attempt count + */ + async fetchWithRetry(missingIds, keys, fetchFunction, parseFunction, retryCount = 0) { + const { + maxRetries: maxRetriesEnv, + retryDelay: retryDelayEnv + } = process.env + + const maxRetries = parseInt(maxRetriesEnv, 10) + const retryDelay = parseInt(retryDelayEnv, 10) + + const response = await fetchFunction(missingIds, keys) + const fetchedItems = parseFunction(response) + + if (missingIds.length === fetchedItems.length) { + return { fetchedItems } + } + + if (retryCount < maxRetries) { + console.log(`Retry ${retryCount + 1}: ${missingIds} were missing. Retrying...`) + + await new Promise((resolve) => { setTimeout(resolve, retryDelay) }) + + return this.fetchWithRetry(missingIds, keys, fetchFunction, parseFunction, retryCount + 1) + } + + throw new Error(`Inconsistent data prevented GraphQL from correctly parsing results (JSON Hits: ${this.jsonItemCount}, UMM Hits: ${this.ummItemCount})`) + } + + /** + * Validates the response from CMR and handles inconsistencies between JSON and UMM data + */ + async validateResponse() { + const response = await this.getResponse() + const { jsonKeys, ummKeys, ummKeyMappings } = this.requestInfo + + const [jsonResponse, ummResponse] = response + + // If both json and umm keys are being requested ensure that each endpoint + // returned the same number of results + if (jsonKeys.length && ummKeys.length) { + if (this.jsonItemCount === this.ummItemCount) { + return true + } + + const parsedJsonResponse = this.parseJsonBody(jsonResponse) + const parsedUmmResponse = this.parseUmmBody(ummResponse) + + const jsonIds = parsedJsonResponse.map((item) => item.concept_id) + const ummIds = parsedUmmResponse.map((item) => item.meta['concept-id']) + + // If the number of concept ids in umm and json match, it mean the same number of items from both endpoint. + // This suggests that the data is consistent, so we can consider the response is valid + if (ummIds.length === jsonIds.length) { + return true + } + + // Handles the case where umm data has missing items. + // Attempts to fetch the missing UMM data and add the missing items to existing items + if (ummIds.length < jsonIds.length) { + const missingUmmIds = jsonIds.filter((id) => !ummIds.includes(id)) + + const { fetchedItems } = await this.fetchWithRetry( + missingUmmIds, + ummKeys, + (params, keys) => this.fetchUmm(params, keys), + (parsedResponse) => this.parseUmmBody(parsedResponse) + ) + + const currentItems = this.getItems() + + this.setUmmItemCount(ummIds.length + fetchedItems.length) + + fetchedItems.forEach((item) => { + const { meta } = item + const conceptId = meta['concept-id'] + + const normalizedItem = this.normalizeUmmItem(item) + const itemKey = Object.keys(currentItems).find((key) => key.startsWith(conceptId)) + + this.setEssentialUmmValues(itemKey, normalizedItem) + + this.setUmmItems(item, itemKey, ummKeys, ummKeyMappings) + }) + } + + // Handles the case where JSON data has missing items. + // Attempts to fetch the missing JSON data and add the missing items to existing items + if (jsonIds.length < ummIds.length) { + const missingJsonIds = ummIds.filter((id) => !jsonIds.includes(id)) + + const { fetchedItems } = await this.fetchWithRetry( + missingJsonIds, + jsonKeys, + (params, keys) => this.fetchJson(params, keys), + (parsedResponse) => this.parseJsonBody(parsedResponse) + ) + + const currentItems = this.getItems() + + this.setJsonItemCount(jsonIds.length + fetchedItems.length) + + fetchedItems.forEach((item) => { + const normalizedItem = this.normalizeJsonItem(item) + + // Find the corresponding item key in the current items + // The key starts with the concept ID (e.g., 'G100000-EDSC-0') + const itemKey = Object.keys(currentItems).find( + (key) => key.startsWith(normalizedItem.concept_id) + ) + + this.setJsonItems(itemKey, jsonKeys, normalizedItem) + }) + } + } + + return true + } + /** * Get the CMR concept type of this object */ @@ -707,6 +825,23 @@ export default class Concept { return `${conceptId}-${itemIndex}` } + /** + * Sets JSON item values in the result set based on requested keys + * @param {String} itemKey - Unique identifier for the item being processed + * @param {Array} jsonKeys - Array of JSON keys requested in the GraphQL query + * @param {Object} item - Raw item data received from the CMR JSON endpoint + */ + setJsonItems(itemKey, jsonKeys, item) { + jsonKeys.forEach((jsonKey) => { + const cmrKey = snakeCase(jsonKey) + + const { [cmrKey]: keyValue } = item + + // Snake case the key requested and any children of that key + this.setItemValue(itemKey, jsonKey, keyValue) + }) + } + /** * Parses the response from the json endpoint * @param {Object} jsonResponse HTTP response from the CMR endpoint @@ -732,14 +867,71 @@ export default class Concept { this.setEssentialJsonValues(itemKey, normalizedItem) - jsonKeys.forEach((jsonKey) => { - const cmrKey = snakeCase(jsonKey) + this.setJsonItems(itemKey, jsonKeys, normalizedItem) + }) + } - const { [cmrKey]: keyValue } = normalizedItem + /** + * Sets UMM item values to the result set + * @param {Object} item - Raw item data received from the CMR UMM endpoint + * @param {String} itemKey - Unique identifier for the item being processed + * @param {Array} ummKeys - Array of UMM keys requested in the GraphQL query + * @param {Object} ummKeyMappings - Object mapping UMM keys to their paths in the CMR response + */ + setUmmItems(item, itemKey, ummKeys, ummKeyMappings) { + ummKeys.forEach((ummKey) => { + // Use lodash.get to retrieve a value from the umm response given the + // path we've defined above + let keyValue = get(item, ummKeyMappings[ummKey]) + + // If the raw `ummMetadata` was requested return that value unaltered + if (ummKey === 'ummMetadata') { + this.setItemValue( + itemKey, + ummKey, + keyValue + ) + + return + } - // Snake case the key requested and any children of that key - this.setItemValue(itemKey, jsonKey, keyValue) - }) + // If the UMM Key is `previewMetadata`, we need to combine the `meta` and `umm` fields + // This ensures all the keys are available for the PreviewMetadata union type + if (ummKey === 'previewMetadata') { + keyValue = { + ...item.umm, + ...item.meta + } + } + + if (keyValue != null) { + const camelCasedObject = camelcaseKeys({ [ummKey]: keyValue }, { + deep: true, + exclude: ['RelatedURLs'] + }) + + // CamelcaseKey converts RelatedURLs to relatedUrLs, so excluding RelatedURLs above. + // This will remove RelatedURLs and create a new + // key called relatedUrls and assign the value to it so MMT and graphql response matches. + if (ummKey === 'previewMetadata') { + const { previewMetadata } = camelCasedObject + camelCasedObject.previewMetadata = { + ...previewMetadata, + relatedUrls: previewMetadata.RelatedURLs + } + + delete camelCasedObject.previewMetadata.RelatedURLs + } + + // Camel case all of the keys of this object (ummKey is already camel cased) + const { [ummKey]: camelCasedValue } = camelCasedObject + + this.setItemValue( + itemKey, + ummKey, + camelCasedValue + ) + } }) } @@ -773,61 +965,7 @@ export default class Concept { this.setEssentialUmmValues(itemKey, normalizedItem) - // Loop through the requested umm keys - ummKeys.forEach((ummKey) => { - // Use lodash.get to retrieve a value from the umm response given the - // path we've defined above - let keyValue = get(item, ummKeyMappings[ummKey]) - - // If the raw `ummMetadata` was requested return that value unaltered - if (ummKey === 'ummMetadata') { - this.setItemValue( - itemKey, - ummKey, - keyValue - ) - - return - } - - // If the UMM Key is `previewMetadata`, we need to combine the `meta` and `umm` fields - // This ensures all the keys are available for the PreviewMetadata union type - if (ummKey === 'previewMetadata') { - keyValue = { - ...item.umm, - ...item.meta - } - } - - if (keyValue != null) { - const camelCasedObject = camelcaseKeys({ [ummKey]: keyValue }, { - deep: true, - exclude: ['RelatedURLs'] - }) - - // CamelcaseKey converts RelatedURLs to relatedUrLs, so excluding RelatedURLs above. - // This will remove RelatedURLs and create a new - // key called relatedUrls and assign the value to it so MMT and graphql response matches. - if (ummKey === 'previewMetadata') { - const { previewMetadata } = camelCasedObject - camelCasedObject.previewMetadata = { - ...previewMetadata, - relatedUrls: previewMetadata.RelatedURLs - } - - delete camelCasedObject.previewMetadata.RelatedURLs - } - - // Camel case all of the keys of this object (ummKey is already camel cased) - const { [ummKey]: camelCasedValue } = camelCasedObject - - this.setItemValue( - itemKey, - ummKey, - camelCasedValue - ) - } - }) + this.setUmmItems(item, itemKey, ummKeys, ummKeyMappings,) }) } diff --git a/src/datasources/__tests__/granule.test.js b/src/datasources/__tests__/granule.test.js index b39fbb277..1c75dae77 100644 --- a/src/datasources/__tests__/granule.test.js +++ b/src/datasources/__tests__/granule.test.js @@ -378,6 +378,9 @@ describe('granule', () => { describe('with json and umm keys', () => { beforeEach(() => { + process.env.maxRetries = '1' + process.env.retryDelay = '1000' + // Overwrite default requestInfo requestInfo = { name: 'granules', @@ -473,6 +476,653 @@ describe('granule', () => { }] }) }) + + describe('when JSON and UMM hit counts differ, but returned items match', () => { + test('return the parsed granule results', async () => { + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 85, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.json/) + .reply(200, { + feed: { + entry: [{ + id: 'G100000-EDSC', + browse_flag: true + }] + } + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 84, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.umm_json/) + .reply(200, { + items: [{ + meta: { + 'concept-id': 'G100000-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.020.nc4' + } + }] + }) + + const response = await granuleDatasource({ + params: { + concept_id: 'G100000-EDSC' + } + }, { + headers: { + 'Client-Id': 'eed-test-graphql', + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + } + }, requestInfo) + + expect(response).toEqual({ + count: 84, + cursor: null, + items: [{ + conceptId: 'G100000-EDSC', + browseFlag: true, + granuleUr: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.020.nc4' + }] + }) + }) + }) + + describe('when JSON and UMM hit counts differ and umm item count differ', () => { + test('should fetch missing UMM data and return combined results with correct counts', async () => { + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 3, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.json/) + .reply(200, { + feed: { + entry: [ + { + id: 'G100000-EDSC', + browse_flag: true + }, + { + id: 'G100001-EDSC', + browse_flag: true + }, + { + id: 'G100002-EDSC', + browse_flag: true + } + ] + } + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 1, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.umm_json/) + .reply(200, { + items: [{ + meta: { + 'concept-id': 'G100000-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.020.nc4' + } + }] + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 2, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.umm_json/) + .reply(200, { + items: [ + { + meta: { + 'concept-id': 'G100001-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.021.nc4' + } + }, + { + meta: { + 'concept-id': 'G100002-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.022.nc4' + } + } + ] + }) + + const response = await granuleDatasource({ + params: { + concept_id: 'G100000-EDSC' + } + }, { + headers: { + 'Client-Id': 'eed-test-graphql', + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + } + }, requestInfo) + + expect(response).toEqual({ + count: 3, + cursor: null, + items: [ + { + conceptId: 'G100000-EDSC', + browseFlag: true, + granuleUr: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.020.nc4' + }, + { + conceptId: 'G100001-EDSC', + browseFlag: true, + granuleUr: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.021.nc4' + }, + { + conceptId: 'G100002-EDSC', + browseFlag: true, + granuleUr: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.022.nc4' + } + ] + }) + }) + }) + + describe('when JSON and UMM hit counts differ and json item count differ', () => { + test('should fetch missing JSON data and return combined results with correct count', async () => { + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 1, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.json/) + .reply(200, { + feed: { + entry: [ + { + id: 'G100000-EDSC', + browse_flag: true + } + ] + } + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 3, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.umm_json/) + .reply(200, { + items: [ + { + meta: { + 'concept-id': 'G100000-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.020.nc4' + } + }, + { + meta: { + 'concept-id': 'G100001-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.021.nc4' + } + }, + { + meta: { + 'concept-id': 'G100002-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.022.nc4' + } + } + + ] + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 3, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.json/) + .reply(200, { + feed: { + entry: [ + { + id: 'G100001-EDSC', + browse_flag: true + }, + { + id: 'G100002-EDSC', + browse_flag: true + } + ] + } + }) + + const response = await granuleDatasource({ + params: { + concept_id: 'G100000-EDSC' + } + }, { + headers: { + 'Client-Id': 'eed-test-graphql', + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + } + }, requestInfo) + + expect(response).toEqual({ + count: 3, + cursor: null, + items: [ + { + conceptId: 'G100000-EDSC', + browseFlag: true, + granuleUr: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.020.nc4' + }, + { + conceptId: 'G100001-EDSC', + browseFlag: true, + granuleUr: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.021.nc4' + }, + { + conceptId: 'G100002-EDSC', + browseFlag: true, + granuleUr: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.022.nc4' + } + ] + }) + }) + }) + + describe('when first retry does not get all of missing umm concepts', () => { + test('should fetch missing concepts again and return combined result', async () => { + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 3, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.json/) + .reply(200, { + feed: { + entry: [ + { + id: 'G100000-EDSC', + browse_flag: true + }, + { + id: 'G100001-EDSC', + browse_flag: true + }, + { + id: 'G100002-EDSC', + browse_flag: true + } + ] + } + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 1, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.umm_json/) + .reply(200, { + items: [{ + meta: { + 'concept-id': 'G100000-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.020.nc4' + } + }] + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 1, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.umm_json/) + .reply(200, { + items: [ + { + meta: { + 'concept-id': 'G100001-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.021.nc4' + } + } + ] + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 2, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.umm_json/) + .reply(200, { + items: [ + { + meta: { + 'concept-id': 'G100001-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.021.nc4' + } + }, + { + meta: { + 'concept-id': 'G100002-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.022.nc4' + } + } + ] + }) + + const response = await granuleDatasource({ + params: { + concept_id: 'G100000-EDSC' + } + }, { + headers: { + 'Client-Id': 'eed-test-graphql', + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + } + }, requestInfo) + + expect(response).toEqual({ + count: 3, + cursor: null, + items: [ + { + conceptId: 'G100000-EDSC', + browseFlag: true, + granuleUr: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.020.nc4' + }, + { + conceptId: 'G100001-EDSC', + browseFlag: true, + granuleUr: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.021.nc4' + }, + { + conceptId: 'G100002-EDSC', + browseFlag: true, + granuleUr: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.022.nc4' + } + ] + }) + }) + }) + + describe('when first retry does not get all of missing json concepts', () => { + test('should fetch missing concepts again and return combined result', async () => { + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 1, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.json/) + .reply(200, { + feed: { + entry: [ + { + id: 'G100000-EDSC', + browse_flag: true + } + ] + } + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 3, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.umm_json/) + .reply(200, { + items: [ + { + meta: { + 'concept-id': 'G100000-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.020.nc4' + } + }, + { + meta: { + 'concept-id': 'G100001-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.021.nc4' + } + }, + { + meta: { + 'concept-id': 'G100002-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.022.nc4' + } + } + + ] + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 1, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.json/) + .reply(200, { + feed: { + entry: [ + { + id: 'G100000-EDSC', + browse_flag: true + } + ] + } + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 1, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.json/) + .reply(200, { + feed: { + entry: [ + { + id: 'G100001-EDSC', + browse_flag: true + }, + { + id: 'G100002-EDSC', + browse_flag: true + } + ] + } + }) + + const response = await granuleDatasource({ + params: { + concept_id: 'G100000-EDSC' + } + }, { + headers: { + 'Client-Id': 'eed-test-graphql', + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + } + }, requestInfo) + + expect(response).toEqual({ + count: 3, + cursor: null, + items: [ + { + conceptId: 'G100000-EDSC', + browseFlag: true, + granuleUr: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.020.nc4' + }, + { + conceptId: 'G100001-EDSC', + browseFlag: true, + granuleUr: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.021.nc4' + }, + { + conceptId: 'G100002-EDSC', + browseFlag: true, + granuleUr: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.022.nc4' + } + ] + }) + }) + }) + + describe('when JSON and UMM hit counts differ and max retry attempts are reached', () => { + test('throws an error after max retries', async () => { + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 1, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.json/) + .reply(200, { + feed: { + entry: [ + { + id: 'G100000-EDSC', + browse_flag: true + } + ] + } + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 3, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.umm_json/) + .reply(200, { + items: [ + { + meta: { + 'concept-id': 'G100000-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.020.nc4' + } + }, + { + meta: { + 'concept-id': 'G100001-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.021.nc4' + } + }, + { + meta: { + 'concept-id': 'G100002-EDSC' + }, + umm: { + GranuleUR: 'GLDAS_CLSM025_D.2.0:GLDAS_CLSM025_D.A19480101.022.nc4' + } + } + + ] + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 1, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.json/) + .reply(200, { + feed: { + entry: [ + { + id: 'G100000-EDSC', + browse_flag: true + } + ] + } + }) + + nock(/example-cmr/) + .defaultReplyHeaders({ + 'CMR-Hits': 1, + 'CMR-Took': 7, + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + }) + .get(/granules\.json/) + .reply(200, { + feed: { + entry: [ + { + id: 'G100001-EDSC', + browse_flag: true + } + ] + } + }) + + await expect(granuleDatasource({ + params: { concept_id: 'G100000-EDSC' } + }, { + headers: { + 'Client-Id': 'eed-test-graphql', + 'CMR-Request-Id': 'abcd-1234-efgh-5678' + } + }, requestInfo)).rejects.toThrow('Inconsistent data prevented GraphQL from correctly parsing results') + }) + }) }) describe('with only umm keys', () => { diff --git a/src/datasources/granule.js b/src/datasources/granule.js index a302dcba1..7e5d7ab6e 100644 --- a/src/datasources/granule.js +++ b/src/datasources/granule.js @@ -17,6 +17,9 @@ export default async (params, context, parsedInfo) => { // Parse the response from CMR await granule.parse(requestInfo) + // Validates consistency between JSON and UMM data, if present + await granule.validateResponse(requestInfo) + // Return a formatted JSON response return granule.getFormattedResponse() }