From ee2e7f06c3c16ec5b1354fe4bd35fdc6a86dfabf Mon Sep 17 00:00:00 2001
From: darnjo <josh@reso.org>
Date: Fri, 5 Apr 2024 09:40:50 -0700
Subject: [PATCH] #89: Addressed termination due to false negative on schema
 validation error

---
 lib/replication/index.js |   4 +-
 lib/replication/utils.js | 113 +++++++++++++++++++--------------------
 2 files changed, 57 insertions(+), 60 deletions(-)

diff --git a/lib/replication/index.js b/lib/replication/index.js
index b75a5fa..1a024aa 100644
--- a/lib/replication/index.js
+++ b/lib/replication/index.js
@@ -137,8 +137,6 @@ const replicate = async ({
       startTimeIsoTimestamp = startTime.toISOString(),
       shouldSaveResults = !!outputPath;
 
-    // TODO - add support for multiple strategies
-
     // Each resource and expansion will have its separate set of requests
     for await (const request of requests) {
       const { requestUri: initialRequestUri, resourceName } = request;
@@ -244,7 +242,7 @@ const replicate = async ({
 
     if (shouldGenerateReports) {
       try {
-        if (jsonSchemaValidation && Object.values(schemaValidationResults)?.length) {
+        if (jsonSchemaValidation && schemaValidationResults?.errorMap?.totalErrors > 0) {
           await writeSchemaValidationErrorReport(schemaValidationResults);
         } else {
           await writeAnalyticsReports({
diff --git a/lib/replication/utils.js b/lib/replication/utils.js
index b5b7fe2..73aace0 100644
--- a/lib/replication/utils.js
+++ b/lib/replication/utils.js
@@ -481,7 +481,7 @@ const processSpecialFields = ({ availabilityMap, fieldName, dateField, value } =
 const isObjectOrArrayOrPrimitive = value => {
   const isObject = typeof value === 'object',
     isArray = Array.isArray(value),
-    isPrimitive = value === null || (!isObject && !isArray);
+    isPrimitive = value === null || !(isObject || isArray);
 
   return {
     isObject,
@@ -698,33 +698,33 @@ const calculateResponseStats = ({ resourceAvailabilityMap = {}, responses = [] }
 
     const expansionStats = expansions?.length
       ? expansions.map(({ resourceName: expandedResourceName, ...remainingExpandedData }) => {
-        const expandedResponseBytes = tallies?.[resourceName]?.expansions?.[expandedResourceName]?.responseBytes ?? [],
-          expandedResponseTimes = tallies?.[resourceName]?.expansions?.[expandedResourceName]?.responseTimes ?? [],
-          expandedRecordCounts = tallies?.[resourceName]?.expansions?.[expandedResourceName]?.expandedRecordCounts ?? [];
-
-        return {
-          resourceName: expandedResourceName,
-          ...remainingExpandedData,
-
-          // payload size
-          averageResponseBytes: calculateMean(expandedResponseBytes),
-          medianResponseBytes: calculateMedian(expandedResponseBytes),
-          stdDevResponseBytes: calculateStdDev(expandedResponseBytes),
-
-          // response times
-          averageResponseTimeMs: calculateMean(expandedResponseTimes),
-          averageResponseTimeMillis: calculateMean(resourceResponseTimes) /* support old value for now */,
-          medianResponseTimeMs: calculateMedian(expandedResponseTimes),
-          stdDevResponseTimeMs: calculateStdDev(expandedResponseTimes),
-
-          // record counts
-          numRecordsFetched: calculateSum(resourceRecordCounts),
-          numExpandedRecordsFetched: calculateSum(expandedRecordCounts),
-
-          // pages fetched - expanded is the same as the parent resource
-          numSamples: resourcePagesFetched
-        };
-      })
+          const expandedResponseBytes = tallies?.[resourceName]?.expansions?.[expandedResourceName]?.responseBytes ?? [],
+            expandedResponseTimes = tallies?.[resourceName]?.expansions?.[expandedResourceName]?.responseTimes ?? [],
+            expandedRecordCounts = tallies?.[resourceName]?.expansions?.[expandedResourceName]?.expandedRecordCounts ?? [];
+
+          return {
+            resourceName: expandedResourceName,
+            ...remainingExpandedData,
+
+            // payload size
+            averageResponseBytes: calculateMean(expandedResponseBytes),
+            medianResponseBytes: calculateMedian(expandedResponseBytes),
+            stdDevResponseBytes: calculateStdDev(expandedResponseBytes),
+
+            // response times
+            averageResponseTimeMs: calculateMean(expandedResponseTimes),
+            averageResponseTimeMillis: calculateMean(resourceResponseTimes) /* support old value for now */,
+            medianResponseTimeMs: calculateMedian(expandedResponseTimes),
+            stdDevResponseTimeMs: calculateStdDev(expandedResponseTimes),
+
+            // record counts
+            numRecordsFetched: calculateSum(resourceRecordCounts),
+            numExpandedRecordsFetched: calculateSum(expandedRecordCounts),
+
+            // pages fetched - expanded is the same as the parent resource
+            numSamples: resourcePagesFetched
+          };
+        })
       : undefined;
 
     return {
@@ -775,7 +775,10 @@ const createRuntimeAvailabilityStats = (resourceAvailabilityMap = {}) =>
  * @param {Object} errorMap a map containing JSON Schema validation errors
  */
 const writeSchemaValidationErrorReport = async (errorMap = {}) => {
-  if (errorMap && Object.values(errorMap)?.length) {
+  const {
+    stats: { totalErrors = 0 }
+  } = errorMap ?? {};
+  if (totalErrors > 0) {
     await writeFile(SCHEMA_VALIDATION_REPORT_FILENAME, JSON.stringify(combineErrors(errorMap)));
     console.log(`Schema validation errors written to: ${SCHEMA_VALIDATION_REPORT_FILENAME}`);
   }
@@ -796,25 +799,21 @@ const writeAnalyticsReports = async ({ version, serviceRootUri, replicationState
     // write responses report
     await writeFile(
       AVAILABILITY_RESPONSES_FILENAME,
-      JSON.stringify(
-        {
-          description: 'RESO Data Availability Responses',
-          version,
-          generatedOn,
-          serviceRootUri,
-          responses: REPLICATION_STATE_SERVICE.getResponses().map(({ requestUri, ...otherResponseInfo }) => {
-            return {
-              requestUri: requestUri.replace(serviceRootUri, ''),
-              ...otherResponseInfo
-            };
-          })
-        },
-        null,
-        '  '
-      )
+      JSON.stringify({
+        description: 'RESO Data Availability Responses',
+        version,
+        generatedOn,
+        serviceRootUri,
+        responses: REPLICATION_STATE_SERVICE.getResponses().map(({ requestUri, ...otherResponseInfo }) => {
+          return {
+            requestUri: requestUri.replace(serviceRootUri, ''),
+            ...otherResponseInfo
+          };
+        })
+      })
     );
 
-    console.log(`Response info written to ${AVAILABILITY_REPORT_FILENAME}`);
+    console.log(`Response info written to ${AVAILABILITY_RESPONSES_FILENAME}`);
 
     // write DA report
     await writeFile(
@@ -1086,11 +1085,11 @@ const createRequestFromParameters = ({ serviceRootUri, resourceName, expansions
   const expansionInfo =
     expansions && expansions?.length
       ? expansions.map(fieldName => {
-        return {
-          fieldName
-          /* TODO: look up type info from reference metadata, when possible */
-        };
-      })
+          return {
+            fieldName
+            /* TODO: look up type info from reference metadata, when possible */
+          };
+        })
       : undefined;
 
   return {
@@ -1212,9 +1211,9 @@ const buildRequestUrlString = ({
       preparedFilter =
         $filter && $filter?.length
           ? $filter
-            .split('and')
-            .flatMap(part => (part?.includes(timestampField) ? [] : part.trim()))
-            .join(' and ')
+              .split('and')
+              .flatMap(part => (part?.includes(timestampField) ? [] : part.trim()))
+              .join(' and ')
           : null;
 
     return new URL(
@@ -1228,9 +1227,9 @@ const buildRequestUrlString = ({
       preparedFilter =
         $filter && $filter?.length
           ? $filter
-            .split('and')
-            .flatMap(part => (part?.includes(timestampField) ? [] : part.trim()))
-            .join(' and ')
+              .split('and')
+              .flatMap(part => (part?.includes(timestampField) ? [] : part.trim()))
+              .join(' and ')
           : null;
 
     return new URL(