Add more validation to Rite Aid API

loader/src/__mocks__/utils.js

@@ -0,0 +1,28 @@
+"use strict";
+const { jest } = require("@jest/globals");
+const originalModule = jest.requireActual("../utils");
+
+const warningLoggers = {};
+const warnings = [];
+
+const mock = {
+  ...originalModule,
+
+  createWarningLogger(prefix) {
+    warningLoggers[prefix] = jest.fn((message) => warnings.push(message));
+    return warningLoggers[prefix];
+  },
+
+  // Only for use in tests: get mock function for a logger.
+  __getWarningLogger(prefix) {
+    prefix = prefix || Object.keys(warningLoggers).pop();
+    return warningLoggers[prefix];
+  },
+
+  // Only for use in tests: get a list of all warnings that were logged.
+  __getWarnings() {
+    return warnings;
+  },
+};
+
+module.exports = mock;

loader/src/schema-validation.js

@@ -47,8 +47,32 @@ function assertSchema(schema, data, message) {
   }
 }
 
+/**
+ * Update an "object" schema to require every property listed in the
+ * `properties` object. Returns the schema so you can just wrap a schema
+ * definition with it.
+ * @param {any} schema The schema to require all properties on.
+ * @returns {any}
+ *
+ * @example
+ * var mySchema = requireAllProperties({
+ *   type: "object",
+ *   properties: {
+ *     a: { type: "number" },
+ *     b: { type: "string" }
+ *   }
+ * });
+ * // Throws an error:
+ * assertSchema(mySchema, { a: 5 });
+ */
+function requireAllProperties(schema) {
+  schema.required = Object.keys(schema.properties);
+  return schema;
+}
+
 module.exports = {
   SchemaError,
-  getValidator,
   assertSchema,
+  getValidator,
+  requireAllProperties,
 };

loader/src/sources/riteaid/api.js

@@ -1,11 +1,25 @@
 const { DateTime } = require("luxon");
+const Sentry = require("@sentry/node");
 const geocoding = require("../../geocoding");
 const { ParseError } = require("../../exceptions");
 const { Available, LocationType } = require("../../model");
-const { createWarningLogger, httpClient, RateLimit } = require("../../utils");
+const {
+  createWarningLogger,
+  httpClient,
+  parseUsPhoneNumber,
+  RateLimit,
+} = require("../../utils");
+const {
+  assertSchema,
+  requireAllProperties,
+} = require("../../schema-validation");
+const { RiteAidApiError } = require("./common");
 
 const warn = createWarningLogger("Rite Aid API");
 
+// Log a warning if a location has more than this many slots in a given day.
+const MAXIMUM_SLOT_COUNT = 500;
+
 // States in which Rite Aid has stores.
 const riteAidStates = new Set([
   "CA",
@@ -28,6 +42,78 @@ const riteAidStates = new Set([
   "WA",
 ]);
 
+const riteAidWrapperSchema = requireAllProperties({
+  type: "object",
+  properties: {
+    Status: { type: "string" },
+    ErrCde: {},
+    ErrMsg: {},
+    ErrMsgDtl: {},
+    Data: requireAllProperties({
+      type: "object",
+      properties: {
+        providerDetails: { type: "array" },
+      },
+      additionalProperties: false,
+    }),
+  },
+});
+
+const riteAidLocationSchema = requireAllProperties({
+  type: "object",
+  properties: {
+    id: { type: "integer", minimum: 1 },
+    // parseUpdateTime() does fancy checking, so no need to check format here.
+    last_updated: { type: "string" },
+    name: { type: "string", pattern: "Rite Aid" },
+    location: requireAllProperties({
+      type: "object",
+      properties: {
+        resourceType: { type: "null" },
+        id: { type: "null" },
+        identifier: { type: "null" },
+        name: { type: "null" },
+        telecom: { type: "null" },
+        address: { type: "null" },
+        position: { type: "null" },
+        meta: { type: "null" },
+        description: { type: "null" },
+        street: { type: "string" },
+        street_line_2: { type: "string", nullable: true },
+        city: { type: "string" },
+        state: { type: "string", pattern: "[A-Z]{2}" },
+        zipcode: { type: "string", pattern: "\\d{1,5}(-\\d{4})?" },
+        county: { type: "null" },
+        identifiers: { type: "null" },
+      },
+      additionalProperties: false,
+    }),
+    contact: requireAllProperties({
+      type: "object",
+      properties: {
+        booking_phone: { type: "string", nullable: true },
+        booking_url: { type: "string", format: "uri" },
+        info_phone: { type: "string", nullable: true },
+        info_url: { type: "string", format: "uri" },
+      },
+      additionalProperties: false,
+    }),
+    availability: {
+      type: "array",
+      items: requireAllProperties({
+        type: "object",
+        properties: {
+          date: { type: "string", format: "date" },
+          total_slots: { type: "integer", minimum: 0 },
+          available_slots: { type: "integer", minimum: 0 },
+        },
+        additionalProperties: false,
+      }),
+    },
+  },
+  additionalProperties: false,
+});
+
 async function queryState(state, rateLimit = null) {
   const RITE_AID_URL = process.env["RITE_AID_URL"];
   const RITE_AID_KEY = process.env["RITE_AID_KEY"];
@@ -40,22 +126,24 @@ async function queryState(state, rateLimit = null) {
 
   if (rateLimit) await rateLimit.ready();
 
-  const body = await httpClient({
+  const response = await httpClient({
     url: RITE_AID_URL,
     headers: { "Proxy-Authorization": "ldap " + RITE_AID_KEY },
     searchParams: { stateCode: state },
-  }).json();
-
-  if (body.Status !== "SUCCESS") {
-    console.error(body.Status);
-    console.error(body.ErrCde);
-    console.error(body.ErrMsg);
-    console.error(body.ErrMsgDtl);
+    responseType: "json",
+  });
 
-    throw new Error("RiteAid API request failed");
+  if (response.body.Status !== "SUCCESS") {
+    throw new RiteAidApiError(response);
   }
 
-  return body.Data.providerDetails.map(formatStore);
+  assertSchema(
+    riteAidWrapperSchema,
+    response.body,
+    "Response did not match schema"
+  );
+
+  return response.body.Data.providerDetails;
 }
 
 /**
@@ -90,6 +178,12 @@ function parseUpdateTime(text) {
 }
 
 function formatStore(provider) {
+  assertSchema(
+    riteAidLocationSchema,
+    provider,
+    "API location did not match schema"
+  );
+
   const address = formatAddress(provider.location);
 
   let county = provider.location.county;
@@ -121,9 +215,13 @@ function formatStore(provider) {
     state: provider.location.state,
     postal_code: provider.location.zipcode,
     county,
-    booking_phone: provider.contact.booking_phone,
+    booking_phone:
+      provider.contact.booking_phone &&
+      parseUsPhoneNumber(provider.contact.booking_phone),
     booking_url: provider.contact.booking_url,
-    info_phone: provider.contact.info_phone,
+    info_phone:
+      provider.contact.info_phone &&
+      parseUsPhoneNumber(provider.contact.info_phone),
     info_url: provider.contact.info_url,
 
     availability: {
@@ -144,12 +242,30 @@ function formatAvailable(provider) {
 }
 
 function formatCapacity(provider) {
-  return provider.availability.map((apiData) => ({
-    date: apiData.date,
-    available: apiData.available_slots > 0 ? Available.yes : Available.no,
-    available_count: apiData.available_slots,
-    unavailable_count: apiData.total_slots - apiData.available_slots,
-  }));
+  let maxDailySlots = 0;
+  const result = provider.availability.map((apiData) => {
+    maxDailySlots = Math.max(maxDailySlots, apiData.total_slots);
+    if (apiData.available_slots > apiData.total_slots) {
+      throw new Error("More available slots than total slots at a Rite Aid");
+    }
+
+    return {
+      date: apiData.date,
+      available: apiData.available_slots > 0 ? Available.yes : Available.no,
+      available_count: apiData.available_slots,
+      unavailable_count: apiData.total_slots - apiData.available_slots,
+    };
+  });
+
+  if (maxDailySlots > MAXIMUM_SLOT_COUNT) {
+    warn(
+      "Unrealistic slot count at a Rite Aid",
+      { slots: maxDailySlots },
+      true
+    );
+  }
+
+  return result;
 }
 
 function formatAddress(location) {
@@ -174,7 +290,7 @@ async function checkAvailability(handler, options) {
 
   if (!states.length) {
     const statesText = Array.from(riteAidStates).join(", ");
-    console.warn(`No states set for riteAidApi (supported: ${statesText})`);
+    warn(`No states set for riteAidApi (supported: ${statesText})`);
   }
 
   if (options.rateLimit != null && isNaN(options.rateLimit)) {
@@ -185,11 +301,26 @@ async function checkAvailability(handler, options) {
 
   let results = [];
   for (const state of states) {
-    let stores;
+    // Sentry's withScope() doesn't work for async code, so we have to manually
+    // track the context data we want to add. :(
+    const errorContext = { state, source: "Rite Aid API" };
+
+    const stores = [];
     try {
-      stores = await queryState(state, rateLimit);
+      const rawData = await queryState(state, rateLimit);
+      for (const rawLocation of rawData) {
+        Sentry.withScope((scope) => {
+          scope.setContext("context", errorContext);
+          scope.setContext("location", { id: rawLocation.id });
+          try {
+            stores.push(formatStore(rawLocation));
+          } catch (error) {
+            warn(error);
+          }
+        });
+      }
     } catch (error) {
-      warn(error, { state, source: "Rite Aid API" }, true);
+      warn(error, errorContext, true);
       continue;
     }
 

loader/src/sources/riteaid/common.js

@@ -0,0 +1,12 @@
+const assert = require("assert/strict");
+const { HttpApiError } = require("../../exceptions");
+
+class RiteAidApiError extends HttpApiError {
+  parse(response) {
+    assert.equal(typeof response.body, "object");
+    this.details = response.body;
+    this.message = `${this.details.Status} ${this.details.ErrCde}: ${this.details.ErrMsg}`;
+  }
+}
+
+module.exports = { RiteAidApiError };

loader/src/sources/riteaid/scraper.js

@@ -11,7 +11,6 @@
 const { mapKeys } = require("lodash");
 const { DateTime } = require("luxon");
 const Sentry = require("@sentry/node");
-const { HttpApiError } = require("../../exceptions");
 const { Available, LocationType, VaccineProduct } = require("../../model");
 const { assertSchema } = require("../../schema-validation");
 const {
@@ -20,6 +19,7 @@ const {
   TIME_ZONE_OFFSET_STRINGS,
   createWarningLogger,
 } = require("../../utils");
+const { RiteAidApiError } = require("./common");
 const { zipCodesCovering100Miles } = require("./zip-codes");
 
 // Load slot-level data in chunks of this many stores at a time.
@@ -50,13 +50,6 @@ const VACCINE_IDS_SHORT = mapKeys(VACCINE_IDS, (_, key) => key.split("-")[0]);
 
 const warn = createWarningLogger("Rite Aid Scraper");
 
-class RiteAidXhrError extends HttpApiError {
-  parse(response) {
-    super.parse(response);
-    this.message = `${this.details.Status} ${this.details.ErrCde}: ${this.details.ErrMsg}`;
-  }
-}
-
 async function queryZipCode(zip, radius = 100, stores = null) {
   const response = await httpClient({
     url: API_URL,
@@ -82,7 +75,7 @@ async function queryZipCode(zip, radius = 100, stores = null) {
   });
 
   if (response.body.Status !== "SUCCESS") {
-    throw new RiteAidXhrError(response);
+    throw new RiteAidApiError(response);
   }
 
   return response.body;
@@ -334,7 +327,7 @@ async function checkAvailability(handler, options) {
   }
 
   if (!states.length) {
-    console.warn(`No states set for riteAidApi`);
+    warn(`No states set for riteAidApi`);
   }
 
   if (options.rateLimit != null && isNaN(options.rateLimit)) {
@@ -366,7 +359,6 @@ module.exports = {
   API_URL,
   VACCINE_IDS,
   VACCINE_IDS_SHORT,
-  RiteAidXhrError,
   checkAvailability,
   queryState,
   queryZipCode,