Server: fix pagination bugs with G4RD

server/src/resolvers/getVariantsResolver/adapters/g4rdAdapter.ts

@@ -1,4 +1,4 @@
-import axios, { AxiosError, AxiosResponse } from 'axios';
+import axios, { AxiosError } from 'axios';
 import jwtDecode from 'jwt-decode';
 import { URLSearchParams } from 'url';
 import { v4 as uuidv4 } from 'uuid';
@@ -23,6 +23,7 @@ import { getFromCache, putInCache } from '../../../utils/cache';
 import { timeit, timeitAsync } from '../../../utils/timeit';
 import resolveAssembly from '../utils/resolveAssembly';
 import fetchPhenotipsVariants from '../utils/fetchPhenotipsVariants';
+import fetchPhenotipsPatients from '../utils/fetchPhenotipsPatients';
 
 /* eslint-disable camelcase */
 
@@ -40,7 +41,7 @@ const _getG4rdNodeQuery = async ({
 }: QueryInput): Promise<VariantQueryResponse> => {
   let G4RDNodeQueryError: G4RDNodeQueryError | null = null;
   let G4RDVariants: null | PTVariantArray = null;
-  let G4RDPatientQueryResponse: null | AxiosResponse<G4RDPatientQueryResult> = null;
+  let G4RDPatientQueryResponse: null | G4RDPatientQueryResult[] = null;
   const FamilyIds: null | Record<string, string> = {}; // <PatientId, FamilyId>
   let Authorization = '';
   try {
@@ -65,30 +66,22 @@ const _getG4rdNodeQuery = async ({
       process.env.G4RD_URL as string,
       geneInput,
       variant,
-      Authorization
+      getAuthHeader
     );
 
     // Get patients info
     if (G4RDVariants) {
+      logger.debug(`G4RDVariants length: ${G4RDVariants.length}`);
       let individualIds = G4RDVariants.flatMap(v => v.individualIds).filter(Boolean); // Filter out undefined and null values.
 
       // Get all unique individual Ids.
       individualIds = [...new Set(individualIds)];
 
       if (individualIds.length > 0) {
-        const patientUrl = `${process.env.G4RD_URL}/rest/patients/fetch?${individualIds
-          .map(id => `id=${id}`)
-          .join('&')}`;
-
-        G4RDPatientQueryResponse = await axios.get<G4RDPatientQueryResult>(
-          new URL(patientUrl).toString(),
-          {
-            headers: {
-              Authorization,
-              'Content-Type': 'application/json',
-              Accept: 'application/json',
-            },
-          }
+        G4RDPatientQueryResponse = await fetchPhenotipsPatients(
+          process.env.G4RD_URL!,
+          individualIds,
+          getAuthHeader
         );
 
         // Get Family Id for each patient.
@@ -100,12 +93,17 @@ const _getG4rdNodeQuery = async ({
           },
         });
 
+        logger.debug('Begin fetching family IDs');
+
         const familyResponses = await Promise.allSettled(
-          individualIds.map(id =>
-            patientFamily.get<G4RDFamilyQueryResult>(
+          individualIds.map((id, i) => {
+            if (i % 50 === 0 || i === individualIds.length - 1) {
+              logger.debug(`Fetching family ${i + 1} of ${individualIds.length}`);
+            }
+            return patientFamily.get<G4RDFamilyQueryResult>(
               new URL(`${process.env.G4RD_URL}/rest/patients/${id}/family`).toString()
-            )
-          )
+            );
+          })
         );
 
         familyResponses.forEach((response, index) => {
@@ -123,7 +121,7 @@ const _getG4rdNodeQuery = async ({
   return {
     data: transformG4RDQueryResponse(
       (G4RDVariants as PTVariantArray) || [],
-      (G4RDPatientQueryResponse?.data as G4RDPatientQueryResult) || [],
+      G4RDPatientQueryResponse,
       FamilyIds
     ),
     error: transformG4RDNodeErrorResponse(G4RDNodeQueryError),

server/src/resolvers/getVariantsResolver/utils/fetchPhenotipsPatients.ts

@@ -0,0 +1,67 @@
+/**
+ * Handles fetching from /rest/patients/match endpoint in Phenotips.
+ */
+import axios from 'axios';
+import { G4RDPatientQueryResult } from '../../../types';
+import logger from '../../../logger';
+
+// How many IDs to query at a time?
+const COUNT = 50;
+
+/**
+ * Fetch patients from /rest/patients/match and return the complete result.
+ *
+ * @param baseUrl The URL where the Phenotips instance is hosted. Used as `{baseUrl}/rest/patients/match`
+ * @param individualIds Individual IDs to query for.
+ * @param authorization Authorization header string to use with query.
+ * @throws
+ */
+const fetchPhenotipsPatients = async (
+  baseUrl: string,
+  individualIds: string[],
+  getAuthorization: () => Promise<string>
+): Promise<G4RDPatientQueryResult[]> => {
+  let currStart = 0;
+  let currEnd = COUNT;
+
+  let finalPatientQueryResponse: G4RDPatientQueryResult[] = [];
+
+  logger.debug(
+    `Begin fetching patients from ${baseUrl}/rest/patients/fetch. Total expected patients: ${individualIds.length}`
+  );
+
+  try {
+    while (currStart < individualIds.length) {
+      const currEndCapped = Math.min(currEnd, individualIds.length);
+      const queryIds = individualIds.slice(currStart, currEndCapped);
+      const patientUrl = `${baseUrl}/rest/patients/fetch?${queryIds
+        .map(id => `id=${id}`)
+        .join('&')}`;
+      const patientQueryResponse = await axios.get<G4RDPatientQueryResult[]>(
+        new URL(patientUrl).toString(),
+        {
+          headers: {
+            Authorization: await getAuthorization(),
+            'Content-Type': 'application/json',
+            Accept: 'application/json',
+          },
+        }
+      );
+      finalPatientQueryResponse = finalPatientQueryResponse.concat(
+        patientQueryResponse?.data || []
+      );
+      logger.debug(
+        `Successful query for patients, received ${finalPatientQueryResponse.length} of ${individualIds.length}.`
+      );
+      currStart += COUNT;
+      currEnd += COUNT;
+    }
+  } catch (error: any) {
+    logger.error(JSON.stringify(error));
+    throw error; // Adapters will need to handle this error
+  }
+
+  return finalPatientQueryResponse;
+};
+
+export default fetchPhenotipsPatients;

server/src/resolvers/getVariantsResolver/utils/fetchPhenotipsVariants.ts

@@ -7,6 +7,7 @@ import logger from '../../../logger';
 import resolveChromosome from './resolveChromosome';
 import { QueryResponseError } from './queryResponseError';
 
+// How many variants to query for at a time?
 const COUNT = 25;
 
 /**
@@ -23,16 +24,25 @@ const fetchPhenotipsVariants = async (
   baseUrl: string,
   gene: GeneQueryInput,
   variant: VariantQueryInput,
-  authorization: string
+  getAuthorization: () => Promise<string>
 ): Promise<PTPaginatedVariantQueryResult['results']> => {
   let currentPage = 1;
   let collectedResults: PTPaginatedVariantQueryResult['results'] = [];
   let maxResults = Infinity;
   const count = COUNT;
-  const position = resolveChromosome(gene.position);
+  const _position = resolveChromosome(gene.position);
+  const chromosome =
+    ['X', 'Y'].indexOf(_position.chromosome) !== -1
+      ? _position.chromosome
+      : Number(_position.chromosome);
+  const position = {
+    chrom: chromosome,
+    start: Number(_position.start),
+    end: Number(_position.end),
+  };
 
   logger.debug(
-    `Begin fetching paginated variants from ${baseUrl}. gene: ${JSON.stringify(
+    `Begin fetching paginated variants from ${baseUrl}/rest/variants/match. gene: ${JSON.stringify(
       gene
     )}, variant: ${JSON.stringify(variant)}`
   );
@@ -45,16 +55,12 @@ const fetchPhenotipsVariants = async (
           limit: count,
           variant: {
             ...variant,
-            position: {
-              chrom: Number(position.chromosome),
-              start: Number(position.start),
-              end: Number(position.end),
-            },
+            position,
           },
         },
         {
           headers: {
-            Authorization: authorization, // TODO: In future, use function instead to get auth?
+            Authorization: await getAuthorization(),
             'Content-Type': 'application/json',
             Accept: 'application/json',
           },