Merge pull request #15 from Agoric/rs-improve-height-api

perf: enhance response time for height API

controllers/heightController.js

@@ -1,7 +1,7 @@
 import path from 'path';
 import { fileURLToPath } from 'url';
 import { processAndConvert } from '../services/fileProcessor.js';
-import { fetchGCPLogsForHeight } from '../services/fetchGCPLogsForHeight.js';
+import { fetchAndStoreHeightLogs } from '../services/fetchAndStoreHeightLogs.js';
 import { networks } from '../helpers/constants.js';
 
 const __filename = fileURLToPath(import.meta.url);
@@ -41,10 +41,8 @@ export const handleHeightLogs = async (req, res) => {
     resource.type="k8s_container"
   `;
 
-  console.log('Fetching data from GCP...');
-  await fetchGCPLogsForHeight({
-    startBlockHeight: height,
-    endBlockHeight: height,
+  await fetchAndStoreHeightLogs({
+    blockHeight: height,
     inputFile,
     network,
     queryfilter,

helpers/getAccessToken.js

@@ -0,0 +1,70 @@
+import crypto from 'crypto';
+import querystring from 'querystring';
+import { getCredentials } from './getGCPCredentials.js';
+
+const credentials = getCredentials();
+
+const createJWT = (scopes) => {
+  const now = Math.floor(Date.now() / 1000);
+
+  const payload = {
+    iss: credentials.client_email,
+    scope: scopes.join(' '),
+    aud: 'https://oauth2.googleapis.com/token',
+    exp: now + 3600,
+    iat: now,
+  };
+
+  const header = JSON.stringify({
+    alg: 'RS256',
+    typ: 'JWT',
+  });
+
+  const base64Header = Buffer.from(header).toString('base64url');
+  const base64Payload = Buffer.from(JSON.stringify(payload)).toString(
+    'base64url'
+  );
+
+  const unsignedJwt = `${base64Header}.${base64Payload}`;
+  const signer = crypto.createSign('RSA-SHA256');
+
+  signer.update(unsignedJwt);
+  const signature = signer.sign(credentials.private_key, 'base64');
+
+  const base64UrlSignature = signature
+    .replace(/\+/g, '-')
+    .replace(/\//g, '_')
+    .replace(/=+$/, '');
+
+  return `${unsignedJwt}.${base64UrlSignature}`;
+};
+
+export const getAccessToken = async (scopes) => {
+  try {
+    const signedJwt = createJWT(scopes);
+
+    const data = querystring.stringify({
+      grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
+      assertion: signedJwt,
+    });
+
+    const response = await fetch('https://oauth2.googleapis.com/token', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+      },
+      body: data,
+    });
+
+    const responseBody = await response.json();
+
+    if (response.ok && responseBody.access_token) {
+      return responseBody.access_token;
+    } else {
+      throw new Error('Failed to get access token');
+    }
+  } catch (error) {
+    console.error('Error getting access token:', error);
+    throw error;
+  }
+};

helpers/getGCPCredentials.js

@@ -0,0 +1,29 @@
+// @ts-check
+import { fs } from 'zx';
+
+let credentials = null;
+
+export const getCredentials = () => {
+  if (credentials) {
+    return credentials;
+  }
+
+  const credentialsPath = process.env.GOOGLE_APPLICATION_CREDENTIALS;
+
+  if (!credentialsPath) {
+    throw new Error(
+      'GOOGLE_APPLICATION_CREDENTIALS environment variable is not set.'
+    );
+  }
+
+  try {
+    const credentialsData = fs.readFileSync(credentialsPath, 'utf8');
+    credentials = JSON.parse(credentialsData);
+  } catch (error) {
+    throw new Error(
+      'Failed to read or parse credentials file: ' + error.message
+    );
+  }
+
+  return credentials;
+};

helpers/logGCPError.js

@@ -0,0 +1,24 @@
+export const logError = (error, verbose = false) => {
+  if (error.response) {
+    console.error(`🚨 Error: ${error.message}`);
+    console.error(`  - Status: ${error.response.status}`);
+    console.error(`  - Stack trace: ${error.stack}`);
+
+    if (verbose) {
+      console.error(
+        `  - Data: ${JSON.stringify(error.response.data, null, 2)}`
+      );
+      console.error(
+        `  - Headers: ${JSON.stringify(error.response.headers, null, 2)}`
+      );
+    }
+  } else if (error.request) {
+    console.error(`📡 No response received for the request:`);
+    console.error(
+      `  - Request details: ${JSON.stringify(error.request, null, 2)}`
+    );
+  } else {
+    console.error(`💥 General Error: ${error.message}`);
+    console.error(`  - Stack trace: ${error.stack}`);
+  }
+};

helpers/utils.js

@@ -1,6 +1,5 @@
 // @ts-check
 import { fs } from 'zx';
-import { createSign } from 'crypto';
 
 export const checkFileExists = async ({ filePath, description = 'File' }) => {
   try {
@@ -34,83 +33,3 @@ export const formatDateString = (dateString) => {
   const date = new Date(dateString);
   return date.toISOString();
 };
-
-export const base64urlEncode = (str) =>
-  Buffer.from(str)
-    .toString('base64')
-    .replace(/=/g, '')
-    .replace(/\+/g, '-')
-    .replace(/\//g, '_');
-
-export const createSignedJWT = (payload, privateKey) => {
-  const signingAlgorithm = 'RSA-SHA256';
-  const jwtTokenRequestHeader = { alg: 'RS256', typ: 'JWT' };
-
-  const encodedHeader = base64urlEncode(JSON.stringify(jwtTokenRequestHeader));
-  const encodedPayload = base64urlEncode(JSON.stringify(payload));
-  const toSign = `${encodedHeader}.${encodedPayload}`;
-  const sign = createSign(signingAlgorithm);
-  sign.update(toSign);
-  const signature = sign
-    .sign(privateKey, 'base64')
-    .replace(/=/g, '')
-    .replace(/\+/g, '-')
-    .replace(/\//g, '_');
-  return `${toSign}.${signature}`;
-};
-
-/**
- * @param {{
- *  auth_provider_x509_cert_url: string;
- *  auth_uri: string;
- *  client_email: string;
- *  client_id: string;
- *  client_x509_cert_url: string;
- *  project_id: string;
- *  private_key: string;
- *  private_key_id: string;
- *  token_uri: string;
- *  type: string;
- *  universe_domain: string;
- * }} serviceAccount
- */
-export const getAccessToken = async (serviceAccount) => {
-  const now = Math.floor(Date.now() / 1000);
-  const AUDIENCE = 'https://oauth2.googleapis.com/token';
-  const SUCCESS_ICON = '✅';
-  const TOKEN_SCOPE = 'https://www.googleapis.com/auth/logging.read';
-  const IN_PROGRESS_ICON = '⏳';
-
-  const payload = {
-    aud: AUDIENCE,
-    exp: now + 3600,
-    iat: now,
-    iss: serviceAccount.client_email,
-    scope: TOKEN_SCOPE,
-  };
-
-  const jwt = createSignedJWT(payload, serviceAccount.private_key);
-
-  const params = new URLSearchParams();
-  params.append('assertion', jwt);
-  params.append('grant_type', 'urn:ietf:params:oauth:grant-type:jwt-bearer');
-
-  console.log(`Fetching JWT access token ${IN_PROGRESS_ICON}`);
-
-  const response = await fetch(AUDIENCE, {
-    body: params.toString(),
-    headers: {
-      'Content-Type': 'application/x-www-form-urlencoded',
-    },
-    method: 'POST',
-  });
-
-  if (!response.ok) {
-    const errorMessage = `Failed to obtain access token due to error: ${await response.text()}`;
-    throw Error(errorMessage);
-  }
-  console.log(`Fetched JWT access token ${SUCCESS_ICON}`);
-
-  const data = await response.json();
-  return String(data.access_token);
-};

index.js

@@ -5,7 +5,6 @@ import { fileURLToPath } from 'url';
 import { fs } from 'zx';
 import router from './router.js';
 import { requestLogger } from './middleware/requestLogger.js';
-import { setupCredentials } from './helpers/credentials.js';
 
 const PORT = 3000;
 const UPLOAD_DIR = 'uploads';
@@ -14,15 +13,6 @@ const __dirname = path.dirname(__filename);
 
 const app = express();
 
-setupCredentials()
-  .then(() => {
-    console.log('Credentials setup successfully');
-  })
-  .catch((error) => {
-    console.error('Failed to set up credentials:', error);
-    process.exit(1);
-  });
-
 if (!fs.existsSync(UPLOAD_DIR)) {
   fs.mkdirSync(UPLOAD_DIR);
 }