Merge branch 'fix/email-parse-logs'

api/src/sendgrid/parseInboundEmail.js

@@ -1,3 +1,4 @@
+// @ts-check
 const { logger } = require('firebase-functions');
 const fs = require('fs/promises');
 const busboy = require('busboy');
@@ -8,47 +9,80 @@ const addrparser = require('address-rfc2822');
 const { MAX_MESSAGE_LENGTH, sendMessageFromEmail } = require('../chat');
 const { sendEmailReplyError } = require('../mail');
 const { auth, db } = require('../firebase');
+const { sendPlausibleEvent } = require('../util/plausible');
 
 /**
  *
  * See https://docs.sendgrid.com/for-developers/parsing-email/setting-up-the-inbound-parse-webhook
  * We use the non-raw webhook.
  * @param {import('express').Request} req
+ * @returns {Promise<{
+ *  envelopeFromEmail?: string,
+ *  headerFrom?: addrparser.Address,
+ *  responseText?: string,
+ *  chatId?: string,
+ *  dkimResult: {
+ *    host?: string,
+ *    result?: string
+ *  }
+ *  senderIP?: string
+ * }>}
  */
 const parseInboundEmailInner = async (req) => {
   let envelopeFromEmail;
+  /**
+   * @type {addrparser.Address | undefined}
+   */
   let headerFrom;
+  let dkimRaw;
   let dkimResult = {};
+  let senderIP;
 
   // Plain-text email text
   let emailPlainText;
 
   // Parse email text
   const bb = busboy({ headers: req.headers });
   bb.on('field', (name, val) => {
-    // Extract processed plain text
-    if (name === 'text') {
-      emailPlainText = val;
-    }
-    if (name === 'envelope') {
-      envelopeFromEmail = JSON.parse(val).from;
-    }
-    if (name === 'from') {
-      [headerFrom] = addrparser.parse(val) || [];
-    }
-    if (name === 'dkim') {
-      // This format isn't really documented, so we can't know how multiple signatures will appear...
-      // TODO: this will only support one signature, since the regex doesn't iterate
-      logger.log('Email DKIM field:', val);
-      const [, host, result] = /@([^\s]+?)\s:\s(pass|fail)/.exec(val);
-      dkimResult = {
-        host,
-        result
-      };
+    try {
+      // Extract processed plain text
+      if (name === 'text') {
+        emailPlainText = val;
+      }
+      if (name === 'envelope') {
+        envelopeFromEmail = JSON.parse(val).from;
+      }
+      if (name === 'from') {
+        [headerFrom] = addrparser.parse(val) || [];
+      }
+      if (name === 'dkim') {
+        // This format isn't really documented, so we can't know how multiple signatures will appear... Example observed values are (on each line):
+        // {@gmail.com : pass}
+        // none
+        //
+        dkimRaw = val;
+        if (val.trim() === 'none') {
+          dkimResult = {};
+          return;
+        }
+        //
+        // TODO: this will only support one DKIM signature, since the regex doesn't iterate. There might be multiple.
+        const [, host, result] = /@([^\s]+?)\s:\s(pass|fail)/.exec(val) || [];
+        dkimResult = {
+          host,
+          result
+        };
+      }
+      if (name === 'sender_ip') {
+        senderIP = val;
+      }
+    } catch (e) {
+      logger.warn(`Error parsing field ${name} with value "${val}"`, e);
     }
   });
 
   bb.on('file', (name, info) => {
+    // @ts-ignore
     const { filename, encoding, mimeType } = info;
     logger.warn(
       `Ignoring attached file: [${name}]: filename: %j, encoding: %j, mimeType: %j`,
@@ -59,20 +93,41 @@ const parseInboundEmailInner = async (req) => {
   });
 
   // Firebase already reads the stream and saves it into a buffer, which we pass here
+  // @ts-ignore this is provided by Firebase functions
   bb.end(req.rawBody);
 
-  const parsedEmail = new EmailReplyParser().read(emailPlainText);
-  // Trim is not done automatically
-  const responseText = parsedEmail.getVisibleText().trim();
-  const quotedText = parsedEmail.getQuotedText();
+  let responseText;
+  let chatId;
+  try {
+    if (emailPlainText) {
+      const parsedEmail = new EmailReplyParser().read(emailPlainText);
+      // Trim is not done automatically
+      responseText = parsedEmail.getVisibleText().trim();
+      const quotedText = parsedEmail.getQuotedText();
+      // Find the chat ID from the quoted text
+      const chatRegex = /\/chat\/.+?\/([a-zA-Z0-9]+)>/.exec(quotedText);
+      chatId = chatRegex ? chatRegex[1] : undefined;
+    }
+  } catch (e) {
+    logger.warn('Error extracting response text from plain email text', e);
+  }
 
-  // Find the chat ID from the quoted text
-  const chatRegex = /\/chat\/.+?\/([a-zA-Z0-9]+)>/.exec(quotedText);
-  const chatId = chatRegex ? chatRegex[1] : undefined;
+  logger.log(`== Parsed email details ==
+Envelope from: ${envelopeFromEmail}
+Header from: ${headerFrom?.address}
+DKIM: ${JSON.stringify(dkimResult)} (raw: ${dkimRaw})
+Response text: ${responseText}
+Chat ID: ${chatId}
+Sender IP: ${senderIP}`);
 
-  return { envelopeFromEmail, headerFrom, responseText, chatId, dkimResult };
+  return { envelopeFromEmail, headerFrom, responseText, chatId, dkimResult, senderIP };
 };
 
+/**
+ * @template T
+ * @typedef {T extends Promise<infer U> ? U : T} Unpacked
+ */
+
 /**
  *
  * See https://docs.sendgrid.com/for-developers/parsing-email/setting-up-the-inbound-parse-webhook
@@ -83,25 +138,23 @@ const parseInboundEmailInner = async (req) => {
 exports.parseInboundEmail = async (req, res) => {
   if (req.method !== 'POST') {
     logger.log('Not a POST');
-    res.status('405').send('Method not allowed');
+    res.status(405).send('Method not allowed');
     return;
   }
 
-  let parsedEmail = {};
+  /**
+   * @type {Unpacked<ReturnType<typeof parseInboundEmailInner>>}
+   */
+  let parsedEmail = { dkimResult: {} };
   try {
     try {
       parsedEmail = await parseInboundEmailInner(req);
     } catch (parseError) {
+      logger.error(parseError);
       throw new Error('Email error: unknown parsing error');
     }
 
-    const { envelopeFromEmail, headerFrom, responseText, chatId, dkimResult } = parsedEmail;
-    logger.log(`== Parsed email details ==
-Envelope from: ${envelopeFromEmail}
-Header from: ${headerFrom.address}
-DKIM: ${JSON.stringify(dkimResult)}
-Response text: ${responseText}
-Chat ID: ${chatId}`);
+    const { headerFrom, responseText, chatId, dkimResult, senderIP } = parsedEmail;
 
     // Verify details
     //
@@ -110,7 +163,12 @@ Chat ID: ${chatId}`);
       throw new Error("Email error: couldn't parse the chat ID");
     }
 
+    if (!headerFrom) {
+      throw new Error("Email error: couldn't parse a valid header rfc2822.from email address");
+    }
+
     const headerFromEmail = headerFrom.address;
+
     //
     // Check DKIM: host must be verified, header host must match verified host
     const headerFromHost = headerFrom.host();
@@ -119,6 +177,14 @@ Chat ID: ${chatId}`);
     }
 
     // Check & truncate max message content. Does not lead to an error.
+    if (!responseText) {
+      throw new Error("Email error: couldn't parse plain email text");
+    }
+
+    if (responseText.trim() === '') {
+      throw new Error('Email error: reply text is empty');
+    }
+
     let message = responseText;
     if (responseText.length > MAX_MESSAGE_LENGTH) {
       message = responseText.substring(0, MAX_MESSAGE_LENGTH);
@@ -131,7 +197,10 @@ Chat ID: ${chatId}`);
       message,
       fromEmail: headerFromEmail
     });
-    logger.log('Reply email processed succcesfully');
+    logger.log('Reply email processed succesfully');
+    // Note: sender IP is the email MTA sender IP, probably not the email client
+    // But it might still be helpful to distinguish "visitors" somewhat in Plausible
+    await sendPlausibleEvent('Send Email Reply', { senderIP });
   } catch (e) {
     // Log error
     if (e instanceof Error) {
@@ -151,17 +220,19 @@ Chat ID: ${chatId}`);
         const user = await auth.getUserByEmail(fromEmail);
         const privateUserProfileDocRef = db.doc(`users-private/${user.uid}`);
         const privateUserProfileData = (await privateUserProfileDocRef.get()).data();
-        language = privateUserProfileData.communicationLanguage;
+        language = privateUserProfileData?.communicationLanguage;
       } catch (langFindError) {
         logger.warn(
           "Couldn't get the communicationLanguage for the user that (supposedly) replied",
           langFindError
         );
       }
+      logger.log(`Sending error email to ${fromEmail} in language ${language}`);
       await sendEmailReplyError(fromEmail, language);
     } else {
-      logger.warn("Couldn't parse an email to send an error message to");
+      logger.warn("Couldn't parse an email to send an error message to, not sending error email");
     }
+    await sendPlausibleEvent('Email Reply Error', { senderIP: parsedEmail?.senderIP });
   }
 
   // Always reply OK to SendGrid

api/src/util/plausible.js

@@ -0,0 +1,39 @@
+// @ts-check
+const { logger } = require('firebase-functions');
+const { default: fetch } = require('node-fetch');
+
+/**
+ * https://plausible.io/docs/events-api
+ * @param {string} name
+ * @param {{url?: string, domain?: string, senderIP?: string}} opts
+ */
+exports.sendPlausibleEvent = async (name, opts = {}) => {
+  const defaults = {
+    url: 'backend://firebase-functions/parseInboundEmail?utm_source=firebase-functions&utm_medium=email',
+    domain: 'welcometomygarden.org'
+  };
+
+  const { senderIP, ...rest } = opts;
+  let senderHeaders = {};
+  if (senderIP) {
+    senderHeaders = { 'X-Forwarded-For': senderIP };
+  }
+
+  try {
+    await fetch('https://visitors.slowby.travel/api/event', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'User-Agent': 'firebase-functions',
+        ...senderHeaders
+      },
+      body: JSON.stringify({
+        name,
+        ...defaults,
+        ...rest
+      })
+    });
+  } catch (e) {
+    logger.warn(`Sending Plausible event ${name} failed`);
+  }
+};