feat(gatsby-source-drupal): Disable caching + add http/2 agent (#32012)

* feat(gatsby-source-drupal): Force revalidation on Drupal API requests Talking to Gatsby/Drupal users — most set long max-age for the Drupal cache to keep their edge http cache as fresh as possible as Drupal can directly purge its edge http cache. But this has the unfortunate side-effect with the recent http client change in #31514 that API calls aren't revalidated. Meaning that a user could change some content in Drupal and not see those updates in their Gatsby site until the Drupal cache expires in the Gatsby cache. This PR removes the `cache-control` header from Drupal API responses so that we only can use `etag` for caching which forces revalidation on every request. * Remove debugging line * Add http/2 agent * Disable cache — it's slower than refetch with revalidation * push don't concat * consistent return * Fix statusCode check for got * Actually this was correct... * Actually use http/2
gatsbyjs · Jun 22, 2021 · 113e43e · 113e43e
1 parent d75b965
commit 113e43e
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 18 deletions.
diff --git a/packages/gatsby-source-drupal/package.json b/packages/gatsby-source-drupal/package.json
@@ -14,6 +14,7 @@
     "fastq": "^1.11.0",
     "gatsby-source-filesystem": "^3.9.0-next.0",
     "got": "^11.8.2",
+    "http2-wrapper": "^2.0.5",
     "lodash": "^4.17.21",
     "tiny-async-pool": "^1.2.0",
     "url-join": "^4.0.1"
@@ -24,6 +25,9 @@
     "babel-preset-gatsby-package": "^1.9.0-next.0",
     "cross-env": "^7.0.3"
   },
+  "engines": {
+    "node": ">=12.13.0"
+  },
   "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-drupal#readme",
   "keywords": [
     "gatsby",
@@ -43,8 +47,5 @@
     "build": "babel src --out-dir . --ignore \"**/__tests__\"",
     "prepare": "cross-env NODE_ENV=production npm run build",
     "watch": "babel -w src --out-dir . --ignore \"**/__tests__\""
-  },
-  "engines": {
-    "node": ">=12.13.0"
   }
 }
diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js
@@ -2,6 +2,7 @@ const got = require(`got`)
 const _ = require(`lodash`)
 const urlJoin = require(`url-join`)
 import HttpAgent from "agentkeepalive"
+const http2wrapper = require(`http2-wrapper`)
 
 const { HttpsAgent } = HttpAgent
 
@@ -18,10 +19,17 @@ const { handleReferences, handleWebhookUpdate } = require(`./utils`)
 const agent = {
   http: new HttpAgent(),
   https: new HttpsAgent(),
+  http2: new http2wrapper.Agent(),
 }
 
 async function worker([url, options]) {
-  return got(url, { agent, ...options })
+  return got(url, {
+    agent,
+    cache: false,
+    request: http2wrapper.auto,
+    http2: true,
+    ...options,
+  })
 }
 
 const requestQueue = require(`fastq`).promise(worker, 20)
@@ -170,10 +178,13 @@ exports.sourceNodes = async (
           headers,
           searchParams: params,
           responseType: `json`,
-          cache,
         },
       ])
 
+      // Fastbuilds returns a -1 if:
+      // - the timestamp has expired
+      // - if old fastbuild logs were purged
+      // - it's been a really long time since you synced so you just do a full fetch.
       if (res.body.status === -1) {
         // The incremental data is expired or this is the first fetch.
         reporter.info(`Unable to pull incremental data changes from Drupal`)
@@ -267,11 +278,11 @@ exports.sourceNodes = async (
         headers,
         searchParams: params,
         responseType: `json`,
-        cache,
       },
     ])
     allData = await Promise.all(
       _.map(res.body.links, async (url, type) => {
+        const dataArray = []
         if (disallowedLinkTypes.includes(type)) return
         if (!url) return
         if (!type) return
@@ -281,7 +292,7 @@ exports.sourceNodes = async (
           entityType => entityType === type
         )
 
-        const getNext = async (url, data = []) => {
+        const getNext = async url => {
           if (typeof url === `object`) {
             // url can be string or object containing href field
             url = url.href
@@ -315,37 +326,33 @@ exports.sourceNodes = async (
                 password: basicAuth.password,
                 headers,
                 responseType: `json`,
-                cache,
               },
             ])
           } catch (error) {
             if (error.response && error.response.statusCode == 405) {
               // The endpoint doesn't support the GET method, so just skip it.
-              return []
+              return
             } else {
               console.error(`Failed to fetch ${url}`, error.message)
               console.log(error)
               throw error
             }
           }
-          data = data.concat(d.body.data)
+          dataArray.push(...d.body.data)
           // Add support for includes. Includes allow entity data to be expanded
           // based on relationships. The expanded data is exposed as `included`
           // in the JSON API response.
           // See https://www.drupal.org/docs/8/modules/jsonapi/includes
           if (d.body.included) {
-            data = data.concat(d.body.included)
+            dataArray.push(...d.body.included)
           }
           if (d.body.links && d.body.links.next) {
-            data = await getNext(d.body.links.next, data)
+            await getNext(d.body.links.next)
           }
-
-          return data
         }
 
-        let data = []
         if (isTranslatable === false) {
-          data = await getNext(url)
+          await getNext(url)
         } else {
           for (let i = 0; i < languageConfig.enabledLanguages.length; i++) {
             let currentLanguage = languageConfig.enabledLanguages[i]
@@ -368,13 +375,13 @@ exports.sourceNodes = async (
             )
             const dataForLanguage = await getNext(joinedUrl)
 
-            data = data.concat(dataForLanguage)
+            dataArray.push(...dataForLanguage)
           }
         }
 
         const result = {
           type,
-          data,
+          data: dataArray,
         }
 
         // eslint-disable-next-line consistent-return

diff --git a/yarn.lock b/yarn.lock
@@ -14446,6 +14446,14 @@ http2-wrapper@^1.0.0-beta.5.2:
     quick-lru "^5.1.1"
     resolve-alpn "^1.0.0"
 
+http2-wrapper@^2.0.5:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-2.0.5.tgz#d4464509df69b6f82125b6cb337dbb80101f406c"
+  integrity sha512-W8+pfYl0iQ27NjvhDrbuQKaMBjBAWIZRHdKvmctV2c8a/naI7SsZgU3e04lCYrgxqnJ2sNPsBBrVI8kBeE/Pig==
+  dependencies:
+    quick-lru "^5.1.1"
+    resolve-alpn "^1.1.1"
+
 httperrors@2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/httperrors/-/httperrors-2.0.1.tgz#02febcaec8d9d6a9e1ae3773915b9fdaa2204672"
@@ -24367,6 +24375,11 @@ resolve-alpn@^1.0.0:
   resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.0.0.tgz#745ad60b3d6aff4b4a48e01b8c0bdc70959e0e8c"
   integrity sha512-rTuiIEqFmGxne4IovivKSDzld2lWW9QCjqv80SYjPgf+gS35eaCAjaP54CCwGAwBtnCsvNLYtqxe1Nw+i6JEmA==
 
+resolve-alpn@^1.1.1:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.1.2.tgz#30b60cfbb0c0b8dc897940fe13fe255afcdd4d28"
+  integrity sha512-8OyfzhAtA32LVUsJSke3auIyINcwdh5l3cvYKdKO0nvsYSKuiLfTM5i78PJswFPT8y6cPW+L1v6/hE95chcpDA==
+
 resolve-cwd@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-2.0.0.tgz#00a9f7387556e27038eae232caa372a6a59b665a"