Merge pull request #1021 from bolt-design-system/fix/handle-github-ap…

…i-throttling

Fix: Improve Handling of Github API Requests

packages/build-tools/package.json

@@ -21,6 +21,7 @@
     }
   ],
   "dependencies": {
+    "@octokit/plugin-throttling": "^2.1.0",
     "@bolt/babel-preset-bolt": "^2.3.0-rc.0",
     "@bolt/config-browserlist": "^2.3.0-rc.0",
     "@bolt/postcss-themify": "^2.3.0-rc.0",

packages/build-tools/utils/bolt-versions.js

@@ -5,15 +5,50 @@ const ora = require('ora');
 const chalk = require('chalk');
 const checkLinks = require('check-links');
 const InCache = require('incache');
-const octokit = require('@octokit/rest')({
+const Octokit = require('@octokit/rest').plugin(
+  require('@octokit/plugin-throttling'),
+);
+
+const octokit = new Octokit({
+  throttle: {
+    onRateLimit: (retryAfter, options) => {
+      console.warn(
+        `Github API Request quota exhausted for request ${options.method} ${
+          options.url
+        }`,
+      );
+
+      // only retry if wait is 15 seconds or less
+      if (options.request.retryCount === 0 && retryAfter <= 15) {
+        // only retries once
+        console.log(`Retrying after ${retryAfter} seconds!`);
+        return true;
+      } else {
+        console.log(
+          `Skipping auto-retry since we don't want to wait ${retryAfter} seconds!`,
+        );
+        return false;
+      }
+    },
+    onAbuseLimit: (retryAfter, options) => {
+      // does not retry, only logs a warning
+      console.warn(
+        `Github API abuse detected for request ${options.method} ${
+          options.url
+        }`,
+      );
+    },
+  },
   debug: false,
   headers: {
     Accept: 'application/vnd.github.v3.raw',
   },
 });
-const { getConfig } = require('./config-store');
 
+const { getConfig } = require('./config-store');
+const { fileExists } = require('./general');
 const store = new InCache();
+let isUsingOldData = false; // remember if we are using up to date version data or older (stale) data as a fallback
 
 // so we don't go over rate limits
 if (process.env.GITHUB_TOKEN) {
@@ -58,14 +93,50 @@ async function gatherBoltVersions() {
   if (store.get('bolt-tags')) {
     tags = await store.get('bolt-tags');
   } else {
-    tags = await octokit.repos.listTags({
-      owner: 'bolt-design-system',
-      repo: 'bolt',
-      per_page: 9999,
-    });
-    tags = tags.data;
-    await store.set('bolt-tags', tags, { maxAge: 900 });
-    await store.save();
+    try {
+      tags = await octokit.repos.listTags({
+        owner: 'bolt-design-system',
+        repo: 'bolt',
+        per_page: 9999,
+      });
+      tags = tags.data;
+      await store.set('bolt-tags', tags, { maxAge: 5 * 60 * 1000 }); // set 5 minute cache expiration
+      await store.save();
+    } catch (err) {
+      // handle expired cached data + not having a GITHUB_TOKEN set as an environmental variable
+
+      // use old stale data if it exists
+      if (fileExists(path.join(process.cwd(), '.incache'))) {
+        let staleData = fs.readFileSync(path.join(process.cwd(), '.incache'));
+        staleData = JSON.parse(staleData);
+        const oldTags = staleData['bolt-tags'].value;
+        const oldUrls = staleData['bolt-urls-to-test'].value;
+
+        await store.set('bolt-tags', oldTags, { maxAge: 5 * 60 * 1000 }); // set 5 minute cache expiration
+        await store.set('bolt-urls-to-test', oldUrls, {
+          maxAge: 5 * 60 * 1000,
+        }); // set 5 minute cache expiration
+        await store.save();
+
+        tags = oldTags;
+        isUsingOldData = true; // remember this is old stale data for later
+      } else {
+        // otherwise just use a static version of the dropdown menu so the docs site doesn't break
+        versionSpinner.warn(
+          chalk.yellow(
+            'Could not generate the list of the latest releases of Bolt due to a missing GITHUB_TOKEN auth token + not finding an old version to fall back on. Skipping for now...',
+          ),
+        );
+
+        return [
+          {
+            label: 'Latest',
+            type: 'option',
+            value: `https://boltdesignsystem.com`,
+          },
+        ];
+      }
+    }
   }
 
   for (index = 0; index < tags.length; index++) {
@@ -88,7 +159,7 @@ async function gatherBoltVersions() {
     results = await store.get('bolt-urls-to-test');
   } else {
     results = await checkLinks(urlsToCheck);
-    await store.set('bolt-urls-to-test', results, { maxAge: 900 });
+    await store.set('bolt-urls-to-test', results, { maxAge: 5 * 60 * 1000 });
     await store.save();
   }
 
@@ -117,9 +188,19 @@ async function gatherBoltVersions() {
     }
   }
 
-  versionSpinner.succeed(
-    chalk.green('Gathered data on the latest Bolt Design System releases!'),
-  );
+  if (isUsingOldData) {
+    versionSpinner.warn(
+      chalk.yellow(
+        'Using Could not find a GITHUB_TOKEN auth token and the cached version of the latest Bolt releases has expired -- using the old (expired) data for now...',
+      ),
+    );
+  } else {
+    versionSpinner.succeed(
+      chalk.green(
+        'Finished gathering data on the latest Bolt Design System releases!',
+      ),
+    );
+  }
 
   return tagUrls;
 }

yarn.lock

@@ -696,9 +696,9 @@
     sleep-promise "^8.0.1"
 
 "@bolt/components-page-footer@file:packages/components/bolt-page-footer":
-  version "2.2.1"
+  version "2.3.0-rc.0"
   dependencies:
-    "@bolt/core" "^2.2.1"
+    "@bolt/core" "^2.3.0-rc.0"
 
 "@ckeditor/ckeditor5-build-classic@^11.1.1":
   version "11.2.0"
@@ -1320,6 +1320,13 @@
     universal-user-agent "^2.0.1"
     url-template "^2.0.8"
 
+"@octokit/plugin-throttling@^2.1.0":
+  version "2.1.0"
+  resolved "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-2.1.0.tgz#0e831be4585eb77cc566096867ca96e269603ba6"
+  integrity sha512-qS32I6LnCHWuIdPsPK2bT25H4XBgb4PBP+q/k0CerlQ4HeAXUkEQmm0IeWMYEyV8qQG3YuK2vMuqco6Rdx+Uuw==
+  dependencies:
+    bottleneck "^2.15.0"
+
 "@octokit/request@2.2.1":
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/@octokit/request/-/request-2.2.1.tgz#1b445e3052842b1f3ab94d68e2606840c85b4265"
@@ -2661,6 +2668,11 @@ boolbase@^1.0.0, boolbase@~1.0.0:
   resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"
   integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24=
 
+bottleneck@^2.15.0:
+  version "2.15.0"
+  resolved "https://registry.npmjs.org/bottleneck/-/bottleneck-2.15.0.tgz#538ec3a32f0e94a06e934bcb2080eb2c4c901c5a"
+  integrity sha512-6acQdvF7HfkbyDQAalnnsY5HGrrDrJ0QYie1/iL/IOch5oxhNVPRjqkIvOeCx4i6QyFi+ubxED/QRB1QWmWCJA==
+
 boxen@^1.2.1:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/boxen/-/boxen-1.3.0.tgz#55c6c39a8ba58d9c61ad22cd877532deb665a20b"