feat: sync all crhome for test binaries (#592)

closes #591

.github/workflows/codeql-analysis.yml

@@ -13,12 +13,9 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ main ]
+    branches: [ master ]
   pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ main ]
-  schedule:
-    - cron: '41 13 * * 3'
+    branches: [ master ]
 
 jobs:
   analyze:

.github/workflows/nodejs.yml

@@ -3,7 +3,11 @@
 
 name: Node.js CI
 
-on: [push, pull_request]
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
 
 jobs:
   test-mysql57-fs-nfs:

.github/workflows/release.yml

@@ -6,7 +6,7 @@ on:
 jobs:
   release:
     name: Node.js
-    uses: node-modules/github-actions/.github/workflows/node-release.yml@master
+    uses: cnpm/github-actions/.github/workflows/node-release.yml@master
     secrets:
       NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
       GIT_TOKEN: ${{ secrets.GIT_TOKEN }}

app/common/adapter/binary/ChromeForTestingBinary.ts

@@ -1,10 +1,13 @@
+import { basename } from 'path';
 import { SingletonProto } from '@eggjs/tegg';
 import { BinaryType } from '../../enum/Binary';
 import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './AbstractBinary';
 
 @SingletonProto()
 @BinaryAdapter(BinaryType.ChromeForTesting)
 export class ChromeForTestingBinary extends AbstractBinary {
+  static lastTimestamp = '';
+
   private dirItems?: {
     [key: string]: BinaryItem[];
   };
@@ -13,57 +16,102 @@ export class ChromeForTestingBinary extends AbstractBinary {
     this.dirItems = undefined;
   }
 
-  async fetch(dir: string): Promise<FetchResult | undefined> {
-    if (!this.dirItems) {
-      this.dirItems = {};
-      this.dirItems['/'] = [];
-      let chromeVersion = '';
-
-      // exports.PUPPETEER_REVISIONS = Object.freeze({
-      //   chrome: '113.0.5672.63',
-      //   firefox: 'latest',
-      // });
-      const unpkgURL = 'https://unpkg.com/puppeteer-core@latest/lib/cjs/puppeteer/revisions.js';
-      const text = await this.requestXml(unpkgURL);
-      const m = /chrome:\s+\'([\d\.]+)\'\,/.exec(text);
-      if (m) {
-        chromeVersion = m[1];
-      }
+  async #syncDirItems() {
+    this.dirItems = {};
+    this.dirItems['/'] = [];
+    const jsonApiEndpoint = 'https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json';
+    const { data } = await this.httpclient.request(jsonApiEndpoint, {
+      dataType: 'json',
+      timeout: 30000,
+      followRedirect: true,
+      gzip: true,
+    });
+    if (data.timestamp === ChromeForTestingBinary.lastTimestamp) return;
 
-      const platforms = [ 'linux64', 'mac-arm64', 'mac-x64', 'win32', 'win64' ];
-      const date = new Date().toISOString();
+    // "timestamp": "2023-09-16T00:21:21.964Z",
+    // "versions": [
+    //   {
+    //     "version": "113.0.5672.0",
+    //     "revision": "1121455",
+    //     "downloads": {
+    //       "chrome": [
+    //         {
+    //           "platform": "linux64",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/linux64/chrome-linux64.zip"
+    //         },
+    //         {
+    //           "platform": "mac-arm64",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/mac-arm64/chrome-mac-arm64.zip"
+    //         },
+    //         {
+    //           "platform": "mac-x64",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/mac-x64/chrome-mac-x64.zip"
+    //         },
+    //         {
+    //           "platform": "win32",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/win32/chrome-win32.zip"
+    //         },
+    //         {
+    //           "platform": "win64",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/win64/chrome-win64.zip"
+    //         }
+    //       ]
+    //     }
+    //   },
+    const versions = data.versions as {
+      version: string;
+      revision: string;
+      downloads: {
+        [key: string]: {
+          platform: string;
+          url: string;
+        }[];
+      };
+    }[];
+    for (const item of versions) {
       this.dirItems['/'].push({
-        name: `${chromeVersion}/`,
-        date,
+        name: `${item.version}/`,
+        date: item.revision,
         size: '-',
         isDir: true,
         url: '',
       });
-      this.dirItems[`/${chromeVersion}/`] = [];
-
-      for (const platform of platforms) {
-        this.dirItems[`/${chromeVersion}/`].push({
-          name: `${platform}/`,
-          date,
-          size: '-',
-          isDir: true,
-          url: '',
-        });
-
-        // https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.63/mac-arm64/chrome-mac-arm64.zip
-        const name = `chrome-${platform}.zip`;
-        this.dirItems[`/${chromeVersion}/${platform}/`] = [
-          {
-            name,
-            date,
+      const versionDir = `/${item.version}/`;
+      if (!this.dirItems[versionDir]) {
+        this.dirItems[versionDir] = [];
+      }
+      for (const category in item.downloads) {
+        const downloads = item.downloads[category];
+        for (const download of downloads) {
+          const platformDir = `${versionDir}${download.platform}/`;
+          if (!this.dirItems[platformDir]) {
+            this.dirItems[platformDir] = [];
+            this.dirItems[versionDir].push({
+              name: `${download.platform}/`,
+              date: item.revision,
+              size: '-',
+              isDir: true,
+              url: '',
+            });
+          }
+          this.dirItems[platformDir].push({
+            name: basename(download.url),
+            date: data.timestamp,
             size: '-',
             isDir: false,
-            url: `https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/${chromeVersion}/${platform}/${name}`,
-          },
-        ];
+            url: download.url,
+          });
+        }
       }
     }
+    ChromeForTestingBinary.lastTimestamp = data.timestamp;
+  }
 
-    return { items: this.dirItems[dir], nextParams: null };
+  async fetch(dir: string): Promise<FetchResult | undefined> {
+    // use https://github.com/GoogleChromeLabs/chrome-for-testing#json-api-endpoints
+    if (!this.dirItems) {
+      await this.#syncDirItems();
+    }
+    return { items: this.dirItems![dir], nextParams: null };
   }
 }

package.json

@@ -120,22 +120,22 @@
     "@cnpmjs/npm-cli-login": "^1.1.0",
     "@elastic/elasticsearch-mock": "^2.0.0",
     "@simplewebauthn/typescript-types": "^7.0.0",
+    "@types/lodash": "^4.14.196",
     "@types/mime-types": "^2.1.1",
     "@types/mocha": "^10.0.1",
     "@types/mysql": "^2.15.21",
+    "@types/npm-package-arg": "^6.1.1",
     "@types/semver": "^7.3.12",
     "@types/tar": "^6.1.4",
+    "@types/ua-parser-js": "^0.7.36",
+    "@types/validate-npm-package-name": "^4.0.0",
     "coffee": "^5.4.0",
     "egg-bin": "^6.0.0",
     "egg-mock": "^5.10.4",
     "eslint": "^8.29.0",
-    "eslint-config-egg": "^12.1.0",
+    "eslint-config-egg": "^13.0.0",
     "git-contributor": "2",
-    "typescript": "^5.0.4",
-    "@types/ua-parser-js": "^0.7.36",
-    "@types/lodash": "^4.14.196",
-    "@types/npm-package-arg": "^6.1.1",
-    "@types/validate-npm-package-name": "^4.0.0"
+    "typescript": "^5.2.2"
   },
   "author": "killagu",
   "license": "MIT",

test/common/adapter/binary/ChromeForTestingBinary.test.ts

@@ -1,6 +1,7 @@
 import assert from 'assert';
 import { app } from 'egg-mock/bootstrap';
 import { ChromeForTestingBinary } from '../../../../app/common/adapter/binary/ChromeForTestingBinary';
+import { TestUtil } from '../../../../test/TestUtil';
 
 describe('test/common/adapter/binary/ChromeForTestingBinary.test.ts', () => {
   let binary: ChromeForTestingBinary;
@@ -9,21 +10,43 @@ describe('test/common/adapter/binary/ChromeForTestingBinary.test.ts', () => {
   });
   describe('fetch()', () => {
     it('should work for chrome binary', async () => {
+      assert.equal(ChromeForTestingBinary.lastTimestamp, '');
+      app.mockHttpclient('https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json', 'GET', {
+        data: await TestUtil.readFixturesFile('chrome-for-testing/known-good-versions-with-downloads.json'),
+        persist: false,
+      });
       const result = await binary.fetch('/');
-      const latestVersion = result?.items?.[0].name;
+      const latestVersion = result!.items![result!.items.length - 1].name;
       assert(latestVersion);
+      assert.equal(latestVersion, '119.0.6008.0/');
 
       const platformRes = await binary.fetch(`/${latestVersion}`);
       const platforms = platformRes?.items.map(item => item.name);
       assert(platforms);
+      assert.deepEqual(platforms, [ 'linux64/', 'mac-arm64/', 'mac-x64/', 'win32/', 'win64/' ]);
 
       for (const platform of platforms) {
         const versionRes = await binary.fetch(`/${latestVersion}${platform}`);
         const versions = versionRes?.items.map(item => item.name);
-        assert.equal(versions?.length, 1);
+        assert.equal(versions?.length, 3);
         assert(versionRes?.items[0].name);
         assert.equal(versionRes?.items[0].isDir, false);
+        assert.match(versionRes?.items[0].name, /^chrome\-/);
+        assert.match(versionRes?.items[1].name, /^chromedriver\-/);
+        assert.match(versionRes?.items[2].name, /^chrome\-headless\-shell\-/);
       }
+      assert(ChromeForTestingBinary.lastTimestamp);
+    });
+
+    it('should return empty when timestamp is not changed', async () => {
+      assert(ChromeForTestingBinary.lastTimestamp);
+      await binary.initFetch();
+      app.mockHttpclient('https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json', 'GET', {
+        data: await TestUtil.readFixturesFile('chrome-for-testing/known-good-versions-with-downloads.json'),
+        persist: false,
+      });
+      const result = await binary.fetch('/');
+      assert.equal(result?.items.length, 0);
     });
   });
 });