chore: generate protocol during browser roll (#2719)

We now commit protocol.ts files during the roll.
New utils/roll_browser.js helps with that.

This makes our installation very shallow:
- build installer;
- download browsers.

.appveyor.yml

@@ -7,6 +7,5 @@ build: off
 install:
   - ps: Install-Product node $env:nodejs_version
   - npm ci
-  - npm run ctest
-  - npm run wtest
-  - npm run ftest
+  - npm run build
+  - npm run test

.circleci/config.yml

@@ -7,18 +7,19 @@ jobs:
       - checkout
 
       - restore_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+          key: dependency-cache-{{ checksum "browsers.json" }}
 
       - run:
           command: npm ci
 
+      - run:
+          command: npm run build
+
       - save_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+          key: dependency-cache-{{ checksum "browsers.json" }}
           paths:
-            - ./.local-browsers
+            - $HOME/.cache/ms-playwright
 
       - run:
           command: |
-            xvfb-run --auto-servernum npm run ctest
-            xvfb-run --auto-servernum npm run wtest
-            xvfb-run --auto-servernum npm run ftest
+            xvfb-run --auto-servernum npm run test

.github/workflows/infra.yml

@@ -28,4 +28,5 @@ jobs:
         sudo apt-get install libgbm-dev
         sudo apt-get install xvfb
     - run: npm ci
+    - run: npm run build
     - run: npm run lint

.github/workflows/publish_canary.yml

@@ -21,6 +21,7 @@ jobs:
         registry-url: 'https://registry.npmjs.org'
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: node utils/update_version.js --next
     - run: utils/publish_all_packages.sh --tip-of-tree
       env:

.github/workflows/publish_release.yml

@@ -19,6 +19,7 @@ jobs:
         registry-url: 'https://registry.npmjs.org'
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: utils/publish_all_packages.sh --release
       env:
         NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/tests.yml

@@ -26,6 +26,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: mkdir -p coredumps
     # Set core dump file name pattern
     - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'
@@ -61,6 +62,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: npm run ctest 2>./chromium-mac-testrun.log
       env:
         DEBUG: "*"
@@ -85,6 +87,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: npm run ctest 2>./chromium-win-testrun.log
       shell: bash
       env:
@@ -110,6 +113,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: mkdir -p coredumps
     # Set core dump file name pattern
     - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'
@@ -145,6 +149,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: npm run wtest 2>./webkit-mac-testrun.log
       env:
         DEBUG: "*,-pw:wrapped*"
@@ -169,6 +174,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: npm run wtest 2>./webkit-win-testrun.log
       shell: bash
       env:
@@ -194,6 +200,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: mkdir -p coredumps
     # Set core dump file name pattern
     - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'
@@ -229,6 +236,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: npm run ftest 2>./firefox-mac-testrun.log
       env:
         DEBUG: "*"
@@ -253,6 +261,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: npm run ftest 2>./firefox-win-testrun.log
       shell: bash
       env:
@@ -283,6 +292,7 @@ jobs:
         node-version: ${{ matrix.node_version }}
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: bash packages/installation-tests/installation-tests.sh
 
   headful_linux:
@@ -295,6 +305,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: mkdir -p coredumps
     # Set core dump file name pattern
     - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'
@@ -326,6 +337,7 @@ jobs:
         node-version: 10
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
+    - run: npm run build
     - run: mkdir -p coredumps
     # Set core dump file name pattern
     - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'

.gitignore

@@ -12,8 +12,5 @@
 yarn.lock
 /node6
 /src/generated/*
-/src/chromium/protocol.ts
-/src/firefox/protocol.ts
-/src/webkit/protocol.ts
 lib/
 /types/*

.travis.yml

@@ -39,16 +39,12 @@ jobs:
           # this is needed for running headful tests
           - xvfb
       script:
-        - npm run lint
-        - xvfb-run --auto-servernum npm run ctest
-        - xvfb-run --auto-servernum npm run ftest
-        - xvfb-run --auto-servernum npm run wtest
+        - npm run build
+        - xvfb-run --auto-servernum npm run test
     - os: osx
       osx_image: xcode11.3
       node_js: '12'
       script:
-        - npm run lint
-        - npm run ctest
-        - npm run ftest
-        - npm run wtest
+        - npm run build
+        - npm run test
 

browser_patches/README.md

@@ -58,12 +58,8 @@ Send a PR to the Playwright repo to be reviewed.
 
 ## 4. Rolling Playwright to the new browser build
 
-Once the patch has been committed, the build bots will kick in, compile and upload a new browser version to all the platforms.
-
-You can check the CDN status:
+Once the patch has been committed, the build bots will kick in, compile and upload a new browser version to all the platforms. Then you can roll the browser:
 
 ```sh
-$ ./browser_patches/tools/check_cdn.sh
+$ node utils/roll_browser.js chromium 123456
 ```
-
-As the builds appear, you can roll to a new browser version in the `./browsers.json` file.

browsers.json

@@ -1,4 +1,5 @@
 {
+  "comment": "Do not edit this file, use utils/roll_browser.js",
   "browsers": [
     {
       "name": "chromium",

install-from-github.js

@@ -18,67 +18,15 @@
 // This file is only run when someone installs via the github repo
 
 const {execSync} = require('child_process');
-const path = require('path');
-const fs = require('fs');
-const util = require('util');
-const rmAsync = util.promisify(require('rimraf'));
 
-(async () => {
-  const SRC_FOLDER = path.join(__dirname, 'src');
-  const LIB_FOLDER = path.join(__dirname, 'lib');
-  const srcTypeScriptFiles = (await listFiles(path.join(__dirname, 'src'))).filter(filepath => filepath.toLowerCase().endsWith('.ts'));
-  const outdatedFiles = await Promise.all(srcTypeScriptFiles.map(async srcFile => {
-    const libFileTS = path.join(LIB_FOLDER, path.relative(SRC_FOLDER, srcFile));
-    const libFile = libFileTS.substring(0, libFileTS.lastIndexOf('.')) + '.js';
-    try {
-      const [srcStat, libStat] = await Promise.all([fs.promises.stat(srcFile), fs.promises.stat(libFile)]);
-      return srcStat.ctimeMs > libStat.ctimeMs;
-    } catch (e) {
-      // Either `.ts` of `.js` file is missing - rebuild is required.
-      return true;
-    }
-  }));
-  if (outdatedFiles.some(Boolean)) {
-    console.log(`Rebuilding playwright...`);
-    try {
-      execSync('npm run build');
-    } catch (e) {
-    }
-  }
-  await downloadAllBrowsersAndGenerateProtocolTypes();
-})();
-
-async function listFiles(dirpath) {
-  const files = [];
-  await dfs(dirpath);
-  return files;
-
-  async function dfs(dirpath) {
-    const entries = await fs.promises.readdir(dirpath, {withFileTypes: true});
-    files.push(...entries.filter(entry => entry.isFile()).map(entry => path.join(dirpath, entry.name)));
-    await Promise.all(entries.filter(entry => entry.isDirectory()).map(entry => dfs(path.join(dirpath, entry.name))));
-  }
+console.log(`Rebuilding installer...`);
+try {
+  execSync('npm run tsc-installer');
+} catch (e) {
 }
 
-async function downloadAllBrowsersAndGenerateProtocolTypes() {
-  const { installBrowsersWithProgressBar } = require('./lib/install/installer');
-  const protocolGenerator = require('./utils/protocol-types-generator');
-  const browserPaths = require('./lib/install/browserPaths');
-  const browsersPath = browserPaths.browsersPath(__dirname);
-  const browsers = require(path.join(__dirname, 'browsers.json'))['browsers'];
-  await installBrowsersWithProgressBar(__dirname);
-  for (const browser of browsers) {
-    const browserPath = browserPaths.browserDirectory(browsersPath, browser);
-    await protocolGenerator.generateProtocol(browser.name, browserPaths.executablePath(browserPath, browser)).catch(console.warn);
-  }
+console.log(`Downloading browsers...`);
+const { installBrowsersWithProgressBar } = require('./lib/install/installer');
+installBrowsersWithProgressBar(__dirname);
 
-  try {
-    console.log('Generating types...');
-    execSync('npm run generate-types');
-  } catch (e) {
-  }
-
-  async function readdirAsync(dirpath) {
-    return fs.promises.readdir(dirpath).then(dirs => dirs.map(dir => path.join(dirpath, dir)));
-  }
-}
+console.log(`Done. Use "npm run watch" to compile.`);

package.json

@@ -26,13 +26,14 @@
     "coverage": "cross-env COVERAGE=true BROWSER=all node --unhandled-rejections=strict test/test.js",
     "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts ./src || eslint --ext js,ts ./src",
     "tsc": "tsc -p .",
+    "tsc-installer": "tsc -p ./src/install/tsconfig.json",
     "doc": "node utils/doclint/cli.js",
     "test-infra": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js && node utils/testrunner/test/test.js",
     "lint": "npm run eslint && npm run tsc && npm run doc && npm run test-types && npm run test-infra",
     "debug-test": "node --inspect-brk test/test.js",
     "clean": "rimraf lib && rimraf types",
     "prepare": "node install-from-github.js",
-    "build": "node utils/runWebpack.js --mode='development' && tsc -p .",
+    "build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run generate-types",
     "watch": "node utils/runWebpack.js --mode='development' --watch --silent | tsc -w -p .",
     "test-types": "npm run generate-types && npx -p typescript@3.7.5 tsc -p utils/generate_types/test/tsconfig.json",
     "generate-types": "node utils/generate_types/"