From 5c4751d5dd7c28804e5d9d76a1cceb7c36521749 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Wed, 1 Jul 2020 15:22:29 -0700 Subject: [PATCH] chore: generate protocol during browser roll (#2719) We now commit protocol.ts files during the roll. New utils/roll_browser.js helps with that. This makes our installation very shallow: - build installer; - download browsers. --- .appveyor.yml | 5 +- .circleci/config.yml | 13 +- .cirrus.yml | 47 - .github/workflows/infra.yml | 1 + .github/workflows/publish_canary.yml | 1 + .github/workflows/publish_release.yml | 1 + .github/workflows/tests.yml | 12 + .gitignore | 3 - .travis.yml | 12 +- browser_patches/README.md | 8 +- browsers.json | 1 + install-from-github.js | 68 +- package.json | 3 +- src/chromium/protocol.ts | 15858 ++++++++++++++++++++++ src/firefox/protocol.ts | 1064 ++ src/install/installer.ts | 4 +- src/install/tsconfig.json | 6 + src/webkit/protocol.ts | 9006 ++++++++++++ utils/protocol-types-generator/index.js | 1 - utils/roll_browser.js | 75 + 20 files changed, 26052 insertions(+), 137 deletions(-) delete mode 100644 .cirrus.yml create mode 100644 src/chromium/protocol.ts create mode 100644 src/firefox/protocol.ts create mode 100644 src/install/tsconfig.json create mode 100644 src/webkit/protocol.ts create mode 100644 utils/roll_browser.js diff --git a/.appveyor.yml b/.appveyor.yml index 4ed10fee8c778..2c2ba6d83a8f0 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,6 +7,5 @@ build: off install: - ps: Install-Product node $env:nodejs_version - npm ci - - npm run ctest - - npm run wtest - - npm run ftest + - npm run build + - npm run test diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a03c05c29fb6..52f4351dca59a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,18 +7,19 @@ jobs: - checkout - restore_cache: - key: dependency-cache-{{ checksum "package.json" }} + key: dependency-cache-{{ checksum "browsers.json" }} - run: command: npm ci + - run: + command: npm run build + - save_cache: - key: dependency-cache-{{ checksum "package.json" }} + key: dependency-cache-{{ checksum "browsers.json" }} paths: - - ./.local-browsers + - $HOME/.cache/ms-playwright - run: command: | - xvfb-run --auto-servernum npm run ctest - xvfb-run --auto-servernum npm run wtest - xvfb-run --auto-servernum npm run ftest + xvfb-run --auto-servernum npm run test diff --git a/.cirrus.yml b/.cirrus.yml deleted file mode 100644 index b7f0975466f38..0000000000000 --- a/.cirrus.yml +++ /dev/null @@ -1,47 +0,0 @@ -env: - DISPLAY: :99.0 - FLAKINESS_DASHBOARD_PASSWORD: ENCRYPTED[b3e207db5d153b543f219d3c3b9123d8321834b783b9e45ac7d380e026ab3a56398bde51b521ac5859e7e45cb95d0992] - FLAKINESS_DASHBOARD_NAME: Cirrus ${CIRRUS_TASK_NAME} - FLAKINESS_DASHBOARD_BUILD_URL: https://cirrus-ci.com/task/${CIRRUS_TASK_ID} - -task: - matrix: - - name: Chromium (node8 + linux) - container: - dockerfile: .ci/node8/Dockerfile.linux - - name: Chromium (node10 + linux) - container: - dockerfile: .ci/node10/Dockerfile.linux - - name: Chromium (node12 + linux) - container: - dockerfile: .ci/node12/Dockerfile.linux - xvfb_start_background_script: Xvfb :99 -ac -screen 0 1024x768x24 - install_script: npm install --unsafe-perm - lint_script: npm run lint - coverage_script: npm run coverage - test_doclint_script: npm run test-doclint - test_types_script: npm run test-types - -task: - matrix: - - name: Firefox (node8 + linux) - container: - dockerfile: .ci/node8/Dockerfile.linux - xvfb_start_background_script: Xvfb :99 -ac -screen 0 1024x768x24 - install_script: npm install --unsafe-perm - test_script: npm run ftest - -task: - osx_instance: - image: high-sierra-base - name: Chromium (node8 + macOS) - env: - HOMEBREW_NO_AUTO_UPDATE: 1 - node_install_script: - - brew install node@8 - - brew link --force node@8 - install_script: npm install --unsafe-perm - lint_script: npm run lint - coverage_script: npm run coverage - test_doclint_script: npm run test-doclint - test_types_script: npm run test-types diff --git a/.github/workflows/infra.yml b/.github/workflows/infra.yml index 5cb69f41dee50..fefe6d5cdb176 100644 --- a/.github/workflows/infra.yml +++ b/.github/workflows/infra.yml @@ -28,4 +28,5 @@ jobs: sudo apt-get install libgbm-dev sudo apt-get install xvfb - run: npm ci + - run: npm run build - run: npm run lint diff --git a/.github/workflows/publish_canary.yml b/.github/workflows/publish_canary.yml index c26265f8d8574..c7d7ca86f7cb8 100644 --- a/.github/workflows/publish_canary.yml +++ b/.github/workflows/publish_canary.yml @@ -21,6 +21,7 @@ jobs: registry-url: 'https://registry.npmjs.org' - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: node utils/update_version.js --next - run: utils/publish_all_packages.sh --tip-of-tree env: diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index e49ab270b8b1f..1bb97211670a0 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -19,6 +19,7 @@ jobs: registry-url: 'https://registry.npmjs.org' - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: utils/publish_all_packages.sh --release env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3f27f81d73b9d..9c15c8b8962d4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,6 +26,7 @@ jobs: node-version: 10 - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: mkdir -p coredumps # Set core dump file name pattern - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern' @@ -61,6 +62,7 @@ jobs: node-version: 10 - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: npm run ctest 2>./chromium-mac-testrun.log env: DEBUG: "*" @@ -85,6 +87,7 @@ jobs: node-version: 10 - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: npm run ctest 2>./chromium-win-testrun.log shell: bash env: @@ -110,6 +113,7 @@ jobs: node-version: 10 - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: mkdir -p coredumps # Set core dump file name pattern - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern' @@ -145,6 +149,7 @@ jobs: node-version: 10 - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: npm run wtest 2>./webkit-mac-testrun.log env: DEBUG: "*,-pw:wrapped*" @@ -169,6 +174,7 @@ jobs: node-version: 10 - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: npm run wtest 2>./webkit-win-testrun.log shell: bash env: @@ -194,6 +200,7 @@ jobs: node-version: 10 - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: mkdir -p coredumps # Set core dump file name pattern - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern' @@ -229,6 +236,7 @@ jobs: node-version: 10 - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: npm run ftest 2>./firefox-mac-testrun.log env: DEBUG: "*" @@ -253,6 +261,7 @@ jobs: node-version: 10 - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: npm run ftest 2>./firefox-win-testrun.log shell: bash env: @@ -283,6 +292,7 @@ jobs: node-version: ${{ matrix.node_version }} - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: bash packages/installation-tests/installation-tests.sh headful_linux: @@ -295,6 +305,7 @@ jobs: node-version: 10 - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: mkdir -p coredumps # Set core dump file name pattern - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern' @@ -326,6 +337,7 @@ jobs: node-version: 10 - uses: microsoft/playwright-github-action@v1 - run: npm ci + - run: npm run build - run: mkdir -p coredumps # Set core dump file name pattern - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern' diff --git a/.gitignore b/.gitignore index e4a3f126fb06e..3e8db130d66b7 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,5 @@ yarn.lock /node6 /src/generated/* -/src/chromium/protocol.ts -/src/firefox/protocol.ts -/src/webkit/protocol.ts lib/ /types/* diff --git a/.travis.yml b/.travis.yml index ea53a4b14865d..7f42bb2653e46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,16 +39,12 @@ jobs: # this is needed for running headful tests - xvfb script: - - npm run lint - - xvfb-run --auto-servernum npm run ctest - - xvfb-run --auto-servernum npm run ftest - - xvfb-run --auto-servernum npm run wtest + - npm run build + - xvfb-run --auto-servernum npm run test - os: osx osx_image: xcode11.3 node_js: '12' script: - - npm run lint - - npm run ctest - - npm run ftest - - npm run wtest + - npm run build + - npm run test diff --git a/browser_patches/README.md b/browser_patches/README.md index 9cf12620b5da1..7c4ac0132e122 100644 --- a/browser_patches/README.md +++ b/browser_patches/README.md @@ -58,12 +58,8 @@ Send a PR to the Playwright repo to be reviewed. ## 4. Rolling Playwright to the new browser build -Once the patch has been committed, the build bots will kick in, compile and upload a new browser version to all the platforms. - -You can check the CDN status: +Once the patch has been committed, the build bots will kick in, compile and upload a new browser version to all the platforms. Then you can roll the browser: ```sh -$ ./browser_patches/tools/check_cdn.sh +$ node utils/roll_browser.js chromium 123456 ``` - -As the builds appear, you can roll to a new browser version in the `./browsers.json` file. diff --git a/browsers.json b/browsers.json index 4aecddf8b701c..a55b1915f0233 100644 --- a/browsers.json +++ b/browsers.json @@ -1,4 +1,5 @@ { + "comment": "Do not edit this file, use utils/roll_browser.js", "browsers": [ { "name": "chromium", diff --git a/install-from-github.js b/install-from-github.js index d9ae8c6126519..5cf94cece1eb4 100644 --- a/install-from-github.js +++ b/install-from-github.js @@ -18,67 +18,15 @@ // This file is only run when someone installs via the github repo const {execSync} = require('child_process'); -const path = require('path'); -const fs = require('fs'); -const util = require('util'); -const rmAsync = util.promisify(require('rimraf')); -(async () => { - const SRC_FOLDER = path.join(__dirname, 'src'); - const LIB_FOLDER = path.join(__dirname, 'lib'); - const srcTypeScriptFiles = (await listFiles(path.join(__dirname, 'src'))).filter(filepath => filepath.toLowerCase().endsWith('.ts')); - const outdatedFiles = await Promise.all(srcTypeScriptFiles.map(async srcFile => { - const libFileTS = path.join(LIB_FOLDER, path.relative(SRC_FOLDER, srcFile)); - const libFile = libFileTS.substring(0, libFileTS.lastIndexOf('.')) + '.js'; - try { - const [srcStat, libStat] = await Promise.all([fs.promises.stat(srcFile), fs.promises.stat(libFile)]); - return srcStat.ctimeMs > libStat.ctimeMs; - } catch (e) { - // Either `.ts` of `.js` file is missing - rebuild is required. - return true; - } - })); - if (outdatedFiles.some(Boolean)) { - console.log(`Rebuilding playwright...`); - try { - execSync('npm run build'); - } catch (e) { - } - } - await downloadAllBrowsersAndGenerateProtocolTypes(); -})(); - -async function listFiles(dirpath) { - const files = []; - await dfs(dirpath); - return files; - - async function dfs(dirpath) { - const entries = await fs.promises.readdir(dirpath, {withFileTypes: true}); - files.push(...entries.filter(entry => entry.isFile()).map(entry => path.join(dirpath, entry.name))); - await Promise.all(entries.filter(entry => entry.isDirectory()).map(entry => dfs(path.join(dirpath, entry.name)))); - } +console.log(`Rebuilding installer...`); +try { + execSync('npm run tsc-installer'); +} catch (e) { } -async function downloadAllBrowsersAndGenerateProtocolTypes() { - const { installBrowsersWithProgressBar } = require('./lib/install/installer'); - const protocolGenerator = require('./utils/protocol-types-generator'); - const browserPaths = require('./lib/install/browserPaths'); - const browsersPath = browserPaths.browsersPath(__dirname); - const browsers = require(path.join(__dirname, 'browsers.json'))['browsers']; - await installBrowsersWithProgressBar(__dirname); - for (const browser of browsers) { - const browserPath = browserPaths.browserDirectory(browsersPath, browser); - await protocolGenerator.generateProtocol(browser.name, browserPaths.executablePath(browserPath, browser)).catch(console.warn); - } +console.log(`Downloading browsers...`); +const { installBrowsersWithProgressBar } = require('./lib/install/installer'); +installBrowsersWithProgressBar(__dirname); - try { - console.log('Generating types...'); - execSync('npm run generate-types'); - } catch (e) { - } - - async function readdirAsync(dirpath) { - return fs.promises.readdir(dirpath).then(dirs => dirs.map(dir => path.join(dirpath, dir))); - } -} +console.log(`Done. Use "npm run watch" to compile.`); diff --git a/package.json b/package.json index d033e969e7ef7..ecf9aeb80a61e 100644 --- a/package.json +++ b/package.json @@ -26,13 +26,14 @@ "coverage": "cross-env COVERAGE=true BROWSER=all node --unhandled-rejections=strict test/test.js", "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts ./src || eslint --ext js,ts ./src", "tsc": "tsc -p .", + "tsc-installer": "tsc -p ./src/install/tsconfig.json", "doc": "node utils/doclint/cli.js", "test-infra": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js && node utils/testrunner/test/test.js", "lint": "npm run eslint && npm run tsc && npm run doc && npm run test-types && npm run test-infra", "debug-test": "node --inspect-brk test/test.js", "clean": "rimraf lib && rimraf types", "prepare": "node install-from-github.js", - "build": "node utils/runWebpack.js --mode='development' && tsc -p .", + "build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run generate-types", "watch": "node utils/runWebpack.js --mode='development' --watch --silent | tsc -w -p .", "test-types": "npm run generate-types && npx -p typescript@3.7.5 tsc -p utils/generate_types/test/tsconfig.json", "generate-types": "node utils/generate_types/" diff --git a/src/chromium/protocol.ts b/src/chromium/protocol.ts new file mode 100644 index 0000000000000..03d337663d54c --- /dev/null +++ b/src/chromium/protocol.ts @@ -0,0 +1,15858 @@ +// This is generated from /utils/protocol-types-generator/index.js +type binary = string; +export module Protocol { + export module Accessibility { + /** + * Unique accessibility node identifier. + */ + export type AXNodeId = string; + /** + * Enum of possible property types. + */ + export type AXValueType = "boolean"|"tristate"|"booleanOrUndefined"|"idref"|"idrefList"|"integer"|"node"|"nodeList"|"number"|"string"|"computedString"|"token"|"tokenList"|"domRelation"|"role"|"internalRole"|"valueUndefined"; + /** + * Enum of possible property sources. + */ + export type AXValueSourceType = "attribute"|"implicit"|"style"|"contents"|"placeholder"|"relatedElement"; + /** + * Enum of possible native property sources (as a subtype of a particular AXValueSourceType). + */ + export type AXValueNativeSourceType = "figcaption"|"label"|"labelfor"|"labelwrapped"|"legend"|"tablecaption"|"title"|"other"; + /** + * A single source for a computed AX property. + */ + export interface AXValueSource { + /** + * What type of source this is. + */ + type: AXValueSourceType; + /** + * The value of this property source. + */ + value?: AXValue; + /** + * The name of the relevant attribute, if any. + */ + attribute?: string; + /** + * The value of the relevant attribute, if any. + */ + attributeValue?: AXValue; + /** + * Whether this source is superseded by a higher priority source. + */ + superseded?: boolean; + /** + * The native markup source for this value, e.g. a