fix(installer): retain browsers installed via Playwrigth CLI (microso…

…ft#5904) Browser registry is responsible for 3 things: 1. Remove downloaded browsers if there are no packages that refer to them 2. Install default browsers needed for the current package 3. Install browsers on-demand when used through Playwright CLI Currently, registry relies on a single "download" field in `browsers.json` to carry both (1) and (2). However, browsers in (3) are marked as `download: false` so that they aren't installed automatically in (2), so auto-remove procedure in (1) removes them on subsequent installation. One possible approach to fix this would be modifying package's `browsers.json` to change `download: false` to `true` when browsers are installed with Playwright CLI. This approach was explored here: microsoft@bc04a51 We decided against this since we have a history of issues related to package modifications after NPM installation. This breaks all sorts of yarn/npm caching mechanisms. Instead, this patch is a two-step refactor: - remove the "download" field in `browsers.json`. Now, all registries (including old ones from previously-released versions) will retain any browsers that are mentioned in the `browsers.json`. - add a new flag "installByDefault", that is **only used** for default installation. With this change, the registry tasks are done like this: - (1) auto-removal: if browser has a back reference, it is retained, otherwise it is removed from registry - (2) default installation: use only `installByDefault` to carry default installations - (3) CLI installation: simply installs browsers. Since we retain everythings that's referenced in (1), browsers aren't removed. Fixes microsoft#5902
aslushnikov · Mar 22, 2021 · a425549 · a425549
1 parent 6dd4d75
commit a425549
Show file tree

Hide file tree

Showing 7 changed files with 96 additions and 37 deletions.
diff --git a/browsers.json b/browsers.json
@@ -4,25 +4,30 @@
     {
       "name": "chromium",
       "revision": "857950",
-      "download": true
+      "installByDefault": true
     },
     {
       "name": "firefox",
       "revision": "1238",
-      "download": true
+      "installByDefault": true
     },
     {
       "name": "webkit",
       "revision": "1446",
-      "download": true,
+      "installByDefault": true,
       "revisionOverrides": {
         "mac10.14": "1443"
       }
     },
+    {
+      "name": "webkit-technology-preview",
+      "revision": "1443",
+      "installByDefault": false
+    },
     {
       "name": "ffmpeg",
       "revision": "1005",
-      "download": true
+      "installByDefault": true
     }
   ]
 }
diff --git a/packages/build_package.js b/packages/build_package.js
@@ -163,7 +163,7 @@ if (!args.some(arg => arg === '--no-cleanup')) {
   // 5. Generate browsers.json
   const browsersJSON = require(path.join(ROOT_PATH, 'browsers.json'));
   for (const browser of browsersJSON.browsers)
-    browser.download = package.browsers.includes(browser.name);
+    browser.installByDefault = package.browsers.includes(browser.name);
   await writeToPackage('browsers.json', JSON.stringify(browsersJSON, null, 2));
 
   // 6. Bake commit SHA into the package

diff --git a/src/cli/cli.ts b/src/cli/cli.ts
@@ -32,6 +32,7 @@ import { BrowserType } from '../client/browserType';
 import { BrowserContextOptions, LaunchOptions } from '../client/types';
 import { spawn } from 'child_process';
 import { installDeps } from '../install/installDeps';
+import { allBrowserNames } from '../utils/registry';
 
 program
     .version('Version ' + require('../../package.json').version)
@@ -85,10 +86,10 @@ program
     .description('ensure browsers necessary for this version of Playwright are installed')
     .action(async function(browserType) {
       try {
-        const allBrowsers = new Set(['chromium', 'firefox', 'webkit', 'ffmpeg']);
+        const allBrowsers = new Set(allBrowserNames);
         for (const type of browserType) {
           if (!allBrowsers.has(type)) {
-            console.log(`Invalid browser name: '${type}'. Expecting 'chromium', 'firefox' or 'webkit'.`);
+            console.log(`Invalid browser name: '${type}'. Expecting one of: ${allBrowserNames.map(name => `'${name}'`).join(', ')}`);
             process.exit(1);
           }
         }

diff --git a/src/install/installer.ts b/src/install/installer.ts
@@ -31,7 +31,7 @@ const fsWriteFileAsync = util.promisify(fs.writeFile.bind(fs));
 
 const PACKAGE_PATH = path.join(__dirname, '..', '..');
 
-export async function installBrowsersWithProgressBar(browserNames: BrowserName[] = allBrowserNames) {
+export async function installBrowsersWithProgressBar(browserNames: BrowserName[] = Registry.currentPackageRegistry().installByDefault()) {
   // PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD should have a value of 0 or 1
   if (getAsBooleanFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) {
     browserFetcher.logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set');
@@ -74,7 +74,7 @@ async function validateCache(linksDir: string, browserNames: BrowserName[]) {
       linkTarget = (await fsReadFileAsync(linkPath)).toString();
       const linkRegistry = new Registry(linkTarget);
       for (const browserName of allBrowserNames) {
-        if (!linkRegistry.shouldDownload(browserName))
+        if (!linkRegistry.shouldRetain(browserName))
           continue;
         const usedBrowserPath = linkRegistry.browserDirectory(browserName);
         const browserRevision = linkRegistry.revision(browserName);
@@ -105,10 +105,8 @@ async function validateCache(linksDir: string, browserNames: BrowserName[]) {
   }
 
   // 3. Install missing browsers for this package.
-  const myRegistry = new Registry(PACKAGE_PATH);
+  const myRegistry = Registry.currentPackageRegistry();
   for (const browserName of browserNames) {
-    if (!myRegistry.shouldDownload(browserName))
-      continue;
     await browserFetcher.downloadBrowserWithProgressBar(myRegistry, browserName).catch(e => {
       throw new Error(`Failed to download ${browserName}, caused by\n${e.stack}`);
     });

diff --git a/src/server/validateDependencies.ts b/src/server/validateDependencies.ts
@@ -43,11 +43,12 @@ export async function validateHostRequirements(registry: registry.Registry, brow
 }
 
 const DL_OPEN_LIBRARIES = {
-  chromium: [],
-  webkit: ['libGLESv2.so.2', 'libx264.so'],
-  firefox: [],
-  clank: [],
-  ffmpeg: [],
+  'chromium': [],
+  'webkit': ['libGLESv2.so.2', 'libx264.so'],
+  'webkit-technology-preview': ['libGLESv2.so.2', 'libx264.so'],
+  'firefox': [],
+  'clank': [],
+  'ffmpeg': [],
 };
 
 function isSupportedWindowsVersion(): boolean {

diff --git a/src/server/webkit/webkit.ts b/src/server/webkit/webkit.ts
@@ -23,12 +23,26 @@ import { BrowserType } from '../browserType';
 import { ConnectionTransport } from '../transport';
 import { BrowserOptions, PlaywrightOptions } from '../browser';
 import * as types from '../types';
+import * as fs from 'fs';
+import { assert } from '../../utils/utils';
 
 export class WebKit extends BrowserType {
   constructor(playwrightOptions: PlaywrightOptions) {
     super('webkit', playwrightOptions);
   }
 
+  executablePath(options?: types.LaunchOptions): string {
+    if (options?.channel) {
+      let executablePath = undefined;
+      if (options.channel === 'technology-preview')
+        executablePath = this._registry.executablePath('webkit-technology-preview');
+      assert(executablePath, `unsupported webkit channel "${options.channel}"`);
+      assert(fs.existsSync(executablePath), `webkit channel "${options.channel}" is not installed. Try running 'npx playwright install webkit-technology-preview'`);
+      return executablePath;
+    }
+    return super.executablePath(options);
+  }
+
   _connectToTransport(transport: ConnectionTransport, options: BrowserOptions): Promise<WKBrowser> {
     return WKBrowser.connect(transport, options);
   }

diff --git a/src/utils/registry.ts b/src/utils/registry.ts
@@ -23,19 +23,21 @@ import * as util from 'util';
 import { getUbuntuVersionSync } from './ubuntuVersion';
 import { assert, getFromENV } from './utils';
 
-export type BrowserName = 'chromium'|'webkit'|'firefox'|'ffmpeg';
-export const allBrowserNames: BrowserName[] = ['chromium', 'webkit', 'firefox', 'ffmpeg'];
+export type BrowserName = 'chromium'|'webkit'|'firefox'|'ffmpeg'|'webkit-technology-preview';
+export const allBrowserNames: BrowserName[] = ['chromium', 'webkit', 'firefox', 'ffmpeg', 'webkit-technology-preview'];
+
+const PACKAGE_PATH = path.join(__dirname, '..', '..');
 
 type BrowserPlatform = 'win32'|'win64'|'mac10.13'|'mac10.14'|'mac10.15'|'mac11'|'mac11-arm64'|'ubuntu18.04'|'ubuntu20.04';
 type BrowserDescriptor = {
   name: BrowserName,
   revision: string,
-  download: boolean,
+  installByDefault: boolean,
   browserDirectory: string,
 };
 
 const EXECUTABLE_PATHS = {
-  chromium: {
+  'chromium': {
     'ubuntu18.04': ['chrome-linux', 'chrome'],
     'ubuntu20.04': ['chrome-linux', 'chrome'],
     'mac10.13': ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium'],
@@ -46,7 +48,7 @@ const EXECUTABLE_PATHS = {
     'win32': ['chrome-win', 'chrome.exe'],
     'win64': ['chrome-win', 'chrome.exe'],
   },
-  firefox: {
+  'firefox': {
     'ubuntu18.04': ['firefox', 'firefox'],
     'ubuntu20.04': ['firefox', 'firefox'],
     'mac10.13': ['firefox', 'Nightly.app', 'Contents', 'MacOS', 'firefox'],
@@ -57,7 +59,7 @@ const EXECUTABLE_PATHS = {
     'win32': ['firefox', 'firefox.exe'],
     'win64': ['firefox', 'firefox.exe'],
   },
-  webkit: {
+  'webkit': {
     'ubuntu18.04': ['pw_run.sh'],
     'ubuntu20.04': ['pw_run.sh'],
     'mac10.13': undefined,
@@ -68,7 +70,18 @@ const EXECUTABLE_PATHS = {
     'win32': ['Playwright.exe'],
     'win64': ['Playwright.exe'],
   },
-  ffmpeg: {
+  'webkit-technology-preview': {
+    'ubuntu18.04': ['pw_run.sh'],
+    'ubuntu20.04': ['pw_run.sh'],
+    'mac10.13': undefined,
+    'mac10.14': ['pw_run.sh'],
+    'mac10.15': ['pw_run.sh'],
+    'mac11': ['pw_run.sh'],
+    'mac11-arm64': ['pw_run.sh'],
+    'win32': ['Playwright.exe'],
+    'win64': ['Playwright.exe'],
+  },
+  'ffmpeg': {
     'ubuntu18.04': ['ffmpeg-linux'],
     'ubuntu20.04': ['ffmpeg-linux'],
     'mac10.13': ['ffmpeg-mac'],
@@ -82,7 +95,7 @@ const EXECUTABLE_PATHS = {
 };
 
 const DOWNLOAD_URLS = {
-  chromium: {
+  'chromium': {
     'ubuntu18.04': '%s/builds/chromium/%s/chromium-linux.zip',
     'ubuntu20.04': '%s/builds/chromium/%s/chromium-linux.zip',
     'mac10.13': '%s/builds/chromium/%s/chromium-mac.zip',
@@ -93,7 +106,7 @@ const DOWNLOAD_URLS = {
     'win32': '%s/builds/chromium/%s/chromium-win32.zip',
     'win64': '%s/builds/chromium/%s/chromium-win64.zip',
   },
-  firefox: {
+  'firefox': {
     'ubuntu18.04': '%s/builds/firefox/%s/firefox-ubuntu-18.04.zip',
     'ubuntu20.04': '%s/builds/firefox/%s/firefox-ubuntu-18.04.zip',
     'mac10.13': '%s/builds/firefox/%s/firefox-mac-10.14.zip',
@@ -104,7 +117,7 @@ const DOWNLOAD_URLS = {
     'win32': '%s/builds/firefox/%s/firefox-win32.zip',
     'win64': '%s/builds/firefox/%s/firefox-win64.zip',
   },
-  webkit: {
+  'webkit': {
     'ubuntu18.04': '%s/builds/webkit/%s/webkit-ubuntu-18.04.zip',
     'ubuntu20.04': '%s/builds/webkit/%s/webkit-ubuntu-20.04.zip',
     'mac10.13': undefined,
@@ -115,7 +128,18 @@ const DOWNLOAD_URLS = {
     'win32': '%s/builds/webkit/%s/webkit-win64.zip',
     'win64': '%s/builds/webkit/%s/webkit-win64.zip',
   },
-  ffmpeg: {
+  'webkit-technology-preview': {
+    'ubuntu18.04': '%s/builds/webkit/%s/webkit-ubuntu-18.04.zip',
+    'ubuntu20.04': '%s/builds/webkit/%s/webkit-ubuntu-20.04.zip',
+    'mac10.13': undefined,
+    'mac10.14': undefined,
+    'mac10.15': '%s/builds/webkit/%s/webkit-mac-10.15.zip',
+    'mac11': '%s/builds/webkit/%s/webkit-mac-10.15.zip',
+    'mac11-arm64': '%s/builds/webkit/%s/webkit-mac-11.0-arm64.zip',
+    'win32': '%s/builds/webkit/%s/webkit-win64.zip',
+    'win64': '%s/builds/webkit/%s/webkit-win64.zip',
+  },
+  'ffmpeg': {
     'ubuntu18.04': '%s/builds/ffmpeg/%s/ffmpeg-linux.zip',
     'ubuntu20.04': '%s/builds/ffmpeg/%s/ffmpeg-linux.zip',
     'mac10.13': '%s/builds/ffmpeg/%s/ffmpeg-mac.zip',
@@ -199,9 +223,17 @@ export function isBrowserDirectory(browserDirectory: string): boolean {
   return false;
 }
 
+let currentPackageRegistry: Registry | undefined = undefined;
+
 export class Registry {
   private _descriptors: BrowserDescriptor[];
 
+  static currentPackageRegistry() {
+    if (!currentPackageRegistry)
+      currentPackageRegistry = new Registry(PACKAGE_PATH);
+    return currentPackageRegistry;
+  }
+
   constructor(packagePath: string) {
     const browsersJSON = JSON.parse(fs.readFileSync(path.join(packagePath, 'browsers.json'), 'utf8'));
     this._descriptors = browsersJSON['browsers'].map((obj: any) => {
@@ -212,7 +244,7 @@ export class Registry {
       return {
         name,
         revision,
-        download: obj.download,
+        installByDefault: !!obj.installByDefault,
         browserDirectory,
       };
     });
@@ -270,10 +302,11 @@ export class Registry {
     const browser = this._descriptors.find(browser => browser.name === browserName);
     assert(browser, `ERROR: Playwright does not support ${browserName}`);
     const envDownloadHost: { [key: string]: string } = {
-      chromium: 'PLAYWRIGHT_CHROMIUM_DOWNLOAD_HOST',
-      firefox: 'PLAYWRIGHT_FIREFOX_DOWNLOAD_HOST',
-      webkit: 'PLAYWRIGHT_WEBKIT_DOWNLOAD_HOST',
-      ffmpeg: 'PLAYWRIGHT_FFMPEG_DOWNLOAD_HOST',
+      'chromium': 'PLAYWRIGHT_CHROMIUM_DOWNLOAD_HOST',
+      'firefox': 'PLAYWRIGHT_FIREFOX_DOWNLOAD_HOST',
+      'webkit': 'PLAYWRIGHT_WEBKIT_DOWNLOAD_HOST',
+      'webkit-technology-preview': 'PLAYWRIGHT_WEBKIT_DOWNLOAD_HOST',
+      'ffmpeg': 'PLAYWRIGHT_FFMPEG_DOWNLOAD_HOST',
     };
     const downloadHost = getFromENV(envDownloadHost[browserName]) ||
                          getFromENV('PLAYWRIGHT_DOWNLOAD_HOST') ||
@@ -283,10 +316,17 @@ export class Registry {
     return util.format(urlTemplate, downloadHost, browser.revision);
   }
 
-  shouldDownload(browserName: BrowserName): boolean {
-    // Older versions do not have "download" field. We assume they need all browsers
-    // from the list. So we want to skip all browsers that are explicitly marked as "download: false".
+  shouldRetain(browserName: BrowserName): boolean {
+    // We retain browsers if they are found in the descriptor.
+    // Note, however, that there are older versions out in the wild that rely on
+    // the "download" field in the browser descriptor and use its value
+    // to retain and download browsers.
+    // As of v1.10, we decided to abandon "download" field.
     const browser = this._descriptors.find(browser => browser.name === browserName);
-    return !!browser && browser.download !== false;
+    return !!browser;
+  }
+
+  installByDefault(): BrowserName[] {
+    return this._descriptors.filter(browser => browser.installByDefault).map(browser => browser.name);
   }
 }