Skip to content

Commit

Permalink
devops: remake downloading logic
Browse files Browse the repository at this point in the history
This patch:
- removes `browserType.downloadBrowserIfNeeded()` method. The method
  turned out to be ill-behaving and could not be used as we'd like to.
- adds a `browserType.setExecutablePath` method to set a browser
  exectuable.

Now, all clients of `playwright-core` should take care of downloading
a browser one way or another and setting the downloaded path with the
`browserType.setExecutablePath` method. The playwright object can then
be returned to client.
  • Loading branch information
aslushnikov committed Mar 18, 2020
1 parent 19dd233 commit 9c6ec4f
Show file tree
Hide file tree
Showing 25 changed files with 360 additions and 506 deletions.
4 changes: 1 addition & 3 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@ jobs:
- save_cache:
key: dependency-cache-{{ checksum "package.json" }}
paths:
- ./.local-chromium
- ./.local-firefox
- ./.local-webkit
- ./.local-browsers

- run:
command: |
Expand Down
5 changes: 2 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,10 @@
/test/output-firefox
/test/output-webkit
/test/test-user-data-dir*
/.local-chromium/
/.local-firefox/
/.local-webkit/
/.local-browsers/
/.dev_profile*
.DS_Store
.downloaded-browsers.json
*.swp
*.pyc
.vscode
Expand Down
13 changes: 5 additions & 8 deletions docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -3602,12 +3602,12 @@ const { chromium } = require('playwright'); // Or 'firefox' or 'webkit'.

<!-- GEN:toc -->
- [browserType.connect(options)](#browsertypeconnectoptions)
- [browserType.downloadBrowserIfNeeded([progress])](#browsertypedownloadbrowserifneededprogress)
- [browserType.executablePath()](#browsertypeexecutablepath)
- [browserType.launch([options])](#browsertypelaunchoptions)
- [browserType.launchPersistentContext(userDataDir, [options])](#browsertypelaunchpersistentcontextuserdatadir-options)
- [browserType.launchServer([options])](#browsertypelaunchserveroptions)
- [browserType.name()](#browsertypename)
- [browserType.setExecutablePath(executablePath)](#browsertypesetexecutablepathexecutablepath)
<!-- GEN:stop -->

#### browserType.connect(options)
Expand All @@ -3618,14 +3618,8 @@ const { chromium } = require('playwright'); // Or 'firefox' or 'webkit'.

This methods attaches Playwright to an existing browser instance.

#### browserType.downloadBrowserIfNeeded([progress])
- `progress` <[function]> If download is initiated, this function is called with two parameters: `downloadedBytes` and `totalBytes`.
- returns: <[Promise]> promise that resolves when browser is successfully downloaded.

Download browser binary if it is missing.

#### browserType.executablePath()
- returns: <[string]> A path where Playwright expects to find a bundled browser.
- returns: <[string]> A path where Playwright expects to find a bundled browser executable.

#### browserType.launch([options])
- `options` <[Object]> Set of configurable options to set on the browser. Can have the following fields:
Expand Down Expand Up @@ -3714,6 +3708,9 @@ const { chromium } = require('playwright'); // Or 'webkit' or 'firefox'.

Returns browser name. For example: `'chromium'`, `'webkit'` or `'firefox'`.

#### browserType.setExecutablePath(executablePath)
- `executablePath` <[string]> An executable path that Playwright will use to launch browser.

### class: ChromiumBrowser

* extends: [Browser]
Expand Down
2 changes: 1 addition & 1 deletion docs/troubleshooting.md
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ done only once per host environment:

```bash
# cd to the downloaded instance
cd <project-dir-path>/node_modules/playwright/.local-chromium/linux-<revision>/chrome-linux/
cd <project-dir-path>/node_modules/playwright/.local-browsers/chromium-<revision>/
sudo chown root:root chrome_sandbox
sudo chmod 4755 chrome_sandbox
# copy sandbox executable to a shared location
Expand Down
39 changes: 24 additions & 15 deletions download-browser.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,37 +13,39 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const fs = require('fs');
const browserFetcher = require('./lib/server/browserFetcher.js');
const packageJSON = require('./package.json');

async function downloadBrowser(browserType) {
const browser = browserType.name();
async function downloadBrowserWithProgressBar(downloadPath, browser, version = '') {
let progressBar = null;
let lastDownloadedBytes = 0;
function onProgress(downloadedBytes, totalBytes) {
const revision = packageJSON.playwright[`${browser}_revision`];
function progress(downloadedBytes, totalBytes) {
if (!progressBar) {
const ProgressBar = require('progress');
progressBar = new ProgressBar(`Downloading ${browser} ${browserType._revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, {
progressBar = new ProgressBar(`Downloading ${browser} r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, {
complete: '=',
incomplete: ' ',
width: 20,
total: totalBytes,
host: getFromENV('PLAYWRIGHT_DOWNLOAD_HOST'),
});
}
const delta = downloadedBytes - lastDownloadedBytes;
lastDownloadedBytes = downloadedBytes;
progressBar.tick(delta);
}

const fetcher = browserType._createBrowserFetcher();
const revisionInfo = fetcher.revisionInfo();
// Do nothing if the revision is already downloaded.
if (revisionInfo.local)
return revisionInfo;
await browserType.downloadBrowserIfNeeded(onProgress);
logPolitely(`${browser} downloaded to ${revisionInfo.folderPath}`);
return revisionInfo;
const executablePath = await browserFetcher.downloadBrowser({
downloadPath,
browser,
revision,
progress,
});
logPolitely(`${browser} downloaded to ${downloadPath}`);
return executablePath;
}


function toMegabytes(bytes) {
const mb = bytes / 1024 / 1024;
return `${Math.round(mb * 10) / 10} Mb`;
Expand All @@ -57,4 +59,11 @@ function logPolitely(toBeLogged) {
console.log(toBeLogged);
}

module.exports = {downloadBrowser};
function getFromENV(name) {
let value = process.env[name];
value = value || process.env[`npm_config_${name.toLowerCase()}`];
value = value || process.env[`npm_package_config_${name.toLowerCase()}`];
return value;
}

module.exports = {downloadBrowserWithProgressBar};
14 changes: 11 additions & 3 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,17 @@
*/
const {Playwright} = require('./lib/server/playwright.js');

module.exports = new Playwright({
downloadPath: __dirname,
const playwright = new Playwright({
browsers: ['webkit', 'chromium', 'firefox'],
respectEnvironmentVariables: false,
});

try {
const downloadedBrowsers = require('./.downloaded-browsers.json');
playwright.chromium.setExecutablePath(downloadedBrowsers.crExecutablePath);
playwright.firefox.setExecutablePath(downloadedBrowsers.ffExecutablePath);
playwright.webkit.setExecutablePath(downloadedBrowsers.wkExecutablePath);
} catch (e) {
}

module.exports = playwright;

70 changes: 49 additions & 21 deletions install-from-github.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,41 +24,69 @@ try {
});
} catch (e) {
}
const {downloadBrowser} = require('./download-browser');
const playwright = require('.');

const path = require('path');
const fs = require('fs');
const util = require('util');
const rmAsync = util.promisify(require('rimraf'));
const existsAsync = path => fs.promises.access(path).then(() => true, e => false);
const {downloadBrowserWithProgressBar} = require('./download-browser');
const protocolGenerator = require('./utils/protocol-types-generator');
const packageJSON = require('./package.json');

const DOWNLOADED_BROWSERS_JSON_PATH = path.join(__dirname, '.downloaded-browsers.json');
const DOWNLOAD_PATHS = {
chromium: path.join(__dirname, '.local-browsers', `chromium-${packageJSON.playwright.chromium_revision}`),
firefox: path.join(__dirname, '.local-browsers', `firefox-${packageJSON.playwright.firefox_revision}`),
webkit: path.join(__dirname, '.local-browsers', `webkit-${packageJSON.playwright.webkit_revision}`),
};

(async function() {
const protocolGenerator = require('./utils/protocol-types-generator');
const downloadedBrowsersJSON = await fs.promises.readFile(DOWNLOADED_BROWSERS_JSON_PATH, 'utf8').then(json => JSON.parse(json)).catch(() => ({}));
try {
const chromeRevision = await downloadAndCleanup(playwright.chromium);
await protocolGenerator.generateChromiunProtocol(chromeRevision);
if (!(await existsAsync(DOWNLOAD_PATHS.chromium))) {
const crExecutablePath = await downloadBrowserWithProgressBar(DOWNLOAD_PATHS.chromium, 'chromium');
downloadedBrowsersJSON.crExecutablePath = crExecutablePath;
await fs.promises.writeFile(DOWNLOADED_BROWSERS_JSON_PATH, JSON.stringify(downloadedBrowsersJSON));
await protocolGenerator.generateChromiumProtocol(crExecutablePath);
}
} catch (e) {
console.warn(e.message);
}

try {
const firefoxRevision = await downloadAndCleanup(playwright.firefox);
await protocolGenerator.generateFirefoxProtocol(firefoxRevision);
if (!(await existsAsync(DOWNLOAD_PATHS.firefox))) {
const ffExecutablePath = await downloadBrowserWithProgressBar(DOWNLOAD_PATHS.firefox, 'firefox');
downloadedBrowsersJSON.ffExecutablePath = ffExecutablePath;
await fs.promises.writeFile(DOWNLOADED_BROWSERS_JSON_PATH, JSON.stringify(downloadedBrowsersJSON));
await protocolGenerator.generateFirefoxProtocol(ffExecutablePath);
}
} catch (e) {
console.warn(e.message);
}

try {
const webkitRevision = await downloadAndCleanup(playwright.webkit);
await protocolGenerator.generateWebKitProtocol(webkitRevision);
if (!(await existsAsync(DOWNLOAD_PATHS.webkit))) {
const wkExecutablePath = await downloadBrowserWithProgressBar(DOWNLOAD_PATHS.webkit, 'webkit');
downloadedBrowsersJSON.wkExecutablePath = wkExecutablePath;
await fs.promises.writeFile(DOWNLOADED_BROWSERS_JSON_PATH, JSON.stringify(downloadedBrowsersJSON));
await protocolGenerator.generateWebKitProtocol(path.dirname(wkExecutablePath));
}
} catch (e) {
console.warn(e.message);
}
})();

async function downloadAndCleanup(browserType) {
const revisionInfo = await downloadBrowser(browserType);
// Cleanup stale revisions.
const directories = new Set(await readdirAsync(path.join(__dirname, '.local-browsers')));
directories.delete(DOWNLOAD_PATHS.chromium);
directories.delete(DOWNLOAD_PATHS.firefox);
directories.delete(DOWNLOAD_PATHS.webkit);
// cleanup old browser directories.
directories.add(path.join(__dirname, '.local-chromium'));
directories.add(path.join(__dirname, '.local-firefox'));
directories.add(path.join(__dirname, '.local-webkit'));
await Promise.all([...directories].map(directory => rmAsync(directory)));

// Remove previous revisions.
const fetcher = browserType._createBrowserFetcher();
const localRevisions = await fetcher.localRevisions();
const cleanupOldVersions = localRevisions.filter(revision => revision !== revisionInfo.revision).map(revision => fetcher.remove(revision));
await Promise.all([...cleanupOldVersions]);
async function readdirAsync(dirpath) {
return fs.promises.readdir(dirpath).then(dirs => dirs.map(dir => path.join(dirpath, dir)));
}
})();

return revisionInfo;
}
15 changes: 11 additions & 4 deletions packages/playwright-chromium/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

const path = require('path');
const {Playwright} = require('playwright-core/lib/server/playwright.js');

module.exports = new Playwright({
downloadPath: __dirname,
const playwright = new Playwright({
browsers: ['chromium'],
respectEnvironmentVariables: true,
});
module.exports = playwright;

try {
const downloadedBrowsers = require(path.join(__dirname, '.downloaded-browsers.json'));
playwright.chromium.setExecutablePath(downloadedBrowsers.crExecutablePath);
} catch (e) {
throw new Error('ERROR: Playwright-Chromium did not download browser');
}


10 changes: 7 additions & 3 deletions packages/playwright-chromium/install.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const {downloadBrowser} = require('playwright-core/download-browser');
const playwright = require('.');
downloadBrowser(playwright.chromium);
const path = require('path');
const fs = require('fs');
const {downloadBrowserWithProgressBar} = require('playwright-core/download-browser');
(async function() {
const crExecutablePath = await downloadBrowserWithProgressBar(path.join(__dirname, '.local-browsers', 'chromium'), 'chromium');
await fs.promises.writeFile(path.join(__dirname, '.downloaded-browsers.json'), JSON.stringify({crExecutablePath}));
})();
15 changes: 11 additions & 4 deletions packages/playwright-firefox/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

const path = require('path');
const {Playwright} = require('playwright-core/lib/server/playwright.js');

module.exports = new Playwright({
downloadPath: __dirname,
const playwright = new Playwright({
browsers: ['firefox'],
respectEnvironmentVariables: true,
});
module.exports = playwright;

try {
const downloadedBrowsers = require(path.join(__dirname, '.downloaded-browsers.json'));
playwright.firefox.setExecutablePath(downloadedBrowsers.ffExecutablePath);
} catch (e) {
throw new Error('ERROR: Playwright-Firefox did not download browser');
}


11 changes: 8 additions & 3 deletions packages/playwright-firefox/install.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const {downloadBrowser} = require('playwright-core/download-browser');
const playwright = require('.');
downloadBrowser(playwright.firefox);
const path = require('path');
const fs = require('fs');
const {downloadBrowserWithProgressBar} = require('playwright-core/download-browser');

(async function() {
const ffExecutablePath = await downloadBrowserWithProgressBar(path.join(__dirname, '.local-browsers', 'firefox'), 'firefox');
await fs.promises.writeFile(path.join(__dirname, '.downloaded-browsers.json'), JSON.stringify({ffExecutablePath, }));
})();
15 changes: 11 additions & 4 deletions packages/playwright-webkit/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

const path = require('path');
const {Playwright} = require('playwright-core/lib/server/playwright.js');

module.exports = new Playwright({
downloadPath: __dirname,
const playwright = new Playwright({
browsers: ['webkit'],
respectEnvironmentVariables: true,
});
module.exports = playwright;

try {
const downloadedBrowsers = require(path.join(__dirname, '.downloaded-browsers.json'));
playwright.webkit.setExecutablePath(downloadedBrowsers.wkExecutablePath);
} catch (e) {
throw new Error('ERROR: Playwright-Webkit did not download browser');
}


11 changes: 8 additions & 3 deletions packages/playwright-webkit/install.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const {downloadBrowser} = require('playwright-core/download-browser');
const playwright = require('.');
downloadBrowser(playwright.webkit);
const path = require('path');
const fs = require('fs');
const {downloadBrowserWithProgressBar} = require('playwright-core/download-browser');

(async function() {
const wkExecutablePath = await downloadBrowserWithProgressBar(path.join(__dirname, '.local-browsers', 'webkit'), 'webkit');
await fs.promises.writeFile(path.join(__dirname, '.downloaded-browsers.json'), JSON.stringify({wkExecutablePath, }));
})();
16 changes: 13 additions & 3 deletions packages/playwright/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const path = require('path');
const {Playwright} = require('playwright-core/lib/server/playwright.js');

module.exports = new Playwright({
downloadPath: __dirname,
const playwright = new Playwright({
browsers: ['webkit', 'chromium', 'firefox'],
respectEnvironmentVariables: true,
});
module.exports = playwright;

try {
const downloadedBrowsers = require(path.join(__dirname, '.downloaded-browsers.json'));
playwright.chromium.setExecutablePath(downloadedBrowsers.crExecutablePath);
playwright.firefox.setExecutablePath(downloadedBrowsers.ffExecutablePath);
playwright.webkit.setExecutablePath(downloadedBrowsers.wkExecutablePath);
} catch (e) {
throw new Error('ERROR: Playwright did not download browsers');
}


Loading

0 comments on commit 9c6ec4f

Please sign in to comment.