Meta: better compression during preview uploads (#2340)

package.json

@@ -32,6 +32,7 @@
     "@alrra/travis-scripts": "^2.1.0",
     "glob": "^7.1.6",
     "jsdom": "^15.0.0",
+    "tar-stream": "^2.2.0",
     "tiny-json-http": "^7.1.2"
   }
 }

scripts/publish-preview.js

@@ -4,7 +4,9 @@ const { join } = require('path');
 const glob = require('glob').sync;
 const tiny = require('tiny-json-http');
 const fs = require('fs');
-const { gzipSync } = require('zlib');
+const zlib = require('zlib');
+const tar = require('tar-stream');
+
 
 async function go() {
 	const { TRAVIS_PULL_REQUEST, TRAVIS_PULL_REQUEST_SHA } = process.env;
@@ -19,21 +21,14 @@ async function go() {
 
 	console.log(`Publishing preview build of PR ${TRAVIS_PULL_REQUEST} (SHA ${TRAVIS_PULL_REQUEST_SHA})`);
 
+	const compressed = await compress(files, dir);
+	console.log(`Packed to ${compressed.length / 1000}kB`);
+
 	const data = {
 		pr: TRAVIS_PULL_REQUEST,
 		sha: TRAVIS_PULL_REQUEST_SHA,
-		files: [],
+		compressed,
 	};
-	for (const file of files) {
-		const filename = file.replace(dir, '').slice(1);
-		const contents = fs.readFileSync(file);
-		const body = gzipSync(contents).toString('base64');
-		console.log(`Packaging: ${filename} (${body.length / 1000}kB)`);
-		data.files.push({
-			filename: filename,
-			body: body,
-		});
-	}
 
 	const url = 'https://ci.tc39.es/preview/tc39/ecma262';
 
@@ -43,9 +38,56 @@ async function go() {
 		throw Error('Payloads must be under 6MB');
 	}
 
-	await tiny.post({ url: url, data: data });
+	await tiny.post({ url, data });
 	console.log('Sent to preview!')
 }
+
+async function compress(files, basedir) {
+	return new Promise((resolve, reject) => {
+		files = [...files];
+		const pack = tar.pack();
+
+		const compressStream = zlib.createBrotliCompress({
+			params: {
+				[zlib.constants.BROTLI_PARAM_QUALITY]: 5, // 30% larger and 100x faster than the default (11)
+			}
+		});
+
+		pack.on('error', reject);
+
+		function packEntry(err) {
+			if (err) {
+				reject(err);
+				return;
+			}
+			if (files.length === 0) {
+				pack.finalize();
+				return;
+			}
+
+			const file = files.pop();
+			const name = file.replace(basedir, '').slice(1);
+			const size = fs.statSync(file).size;
+			console.log(`Packaging: ${name} (${size / 1000}kB)`);
+
+			const entry = pack.entry({ name, size }, packEntry);
+
+			const readStream = fs.createReadStream(file);
+
+			readStream.on('error', reject);
+			readStream.pipe(entry);
+		}
+		packEntry();
+
+		const stream = pack.pipe(compressStream);
+		const chunks = [];
+		stream.on('data', (chunk) => chunks.push(chunk));
+		stream.on('error', reject);
+		stream.on('end', () => resolve(Buffer.concat(chunks).toString('base64')));
+	});
+}
+
+
 go().catch((err) => {
 	console.error(err);
 	process.exitCode = 1;