Skip to content

Commit

Permalink
Alter build process to rely on canaries only
Browse files Browse the repository at this point in the history 
With the react-sdk and js-sdk having their `npm start`s split out (as per matrix-org/matrix-react-sdk#2175 and matrix-org/matrix-js-sdk#742) we can trigger an initial build ourselves and start the watcher afterwards. This canary approach has a very slight speed increase over serially running all the commands as the watcher can be started as early as possible.

This all can be improved and potentially eliminated with a bit more planning, however: #7386
  • Loading branch information
@turt2live
turt2live committed Sep 25, 2018
1 parent 8d7cec2 commit 27c2305
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 80 deletions.
32 changes: 19 additions & 13 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@
"url": "^0.11.0"
},
"devDependencies": {
"async-lock": "^1.1.3",
"autoprefixer": "^6.6.0",
"babel-cli": "^6.5.2",
"babel-core": "^6.14.0",
Expand Down
57 changes: 8 additions & 49 deletions scripts/block-on-sdk-build.js
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,12 @@
const path = require('path');
const chokidar = require('chokidar');
const AsyncLock = require('async-lock');


// This script sits and waits for a build of an underlying SDK (js or react)
// to complete before exiting. This is done by cooperating with build-watch-sdk.js
// by waiting for it's signal to start watching for file changes, then watching
// the SDK's build output for a storm of file changes to stop. Both the js-sdk
// and react-sdk compile each file one by one, so by waiting for file changes to
// stop we know it is safe to continue and therefore exit this script. We give
// some leeway to the SDK's build process to handle larger/more complex files
// through use of a reset-on-touch countdown timer. When a file change occurs,
// we reset the countdown to WAIT_TIME and let it count down. If the count down
// completes, we consider ourselves having left the file system update storm and
// therefore can consider a build of the SDK to be fully completed.
// This script waits for a signal that an underlying SDK (js or react) has finished
// enough of the build process to be safe to rely on. In riot-web's case, this means
// that the underlying SDK has finished an initial build and is getting ready to watch
// for changes. This is done through use of a canary file that is deleted when it is
// safe to continue (see build-watch-sdk.js for why we listen for a delete event).

// Why do we block in the first place? Because if riot-web starts it's initial
// build (via webpack-dev-server) and the react-sdk or js-sdk are halfway through
Expand All @@ -26,21 +19,12 @@ const AsyncLock = require('async-lock');
// build which riot-web is waiting for. When complete, riot-web starts building as
// per normal.

// Why the canary to begin watching? Because we can't reliably determine that the
// build triggered by `npm install` in each SDK is actually the process we need to
// be watching for. To work around this, build-watch-sdk.js does the `npm install`
// and follows through with a canary to signal to this script that it should start
// watching for changes produced by that SDK's `npm start` (run immediately after
// the canary is sent).


const WAIT_TIME = 5000; // ms

function waitForCanary(canaryName) {
return new Promise((resolve, reject) => {
const filename = path.resolve(path.join(".tmp", canaryName));
const filename = path.resolve(path.join(".tmp", canaryName + ".canary"));

// See triggerCanarySignal in build-watch-sdk.js for why we watch for `unlink`
// See build-watch-sdk.js for why we listen for 'unlink' specifically.
const watcher = chokidar.watch(filename).on('unlink', (path) => {
console.log("[block-on-build] Received signal to start watching for builds");
watcher.close();
Expand All @@ -49,39 +33,14 @@ function waitForCanary(canaryName) {
});
}

function waitOnSdkBuild(sdkName) {
// First we wait for a local canary file to be changed
return waitForCanary(sdkName).then(() => new Promise((resolve, reject) => {
const buildDirectory = path.dirname(require.resolve(`matrix-${sdkName}-sdk`));
const lock = new AsyncLock();
let timerId = null;

const watcher = chokidar.watch(buildDirectory).on('all', (event, path) => {
lock.acquire("timer", (done) => {
if (timerId !== null) {
//console.log("Resetting countdown");
clearTimeout(timerId);
}
//console.log(`Waiting ${WAIT_TIME}ms for another file update...`);
timerId = setTimeout(() => {
console.log("[block-on-build] No updates - unblocking");
watcher.close();
resolve();
}, WAIT_TIME);
done();
}, null, null);
});
}));
}

const sdkName = process.argv[2];
if (!sdkName) {
console.error("[block-on-build] No SDK name provided");
process.exit(1);
}

console.log("[block-on-build] Waiting for SDK: " + sdkName);
waitOnSdkBuild(sdkName).then(() => {
waitForCanary(sdkName).then(() => {
console.log("[block-on-build] Unblocked");
process.exit(0);
});
59 changes: 42 additions & 17 deletions scripts/build-watch-sdk.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,15 @@ if (!sdkName) {
}

const sdkPath = path.dirname(require.resolve(`matrix-${sdkName}-sdk/package.json`));
console.log(sdkPath);

// Note: we intentionally create then delete the canary file to work
// around a file watching problem where if the file exists on startup it
// may fire a "created" event for the file. By having the behaviour be "do
// something on delete" we avoid accidentally firing the signal too early.
// We also need to ensure the create and delete events are not too close
// together, otherwise the filesystem may not fire the watcher. Therefore
// we create the canary as early as possible and delete it as late as possible.
prepareCanarySignal(sdkName);

// We only want to build the SDK if it looks like it was `npm link`ed
if (fs.existsSync(path.join(sdkPath, '.git'))) {
Expand All @@ -37,38 +45,55 @@ if (fs.existsSync(path.join(sdkPath, '.git'))) {
});
}

// Send a signal so that the various blocks can unblock. See the top of
// block-on-sdk-build.js for more information on how this is used.
// Prepare an initial build of the SDK
child_process.execSync("npm run start:init", {
env: process.env,
cwd: sdkPath,
});

// Send a signal to unblock the build for other processes. Used by block-on-sdk-build.js
console.log("Sending signal that other processes may unblock");
triggerCanarySignal(sdkName);

// Actually start the watcher process for the sdk. This is what block-on-sdk-build.js
// is going to monitor.
// Actually start the watcher process for the SDK (without an initial build)
console.log("Performing task: " + task);
child_process.execSync(`npm ${task === "build" ? "run build" : "start"}`, {
const watchTask = sdkName === 'js' ? "start:watch" : "start:all";
const buildTask = "build";
child_process.execSync(`npm run ${task === "build" ? buildTask : watchTask}`, {
env: process.env,
cwd: sdkPath,
});
}
} else triggerCanarySignal(sdkName);

function triggerCanarySignal(sdkName) {
const tmpPath = path.resolve(".tmp");
fs.unlinkSync(getCanaryPath(sdkName));
}

function prepareCanarySignal(sdkName) {
const canaryPath = getCanaryPath(sdkName);
const canaryDir = path.dirname(canaryPath);

try {
fs.mkdirSync(tmpPath);
console.log("Creating canary temp path...");
fs.mkdirSync(canaryDir);
} catch (e) {
if (e.code !== 'EEXIST') {
console.error(e);
throw "Failed to create temporary directory";
}
}

// Note: we intentionally create then delete the file to work around
// a file watching problem where if the file exists on startup it may
// fire a "created" event for the file. By having the behaviour be "do
// something on delete" we avoid accidentally firing the signal too
// early.
const canaryPath = path.join(tmpPath, sdkName);
fs.closeSync(fs.openSync(canaryPath, 'w'));
fs.unlinkSync(canaryPath);
try {
console.log("Creating canary file: " + canaryPath);
fs.closeSync(fs.openSync(canaryPath, 'w'));
} catch (e) {
if (e.code !== 'EEXIST') {
console.error(e);
throw "Failed to create canary file";
}
}
}

function getCanaryPath(sdkName) {
return path.join(path.resolve(".tmp"), sdkName + ".canary");
}

0 comments on commit 27c2305

Please sign in to comment.