use fyn to bootstrap packages

.travis.yml

@@ -8,6 +8,7 @@ addons:
 notifications:
   email:
     - XChen@walmartlabs.com
+install: eval `fyn bash` && fyn --pg none install
+script: eval `fyn bash` && npm test
 before_install:
-  - if [[ `npm -v` != 3* ]]; then npm i -g npm@3; npm --version; fi
   - npm install -g xclap-cli fyn

CONTRIBUTING.md

@@ -6,12 +6,12 @@ There are [few guidelines](#contributing-guidelines) that we request contributor
 
 ## Getting Started
 
-This repo uses [Lerna] as a top level setup.
+This repo uses [Lerna] as a top level setup and [fyn] to manage Node Modules.
 
--   Install the `clap` command globally if you don't want to invoke from `node_modules/.bin`.
+-   Install these CLI tools globally: [xclap-cli] and [fyn]
 
 ```bash
-$ npm install -g xclap-cli
+$ npm install -g xclap-cli fyn
 ```
 
 -   Fork and clone the repo at <https://github.com/electrode-io/electrode.git>
@@ -56,7 +56,7 @@ We love PRs and appreciate any help you can offer.  Please follow the guidelines
 
 #### Styling
 
-We've now switched to use [prettier] to format all our code.  
+We've now switched to use [prettier] to format all our code.
 
 Our [prettier] settings are: `--print-width 100`
 
@@ -127,3 +127,7 @@ Here is the documentation on a [gitbook] structure: <https://toolchain.gitbook.c
 [lerna]: https://lernajs.io/
 
 [gitbook]: https://www.gitbook.com
+
+[xclap-cli]: https://www.npmjs.com/package/xclap-cli
+
+[fyn]: https://www.npmjs.com/package/fyn

package.json

@@ -4,23 +4,25 @@
   "description": "Electrode Platform for NodeJS/React Universal Application",
   "homepage": "http://www.electrode.io",
   "scripts": {
-    "test": "npm run bootstrap && lerna run test --ignore=electrode-webpack-reporter && npm run test-reporter && clap build-test",
-    "test-generator": "clap test-generator",
-    "test-boilerplate": "clap test-boilerplate",
-    "bootstrap": "lerna bootstrap --ignore=electrode-webpack-reporter",
+    "test": "eval `fyn bash` && npm run bootstrap && lerna run test --ignore=electrode-webpack-reporter && npm run test-reporter && clap build-test",
+    "test-generator": "eval `fyn bash` && clap test-generator",
+    "test-boilerplate": "eval `fyn bash` && clap test-boilerplate",
+    "bootstrap": "eval `fyn bash` && node tools/fynpo-bootstrap",
     "clean": "npm run nuke && npm run nuke-packages && npm run nuke-samples",
     "nuke": "rm -rf node_modules tmp lerna-debug.log npm-debug.log",
     "nuke-packages": "rm -rf packages/*/node_modules packages/*/coverage",
     "nuke-samples": "rm -rf samples/*/node_modules samples/*/coverage samples/*/dist samples/*/.isomorphic-loader-config.json samples/*/.etmp",
-    "test-reporter": "if lerna updated | grep electrode-webpack-reporter; then cd packages/electrode-webpack-reporter && npm install && npm test; fi",
-    "update-changelog": "node tools/update-changelog.js",
-    "gitbook-serve": "gitbook serve --no-watch --no-live"
+    "test-reporter": "eval `fyn bash` && if lerna updated | grep electrode-webpack-reporter; then cd packages/electrode-webpack-reporter && fyn install && npm test; fi",
+    "update-changelog": "eval `fyn bash` && node tools/update-changelog.js",
+    "gitbook-serve": "eval `fyn bash` && gitbook serve --no-watch --no-live"
   },
   "devDependencies": {
     "bluebird": "^3.5.0",
+    "item-queue": "^0.1.0",
     "lerna": "^2.0.0",
     "lodash": "^4.17.2",
     "semver": "^5.3.0",
+    "visual-logger": "^0.1.3",
     "xclap": "^0.2.18",
     "xsh": "^0.4.0",
     "yeoman-test": "^1.6.0"

tools/fynpo-bootstrap.js

@@ -0,0 +1,201 @@
+"use strict";
+
+const Promise = require("bluebird");
+
+const Fs = require("fs");
+const Path = require("path");
+const _ = require("lodash");
+const pkgPath = Path.resolve("packages");
+const ItemQueue = require("item-queue");
+const xsh = require("xsh");
+
+const packages = Fs.readdirSync(pkgPath).reduce((acc, k) => {
+  try {
+    const path = Path.join(pkgPath, k);
+    const pkgFile = Path.join(path, "package.json");
+    const pkgStr = Fs.readFileSync(pkgFile);
+    const pkgJson = JSON.parse(pkgStr);
+    acc[pkgJson.name] = Object.assign(
+      _.pick(pkgJson, [
+        "name",
+        "version",
+        "dependencies",
+        "devDependencies",
+        "optionalDependencies",
+        "peerDependencies"
+      ]),
+      {
+        localDeps: [],
+        dependents: [],
+        indirectDeps: [],
+        path,
+        pkgFile,
+        pkgStr,
+        pkgJson,
+        installed: false
+      }
+    );
+  } catch (e) {}
+  return acc;
+}, {});
+
+const circulars = [];
+
+function listDeps() {
+  const add = (name, deps) => {
+    const depPkg = packages[name];
+
+    _.each(deps, (semver, depName) => {
+      if (!packages.hasOwnProperty(depName)) return;
+      depPkg.localDeps.push(depName);
+      packages[depName].dependents.push(name);
+    });
+  };
+
+  _.each(packages, (pkg, name) => {
+    add(name, pkg.dependencies);
+    add(name, pkg.devDependencies);
+    add(name, pkg.optionalDependencies);
+  });
+}
+
+function listIndirectDeps() {
+  let change = 0;
+
+  const add = (info, deps) => {
+    _.each(deps, dep => {
+      const depPkg = packages[dep];
+      if (info.localDeps.indexOf(dep) < 0 && info.indirectDeps.indexOf(dep) < 0) {
+        change++;
+        info.indirectDeps.push(dep);
+        depPkg.dependents.push(info.name);
+      }
+      if (depPkg.localDeps.indexOf(info.name) >= 0) {
+        const x = [info.name, depPkg.name].sort().join(",");
+        if (circulars.indexOf(x) < 0) {
+          circulars.push(x);
+        }
+        return;
+      }
+      add(info, depPkg.localDeps.concat(depPkg.indirectDeps));
+    });
+  };
+
+  _.each(packages, (pkg, name) => {
+    add(pkg, pkg.localDeps.concat(pkg.indirectDeps));
+  });
+
+  if (change > 0) {
+    listIndirectDeps();
+  }
+}
+
+listDeps();
+listIndirectDeps();
+
+const depMap = _.mapValues(packages, pkg => {
+  return _.pick(pkg, ["name", "localDeps", "indirectDeps", "dependents"]);
+});
+
+const uniqCirculars = _.uniq(circulars).map(x => x.split(","));
+const ignores = _.map(uniqCirculars, pair => {
+  const depA = packages[pair[0]].dependents.length;
+  const depB = packages[pair[1]].dependents.length;
+  return depA > depB ? pair[1] : pair[0];
+});
+
+packages["electrode-webpack-reporter"].ignore = true;
+
+function install(pkg, queue) {
+  if (pkg.ignore) return true;
+  if (pkg.installed === "pending") return false;
+  if (pkg.installed) return true;
+  let pending = 0;
+  _.each(pkg.localDeps, depName => {
+    if (!install(packages[depName], queue)) pending++;
+  });
+
+  if (pending === 0 && !pkg.installed) {
+    queue.push(pkg);
+    pkg.installed = "pending";
+  }
+  return false;
+}
+
+const VisualExec = require("./visual-exec");
+
+function updatePkgToLocal(pkg) {
+  if (pkg.ignore) return;
+  const json = pkg.pkgJson;
+  if (!json) return;
+  ["dependencies", "devDependencies", "optionalDependencies"].forEach(sec => {
+    const deps = json[sec];
+    if (!deps) return;
+    _.each(pkg.localDeps, depName => {
+      if (!packages[depName].ignore && deps.hasOwnProperty(depName))
+        deps[depName] = `../${depName}`;
+    });
+  });
+  Fs.writeFileSync(pkg.pkgFile, `${JSON.stringify(json, null, 2)}\n`);
+}
+
+function restorePkgJson() {
+  _.each(packages, pkg => {
+    if (!pkg.ignore) Fs.writeFileSync(pkg.pkgFile, pkg.pkgStr);
+  });
+}
+
+_.each(packages, updatePkgToLocal);
+
+const itemQ = new ItemQueue({
+  processItem: item => {
+    const command = [`eval "$(fyn bash)"`, `fyn -q i install`];
+    if (_.get(item, "pkgJson.scripts.prepublish")) command.push("npm run prepublish");
+    if (_.get(item, "pkgJson.scripts.prepare")) command.push("npm run prepare");
+    return new VisualExec({
+      title: `bootstrap ${item.name}`,
+      cwd: item.path,
+      command: command.join(" && ")
+    }).execute();
+  },
+  concurrency: 3,
+  stopOnError: false,
+  Promise: require("bluebird")
+});
+
+function getMoreInstall(item) {
+  if (item) item.installed = true;
+
+  const queue = [];
+
+  _.each(packages, (pkg, name) => {
+    install(pkg, queue);
+  });
+
+  queue.forEach(x => itemQ.addItem(x, true));
+}
+
+itemQ.on("doneItem", data => getMoreInstall(data.item));
+
+let failed = 0;
+itemQ.on("done", restorePkgJson);
+itemQ.on("failItem", () => {
+  failed = 1;
+});
+itemQ.on("fail", () => {
+  failed = 1;
+  restorePkgJson();
+});
+
+getMoreInstall();
+itemQ.resume();
+
+itemQ
+  .wait()
+  .then(() => {
+    process.exit(failed);
+  })
+  .catch(err => {
+    console.log(err);
+    process.exit(1);
+  });

tools/logger.js

@@ -0,0 +1,9 @@
+"use strict";
+
+const VisualLogger = require("visual-logger");
+
+if (process.env.CI) {
+  console.log("CI env detected");
+}
+
+module.exports = new VisualLogger().setItemType(process.env.CI ? "none" : "normal");