Merge pull request #5 from Polymer/dependency-stream

adds smarter dependency stream via new analyzer architecture

src/bundle.ts

@@ -15,7 +15,7 @@ import {posix as posixPath} from 'path';
 import {Transform} from 'stream';
 import File = require('vinyl');
 import * as logging from 'plylog';
-import urlFromPath from './url-from-path';
+import {urlFromPath} from './path-transformers';
 import {StreamAnalyzer, DepsIndex} from './analyzer';
 import {compose} from './streams';
 
@@ -254,7 +254,7 @@ export class Bundler extends Transform {
   }
 
   _getBundles() {
-    return this.analyzer.analyze.then((indexes) => {
+    return this.analyzer.analyzeDependencies.then((indexes) => {
       let depsToEntrypoints = indexes.depsToFragments;
       let fragmentToDeps = indexes.fragmentToDeps;
       let bundles = new Map<string, string[]>();

src/get-dependencies-from-document.ts

@@ -0,0 +1,71 @@
+/**
+ * @license
+ * Copyright (c) 2016 The Polymer Project Authors. All rights reserved.
+ * This code may only be used under the BSD style license found at http://polymer.github.io/LICENSE.txt
+ * The complete set of authors may be found at http://polymer.github.io/AUTHORS.txt
+ * The complete set of contributors may be found at http://polymer.github.io/CONTRIBUTORS.txt
+ * Code distributed by Google as part of the polymer project is also
+ * subject to an additional IP rights grant found at http://polymer.github.io/PATENTS.txt
+ */
+
+import {DocumentDescriptor} from 'hydrolysis';
+import {posix as posixPath} from 'path';
+import {Node, queryAll, predicates, getAttribute} from 'dom5';
+
+export interface DocumentDeps {
+  imports?: Array<string>;
+  scripts?: Array<string>;
+  styles?: Array<string>;
+}
+
+function collectScriptsAndStyles(tree: DocumentDescriptor): DocumentDeps {
+  let scripts: string[] = [];
+  let styles: string[] = [];
+  tree.html.script.forEach((script: Node) => {
+    // TODO(justinfagnani): stop patching Nodes in Hydrolysis
+    let __hydrolysisInlined = (<any>script).__hydrolysisInlined;
+    if (__hydrolysisInlined) {
+      scripts.push(__hydrolysisInlined);
+    }
+  });
+  tree.html.style.forEach((style: Node) => {
+    let href = getAttribute(style, 'href');
+    if (href) {
+      styles.push(href);
+    }
+  });
+  return {
+    scripts,
+    styles
+  };
+}
+
+export function getDependenciesFromDocument(descriptor: DocumentDescriptor, dir: string): DocumentDeps {
+  let allHtmlDeps: string[] = [];
+  let allScriptDeps = new Set<string>();
+  let allStyleDeps = new Set<string>();
+
+  let deps: DocumentDeps = collectScriptsAndStyles(descriptor);
+  deps.scripts.forEach((s) => allScriptDeps.add(posixPath.join(dir, s)));
+  deps.styles.forEach((s) => allStyleDeps.add(posixPath.join(dir, s)));
+  if (descriptor.imports) {
+    let queue = descriptor.imports.slice();
+    let next;
+    while (next = queue.shift()) {
+      if (!next.href) {
+        continue;
+      }
+      allHtmlDeps.push(next.href);
+      let childDeps = getDependenciesFromDocument(next, posixPath.dirname(next.href));
+      allHtmlDeps = allHtmlDeps.concat(childDeps.imports);
+      childDeps.scripts.forEach((s) => allScriptDeps.add(s));
+      childDeps.styles.forEach((s) => allStyleDeps.add(s));
+    }
+  }
+
+  return {
+    scripts: Array.from(allScriptDeps),
+    styles: Array.from(allStyleDeps),
+    imports: allHtmlDeps,
+  };
+}

src/url-from-path.ts → src/path-transformers.ts

@@ -38,7 +38,7 @@
 
 import * as path from 'path';
 
-export default function urlFromPath(root: string, filepath: string) {
+export function urlFromPath(root: string, filepath: string) {
   if (!filepath.startsWith(root)) {
     throw new Error(`file path is not in root: ${filepath} (${root})`);
   }
@@ -54,3 +54,18 @@ export default function urlFromPath(root: string, filepath: string) {
   // Otherwise, just return the relative path between the two
   return path.relative(root, filepath);
 }
+
+export function pathFromUrl(root: string, url: string) {
+  let isPlatformWin = /^win/.test(process.platform);
+  let filepath: string;
+
+  // On windows systems, convert URL to filesystem path by replacing slashes
+  if (isPlatformWin) {
+    filepath = url.replace(/\//g, '\\');
+  } else {
+    filepath = url;
+  }
+
+  // Otherwise, just return the relative path between the two
+  return path.join(root, url);
+}

src/polymer-project.ts

@@ -12,14 +12,15 @@ import * as dom5 from 'dom5';
 import {posix as posixPath} from 'path';
 import * as osPath from 'path';
 import * as logging from 'plylog';
-import {Transform, Readable} from 'stream';
+import {Transform} from 'stream';
 import File = require('vinyl');
 import * as vfs from 'vinyl-fs';
-
 import {StreamAnalyzer} from './analyzer';
 import {Bundler} from './bundle';
 import {optimize, OptimizeOptions} from './optimize';
 import {FileCB} from './streams';
+import {forkStream} from './fork-stream';
+const mergeStream = require('merge-stream');
 
 const logger = logging.getLogger('polymer-project');
 const pred = dom5.predicates;
@@ -62,20 +63,17 @@ export interface ProjectOptions {
   sourceGlobs?: string[];
 
   /**
-   * List of glob patterns, relative to root, of dependencies to read from the
-   * file system. For example node_modules\/**\/* and bower_components\/**\/*
+   * List of file paths, relative to the project directory, that should be included
+   * as dependencies in the build target.
    */
-  dependencyGlobs?: string[];
+  includeDependencies?: string[];
 }
 
 export const defaultSourceGlobs = [
-  '**/*',
-  '!build/**/*',
-];
-
-export const defaultDependencyGlobs = [
-  'bower_components/**/*',
-  'node_modules/**/*',
+  'src/**/*',
+  // NOTE(fks) 06-29-2016: `polymer-cli serve` uses a bower.json file to display
+  // information about the project. The file is included here by default.
+  'bower.json',
 ];
 
 function resolveGlob(fromPath: string, glob: string) {
@@ -105,7 +103,7 @@ export class PolymerProject {
   shell: string;
   fragments: string[];
   sourceGlobs: string[];
-  dependencyGlobs: string[];
+  includeDependencies: string[];
 
   _splitFiles: Map<string, SplitFile> = new Map();
   _parts: Map<string, SplitFile> = new Map();
@@ -119,15 +117,16 @@ export class PolymerProject {
     this.fragments = (options.fragments || [])
         .map((f) => osPath.resolve(this.root, f));
     this.sourceGlobs = (options.sourceGlobs || defaultSourceGlobs)
-        .map((g) => resolveGlob(this.root, g));
-    this.dependencyGlobs = (options.dependencyGlobs || defaultDependencyGlobs)
-        .map((g) => resolveGlob(this.root, g));
+        .map((glob) => resolveGlob(this.root, glob));
+    this.includeDependencies = (options.includeDependencies || [])
+        .map((path) => osPath.resolve(this.root, path));
 
     this._analyzer = new StreamAnalyzer(
       this.root,
       this.entrypoint,
       this.shell,
-      this.fragments);
+      this.fragments,
+      this.allSourceGlobs);
 
     this._bundler = new Bundler(
       this.root,
@@ -141,12 +140,12 @@ export class PolymerProject {
     logger.debug(`entrypoint: ${this.entrypoint}`);
     logger.debug(`fragments: ${this.entrypoint}`);
     logger.debug(`sources: ${this.sourceGlobs}`);
-    logger.debug(`dependencies: \n\t${this.dependencyGlobs}`);
+    logger.debug(`includeDependencies: ${this.includeDependencies}`);
   }
 
   /**
    * An array of globs composed of `entrypoint`, `shell`, `fragments`,
-   * `sourceGlobs`, and the inverted array of `dependencyGlobs`.
+   * and `sourceGlobs`.
    */
   get allSourceGlobs(): string[] {
     let globs: string[] = [];
@@ -158,11 +157,6 @@ export class PolymerProject {
     if (this.sourceGlobs && this.sourceGlobs.length > 0) {
       globs = globs.concat(this.sourceGlobs);
     }
-    if (this.dependencyGlobs && this.dependencyGlobs.length > 0) {
-      let excludes = this.dependencyGlobs.map((g) => invertGlob(g));
-      logger.debug(`excludes: \n\t${excludes.join('\n\t')}`);
-      globs = globs.concat(excludes);
-    }
     logger.debug(`sourceGlobs: \n\t${globs.join('\n\t')}`);
     return globs;
   }
@@ -181,17 +175,25 @@ export class PolymerProject {
     });
   }
 
-  // TODO(justinfagnani): add options, pass to vfs.src()
   dependencies(): NodeJS.ReadableStream {
-    let deps = this.dependencyGlobs;
-    return vfs.src(deps, {
-      allowEmpty: true,
-      cwdbase: true,
-      nodir: true,
-    });
-  }
+    let dependenciesStream: NodeJS.ReadableStream = forkStream(
+      this._analyzer.dependencies
+    );
+
+    // If we need to include additional dependencies, create a new vfs.src
+    // stream and pipe our default dependencyStream through it to combine.
+    if (this.includeDependencies.length > 0) {
+      let includeStream = vfs.src(this.includeDependencies, {
+         allowEmpty: true,
+         cwdbase: true,
+         nodir: true,
+         passthrough: true,
+      });
+      dependenciesStream = dependenciesStream.pipe(includeStream);
+    }
 
-  // TODO(justinfagnani): add allFiles()
+    return dependenciesStream;
+  }
 
   /**
    * Returns a new `Transform` that splits inline script into separate files.
@@ -359,6 +361,7 @@ class HtmlSplitter extends Transform {
   }
 }
 
+
 /**
  * Joins HTML files split by `Splitter`.
  */

src/prefetch.ts

@@ -116,7 +116,7 @@ export class PrefetchTransform extends Transform {
     if (this.fileMap.size === 0) {
       return done();
     }
-    this.analyzer.analyze.then((depsIndex: DepsIndex) => {
+    this.analyzer.analyzeDependencies.then((depsIndex: DepsIndex) => {
       let fragmentToDeps = new Map(depsIndex.fragmentToDeps);
 
       if (this.entrypoint && this.shell) {

src/streams.ts

@@ -10,6 +10,7 @@
 
 import {PassThrough, Readable, Transform} from 'stream';
 import File = require('vinyl');
+import * as fs from 'fs';
 
 const multipipe = require('multipipe');
 
@@ -42,3 +43,32 @@ export function compose(streams: NodeJS.ReadWriteStream[]) {
     return new PassThrough({objectMode: true});
   }
 }
+
+/**
+ * A stream that takes file path strings, and outputs full Vinyl file objects
+ * for the file at each location.
+ */
+export class VinylReaderTransform extends Transform {
+
+  constructor() {
+    super({ objectMode: true });
+  }
+
+  _transform(
+    filePath: string,
+    encoding: string,
+    callback: (error?: Error, data?: File) => void
+  ): void {
+    fs.readFile(filePath, (err?: Error, data?: Buffer) => {
+      if (err) {
+        callback(err);
+        return;
+      }
+      callback(null, new File({
+        path: filePath,
+        contents: data
+      }));
+    });
+  }
+
+}