diff --git a/packages/pundle-job-transformer-js-dedupe/package.json b/packages/pundle-job-transformer-js-dedupe/package.json index 21a451d1..4d4e8f66 100644 --- a/packages/pundle-job-transformer-js-dedupe/package.json +++ b/packages/pundle-job-transformer-js-dedupe/package.json @@ -10,8 +10,12 @@ "flow:main": "src/index.js", "scripts": {}, "dependencies": { + "escape-regexp": "^0.0.1", + "lodash": "^4.17.10", "p-map": "^1.2.0", - "sb-fs": "^3.0.0" + "sb-fs": "^3.0.0", + "sb-memoize": "^2.1.0", + "semver": "^5.5.1" }, "peerDependencies": { "pundle-api": "3.0.0-beta8" diff --git a/packages/pundle-job-transformer-js-dedupe/src/index.js b/packages/pundle-job-transformer-js-dedupe/src/index.js index 74ceabbe..ed5888aa 100644 --- a/packages/pundle-job-transformer-js-dedupe/src/index.js +++ b/packages/pundle-job-transformer-js-dedupe/src/index.js @@ -1,18 +1,22 @@ // @flow -import { createJobTransformer } from 'pundle-api' -import { name, version } from '../package.json' +import path from 'path' +import escapeRegexp from 'escape-regexp' +import cloneDeep from 'lodash/cloneDeep' +import { createJobTransformer, getUniqueHash } from 'pundle-api' +import { name, version } from '../package.json' import ManifestRegistry from './manifest-registry' function createComponent() { + const DEBUG = process.env.PUNDLE_DEBUG_DEDUPE === '1' const manifestRegistry = new ManifestRegistry() return createJobTransformer({ name, version, - async callback({ context, worker, job: jobOrig }) { - const job = jobOrig.clone() + async callback({ job: jobOrig }) { + const job: typeof jobOrig = jobOrig.clone() const directories = new Set() job.files.forEach(({ meta }) => { @@ -22,8 +26,93 @@ function createComponent() { }) await manifestRegistry.load(Array.from(directories)) - console.log('dedupe transformer') - return null + + job.files.forEach((file, fileKey) => { + const { meta } = file + if (!meta.directory) return + + const fileManifest = manifestRegistry.getManifestForPath(path.dirname(file.filePath)) + if (!fileManifest) { + if (DEBUG) { + console.log(`No manifest found for '${file.filePath}'. Was this file loaded from a resolver alias?`) + } + return + } + + const newFile: typeof file = cloneDeep(file) + job.files.set(fileKey, newFile) + + newFile.imports.forEach((fileImport, index) => { + const importManifest = manifestRegistry.getManifestForPath( + fileImport.meta.directory || path.dirname(fileImport.filePath), + ) + if (!importManifest) { + if (DEBUG) { + console.log( + `No manifest found for import '${fileImport.filePath}' from '${ + newFile.filePath + }'. Was this file loaded from a resolver alias?`, + ) + } + return + } + + if (importManifest === fileManifest) { + return + } + + const fileImportSemver = fileManifest.dependencies[importManifest.name] + + if (!fileImportSemver) { + if (DEBUG) { + console.log( + `Ignoring import '${fileImport.filePath}' because it was not found in '${newFile.filePath}' dependencies`, + ) + } + + return + } + const latestManifest = manifestRegistry.getPackageMatchingSemver(importManifest.name, fileImportSemver) + if (!latestManifest) { + if (DEBUG) { + console.log( + `Unable to get semver-compatible alternatives for '${importManifest.directory}' in '${newFile.filePath}'`, + ) + } + return + } + if (latestManifest === importManifest) { + return + } + + if (DEBUG) { + console.log( + `Using '${latestManifest.directory}' instead of '${importManifest.directory}' for '${file.filePath}'`, + ) + } + + const updatedImport = { + format: fileImport.format, + filePath: fileImport.filePath.replace(importManifest.directory, latestManifest.directory), + meta: { + ...fileImport.meta, + directory: latestManifest.directory, + }, + } + newFile.imports[index] = updatedImport + + if (typeof newFile.contents === 'string') { + newFile.contents = newFile.contents.replace( + new RegExp(escapeRegexp(getUniqueHash(fileImport)), 'g'), + getUniqueHash(updatedImport), + ) + } + }) + }) + + return { + job, + } }, }) } diff --git a/packages/pundle-job-transformer-js-dedupe/src/manifest-registry.js b/packages/pundle-job-transformer-js-dedupe/src/manifest-registry.js index 721e27b7..e8aebced 100644 --- a/packages/pundle-job-transformer-js-dedupe/src/manifest-registry.js +++ b/packages/pundle-job-transformer-js-dedupe/src/manifest-registry.js @@ -3,10 +3,51 @@ import fs from 'sb-fs' import path from 'path' import pMap from 'p-map' +import semver from 'semver' +import memoize from 'sb-memoize' +type Package = {| + name: string, + version: string, + directory: string, + lastModified: number, + dependencies: Object, +|} export default class ManifestRegistry { - cache: Map = new Map() + cache: Map = new Map() sortedKeys: Array = [] + + updateCache() { + this.sortedKeys = Array.from(this.cache.keys()).sort((a, b) => b.length - a.length) + this.getManifestForPath.clearCache() + this.getPackageMatchingSemver.clearCache() + } + + getManifestForPath: string => ?Package = memoize((filePath: string) => { + const rootDirectory = this.sortedKeys.find(item => filePath.startsWith(item)) + if (rootDirectory) { + // Avoid node_modules with no rootDirectory to be associated with root level manifest + if (filePath.includes('node_modules') && !rootDirectory.includes('node_modules')) { + return null + } + return this.cache.get(rootDirectory) + } + return null + }) + + getPackageMatchingSemver: (string, string) => ?Package = memoize((name: string, requestedVersion: string) => { + const matching = [] + this.cache.forEach(item => { + if (item.name === name && item.version && semver.satisfies(item.version, requestedVersion)) { + matching.push(item) + } + }) + if (matching.length <= 1) { + return null + } + return matching.sort((a, b) => semver.compare(b.version, a.version))[0] + }) + async load(directories: Array) { let needsSorting = false @@ -38,19 +79,21 @@ export default class ManifestRegistry { throw new Error(`Malformed JSON file found at '${manifestPath}'`) } - if (!manifest.name || !manifest.version) { + if (!manifest) { if (oldValue) { needsSorting = true this.cache.delete(directory) } return } + needsSorting = true this.cache.set(directory, { - name: manifest.name, + name: manifest.name || path.basename(directory), version: manifest.version, + directory, lastModified, - manifest, + dependencies: { ...manifest.devDependencies, ...manifest.peerDependencies, ...manifest.dependencies }, }) } }, @@ -58,7 +101,7 @@ export default class ManifestRegistry { ) if (needsSorting) { - this.sortedKeys = Array.from(this.cache.keys()).sort((a, b) => b.length - a.length) + this.updateCache() } } } diff --git a/packages/pundle-resolver-default/src/index.js b/packages/pundle-resolver-default/src/index.js index 2b9ef822..9d2a3110 100644 --- a/packages/pundle-resolver-default/src/index.js +++ b/packages/pundle-resolver-default/src/index.js @@ -30,6 +30,8 @@ function createComponent({ throw new Error("options.mainFields must not include 'browser'") } + const DEBUG = process.env.PUNDLE_DEBUG_RESOLVER === '1' + return createFileResolver({ name: manifest.name, version: manifest.version, @@ -78,7 +80,7 @@ function createComponent({ if (!response) return null - if (process.env.PUNDLE_DEBUG_RESOLVER === '1') { + if (DEBUG) { console.log('From', requestFile, 'processed', request, 'to', response) } diff --git a/packages/pundle-transformer-babel/src/index.js b/packages/pundle-transformer-babel/src/index.js index 4f21c472..fcfe533a 100644 --- a/packages/pundle-transformer-babel/src/index.js +++ b/packages/pundle-transformer-babel/src/index.js @@ -9,7 +9,7 @@ const ALLOWED_VERSIONS = [6, 7] function createComponent({ exclude: givenExclude, - ignoreInNodeModules = false, + ignoreInNodeModules = true, ignoreOutsideProjectRoot = true, options = {}, version, diff --git a/yarn.lock b/yarn.lock index 951704b2..4ecb9d80 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2773,6 +2773,10 @@ escape-html@~1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" +escape-regexp@^0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/escape-regexp/-/escape-regexp-0.0.1.tgz#f44bda12d45bbdf9cb7f862ee7e4827b3dd32254" + escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" @@ -6374,6 +6378,10 @@ sb-memoize@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/sb-memoize/-/sb-memoize-1.0.2.tgz#128375c62ddb9cc4ffa905d0c5a597c19baead8e" +sb-memoize@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/sb-memoize/-/sb-memoize-2.1.0.tgz#f68d92a9a7afd95c76403d316002e91008b2ea2f" + sb-npm-path@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/sb-npm-path/-/sb-npm-path-2.0.0.tgz#0f6cc2cf371debca7d936ed76b7d4c3cc1eb3d58" @@ -6404,7 +6412,7 @@ scss-tokenizer@^0.2.3: version "5.5.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" -"semver@2.x || 3.x || 4 || 5": +"semver@2.x || 3.x || 4 || 5", semver@^5.5.1: version "5.5.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.1.tgz#7dfdd8814bdb7cabc7be0fb1d734cfb66c940477"