From e223c35f1a51c1bec6f1dd18603731695b2df4e4 Mon Sep 17 00:00:00 2001 From: halfnelson Date: Sat, 6 Jun 2020 17:14:10 +1000 Subject: [PATCH 01/25] add source map support for preprocessors --- package-lock.json | 28 ++- package.json | 4 +- src/compiler/compile/Component.ts | 30 +++ src/compiler/compile/index.ts | 1 + src/compiler/interfaces.ts | 1 + src/compiler/preprocess/index.ts | 128 ++++++++---- src/compiler/utils/string_with_map.ts | 189 ++++++++++++++++++ test/preprocess/index.js | 3 + test/sourcemaps/index.js | 18 +- .../preprocessed-markup/_preprocessor.js | 16 ++ .../samples/preprocessed-markup/input.svelte | 5 + .../samples/preprocessed-markup/test.js | 32 +++ .../preprocessed-multiple/_preprocessor.js | 51 +++++ .../preprocessed-multiple/input.svelte | 9 + .../samples/preprocessed-multiple/test.js | 32 +++ .../preprocessed-script/_preprocessor.js | 17 ++ .../samples/preprocessed-script/input.svelte | 9 + .../samples/preprocessed-script/test.js | 32 +++ .../preprocessed-styles/_preprocessor.js | 17 ++ .../samples/preprocessed-styles/input.svelte | 13 ++ .../samples/preprocessed-styles/test.js | 32 +++ 21 files changed, 619 insertions(+), 48 deletions(-) create mode 100644 src/compiler/utils/string_with_map.ts create mode 100644 test/sourcemaps/samples/preprocessed-markup/_preprocessor.js create mode 100644 test/sourcemaps/samples/preprocessed-markup/input.svelte create mode 100644 test/sourcemaps/samples/preprocessed-markup/test.js create mode 100644 test/sourcemaps/samples/preprocessed-multiple/_preprocessor.js create mode 100644 test/sourcemaps/samples/preprocessed-multiple/input.svelte create mode 100644 test/sourcemaps/samples/preprocessed-multiple/test.js create mode 100644 test/sourcemaps/samples/preprocessed-script/_preprocessor.js create mode 100644 test/sourcemaps/samples/preprocessed-script/input.svelte create mode 100644 test/sourcemaps/samples/preprocessed-script/test.js create mode 100644 test/sourcemaps/samples/preprocessed-styles/_preprocessor.js create mode 100644 test/sourcemaps/samples/preprocessed-styles/input.svelte create mode 100644 test/sourcemaps/samples/preprocessed-styles/test.js diff --git a/package-lock.json b/package-lock.json index 2e2a95b362bc..d6d72944b8bb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,16 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@ampproject/remapping": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-0.3.0.tgz", + "integrity": "sha512-dqmASpaTCavldZqwdEpokgG4yOXmEiEGPP3ATTsBbdXXSKf6kx8jt2fPcKhodABdZlYe82OehR2oFK1y9gwZxw==", + "dev": true, + "requires": { + "@jridgewell/resolve-uri": "1.0.0", + "sourcemap-codec": "1.4.8" + } + }, "@babel/code-frame": { "version": "7.10.1", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.1.tgz", @@ -36,6 +46,12 @@ "integrity": "sha512-KioOCsSvSvXx6xUNLiJz+P+VMb7NRcePjoefOr74Y5P6lEKsiOn35eZyZzgpK4XCNJdXTDR7+zykj0lwxRvZ2g==", "dev": true }, + "@jridgewell/resolve-uri": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-1.0.0.tgz", + "integrity": "sha512-9oLAnygRMi8Q5QkYEU4XWK04B+nuoXoxjRvRxgjuChkLZFBja0YPSgdZ7dZtwhncLBcQe/I/E+fLuk5qxcYVJA==", + "dev": true + }, "@rollup/plugin-commonjs": { "version": "11.0.0", "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-11.0.0.tgz", @@ -2645,9 +2661,9 @@ } }, "magic-string": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.3.tgz", - "integrity": "sha512-6QK0OpF/phMz0Q2AxILkX2mFhi7m+WMwTRg0LQKq/WBB0cDP4rYH3Wp4/d3OTXlrPLVJT/RFqj8tFeAR4nk8AA==", + "version": "0.25.7", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.7.tgz", + "integrity": "sha512-4CrMT5DOHTDk4HYDlzmwu4FVCcIYI8gauveasrdCu2IKIFOJ3f0v/8MDGJCDL9oD2ppz/Av1b0Nj345H9M+XIA==", "dev": true, "requires": { "sourcemap-codec": "^1.4.4" @@ -3737,9 +3753,9 @@ } }, "sourcemap-codec": { - "version": "1.4.6", - "resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.6.tgz", - "integrity": "sha512-1ZooVLYFxC448piVLBbtOxFcXwnymH9oUF8nRd3CuYDVvkRBxRl6pB4Mtas5a4drtL+E8LDgFkQNcgIw6tc8Hg==", + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz", + "integrity": "sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==", "dev": true }, "spdx-correct": { diff --git a/package.json b/package.json index 7ebb41fdfb7a..f13c54b1ac51 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ }, "homepage": "https://github.com/sveltejs/svelte#README", "devDependencies": { + "@ampproject/remapping": "^0.3.0", "@rollup/plugin-commonjs": "^11.0.0", "@rollup/plugin-json": "^4.0.1", "@rollup/plugin-node-resolve": "^6.0.0", @@ -91,7 +92,8 @@ "source-map-support": "^0.5.13", "tiny-glob": "^0.2.6", "tslib": "^1.10.0", - "typescript": "^3.5.3" + "typescript": "^3.5.3", + "sourcemap-codec": "^1.4.8" }, "nyc": { "include": [ diff --git a/src/compiler/compile/Component.ts b/src/compiler/compile/Component.ts index 078ecb8869b2..8f40f65b76a1 100644 --- a/src/compiler/compile/Component.ts +++ b/src/compiler/compile/Component.ts @@ -29,6 +29,7 @@ import add_to_set from './utils/add_to_set'; import check_graph_for_cycles from './utils/check_graph_for_cycles'; import { print, x, b } from 'code-red'; import { is_reserved_keyword } from './utils/reserved_keywords'; +import remapping from '@ampproject/remapping'; interface ComponentOptions { namespace?: string; @@ -324,6 +325,35 @@ export default class Component { js.map.sourcesContent = [ this.source ]; + + if (compile_options.sourceMap) { + if (js.map) { + const pre_remap_sources = js.map.sources; + js.map = remapping([js.map, compile_options.sourceMap], () => null); + // remapper can remove our source if it isn't used (no segments map back to it). It is still handy to have a source + // so we add it back + if (js.map.sources && js.map.sources.length == 0) { + js.map.sources = pre_remap_sources; + } + Object.defineProperties(js.map, { + toString: { + enumerable: false, + value: function toString() { + return JSON.stringify(this); + } + }, + toUrl: { + enumerable: false, + value: function toUrl() { + return 'data:application/json;charset=utf-8;base64,' + btoa(this.toString()); + } + } + }); + } + if (css.map) { + css.map = remapping([css.map, compile_options.sourceMap], () => null); + } + } } return { diff --git a/src/compiler/compile/index.ts b/src/compiler/compile/index.ts index 9d56dfae0252..297e19a03982 100644 --- a/src/compiler/compile/index.ts +++ b/src/compiler/compile/index.ts @@ -12,6 +12,7 @@ const valid_options = [ 'format', 'name', 'filename', + 'sourceMap', 'generate', 'outputFilename', 'cssOutputFilename', diff --git a/src/compiler/interfaces.ts b/src/compiler/interfaces.ts index 6e96e340adee..8003b60c0a32 100644 --- a/src/compiler/interfaces.ts +++ b/src/compiler/interfaces.ts @@ -110,6 +110,7 @@ export interface CompileOptions { filename?: string; generate?: 'dom' | 'ssr' | false; + sourceMap?: object | string; outputFilename?: string; cssOutputFilename?: string; sveltePath?: string; diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index 5371baf4b3cd..4855b8a9f772 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -1,3 +1,9 @@ +import remapper from '@ampproject/remapping'; +import { decode } from 'sourcemap-codec'; +import { getLocator } from 'locate-character'; +import { GeneratedStringWithMap, offset_source_location } from '../utils/string_with_map'; + + export interface Processed { code: string; map?: object | string; @@ -37,34 +43,7 @@ function parse_attributes(str: string) { interface Replacement { offset: number; length: number; - replacement: string; -} - -async function replace_async(str: string, re: RegExp, func: (...any) => Promise) { - const replacements: Array> = []; - str.replace(re, (...args) => { - replacements.push( - func(...args).then( - res => - ({ - offset: args[args.length - 2], - length: args[0].length, - replacement: res - }) as Replacement - ) - ); - return ''; - }); - let out = ''; - let last_end = 0; - for (const { offset, length, replacement } of await Promise.all( - replacements - )) { - out += str.slice(last_end, offset) + replacement; - last_end = offset + length; - } - out += str.slice(last_end); - return out; + replacement: GeneratedStringWithMap; } export default async function preprocess( @@ -81,7 +60,58 @@ export default async function preprocess( const markup = preprocessors.map(p => p.markup).filter(Boolean); const script = preprocessors.map(p => p.script).filter(Boolean); const style = preprocessors.map(p => p.style).filter(Boolean); + const source_maps: Array = []; + + let source_locator: ReturnType; + + function get_replacement(offset: number, original: string, processed: Processed, prefix: string, suffix: string): GeneratedStringWithMap { + const generated_prefix = GeneratedStringWithMap.from_source(filename, prefix, source_locator(offset)); + const generated_suffix = GeneratedStringWithMap.from_source(filename, suffix, source_locator(offset + prefix.length + original.length)); + + let generated; + if (processed.map) { + const full_map = typeof processed.map === "string" ? JSON.parse(processed.map) : processed.map; + const decoded_map = { ...full_map, mappings: decode(full_map.mappings) }; + const processed_offset = source_locator(offset + prefix.length); + generated = GeneratedStringWithMap.from_generated(processed.code, offset_source_location(processed_offset, decoded_map)); + } else { + generated = GeneratedStringWithMap.from_generated(processed.code); + } + const map = generated_prefix.concat(generated).concat(generated_suffix); + return map; + } + + async function replace_async(str: string, re: RegExp, func: (...any) => Promise): Promise { + const replacement_promises: Array> = []; + str.replace(re, (...args) => { + replacement_promises.push( + func(...args).then( + (replacement) => + ({ + offset: args[args.length - 2], + length: args[0].length, + replacement + }) as Replacement + ) + ); + return ''; + }); + const replacements = await Promise.all(replacement_promises); + let out: GeneratedStringWithMap; + let last_end = 0; + for (const { offset, length, replacement } of replacements) + { + const content = GeneratedStringWithMap.from_source(filename, str.slice(last_end, offset), source_locator(last_end)); + out = out ? out.concat(content) : content; + out = out.concat(replacement); + last_end = offset + length; + } + const final_content = GeneratedStringWithMap.from_source(filename, str.slice(last_end), source_locator(last_end)); + out = out.concat(final_content); + return out; + } + for (const fn of markup) { const processed = await fn({ content: source, @@ -89,47 +119,65 @@ export default async function preprocess( }); if (processed && processed.dependencies) dependencies.push(...processed.dependencies); source = processed ? processed.code : source; + if (processed && processed.map) source_maps.unshift(processed.map); } for (const fn of script) { - source = await replace_async( + source_locator = getLocator(source); + const res = await replace_async( source, /|([^]*?)<\/script>|\/>)/gi, - async (match, attributes = '', content = '') => { + async (match, attributes = '', content, offset) => { + const no_change = () => GeneratedStringWithMap.from_source(filename, match, source_locator(offset)); + if (!attributes && !content) { - return match; + return no_change(); } + attributes = attributes || ''; const processed = await fn({ content, attributes: parse_attributes(attributes), filename }); - if (processed && processed.dependencies) dependencies.push(...processed.dependencies); - return processed ? `${processed.code}` : match; + + if (!processed) return no_change(); + if (processed.dependencies) dependencies.push(...processed.dependencies); + return get_replacement(offset, content, processed, ``, ``); } ); + source = res.generated; + source_maps.unshift(res.as_sourcemap()); } for (const fn of style) { - source = await replace_async( + source_locator = getLocator(source); + const res = await replace_async( source, - /|([^]*?)<\/style>|\/>)/gi, - async (match, attributes = '', content = '') => { + /|([^]*?)<\/style>/gi, + async (match, attributes = '', content, offset) => { + const no_change = () => GeneratedStringWithMap.from_source(filename, match, source_locator(offset)); if (!attributes && !content) { - return match; + return no_change(); } + const processed: Processed = await fn({ content, attributes: parse_attributes(attributes), filename }); - if (processed && processed.dependencies) dependencies.push(...processed.dependencies); - return processed ? `${processed.code}` : match; + + if (!processed) return no_change(); + if (processed.dependencies) dependencies.push(...processed.dependencies); + return get_replacement(offset, content, processed, ``, ``); } ); + + source = res.generated; + source_maps.unshift(res.as_sourcemap()); } + const map: ReturnType = source_maps.length == 0 ? null : remapper(source_maps as any, () => null); return { // TODO return separated output, in future version where svelte.compile supports it: // style: { code: styleCode, map: styleMap }, @@ -138,7 +186,7 @@ export default async function preprocess( code: source, dependencies: [...new Set(dependencies)], - + map, toString() { return source; } diff --git a/src/compiler/utils/string_with_map.ts b/src/compiler/utils/string_with_map.ts new file mode 100644 index 000000000000..0a45b404ebdc --- /dev/null +++ b/src/compiler/utils/string_with_map.ts @@ -0,0 +1,189 @@ +import { encode } from "sourcemap-codec"; + +type MappingSegment = [ number ] | [ number, number, number, number ] | [ number, number, number, number, number ] + +type SourceMappings = { + sources: string[]; + names: string[]; + mappings: MappingSegment[][]; +} + +type SourceLocation = { + line: number; + column: number; +} + +function get_end_location(s: string): SourceLocation { + const parts = s.split('\n'); + return { + line: parts.length - 1, + column: parts[parts.length - 1].length - 1 + }; +} + + +export function offset_source_location(offset: SourceLocation, map: SourceMappings): SourceMappings { + + const new_mappings = map.mappings.map(line => line.map(seg => { + if (seg.length < 3) return seg; + const new_seg = seg.slice() as MappingSegment; + new_seg[2] = new_seg[2] + offset.line; + return new_seg; + })); + + // first line has column altered + if (new_mappings.length > 0) { + new_mappings[0] = new_mappings[0].map(seg => { + if (seg.length < 4) return seg; + const newSeg = seg.slice() as MappingSegment; + newSeg[3] = newSeg[3] + offset.column; + return newSeg; + }); + } + + return { + sources: map.sources, + mappings: new_mappings + } as SourceMappings; +} + + + +function merge_tables( original: T[], extended: T[]): { table: T[]; new_idx: number[] } { + const table = original.slice(); + const new_idx = []; + for (let j = 0; j < original.length; j++) { + const current = extended[j]; + const existing = table.indexOf(current); + if (existing < 0) { + table.push(current); + new_idx[j] = table.length - 1; + } else { + new_idx[j] = existing; + } + } + return { table, new_idx }; +} + + + +export class GeneratedStringWithMap { + readonly generated: string; + readonly map: SourceMappings; + + constructor(generated: string , map: SourceMappings) { + this.generated = generated; + this.map = map; + } + + as_sourcemap() { + return { + version: 3, + sources: this.map.sources, + names: [], + mappings: encode(this.map.mappings as any) + }; + } + + concat(other: GeneratedStringWithMap): GeneratedStringWithMap { + // if one is empty, return the other + if (this.generated.length == 0) return other; + if (other.generated.length == 0) return this; + + //combine sources + const { table: new_sources, new_idx: other_source_idx } = merge_tables(this.map.sources, other.map.sources); + const { table: new_names, new_idx: other_name_idx } = merge_tables(this.map.names, other.map.names); + + //update source and name references in segments + const other_mappings = other.map.mappings.map(line => line.map(seg => { + //to reduce allocations, we only return a new segment if a value has changed + if ( + (seg.length > 1 && other_source_idx[seg[1]] != seg[1]) // has source idx that has been updated + || (seg.length == 5 && other_name_idx[seg[4]] != seg[4])) // has name idx that has been updated + { + const new_seg = seg.slice() as MappingSegment; + new_seg[1] = other_source_idx[seg[1]]; + if (seg.length == 5) { + new_seg[4] = other_name_idx[seg[4]]; + } + return new_seg; + } else { + return seg; + } + })); + + //combine the mappings + let new_mappings = this.map.mappings.slice(); + + //shift the first line of the second mapping by the number of columns in the last line of the first + const end = get_end_location(this.generated); + const col_offset = end.column + 1; + const first_line = other_mappings.length == 0 ? [] : other_mappings[0].map(seg => { + const new_seg = seg.slice() as MappingSegment; + new_seg[0] = seg[0] + col_offset; + return new_seg; + }); + new_mappings[new_mappings.length - 1] = new_mappings[new_mappings.length - 1].concat(first_line); + + //the rest don't need modification and can just be appended + new_mappings = new_mappings.concat(other_mappings.slice(1) as MappingSegment[][]); + + return new GeneratedStringWithMap(this.generated + other.generated, { + sources: new_sources, + names: new_names, + mappings: new_mappings + }); + } + + + static from_generated(generated: string, map?: SourceMappings): GeneratedStringWithMap { + if (map) return new GeneratedStringWithMap(generated, map); + + const replacement_map: SourceMappings = { + names: [], + sources: [], + mappings: [] + }; + + if (generated.length == 0) return new GeneratedStringWithMap(generated, replacement_map); + + // we generate a mapping where the source was overwritten by the generated + const end = get_end_location(generated); + for (let i = 0; i <= end.line; i++) { + replacement_map.mappings.push([]); // unmapped line + } + + return new GeneratedStringWithMap(generated, replacement_map); + } + + + + static from_source(source_file: string, source: string, offset_in_source?: SourceLocation): GeneratedStringWithMap { + const offset = offset_in_source || { line: 0, column: 0 }; + const map: SourceMappings = { + names: [], + sources: [ source_file ], + mappings: [] + }; + + if (source.length == 0) return new GeneratedStringWithMap(source, map); + + // we create a high resolution identity map here, we know that it will eventually be + // merged with svelte's map, at which stage the resolution will decrease. + const lines = source.split('\n'); + let pos = 0; + const identity_map = lines.map((line, line_idx) => { + const segs = line.split(/([^\d\w\s]|\s+)/g).filter(x => x !== "").map(s => { + const seg: MappingSegment = [pos, 0, offset.line + line_idx, pos + (line_idx == 0 ? offset.column : 0)]; + pos = pos + s.length; + return seg; + }); + pos = 0; + return segs; + }); + + map.mappings = identity_map; + + return new GeneratedStringWithMap(source, map); + } +} \ No newline at end of file diff --git a/test/preprocess/index.js b/test/preprocess/index.js index 5d83bb60590d..ac096bad2e94 100644 --- a/test/preprocess/index.js +++ b/test/preprocess/index.js @@ -19,6 +19,9 @@ describe('preprocess', () => { const result = await svelte.preprocess(input, config.preprocess); fs.writeFileSync(`${__dirname}/samples/${dir}/_actual.html`, result.code); + if (result.map) { + fs.writeFileSync(`${__dirname}/samples/${dir}/_actual.html.map`, JSON.stringify(result.map, null, 2)); + } assert.equal(result.code, expected); diff --git a/test/sourcemaps/index.js b/test/sourcemaps/index.js index 0b0424a764ff..428c9b8000f4 100644 --- a/test/sourcemaps/index.js +++ b/test/sourcemaps/index.js @@ -25,9 +25,24 @@ describe("sourcemaps", () => { `${__dirname}/samples/${dir}/output` ); + const preprocessorFilename = path.resolve( + `${__dirname}/samples/${dir}/_preprocessor.js` + ) + const input = fs.readFileSync(filename, "utf-8").replace(/\s+$/, ""); - const { js, css } = svelte.compile(input, { + let processed_input = input; + let processed_map = null; + + if (fs.existsSync(preprocessorFilename)) { + let { preprocessors } = require(preprocessorFilename); + if (preprocessors.length > 0) { + ({ code: processed_input, map: processed_map } = await svelte.preprocess(input, preprocessors, { filename: 'input.svelte' })); + } + } + + const { js, css } = svelte.compile(processed_input, { filename, + sourceMap: processed_map, outputFilename: `${outputFilename}.js`, cssOutputFilename: `${outputFilename}.css` }); @@ -55,6 +70,7 @@ describe("sourcemaps", () => { } assert.deepEqual(js.map.sources, ["input.svelte"]); + if (css.map) assert.deepEqual(css.map.sources, ["input.svelte"]); const { test } = require(`./samples/${dir}/test.js`); diff --git a/test/sourcemaps/samples/preprocessed-markup/_preprocessor.js b/test/sourcemaps/samples/preprocessed-markup/_preprocessor.js new file mode 100644 index 000000000000..db3103446b8f --- /dev/null +++ b/test/sourcemaps/samples/preprocessed-markup/_preprocessor.js @@ -0,0 +1,16 @@ +import MagicString from 'magic-string'; + +export const preprocessors = [{ + markup: ({content, filename}) => { + const src = new MagicString(content); + const idx = content.indexOf("baritone"); + src.overwrite(idx, idx+"baritone".length, "bar"); + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + includeContent: false + }) + }; + } +}]; \ No newline at end of file diff --git a/test/sourcemaps/samples/preprocessed-markup/input.svelte b/test/sourcemaps/samples/preprocessed-markup/input.svelte new file mode 100644 index 000000000000..ee4b90372acd --- /dev/null +++ b/test/sourcemaps/samples/preprocessed-markup/input.svelte @@ -0,0 +1,5 @@ + + +{foo.baritone.baz} diff --git a/test/sourcemaps/samples/preprocessed-markup/test.js b/test/sourcemaps/samples/preprocessed-markup/test.js new file mode 100644 index 000000000000..b398dcc3dd62 --- /dev/null +++ b/test/sourcemaps/samples/preprocessed-markup/test.js @@ -0,0 +1,32 @@ +export function test({ assert, smc, locateInSource, locateInGenerated }) { + const expectedBar = locateInSource('baritone.baz'); + const expectedBaz = locateInSource('.baz'); + + let start = locateInGenerated('bar.baz'); + + const actualbar = smc.originalPositionFor({ + line: start.line + 1, + column: start.column + }); + + assert.deepEqual(actualbar, { + source: 'input.svelte', + name: null, + line: expectedBar.line + 1, + column: expectedBar.column + }); + + start = locateInGenerated('.baz'); + + const actualbaz = smc.originalPositionFor({ + line: start.line + 1, + column: start.column + }); + + assert.deepEqual(actualbaz, { + source: 'input.svelte', + name: null, + line: expectedBaz.line + 1, + column: expectedBaz.column + }); +} diff --git a/test/sourcemaps/samples/preprocessed-multiple/_preprocessor.js b/test/sourcemaps/samples/preprocessed-multiple/_preprocessor.js new file mode 100644 index 000000000000..4c7d023382c6 --- /dev/null +++ b/test/sourcemaps/samples/preprocessed-multiple/_preprocessor.js @@ -0,0 +1,51 @@ +import MagicString from 'magic-string'; + +export const preprocessors = [{ + markup: ({ content, filename }) => { + const src = new MagicString(content); + const idx = content.indexOf("baritone"); + src.overwrite(idx, idx + "baritone".length, "bar"); + + const css_idx = content.indexOf("--bazitone"); + src.overwrite(css_idx, css_idx + "--bazitone".length, "--baz"); + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + hires: true, + includeContent: false + }) + }; + } +}, +{ + script: ({ content, filename }) => { + const src = new MagicString(content); + const idx = content.indexOf("bar"); + src.prependLeft(idx, " "); + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + hires: true, + includeContent: false + }) + }; + } +}, +{ + style: ({ content, filename }) => { + const src = new MagicString(content); + const idx = content.indexOf("--baz"); + src.prependLeft(idx, " "); + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + hires: true, + includeContent: false + }) + }; + } +} +]; \ No newline at end of file diff --git a/test/sourcemaps/samples/preprocessed-multiple/input.svelte b/test/sourcemaps/samples/preprocessed-multiple/input.svelte new file mode 100644 index 000000000000..e656d399ae04 --- /dev/null +++ b/test/sourcemaps/samples/preprocessed-multiple/input.svelte @@ -0,0 +1,9 @@ + + +

multiple {foo}

diff --git a/test/sourcemaps/samples/preprocessed-multiple/test.js b/test/sourcemaps/samples/preprocessed-multiple/test.js new file mode 100644 index 000000000000..d485695ba4fa --- /dev/null +++ b/test/sourcemaps/samples/preprocessed-multiple/test.js @@ -0,0 +1,32 @@ +export function test({ assert, smc, smcCss, locateInSource, locateInGenerated, locateInGeneratedCss }) { + const expectedBar = locateInSource('baritone'); + const expectedBaz = locateInSource('--bazitone'); + + let start = locateInGenerated('bar'); + + const actualbar = smc.originalPositionFor({ + line: start.line + 1, + column: start.column + }); + + assert.deepEqual(actualbar, { + source: 'input.svelte', + name: null, + line: expectedBar.line + 1, + column: expectedBar.column + }); + + start = locateInGeneratedCss('--baz'); + + const actualbaz = smcCss.originalPositionFor({ + line: start.line + 1, + column: start.column + }); + + assert.deepEqual(actualbaz, { + source: 'input.svelte', + name: null, + line: expectedBaz.line + 1, + column: expectedBaz.column + }, `couldn't find baz in css,\n gen:${JSON.stringify(start)}\n actual:${JSON.stringify(actualbaz)}\n expected:${JSON.stringify(expectedBaz)}`); +} diff --git a/test/sourcemaps/samples/preprocessed-script/_preprocessor.js b/test/sourcemaps/samples/preprocessed-script/_preprocessor.js new file mode 100644 index 000000000000..bf895e0943c0 --- /dev/null +++ b/test/sourcemaps/samples/preprocessed-script/_preprocessor.js @@ -0,0 +1,17 @@ +import MagicString from 'magic-string'; + +export const preprocessors = [{ + script: ({content, filename}) => { + const src = new MagicString(content); + const idx = content.indexOf("baritone"); + src.overwrite(idx, idx+"baritone".length, "bar"); + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + hires: true, + includeContent: false + }) + }; + } +}]; \ No newline at end of file diff --git a/test/sourcemaps/samples/preprocessed-script/input.svelte b/test/sourcemaps/samples/preprocessed-script/input.svelte new file mode 100644 index 000000000000..11586619e1a0 --- /dev/null +++ b/test/sourcemaps/samples/preprocessed-script/input.svelte @@ -0,0 +1,9 @@ + + +

{foo.bar.baz}

diff --git a/test/sourcemaps/samples/preprocessed-script/test.js b/test/sourcemaps/samples/preprocessed-script/test.js new file mode 100644 index 000000000000..a0ce8732b3c0 --- /dev/null +++ b/test/sourcemaps/samples/preprocessed-script/test.js @@ -0,0 +1,32 @@ +export function test({ assert, smc, locateInSource, locateInGenerated }) { + const expectedBar = locateInSource('baritone:'); + const expectedBaz = locateInSource('baz:'); + + let start = locateInGenerated('bar:'); + + const actualbar = smc.originalPositionFor({ + line: start.line + 1, + column: start.column + }); + + assert.deepEqual(actualbar, { + source: 'input.svelte', + name: null, + line: expectedBar.line + 1, + column: expectedBar.column + }, `couldn't find bar: in source` ); + + start = locateInGenerated('baz:'); + + const actualbaz = smc.originalPositionFor({ + line: start.line + 1, + column: start.column + }); + + assert.deepEqual(actualbaz, { + source: 'input.svelte', + name: null, + line: expectedBaz.line + 1, + column: expectedBaz.column + }, `couldn't find baz: in source` ); +} diff --git a/test/sourcemaps/samples/preprocessed-styles/_preprocessor.js b/test/sourcemaps/samples/preprocessed-styles/_preprocessor.js new file mode 100644 index 000000000000..42166453e36f --- /dev/null +++ b/test/sourcemaps/samples/preprocessed-styles/_preprocessor.js @@ -0,0 +1,17 @@ +import MagicString from 'magic-string'; + +export const preprocessors = [{ + style: ({content, filename}) => { + const src = new MagicString(content); + const idx = content.indexOf("baritone"); + src.overwrite(idx, idx+"baritone".length, "bar"); + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + hires: true, + includeContent: false + }) + }; + } +}]; \ No newline at end of file diff --git a/test/sourcemaps/samples/preprocessed-styles/input.svelte b/test/sourcemaps/samples/preprocessed-styles/input.svelte new file mode 100644 index 000000000000..5ef6a3d443a0 --- /dev/null +++ b/test/sourcemaps/samples/preprocessed-styles/input.svelte @@ -0,0 +1,13 @@ +

Testing Styles

+

Testing Styles 2

+ + + diff --git a/test/sourcemaps/samples/preprocessed-styles/test.js b/test/sourcemaps/samples/preprocessed-styles/test.js new file mode 100644 index 000000000000..7c6989588a74 --- /dev/null +++ b/test/sourcemaps/samples/preprocessed-styles/test.js @@ -0,0 +1,32 @@ +export function test({ assert, smcCss, locateInSource, locateInGeneratedCss }) { + const expectedBar = locateInSource('--baritone'); + const expectedBaz = locateInSource('--baz'); + + let start = locateInGeneratedCss('--bar'); + + const actualbar = smcCss.originalPositionFor({ + line: start.line + 1, + column: start.column + }); + + assert.deepEqual(actualbar, { + source: 'input.svelte', + name: null, + line: expectedBar.line + 1, + column: expectedBar.column + }, `couldn't find bar in source` ); + + start = locateInGeneratedCss('--baz'); + + const actualbaz = smcCss.originalPositionFor({ + line: start.line + 1, + column: start.column + }); + + assert.deepEqual(actualbaz, { + source: 'input.svelte', + name: null, + line: expectedBaz.line + 1, + column: expectedBaz.column + }, `couldn't find baz in source` ); +} From 43c5d5ecfc3c1c2e279699ef40d60920b01bfd6c Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Sat, 5 Sep 2020 11:22:10 +0200 Subject: [PATCH 02/25] preprocessor sourcemaps: prettify code sort devDependencies rename to compile_options.sourcemap file src/compiler/utils/string_with_map.ts * indent with tabs * run prettier * remove extra whitespace * add newline at end of file * wrap long lines * rephrase some comments --- package.json | 4 +- src/compiler/compile/Component.ts | 6 +- src/compiler/compile/index.ts | 2 +- src/compiler/interfaces.ts | 2 +- src/compiler/utils/string_with_map.ts | 394 ++++++++++++++------------ test/sourcemaps/index.js | 2 +- 6 files changed, 228 insertions(+), 182 deletions(-) diff --git a/package.json b/package.json index f13c54b1ac51..a16e849d6545 100644 --- a/package.json +++ b/package.json @@ -90,10 +90,10 @@ "rollup": "^1.27.14", "source-map": "^0.7.3", "source-map-support": "^0.5.13", + "sourcemap-codec": "^1.4.8", "tiny-glob": "^0.2.6", "tslib": "^1.10.0", - "typescript": "^3.5.3", - "sourcemap-codec": "^1.4.8" + "typescript": "^3.5.3" }, "nyc": { "include": [ diff --git a/src/compiler/compile/Component.ts b/src/compiler/compile/Component.ts index 8f40f65b76a1..4056949b2537 100644 --- a/src/compiler/compile/Component.ts +++ b/src/compiler/compile/Component.ts @@ -326,10 +326,10 @@ export default class Component { this.source ]; - if (compile_options.sourceMap) { + if (compile_options.sourcemap) { if (js.map) { const pre_remap_sources = js.map.sources; - js.map = remapping([js.map, compile_options.sourceMap], () => null); + js.map = remapping([js.map, compile_options.sourcemap], () => null); // remapper can remove our source if it isn't used (no segments map back to it). It is still handy to have a source // so we add it back if (js.map.sources && js.map.sources.length == 0) { @@ -351,7 +351,7 @@ export default class Component { }); } if (css.map) { - css.map = remapping([css.map, compile_options.sourceMap], () => null); + css.map = remapping([css.map, compile_options.sourcemap], () => null); } } } diff --git a/src/compiler/compile/index.ts b/src/compiler/compile/index.ts index 297e19a03982..96a67e2a8d15 100644 --- a/src/compiler/compile/index.ts +++ b/src/compiler/compile/index.ts @@ -12,7 +12,7 @@ const valid_options = [ 'format', 'name', 'filename', - 'sourceMap', + 'sourcemap', 'generate', 'outputFilename', 'cssOutputFilename', diff --git a/src/compiler/interfaces.ts b/src/compiler/interfaces.ts index 8003b60c0a32..7843cf8938db 100644 --- a/src/compiler/interfaces.ts +++ b/src/compiler/interfaces.ts @@ -110,7 +110,7 @@ export interface CompileOptions { filename?: string; generate?: 'dom' | 'ssr' | false; - sourceMap?: object | string; + sourcemap?: object | string; outputFilename?: string; cssOutputFilename?: string; sveltePath?: string; diff --git a/src/compiler/utils/string_with_map.ts b/src/compiler/utils/string_with_map.ts index 0a45b404ebdc..5c46eddecf3d 100644 --- a/src/compiler/utils/string_with_map.ts +++ b/src/compiler/utils/string_with_map.ts @@ -1,189 +1,235 @@ import { encode } from "sourcemap-codec"; -type MappingSegment = [ number ] | [ number, number, number, number ] | [ number, number, number, number, number ] +type MappingSegment = + | [number] + | [number, number, number, number] + | [number, number, number, number, number]; type SourceMappings = { - sources: string[]; - names: string[]; - mappings: MappingSegment[][]; -} + sources: string[]; + names: string[]; + mappings: MappingSegment[][]; +}; type SourceLocation = { - line: number; - column: number; -} + line: number; + column: number; +}; function get_end_location(s: string): SourceLocation { - const parts = s.split('\n'); - return { - line: parts.length - 1, - column: parts[parts.length - 1].length - 1 - }; + const parts = s.split("\n"); + return { + line: parts.length - 1, + column: parts[parts.length - 1].length - 1, + }; } - -export function offset_source_location(offset: SourceLocation, map: SourceMappings): SourceMappings { - - const new_mappings = map.mappings.map(line => line.map(seg => { - if (seg.length < 3) return seg; - const new_seg = seg.slice() as MappingSegment; - new_seg[2] = new_seg[2] + offset.line; - return new_seg; - })); - - // first line has column altered - if (new_mappings.length > 0) { - new_mappings[0] = new_mappings[0].map(seg => { - if (seg.length < 4) return seg; - const newSeg = seg.slice() as MappingSegment; - newSeg[3] = newSeg[3] + offset.column; - return newSeg; - }); - } - - return { - sources: map.sources, - mappings: new_mappings - } as SourceMappings; +export function offset_source_location( + offset: SourceLocation, + map: SourceMappings +): SourceMappings { + const new_mappings = map.mappings.map((line) => + line.map((seg) => { + if (seg.length < 3) return seg; + const new_seg = seg.slice() as MappingSegment; + new_seg[2] = new_seg[2] + offset.line; + return new_seg; + }) + ); + + // column changed in first line + if (new_mappings.length > 0) { + new_mappings[0] = new_mappings[0].map((seg) => { + if (seg.length < 4) return seg; + const newSeg = seg.slice() as MappingSegment; + newSeg[3] = newSeg[3] + offset.column; + return newSeg; + }); + } + + return { + sources: map.sources, + mappings: new_mappings, + } as SourceMappings; } - - -function merge_tables( original: T[], extended: T[]): { table: T[]; new_idx: number[] } { - const table = original.slice(); - const new_idx = []; - for (let j = 0; j < original.length; j++) { - const current = extended[j]; - const existing = table.indexOf(current); - if (existing < 0) { - table.push(current); - new_idx[j] = table.length - 1; - } else { - new_idx[j] = existing; - } - } - return { table, new_idx }; +function merge_tables( + original: T[], + extended: T[] +): { table: T[]; new_idx: number[] } { + const table = original.slice(); + const new_idx = []; + for (let j = 0; j < original.length; j++) { + const current = extended[j]; + const existing = table.indexOf(current); + if (existing < 0) { + table.push(current); + new_idx[j] = table.length - 1; + } else { + new_idx[j] = existing; + } + } + return { table, new_idx }; } - - export class GeneratedStringWithMap { - readonly generated: string; - readonly map: SourceMappings; - - constructor(generated: string , map: SourceMappings) { - this.generated = generated; - this.map = map; - } - - as_sourcemap() { - return { - version: 3, - sources: this.map.sources, - names: [], - mappings: encode(this.map.mappings as any) - }; - } - - concat(other: GeneratedStringWithMap): GeneratedStringWithMap { - // if one is empty, return the other - if (this.generated.length == 0) return other; - if (other.generated.length == 0) return this; - - //combine sources - const { table: new_sources, new_idx: other_source_idx } = merge_tables(this.map.sources, other.map.sources); - const { table: new_names, new_idx: other_name_idx } = merge_tables(this.map.names, other.map.names); - - //update source and name references in segments - const other_mappings = other.map.mappings.map(line => line.map(seg => { - //to reduce allocations, we only return a new segment if a value has changed - if ( - (seg.length > 1 && other_source_idx[seg[1]] != seg[1]) // has source idx that has been updated - || (seg.length == 5 && other_name_idx[seg[4]] != seg[4])) // has name idx that has been updated - { - const new_seg = seg.slice() as MappingSegment; - new_seg[1] = other_source_idx[seg[1]]; - if (seg.length == 5) { - new_seg[4] = other_name_idx[seg[4]]; - } - return new_seg; - } else { - return seg; - } - })); - - //combine the mappings - let new_mappings = this.map.mappings.slice(); - - //shift the first line of the second mapping by the number of columns in the last line of the first - const end = get_end_location(this.generated); - const col_offset = end.column + 1; - const first_line = other_mappings.length == 0 ? [] : other_mappings[0].map(seg => { - const new_seg = seg.slice() as MappingSegment; - new_seg[0] = seg[0] + col_offset; - return new_seg; - }); - new_mappings[new_mappings.length - 1] = new_mappings[new_mappings.length - 1].concat(first_line); - - //the rest don't need modification and can just be appended - new_mappings = new_mappings.concat(other_mappings.slice(1) as MappingSegment[][]); - - return new GeneratedStringWithMap(this.generated + other.generated, { - sources: new_sources, - names: new_names, - mappings: new_mappings - }); - } - - - static from_generated(generated: string, map?: SourceMappings): GeneratedStringWithMap { - if (map) return new GeneratedStringWithMap(generated, map); - - const replacement_map: SourceMappings = { - names: [], - sources: [], - mappings: [] - }; - - if (generated.length == 0) return new GeneratedStringWithMap(generated, replacement_map); - - // we generate a mapping where the source was overwritten by the generated - const end = get_end_location(generated); - for (let i = 0; i <= end.line; i++) { - replacement_map.mappings.push([]); // unmapped line - } - - return new GeneratedStringWithMap(generated, replacement_map); - } - - - - static from_source(source_file: string, source: string, offset_in_source?: SourceLocation): GeneratedStringWithMap { - const offset = offset_in_source || { line: 0, column: 0 }; - const map: SourceMappings = { - names: [], - sources: [ source_file ], - mappings: [] - }; - - if (source.length == 0) return new GeneratedStringWithMap(source, map); - - // we create a high resolution identity map here, we know that it will eventually be - // merged with svelte's map, at which stage the resolution will decrease. - const lines = source.split('\n'); - let pos = 0; - const identity_map = lines.map((line, line_idx) => { - const segs = line.split(/([^\d\w\s]|\s+)/g).filter(x => x !== "").map(s => { - const seg: MappingSegment = [pos, 0, offset.line + line_idx, pos + (line_idx == 0 ? offset.column : 0)]; - pos = pos + s.length; - return seg; - }); - pos = 0; - return segs; - }); - - map.mappings = identity_map; - - return new GeneratedStringWithMap(source, map); - } -} \ No newline at end of file + readonly generated: string; + readonly map: SourceMappings; + + constructor(generated: string, map: SourceMappings) { + this.generated = generated; + this.map = map; + } + + as_sourcemap() { + return { + version: 3, + sources: this.map.sources, + names: [], + mappings: encode(this.map.mappings as any), + }; + } + + concat(other: GeneratedStringWithMap): GeneratedStringWithMap { + // if one is empty, return the other + if (this.generated.length == 0) return other; + if (other.generated.length == 0) return this; + + // combine sources + const { + table: new_sources, + new_idx: other_source_idx + } = merge_tables( + this.map.sources, + other.map.sources + ); + const { + table: new_names, + new_idx: other_name_idx + } = merge_tables( + this.map.names, + other.map.names + ); + + // update source refs and name refs in segments + const other_mappings = other.map.mappings.map((line) => + line.map((seg) => { + // to reduce allocations, + // we only return a new segment if a value has changed + if ( + // new source idx + (seg.length > 1 && other_source_idx[seg[1]] != seg[1]) || + // new name idx + (seg.length == 5 && other_name_idx[seg[4]] != seg[4]) + ) { + const new_seg = seg.slice() as MappingSegment; + new_seg[1] = other_source_idx[seg[1]]; + if (seg.length == 5) { + new_seg[4] = other_name_idx[seg[4]]; + } + return new_seg; + } else { + return seg; + } + }) + ); + + // combine the mappings + let new_mappings = this.map.mappings.slice(); + + // shift the first line of the second mapping + // by the number of columns in the last line of the first mapping + const end = get_end_location(this.generated); + const col_offset = end.column + 1; + const first_line = + other_mappings.length == 0 + ? [] + : other_mappings[0].map((seg) => { + const new_seg = seg.slice() as MappingSegment; + new_seg[0] = seg[0] + col_offset; + return new_seg; + }); + new_mappings[new_mappings.length - 1] = new_mappings[ + new_mappings.length - 1 + ].concat(first_line); + + // the rest don't need modification and can just be appended + new_mappings = new_mappings.concat( + other_mappings.slice(1) as MappingSegment[][] + ); + + return new GeneratedStringWithMap( + this.generated + other.generated, { + sources: new_sources, + names: new_names, + mappings: new_mappings, + }); + } + + static from_generated( + generated: string, + map?: SourceMappings + ): GeneratedStringWithMap { + if (map) return new GeneratedStringWithMap(generated, map); + + const replacement_map: SourceMappings = { + names: [], + sources: [], + mappings: [], + }; + + if (generated.length == 0) + return new GeneratedStringWithMap(generated, replacement_map); + + // we generate a mapping + // where the source was overwritten by the generated + const end = get_end_location(generated); + for (let i = 0; i <= end.line; i++) { + replacement_map.mappings.push([]); // unmapped line + } + + return new GeneratedStringWithMap(generated, replacement_map); + } + + static from_source( + source_file: string, + source: string, + offset_in_source?: SourceLocation + ): GeneratedStringWithMap { + const offset = offset_in_source || { line: 0, column: 0 }; + const map: SourceMappings = { + names: [], + sources: [source_file], + mappings: [], + }; + + if (source.length == 0) return new GeneratedStringWithMap(source, map); + + // we create a high resolution identity map here, + // we know that it will eventually be merged with svelte's map, + // at which stage the resolution will decrease. + const lines = source.split("\n"); + let pos = 0; + const identity_map = lines.map((line, line_idx) => { + const segs = line + .split(/([^\d\w\s]|\s+)/g) + .filter((x) => x !== "") + .map((s) => { + const seg: MappingSegment = [ + pos, + 0, + offset.line + line_idx, + pos + (line_idx == 0 ? offset.column : 0), + ]; + pos = pos + s.length; + return seg; + }); + pos = 0; + return segs; + }); + + map.mappings = identity_map; + + return new GeneratedStringWithMap(source, map); + } +} diff --git a/test/sourcemaps/index.js b/test/sourcemaps/index.js index 428c9b8000f4..c54e384d0127 100644 --- a/test/sourcemaps/index.js +++ b/test/sourcemaps/index.js @@ -42,7 +42,7 @@ describe("sourcemaps", () => { const { js, css } = svelte.compile(processed_input, { filename, - sourceMap: processed_map, + sourcemap: processed_map, outputFilename: `${outputFilename}.js`, cssOutputFilename: `${outputFilename}.css` }); From b8e9b68c0d5e506f2ce05f47e62ac8f417a2247a Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Sat, 5 Sep 2020 13:26:41 +0200 Subject: [PATCH 03/25] preprocessor sourcemaps: prettify code 2 move string_with_map.ts to string_with_sourcemap.ts rename GeneratedStringWithMap to StringWithSourcemap rename offset_source_location to sourcemap_add_offset rename source_maps to sourcemap_list rename source_locator to get_location rename as_sourcemap to get_sourcemap rename encode to sourcemap_encode move { brace to line before add some comments --- src/compiler/preprocess/index.ts | 112 ++++++++++++------ ...g_with_map.ts => string_with_sourcemap.ts} | 54 +++++---- 2 files changed, 109 insertions(+), 57 deletions(-) rename src/compiler/utils/{string_with_map.ts => string_with_sourcemap.ts} (80%) diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index 4855b8a9f772..7d146fdbe5fb 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -1,7 +1,7 @@ import remapper from '@ampproject/remapping'; import { decode } from 'sourcemap-codec'; import { getLocator } from 'locate-character'; -import { GeneratedStringWithMap, offset_source_location } from '../utils/string_with_map'; +import { StringWithSourcemap, sourcemap_add_offset } from '../utils/string_with_sourcemap'; export interface Processed { @@ -43,7 +43,7 @@ function parse_attributes(str: string) { interface Replacement { offset: number; length: number; - replacement: GeneratedStringWithMap; + replacement: StringWithSourcemap; } export default async function preprocess( @@ -60,28 +60,41 @@ export default async function preprocess( const markup = preprocessors.map(p => p.markup).filter(Boolean); const script = preprocessors.map(p => p.script).filter(Boolean); const style = preprocessors.map(p => p.style).filter(Boolean); - const source_maps: Array = []; - let source_locator: ReturnType; + const sourcemap_list: Array = []; - function get_replacement(offset: number, original: string, processed: Processed, prefix: string, suffix: string): GeneratedStringWithMap { - const generated_prefix = GeneratedStringWithMap.from_source(filename, prefix, source_locator(offset)); - const generated_suffix = GeneratedStringWithMap.from_source(filename, suffix, source_locator(offset + prefix.length + original.length)); + let get_location: ReturnType; + + function get_replacement( + offset: number, + original: string, + processed: Processed, + prefix: string, + suffix: string + ): StringWithSourcemap { + const generated_prefix = StringWithSourcemap.from_source( + filename, prefix, get_location(offset)); + const generated_suffix = StringWithSourcemap.from_source( + filename, suffix, get_location(offset + prefix.length + original.length)); let generated; if (processed.map) { const full_map = typeof processed.map === "string" ? JSON.parse(processed.map) : processed.map; const decoded_map = { ...full_map, mappings: decode(full_map.mappings) }; - const processed_offset = source_locator(offset + prefix.length); - generated = GeneratedStringWithMap.from_generated(processed.code, offset_source_location(processed_offset, decoded_map)); + const processed_offset = get_location(offset + prefix.length); + generated = StringWithSourcemap.from_generated(processed.code, sourcemap_add_offset(processed_offset, decoded_map)); } else { - generated = GeneratedStringWithMap.from_generated(processed.code); + generated = StringWithSourcemap.from_generated(processed.code); } const map = generated_prefix.concat(generated).concat(generated_suffix); return map; } - async function replace_async(str: string, re: RegExp, func: (...any) => Promise): Promise { + async function replace_async( + str: string, + re: RegExp, + func: (...any) => Promise + ): Promise { const replacement_promises: Array> = []; str.replace(re, (...args) => { replacement_promises.push( @@ -98,86 +111,117 @@ export default async function preprocess( }); const replacements = await Promise.all(replacement_promises); - let out: GeneratedStringWithMap; + let out: StringWithSourcemap; let last_end = 0; - for (const { offset, length, replacement } of replacements) - { - const content = GeneratedStringWithMap.from_source(filename, str.slice(last_end, offset), source_locator(last_end)); + for (const { offset, length, replacement } of replacements) { + // content = source before replacement + const content = StringWithSourcemap.from_source( + filename, str.slice(last_end, offset), get_location(last_end)); out = out ? out.concat(content) : content; out = out.concat(replacement); last_end = offset + length; } - const final_content = GeneratedStringWithMap.from_source(filename, str.slice(last_end), source_locator(last_end)); + // final_content = source after last replacement + const final_content = StringWithSourcemap.from_source( + filename, str.slice(last_end), get_location(last_end)); out = out.concat(final_content); return out; } - + for (const fn of markup) { + + // run markup preprocessor const processed = await fn({ content: source, filename }); - if (processed && processed.dependencies) dependencies.push(...processed.dependencies); + + if (processed && processed.dependencies) { + dependencies.push(...processed.dependencies); + } source = processed ? processed.code : source; - if (processed && processed.map) source_maps.unshift(processed.map); + if (processed && processed.map) { + sourcemap_list.unshift(processed.map); + } } for (const fn of script) { - source_locator = getLocator(source); + get_location = getLocator(source); const res = await replace_async( source, /|([^]*?)<\/script>|\/>)/gi, async (match, attributes = '', content, offset) => { - const no_change = () => GeneratedStringWithMap.from_source(filename, match, source_locator(offset)); - + const no_change = () => StringWithSourcemap.from_source( + filename, match, get_location(offset)); if (!attributes && !content) { return no_change(); } - attributes = attributes || ''; + + // run script preprocessor const processed = await fn({ content, attributes: parse_attributes(attributes), filename }); - if (!processed) return no_change(); - if (processed.dependencies) dependencies.push(...processed.dependencies); - return get_replacement(offset, content, processed, ``, ``); + if (!processed) { + return no_change(); + } + if (processed.dependencies) { + dependencies.push(...processed.dependencies); + } + return get_replacement( + offset, content, processed, + ``, `` + ); } ); source = res.generated; - source_maps.unshift(res.as_sourcemap()); + sourcemap_list.unshift(res.get_sourcemap()); } for (const fn of style) { - source_locator = getLocator(source); + get_location = getLocator(source); const res = await replace_async( source, /|([^]*?)<\/style>/gi, async (match, attributes = '', content, offset) => { - const no_change = () => GeneratedStringWithMap.from_source(filename, match, source_locator(offset)); + const no_change = () => StringWithSourcemap.from_source( + filename, match, get_location(offset)); if (!attributes && !content) { return no_change(); } + // run style preprocessor const processed: Processed = await fn({ content, attributes: parse_attributes(attributes), filename }); - if (!processed) return no_change(); - if (processed.dependencies) dependencies.push(...processed.dependencies); - return get_replacement(offset, content, processed, ``, ``); + if (!processed) { + return no_change(); + } + if (processed.dependencies) { + dependencies.push(...processed.dependencies); + } + return get_replacement( + offset, content, processed, + ``, `` + ); } ); source = res.generated; - source_maps.unshift(res.as_sourcemap()); + sourcemap_list.unshift(res.get_sourcemap()); } - const map: ReturnType = source_maps.length == 0 ? null : remapper(source_maps as any, () => null); + const map: ReturnType = + sourcemap_list.length == 0 + ? null + : remapper(sourcemap_list as any, () => null); + return { // TODO return separated output, in future version where svelte.compile supports it: // style: { code: styleCode, map: styleMap }, diff --git a/src/compiler/utils/string_with_map.ts b/src/compiler/utils/string_with_sourcemap.ts similarity index 80% rename from src/compiler/utils/string_with_map.ts rename to src/compiler/utils/string_with_sourcemap.ts index 5c46eddecf3d..f899b03927d7 100644 --- a/src/compiler/utils/string_with_map.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -1,4 +1,4 @@ -import { encode } from "sourcemap-codec"; +import { encode as sourcemap_encode } from "sourcemap-codec"; type MappingSegment = | [number] @@ -24,7 +24,7 @@ function get_end_location(s: string): SourceLocation { }; } -export function offset_source_location( +export function sourcemap_add_offset( offset: SourceLocation, map: SourceMappings ): SourceMappings { @@ -72,7 +72,7 @@ function merge_tables( return { table, new_idx }; } -export class GeneratedStringWithMap { +export class StringWithSourcemap { readonly generated: string; readonly map: SourceMappings; @@ -81,16 +81,16 @@ export class GeneratedStringWithMap { this.map = map; } - as_sourcemap() { + get_sourcemap() { return { version: 3, sources: this.map.sources, names: [], - mappings: encode(this.map.mappings as any), + mappings: sourcemap_encode(this.map.mappings as any), }; } - concat(other: GeneratedStringWithMap): GeneratedStringWithMap { + concat(other: StringWithSourcemap): StringWithSourcemap { // if one is empty, return the other if (this.generated.length == 0) return other; if (other.generated.length == 0) return this; @@ -103,6 +103,8 @@ export class GeneratedStringWithMap { this.map.sources, other.map.sources ); + + // combine names const { table: new_names, new_idx: other_name_idx @@ -135,10 +137,14 @@ export class GeneratedStringWithMap { ); // combine the mappings + + // this.map is readonly, so we copy let new_mappings = this.map.mappings.slice(); - // shift the first line of the second mapping - // by the number of columns in the last line of the first mapping + // combine: + // 1. last line of first map + // 2. first line of second map + // columns of 2 must be shifted const end = get_end_location(this.generated); const col_offset = end.column + 1; const first_line = @@ -148,17 +154,18 @@ export class GeneratedStringWithMap { const new_seg = seg.slice() as MappingSegment; new_seg[0] = seg[0] + col_offset; return new_seg; - }); - new_mappings[new_mappings.length - 1] = new_mappings[ - new_mappings.length - 1 - ].concat(first_line); + }); + + // append segments to last line of first map + new_mappings[new_mappings.length - 1] = + new_mappings[new_mappings.length - 1].concat(first_line); - // the rest don't need modification and can just be appended + // the other lines don't need modification and can just be appended new_mappings = new_mappings.concat( other_mappings.slice(1) as MappingSegment[][] ); - return new GeneratedStringWithMap( + return new StringWithSourcemap( this.generated + other.generated, { sources: new_sources, names: new_names, @@ -169,8 +176,8 @@ export class GeneratedStringWithMap { static from_generated( generated: string, map?: SourceMappings - ): GeneratedStringWithMap { - if (map) return new GeneratedStringWithMap(generated, map); + ): StringWithSourcemap { + if (map) return new StringWithSourcemap(generated, map); const replacement_map: SourceMappings = { names: [], @@ -179,7 +186,7 @@ export class GeneratedStringWithMap { }; if (generated.length == 0) - return new GeneratedStringWithMap(generated, replacement_map); + return new StringWithSourcemap(generated, replacement_map); // we generate a mapping // where the source was overwritten by the generated @@ -188,14 +195,14 @@ export class GeneratedStringWithMap { replacement_map.mappings.push([]); // unmapped line } - return new GeneratedStringWithMap(generated, replacement_map); + return new StringWithSourcemap(generated, replacement_map); } static from_source( source_file: string, source: string, offset_in_source?: SourceLocation - ): GeneratedStringWithMap { + ): StringWithSourcemap { const offset = offset_in_source || { line: 0, column: 0 }; const map: SourceMappings = { names: [], @@ -203,7 +210,7 @@ export class GeneratedStringWithMap { mappings: [], }; - if (source.length == 0) return new GeneratedStringWithMap(source, map); + if (source.length == 0) return new StringWithSourcemap(source, map); // we create a high resolution identity map here, // we know that it will eventually be merged with svelte's map, @@ -213,12 +220,13 @@ export class GeneratedStringWithMap { const identity_map = lines.map((line, line_idx) => { const segs = line .split(/([^\d\w\s]|\s+)/g) - .filter((x) => x !== "") + .filter((s) => s !== "") .map((s) => { const seg: MappingSegment = [ pos, 0, - offset.line + line_idx, + line_idx + offset.line, + // shift first line pos + (line_idx == 0 ? offset.column : 0), ]; pos = pos + s.length; @@ -230,6 +238,6 @@ export class GeneratedStringWithMap { map.mappings = identity_map; - return new GeneratedStringWithMap(source, map); + return new StringWithSourcemap(source, map); } } From 4a03b104f2a8b4483f2156a01c58a93e357e6ccb Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Sat, 5 Sep 2020 14:45:43 +0200 Subject: [PATCH 04/25] preprocessor sourcemaps: prettify code 3 rename decode to sourcemap_decode allow self-closing ` - ); + if (processed && processed.dependencies) dependencies.push(...processed.dependencies); + return processed + ? get_replacement(filename, offset, get_location, content, processed, ``, ``) + : no_change(); } ); - source = res.generated; sourcemap_list.unshift(res.get_sourcemap()); } + // HACK + // remove `undefined` sources in first sourcemap + // otherwise remapper throws error: + // Error: Transformation map 0 must have exactly one source file. + // Did you specify these with the most recent transformation maps first? + // test: preprocess comments + (firstMap => { + if (firstMap && firstMap.sources) + firstMap.sources = firstMap.sources.filter(Boolean); + })(sourcemap_list[0] as any); + + // https://github.com/ampproject/remapping#usage + // https://github.com/mozilla/source-map#new-sourcemapconsumerrawsourcemap const map: ReturnType = sourcemap_list.length == 0 ? null - : remapper(sourcemap_list as any, () => null); + : remapper(sourcemap_list as any, () => null, true); // true: skip optional field `sourcesContent` + + if (map) delete map.file; // skip optional field `file` return { // TODO return separated output, in future version where svelte.compile supports it: @@ -234,6 +244,7 @@ export default async function preprocess( code: source, dependencies: [...new Set(dependencies)], map, + toString() { return source; } diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index f899b03927d7..e9340a20804f 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -138,7 +138,7 @@ export class StringWithSourcemap { // combine the mappings - // this.map is readonly, so we copy + // this.map is read-only, so we copy let new_mappings = this.map.mappings.slice(); // combine: @@ -147,14 +147,18 @@ export class StringWithSourcemap { // columns of 2 must be shifted const end = get_end_location(this.generated); const col_offset = end.column + 1; + const first_line = other_mappings.length == 0 ? [] - : other_mappings[0].map((seg) => { - const new_seg = seg.slice() as MappingSegment; - new_seg[0] = seg[0] + col_offset; - return new_seg; - }); + : col_offset == 0 + ? other_mappings[0] + : other_mappings[0].map((seg) => { + // shift columns + const new_seg = seg.slice() as MappingSegment; + new_seg[0] = seg[0] + col_offset; + return new_seg; + }); // append segments to last line of first map new_mappings[new_mappings.length - 1] = diff --git a/test/sourcemaps/samples/preprocessed-markup/_preprocessor.js b/test/sourcemaps/samples/preprocessed-markup/_config.js similarity index 100% rename from test/sourcemaps/samples/preprocessed-markup/_preprocessor.js rename to test/sourcemaps/samples/preprocessed-markup/_config.js diff --git a/test/sourcemaps/samples/preprocessed-multiple/_preprocessor.js b/test/sourcemaps/samples/preprocessed-multiple/_config.js similarity index 100% rename from test/sourcemaps/samples/preprocessed-multiple/_preprocessor.js rename to test/sourcemaps/samples/preprocessed-multiple/_config.js diff --git a/test/sourcemaps/samples/preprocessed-script/_preprocessor.js b/test/sourcemaps/samples/preprocessed-script/_config.js similarity index 100% rename from test/sourcemaps/samples/preprocessed-script/_preprocessor.js rename to test/sourcemaps/samples/preprocessed-script/_config.js diff --git a/test/sourcemaps/samples/preprocessed-styles/_preprocessor.js b/test/sourcemaps/samples/preprocessed-styles/_config.js similarity index 100% rename from test/sourcemaps/samples/preprocessed-styles/_preprocessor.js rename to test/sourcemaps/samples/preprocessed-styles/_config.js From e7534894556ec7621ffe63d54626731cf94c7c78 Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Sat, 19 Sep 2020 21:19:51 +0200 Subject: [PATCH 07/25] fix test/preprocess --- test/preprocess/index.js | 9 +++++++-- test/preprocess/samples/filename/_config.js | 10 ++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/test/preprocess/index.js b/test/preprocess/index.js index ac096bad2e94..09ab03fbbd6c 100644 --- a/test/preprocess/index.js +++ b/test/preprocess/index.js @@ -8,16 +8,21 @@ describe('preprocess', () => { const config = loadConfig(`${__dirname}/samples/${dir}/_config.js`); const solo = config.solo || /\.solo/.test(dir); + const skip = config.skip || /\.skip/.test(dir); if (solo && process.env.CI) { throw new Error('Forgot to remove `solo: true` from test'); } - (config.skip ? it.skip : solo ? it.only : it)(dir, async () => { + (skip ? it.skip : solo ? it.only : it)(dir, async () => { const input = fs.readFileSync(`${__dirname}/samples/${dir}/input.svelte`, 'utf-8'); const expected = fs.readFileSync(`${__dirname}/samples/${dir}/output.svelte`, 'utf-8'); - const result = await svelte.preprocess(input, config.preprocess); + const result = await svelte.preprocess( + input, + config.preprocess, + config.options || { filename: 'input.svelte' } + ); fs.writeFileSync(`${__dirname}/samples/${dir}/_actual.html`, result.code); if (result.map) { fs.writeFileSync(`${__dirname}/samples/${dir}/_actual.html.map`, JSON.stringify(result.map, null, 2)); diff --git a/test/preprocess/samples/filename/_config.js b/test/preprocess/samples/filename/_config.js index c71cdafcac9a..a3763f353fba 100644 --- a/test/preprocess/samples/filename/_config.js +++ b/test/preprocess/samples/filename/_config.js @@ -1,6 +1,7 @@ export default { preprocess: { - filename: 'file.svelte', + // this is ignored cos filename is set in options + //filename: 'file.svelte', markup: ({ content, filename }) => { return { code: content.replace('__MARKUP_FILENAME__', filename) @@ -16,5 +17,10 @@ export default { code: content.replace('__SCRIPT_FILENAME__', filename) }; } + }, + options: { + // options.filename is preferred over preprocessor.filename + // see function preprocess + filename: 'file.svelte', } -}; \ No newline at end of file +}; From 6d06b7b8da6422c332f0a2de748dde9a4dbdb198 Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Sat, 19 Sep 2020 21:22:52 +0200 Subject: [PATCH 08/25] refactor test/sourcemaps use _config.js, indent with tabs, rename variables, --- test/sourcemaps/index.js | 87 ++++++++--------- test/sourcemaps/samples/basic/test.js | 12 +-- .../samples/binding-shorthand.skip/test.js | 13 +-- test/sourcemaps/samples/binding/test.js | 12 +-- test/sourcemaps/samples/css/input.svelte | 2 +- test/sourcemaps/samples/css/test.js | 10 +- .../samples/each-block/input.svelte | 2 +- test/sourcemaps/samples/each-block/test.js | 10 +- .../samples/preprocessed-markup/_config.js | 30 +++--- .../samples/preprocessed-markup/test.js | 14 +-- .../samples/preprocessed-multiple/_config.js | 93 +++++++++---------- .../samples/preprocessed-multiple/test.js | 23 +++-- .../samples/preprocessed-script/_config.js | 32 ++++--- .../samples/preprocessed-script/test.js | 18 ++-- .../samples/preprocessed-styles/_config.js | 32 ++++--- .../samples/preprocessed-styles/input.svelte | 1 - .../samples/preprocessed-styles/test.js | 20 ++-- .../samples/script-after-comment/test.js | 10 +- test/sourcemaps/samples/script/test.js | 8 +- .../samples/static-no-script/input.svelte | 2 +- .../samples/static-no-script/test.js | 12 +-- test/sourcemaps/samples/two-scripts/test.js | 8 +- 22 files changed, 230 insertions(+), 221 deletions(-) diff --git a/test/sourcemaps/index.js b/test/sourcemaps/index.js index c54e384d0127..adaac60ebf14 100644 --- a/test/sourcemaps/index.js +++ b/test/sourcemaps/index.js @@ -1,7 +1,9 @@ import * as fs from "fs"; import * as path from "path"; import * as assert from "assert"; -import { svelte } from "../helpers.js"; +import { loadConfig, svelte } from "../helpers.js"; +// keep source-map at version 0.7.x +// https://github.com/mozilla/source-map/issues/400 import { SourceMapConsumer } from "source-map"; import { getLocator } from "locate-character"; @@ -9,81 +11,80 @@ describe("sourcemaps", () => { fs.readdirSync(`${__dirname}/samples`).forEach(dir => { if (dir[0] === ".") return; + const config = loadConfig(`${__dirname}/samples/${dir}/_config.js`); + // add .solo to a sample directory name to only run that test - const solo = /\.solo/.test(dir); - const skip = /\.skip/.test(dir); + const solo = config.solo || /\.solo/.test(dir); + const skip = config.skip || /\.skip/.test(dir); if (solo && process.env.CI) { throw new Error("Forgot to remove `solo: true` from test"); } (solo ? it.only : skip ? it.skip : it)(dir, async () => { - const filename = path.resolve( - `${__dirname}/samples/${dir}/input.svelte` - ); - const outputFilename = path.resolve( - `${__dirname}/samples/${dir}/output` - ); + const inputFile = path.resolve(`${__dirname}/samples/${dir}/input.svelte`); + const outputBase = path.resolve(`${__dirname}/samples/${dir}/_actual`); - const preprocessorFilename = path.resolve( - `${__dirname}/samples/${dir}/_preprocessor.js` - ) + const input = {}; + input.code = fs.readFileSync(inputFile, "utf-8"); + input.locate = getLocator(input.code); - const input = fs.readFileSync(filename, "utf-8").replace(/\s+$/, ""); - let processed_input = input; - let processed_map = null; + const preprocessed = await svelte.preprocess( + input.code, + config.preprocess, { + filename: "input.svelte" + }); - if (fs.existsSync(preprocessorFilename)) { - let { preprocessors } = require(preprocessorFilename); - if (preprocessors.length > 0) { - ({ code: processed_input, map: processed_map } = await svelte.preprocess(input, preprocessors, { filename: 'input.svelte' })); - } - } - - const { js, css } = svelte.compile(processed_input, { - filename, - sourcemap: processed_map, - outputFilename: `${outputFilename}.js`, - cssOutputFilename: `${outputFilename}.css` + const { js, css } = svelte.compile( + preprocessed.code, { + filename: "input.svelte", + sourcemap: preprocessed.map, + // filenames for sourcemaps + outputFilename: "output.js", + cssOutputFilename: "output.css", }); - const _code = js.code.replace(/Svelte v\d+\.\d+\.\d+/, match => match.replace(/\d/g, 'x')); + js.code = js.code.replace( + /generated by Svelte v\d+\.\d+\.\d+/, + match => match.replace(/\d/g, "x") + ); + fs.writeFileSync(`${outputBase}.html`, preprocessed.code); + if (preprocessed.map) { + fs.writeFileSync(`${outputBase}.html.map`, JSON.stringify(preprocessed.map, null, 2)); + } fs.writeFileSync( - `${outputFilename}.js`, - `${_code}\n//# sourceMappingURL=output.js.map` + `${outputBase}.js`, + `${js.code}\n//# sourceMappingURL=output.js.map` ); fs.writeFileSync( - `${outputFilename}.js.map`, - JSON.stringify(js.map, null, " ") + `${outputBase}.js.map`, + JSON.stringify(js.map, 0, 2) ); - if (css.code) { fs.writeFileSync( - `${outputFilename}.css`, + `${outputBase}.css`, `${css.code}\n/*# sourceMappingURL=output.css.map */` ); fs.writeFileSync( - `${outputFilename}.css.map`, - JSON.stringify(css.map, null, " ") + `${outputBase}.css.map`, + JSON.stringify(css.map, 0, 2) ); } assert.deepEqual(js.map.sources, ["input.svelte"]); - if (css.map) assert.deepEqual(css.map.sources, ["input.svelte"]); const { test } = require(`./samples/${dir}/test.js`); - const locateInSource = getLocator(input); + js.mapConsumer = await new SourceMapConsumer(js.map); + js.locate = getLocator(js.code); - const smc = await new SourceMapConsumer(js.map); - const locateInGenerated = getLocator(_code); + css.mapConsumer = css.map && await new SourceMapConsumer(css.map); + css.locate = getLocator(css.code || ""); - const smcCss = css.map && await new SourceMapConsumer(css.map); - const locateInGeneratedCss = getLocator(css.code || ''); + test({ assert, input, js, css }); - test({ assert, code: _code, map: js.map, smc, smcCss, locateInSource, locateInGenerated, locateInGeneratedCss }); }); }); }); diff --git a/test/sourcemaps/samples/basic/test.js b/test/sourcemaps/samples/basic/test.js index 705976167461..34c619c128ca 100644 --- a/test/sourcemaps/samples/basic/test.js +++ b/test/sourcemaps/samples/basic/test.js @@ -1,12 +1,12 @@ -export function test({ assert, smc, locateInSource, locateInGenerated }) { - const expected = locateInSource('foo.bar.baz'); +export function test({ assert, input, js }) { + const expected = input.locate('foo.bar.baz'); let start; let actual; - start = locateInGenerated('ctx[0].bar.baz'); + start = js.locate('ctx[0].bar.baz'); - actual = smc.originalPositionFor({ + actual = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); @@ -18,9 +18,9 @@ export function test({ assert, smc, locateInSource, locateInGenerated }) { column: expected.column }); - start = locateInGenerated('ctx[0].bar.baz', start.character + 1); + start = js.locate('ctx[0].bar.baz', start.character + 1); - actual = smc.originalPositionFor({ + actual = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); diff --git a/test/sourcemaps/samples/binding-shorthand.skip/test.js b/test/sourcemaps/samples/binding-shorthand.skip/test.js index cdfbbdc091ac..591af0998782 100644 --- a/test/sourcemaps/samples/binding-shorthand.skip/test.js +++ b/test/sourcemaps/samples/binding-shorthand.skip/test.js @@ -1,13 +1,14 @@ -export function test({ assert, smc, locateInSource, locateInGenerated }) { - const expected = locateInSource('potato'); +export function test({ assert, input, js }) { + const expected = input.locate('potato'); let start; - start = locateInGenerated('potato'); - start = locateInGenerated('potato', start.character + 1); - start = locateInGenerated('potato', start.character + 1); // we need the third instance of 'potato' + start = js.locate('potato'); + start = js.locate('potato', start.character + 1); + start = js.locate('potato', start.character + 1); + // we need the third instance of 'potato' - const actual = smc.originalPositionFor({ + const actual = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); diff --git a/test/sourcemaps/samples/binding/test.js b/test/sourcemaps/samples/binding/test.js index 31c0e98442d5..3cb3246e5089 100644 --- a/test/sourcemaps/samples/binding/test.js +++ b/test/sourcemaps/samples/binding/test.js @@ -1,12 +1,12 @@ -export function test({ assert, smc, locateInSource, locateInGenerated }) { - const expected = locateInSource('bar.baz'); +export function test({ assert, input, js }) { + const expected = input.locate('bar.baz'); let start; let actual; - start = locateInGenerated('bar.baz'); + start = js.locate('bar.baz'); - actual = smc.originalPositionFor({ + actual = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); @@ -18,9 +18,9 @@ export function test({ assert, smc, locateInSource, locateInGenerated }) { column: expected.column }); - start = locateInGenerated('bar.baz', start.character + 1); + start = js.locate('bar.baz', start.character + 1); - actual = smc.originalPositionFor({ + actual = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); diff --git a/test/sourcemaps/samples/css/input.svelte b/test/sourcemaps/samples/css/input.svelte index ad0845d15b73..eba8f758a719 100644 --- a/test/sourcemaps/samples/css/input.svelte +++ b/test/sourcemaps/samples/css/input.svelte @@ -4,4 +4,4 @@ .foo { color: red; } - \ No newline at end of file + diff --git a/test/sourcemaps/samples/css/test.js b/test/sourcemaps/samples/css/test.js index 54082bd4ee66..1e0dda1dff43 100644 --- a/test/sourcemaps/samples/css/test.js +++ b/test/sourcemaps/samples/css/test.js @@ -1,14 +1,14 @@ -export function test({ assert, smcCss, locateInSource, locateInGeneratedCss }) { - const expected = locateInSource( '.foo' ); +export function test({ assert, input, css }) { + const expected = input.locate('.foo'); - const start = locateInGeneratedCss( '.foo' ); + const start = css.locate('.foo'); - const actual = smcCss.originalPositionFor({ + const actual = css.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); - assert.deepEqual( actual, { + assert.deepEqual(actual, { source: 'input.svelte', name: null, line: expected.line + 1, diff --git a/test/sourcemaps/samples/each-block/input.svelte b/test/sourcemaps/samples/each-block/input.svelte index bf2f0609bd7e..bf33a18b0d94 100644 --- a/test/sourcemaps/samples/each-block/input.svelte +++ b/test/sourcemaps/samples/each-block/input.svelte @@ -1,3 +1,3 @@ {#each foo as bar} {bar} -{/each} \ No newline at end of file +{/each} diff --git a/test/sourcemaps/samples/each-block/test.js b/test/sourcemaps/samples/each-block/test.js index 35479986a546..08b37686a462 100644 --- a/test/sourcemaps/samples/each-block/test.js +++ b/test/sourcemaps/samples/each-block/test.js @@ -1,10 +1,10 @@ -export function test({ assert, code, smc, map, locateInSource, locateInGenerated }) { - const startIndex = code.indexOf('create_main_fragment'); +export function test({ assert, input, js }) { + const startIndex = js.code.indexOf('create_main_fragment'); - const expected = locateInSource('each'); - const start = locateInGenerated('length', startIndex); + const expected = input.locate('each'); + const start = js.locate('length', startIndex); - const actual = smc.originalPositionFor({ + const actual = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); diff --git a/test/sourcemaps/samples/preprocessed-markup/_config.js b/test/sourcemaps/samples/preprocessed-markup/_config.js index db3103446b8f..f515cc6d1256 100644 --- a/test/sourcemaps/samples/preprocessed-markup/_config.js +++ b/test/sourcemaps/samples/preprocessed-markup/_config.js @@ -1,16 +1,18 @@ import MagicString from 'magic-string'; -export const preprocessors = [{ - markup: ({content, filename}) => { - const src = new MagicString(content); - const idx = content.indexOf("baritone"); - src.overwrite(idx, idx+"baritone".length, "bar"); - return { - code: src.toString(), - map: src.generateMap({ - source: filename, - includeContent: false - }) - }; - } -}]; \ No newline at end of file +export default { + preprocess: { + markup: ({ content, filename }) => { + const src = new MagicString(content); + const idx = content.indexOf("baritone"); + src.overwrite(idx, idx+"baritone".length, "bar"); + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + includeContent: false + }) + }; + } + } +}; diff --git a/test/sourcemaps/samples/preprocessed-markup/test.js b/test/sourcemaps/samples/preprocessed-markup/test.js index b398dcc3dd62..9c3f0ef06d6c 100644 --- a/test/sourcemaps/samples/preprocessed-markup/test.js +++ b/test/sourcemaps/samples/preprocessed-markup/test.js @@ -1,10 +1,10 @@ -export function test({ assert, smc, locateInSource, locateInGenerated }) { - const expectedBar = locateInSource('baritone.baz'); - const expectedBaz = locateInSource('.baz'); +export function test({ assert, input, js }) { + const expectedBar = input.locate('baritone.baz'); + const expectedBaz = input.locate('.baz'); - let start = locateInGenerated('bar.baz'); + let start = js.locate('bar.baz'); - const actualbar = smc.originalPositionFor({ + const actualbar = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); @@ -16,9 +16,9 @@ export function test({ assert, smc, locateInSource, locateInGenerated }) { column: expectedBar.column }); - start = locateInGenerated('.baz'); + start = js.locate('.baz'); - const actualbaz = smc.originalPositionFor({ + const actualbaz = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); diff --git a/test/sourcemaps/samples/preprocessed-multiple/_config.js b/test/sourcemaps/samples/preprocessed-multiple/_config.js index 4c7d023382c6..c587a71bbd99 100644 --- a/test/sourcemaps/samples/preprocessed-multiple/_config.js +++ b/test/sourcemaps/samples/preprocessed-multiple/_config.js @@ -1,51 +1,48 @@ import MagicString from 'magic-string'; -export const preprocessors = [{ - markup: ({ content, filename }) => { - const src = new MagicString(content); - const idx = content.indexOf("baritone"); - src.overwrite(idx, idx + "baritone".length, "bar"); +export default { + preprocess: { + markup: ({ content, filename }) => { + const src = new MagicString(content); + const idx = content.indexOf("baritone"); + src.overwrite(idx, idx + "baritone".length, "bar"); - const css_idx = content.indexOf("--bazitone"); - src.overwrite(css_idx, css_idx + "--bazitone".length, "--baz"); - return { - code: src.toString(), - map: src.generateMap({ - source: filename, - hires: true, - includeContent: false - }) - }; - } -}, -{ - script: ({ content, filename }) => { - const src = new MagicString(content); - const idx = content.indexOf("bar"); - src.prependLeft(idx, " "); - return { - code: src.toString(), - map: src.generateMap({ - source: filename, - hires: true, - includeContent: false - }) - }; - } -}, -{ - style: ({ content, filename }) => { - const src = new MagicString(content); - const idx = content.indexOf("--baz"); - src.prependLeft(idx, " "); - return { - code: src.toString(), - map: src.generateMap({ - source: filename, - hires: true, - includeContent: false - }) - }; - } -} -]; \ No newline at end of file + const css_idx = content.indexOf("--bazitone"); + src.overwrite(css_idx, css_idx + "--bazitone".length, "--baz"); + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + hires: true, + includeContent: false + }) + }; + }, + script: ({ content, filename }) => { + const src = new MagicString(content); + const idx = content.indexOf("bar"); + src.prependLeft(idx, " "); + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + hires: true, + includeContent: false + }) + }; + }, + style: ({ content, filename }) => { + const src = new MagicString(content); + const idx = content.indexOf("--baz"); + src.prependLeft(idx, " "); + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + hires: true, + includeContent: false + }) + }; + } + } +}; diff --git a/test/sourcemaps/samples/preprocessed-multiple/test.js b/test/sourcemaps/samples/preprocessed-multiple/test.js index d485695ba4fa..64b215677306 100644 --- a/test/sourcemaps/samples/preprocessed-multiple/test.js +++ b/test/sourcemaps/samples/preprocessed-multiple/test.js @@ -1,10 +1,10 @@ -export function test({ assert, smc, smcCss, locateInSource, locateInGenerated, locateInGeneratedCss }) { - const expectedBar = locateInSource('baritone'); - const expectedBaz = locateInSource('--bazitone'); - - let start = locateInGenerated('bar'); +export function test({ assert, input, js, css }) { + const expectedBar = input.locate('baritone'); + const expectedBaz = input.locate('--bazitone'); - const actualbar = smc.originalPositionFor({ + let start = js.locate('bar'); + + const actualbar = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); @@ -16,9 +16,9 @@ export function test({ assert, smc, smcCss, locateInSource, locateInGenerated, l column: expectedBar.column }); - start = locateInGeneratedCss('--baz'); + start = css.locate('--baz'); - const actualbaz = smcCss.originalPositionFor({ + const actualbaz = css.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); @@ -28,5 +28,10 @@ export function test({ assert, smc, smcCss, locateInSource, locateInGenerated, l name: null, line: expectedBaz.line + 1, column: expectedBaz.column - }, `couldn't find baz in css,\n gen:${JSON.stringify(start)}\n actual:${JSON.stringify(actualbaz)}\n expected:${JSON.stringify(expectedBaz)}`); + }, `\ +couldn't find baz in css, + gen: ${JSON.stringify(start)} + actual: ${JSON.stringify(actualbaz)} + expected: ${JSON.stringify(expectedBaz)}\ +`); } diff --git a/test/sourcemaps/samples/preprocessed-script/_config.js b/test/sourcemaps/samples/preprocessed-script/_config.js index bf895e0943c0..8781e9d7f2c1 100644 --- a/test/sourcemaps/samples/preprocessed-script/_config.js +++ b/test/sourcemaps/samples/preprocessed-script/_config.js @@ -1,17 +1,19 @@ import MagicString from 'magic-string'; -export const preprocessors = [{ - script: ({content, filename}) => { - const src = new MagicString(content); - const idx = content.indexOf("baritone"); - src.overwrite(idx, idx+"baritone".length, "bar"); - return { - code: src.toString(), - map: src.generateMap({ - source: filename, - hires: true, - includeContent: false - }) - }; - } -}]; \ No newline at end of file +export default { + preprocess: { + script: ({ content, filename }) => { + const src = new MagicString(content); + const idx = content.indexOf("baritone"); + src.overwrite(idx, idx+"baritone".length, "bar"); + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + hires: true, + includeContent: false + }) + }; + } + } +}; diff --git a/test/sourcemaps/samples/preprocessed-script/test.js b/test/sourcemaps/samples/preprocessed-script/test.js index a0ce8732b3c0..a7e53a96e70b 100644 --- a/test/sourcemaps/samples/preprocessed-script/test.js +++ b/test/sourcemaps/samples/preprocessed-script/test.js @@ -1,10 +1,10 @@ -export function test({ assert, smc, locateInSource, locateInGenerated }) { - const expectedBar = locateInSource('baritone:'); - const expectedBaz = locateInSource('baz:'); +export function test({ assert, input, js }) { + const expectedBar = input.locate('baritone:'); + const expectedBaz = input.locate('baz:'); - let start = locateInGenerated('bar:'); + let start = js.locate('bar:'); - const actualbar = smc.originalPositionFor({ + const actualbar = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); @@ -14,11 +14,11 @@ export function test({ assert, smc, locateInSource, locateInGenerated }) { name: null, line: expectedBar.line + 1, column: expectedBar.column - }, `couldn't find bar: in source` ); + }, "couldn't find bar: in source"); - start = locateInGenerated('baz:'); + start = js.locate('baz:'); - const actualbaz = smc.originalPositionFor({ + const actualbaz = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); @@ -28,5 +28,5 @@ export function test({ assert, smc, locateInSource, locateInGenerated }) { name: null, line: expectedBaz.line + 1, column: expectedBaz.column - }, `couldn't find baz: in source` ); + }, "couldn't find baz: in source"); } diff --git a/test/sourcemaps/samples/preprocessed-styles/_config.js b/test/sourcemaps/samples/preprocessed-styles/_config.js index 42166453e36f..4e199e6d70c4 100644 --- a/test/sourcemaps/samples/preprocessed-styles/_config.js +++ b/test/sourcemaps/samples/preprocessed-styles/_config.js @@ -1,17 +1,19 @@ import MagicString from 'magic-string'; -export const preprocessors = [{ - style: ({content, filename}) => { - const src = new MagicString(content); - const idx = content.indexOf("baritone"); - src.overwrite(idx, idx+"baritone".length, "bar"); - return { - code: src.toString(), - map: src.generateMap({ - source: filename, - hires: true, - includeContent: false - }) - }; - } -}]; \ No newline at end of file +export default { + preprocess: { + style: ({ content, filename }) => { + const src = new MagicString(content); + const idx = content.indexOf("baritone"); + src.overwrite(idx, idx+"baritone".length, "bar"); + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + hires: true, + includeContent: false + }) + }; + } + } +}; diff --git a/test/sourcemaps/samples/preprocessed-styles/input.svelte b/test/sourcemaps/samples/preprocessed-styles/input.svelte index 5ef6a3d443a0..0d942390f4b8 100644 --- a/test/sourcemaps/samples/preprocessed-styles/input.svelte +++ b/test/sourcemaps/samples/preprocessed-styles/input.svelte @@ -10,4 +10,3 @@ --baz: blue; } - diff --git a/test/sourcemaps/samples/preprocessed-styles/test.js b/test/sourcemaps/samples/preprocessed-styles/test.js index 7c6989588a74..5b28a1251481 100644 --- a/test/sourcemaps/samples/preprocessed-styles/test.js +++ b/test/sourcemaps/samples/preprocessed-styles/test.js @@ -1,10 +1,10 @@ -export function test({ assert, smcCss, locateInSource, locateInGeneratedCss }) { - const expectedBar = locateInSource('--baritone'); - const expectedBaz = locateInSource('--baz'); - - let start = locateInGeneratedCss('--bar'); +export function test({ assert, input, css }) { + const expectedBar = input.locate('--baritone'); + const expectedBaz = input.locate('--baz'); - const actualbar = smcCss.originalPositionFor({ + let start = css.locate('--bar'); + + const actualbar = css.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); @@ -14,11 +14,11 @@ export function test({ assert, smcCss, locateInSource, locateInGeneratedCss }) { name: null, line: expectedBar.line + 1, column: expectedBar.column - }, `couldn't find bar in source` ); + }, "couldn't find bar in source"); - start = locateInGeneratedCss('--baz'); + start = css.locate('--baz'); - const actualbaz = smcCss.originalPositionFor({ + const actualbaz = css.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); @@ -28,5 +28,5 @@ export function test({ assert, smcCss, locateInSource, locateInGeneratedCss }) { name: null, line: expectedBaz.line + 1, column: expectedBaz.column - }, `couldn't find baz in source` ); + }, "couldn't find baz in source"); } diff --git a/test/sourcemaps/samples/script-after-comment/test.js b/test/sourcemaps/samples/script-after-comment/test.js index b4739701122a..06ecc46929df 100644 --- a/test/sourcemaps/samples/script-after-comment/test.js +++ b/test/sourcemaps/samples/script-after-comment/test.js @@ -1,13 +1,13 @@ -export function test({ assert, smc, locateInSource, locateInGenerated }) { - const expected = locateInSource( 'assertThisLine' ); - const start = locateInGenerated( 'assertThisLine' ); +export function test({ assert, input, js }) { + const expected = input.locate('assertThisLine'); + const start = js.locate('assertThisLine'); - const actual = smc.originalPositionFor({ + const actual = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); - assert.deepEqual( actual, { + assert.deepEqual(actual, { source: 'input.svelte', name: null, line: expected.line + 1, diff --git a/test/sourcemaps/samples/script/test.js b/test/sourcemaps/samples/script/test.js index 73971ef48763..e6a91f51e1c1 100644 --- a/test/sourcemaps/samples/script/test.js +++ b/test/sourcemaps/samples/script/test.js @@ -1,8 +1,8 @@ -export function test({ assert, smc, locateInSource, locateInGenerated }) { - const expected = locateInSource( '42' ); - const start = locateInGenerated( '42' ); +export function test({ assert, input, js }) { + const expected = input.locate('42'); + const start = js.locate('42'); - const actual = smc.originalPositionFor({ + const actual = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); diff --git a/test/sourcemaps/samples/static-no-script/input.svelte b/test/sourcemaps/samples/static-no-script/input.svelte index 1e2d59797188..a6a67113e10f 100644 --- a/test/sourcemaps/samples/static-no-script/input.svelte +++ b/test/sourcemaps/samples/static-no-script/input.svelte @@ -1 +1 @@ -

no moving parts

\ No newline at end of file +

no moving parts

diff --git a/test/sourcemaps/samples/static-no-script/test.js b/test/sourcemaps/samples/static-no-script/test.js index a8f4d89ab822..c683c94d6b31 100644 --- a/test/sourcemaps/samples/static-no-script/test.js +++ b/test/sourcemaps/samples/static-no-script/test.js @@ -1,9 +1,9 @@ -const fs = require( 'fs' ); -const path = require( 'path' ); +const fs = require('fs'); +const path = require('path'); -export function test({ assert, map }) { - assert.deepEqual( map.sources, [ 'input.svelte' ]); - assert.deepEqual( map.sourcesContent, [ - fs.readFileSync( path.join( __dirname, 'input.svelte' ), 'utf-8' ) +export function test({ assert, js }) { + assert.deepEqual(js.map.sources, ['input.svelte']); + assert.deepEqual(js.map.sourcesContent, [ + fs.readFileSync(path.join(__dirname, 'input.svelte'), 'utf-8') ]); } diff --git a/test/sourcemaps/samples/two-scripts/test.js b/test/sourcemaps/samples/two-scripts/test.js index b4739701122a..70901af8c9d2 100644 --- a/test/sourcemaps/samples/two-scripts/test.js +++ b/test/sourcemaps/samples/two-scripts/test.js @@ -1,8 +1,8 @@ -export function test({ assert, smc, locateInSource, locateInGenerated }) { - const expected = locateInSource( 'assertThisLine' ); - const start = locateInGenerated( 'assertThisLine' ); +export function test({ assert, input, js }) { + const expected = input.locate( 'assertThisLine' ); + const start = js.locate( 'assertThisLine' ); - const actual = smc.originalPositionFor({ + const actual = js.mapConsumer.originalPositionFor({ line: start.line + 1, column: start.column }); From 433213a5e46e8d557a64eac3f63ff26f5cf6c80c Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Sat, 19 Sep 2020 21:33:50 +0200 Subject: [PATCH 09/25] lint commas --- src/compiler/utils/string_with_sourcemap.ts | 14 +++++++------- test/preprocess/samples/filename/_config.js | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index e9340a20804f..2caa769693e6 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -20,7 +20,7 @@ function get_end_location(s: string): SourceLocation { const parts = s.split("\n"); return { line: parts.length - 1, - column: parts[parts.length - 1].length - 1, + column: parts[parts.length - 1].length - 1 }; } @@ -49,7 +49,7 @@ export function sourcemap_add_offset( return { sources: map.sources, - mappings: new_mappings, + mappings: new_mappings } as SourceMappings; } @@ -86,7 +86,7 @@ export class StringWithSourcemap { version: 3, sources: this.map.sources, names: [], - mappings: sourcemap_encode(this.map.mappings as any), + mappings: sourcemap_encode(this.map.mappings as any) }; } @@ -173,7 +173,7 @@ export class StringWithSourcemap { this.generated + other.generated, { sources: new_sources, names: new_names, - mappings: new_mappings, + mappings: new_mappings }); } @@ -186,7 +186,7 @@ export class StringWithSourcemap { const replacement_map: SourceMappings = { names: [], sources: [], - mappings: [], + mappings: [] }; if (generated.length == 0) @@ -211,7 +211,7 @@ export class StringWithSourcemap { const map: SourceMappings = { names: [], sources: [source_file], - mappings: [], + mappings: [] }; if (source.length == 0) return new StringWithSourcemap(source, map); @@ -231,7 +231,7 @@ export class StringWithSourcemap { 0, line_idx + offset.line, // shift first line - pos + (line_idx == 0 ? offset.column : 0), + pos + (line_idx == 0 ? offset.column : 0) ]; pos = pos + s.length; return seg; diff --git a/test/preprocess/samples/filename/_config.js b/test/preprocess/samples/filename/_config.js index a3763f353fba..1563ac71fd7a 100644 --- a/test/preprocess/samples/filename/_config.js +++ b/test/preprocess/samples/filename/_config.js @@ -21,6 +21,6 @@ export default { options: { // options.filename is preferred over preprocessor.filename // see function preprocess - filename: 'file.svelte', + filename: 'file.svelte' } }; From e76f37d8a813a84b981d7302becd6834571862f4 Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Sun, 20 Sep 2020 16:45:41 +0200 Subject: [PATCH 10/25] move fn get_replacement --- src/compiler/preprocess/index.ts | 55 ++++++++++++++++---------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index e804aa4919ae..357c63603cd4 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -86,6 +86,33 @@ async function replace_async( return out; } +function get_replacement( + filename: string, + offset: number, + get_location: ReturnType, + original: string, + processed: Processed, + prefix: string, + suffix: string +): StringWithSourcemap { + const generated_prefix = StringWithSourcemap.from_source( + filename, prefix, get_location(offset)); + const generated_suffix = StringWithSourcemap.from_source( + filename, suffix, get_location(offset + prefix.length + original.length)); + + let generated; + if (processed.map) { + const full_map = typeof processed.map === "string" ? JSON.parse(processed.map) : processed.map; + const decoded_map = { ...full_map, mappings: sourcemap_decode(full_map.mappings) }; + const processed_offset = get_location(offset + prefix.length); + generated = StringWithSourcemap.from_generated(processed.code, sourcemap_add_offset(processed_offset, decoded_map)); + } else { + generated = StringWithSourcemap.from_generated(processed.code); + } + const map = generated_prefix.concat(generated).concat(generated_suffix); + return map; +} + export default async function preprocess( source: string, preprocessor: PreprocessorGroup | PreprocessorGroup[], @@ -108,34 +135,6 @@ export default async function preprocess( // https://github.com/ampproject/remapping#multiple-transformations-of-a-file const sourcemap_list: Array = []; - function get_replacement( - filename: string, - offset: number, - get_location: ReturnType, - original: string, - processed: Processed, - prefix: string, - suffix: string - ): StringWithSourcemap { - - const generated_prefix = StringWithSourcemap.from_source( - filename, prefix, get_location(offset)); - const generated_suffix = StringWithSourcemap.from_source( - filename, suffix, get_location(offset + prefix.length + original.length)); - - let generated; - if (processed.map) { - const full_map = typeof processed.map === "string" ? JSON.parse(processed.map) : processed.map; - const decoded_map = { ...full_map, mappings: sourcemap_decode(full_map.mappings) }; - const processed_offset = get_location(offset + prefix.length); - generated = StringWithSourcemap.from_generated(processed.code, sourcemap_add_offset(processed_offset, decoded_map)); - } else { - generated = StringWithSourcemap.from_generated(processed.code); - } - const map = generated_prefix.concat(generated).concat(generated_suffix); - return map; - } - for (const fn of markup) { // run markup preprocessor From b7d5974a513a3138e3219decf7442552d56a6b98 Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Sun, 20 Sep 2020 18:05:40 +0200 Subject: [PATCH 11/25] bugfix in fn merge_tables --- src/compiler/utils/string_with_sourcemap.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index 2caa769693e6..fa384d18bda0 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -59,14 +59,16 @@ function merge_tables( ): { table: T[]; new_idx: number[] } { const table = original.slice(); const new_idx = []; - for (let j = 0; j < original.length; j++) { - const current = extended[j]; - const existing = table.indexOf(current); - if (existing < 0) { - table.push(current); - new_idx[j] = table.length - 1; - } else { - new_idx[j] = existing; + if (extended) { + for (let j = 0; j < extended.length; j++) { + const current = extended[j]; + const existing = table.indexOf(current); + if (existing == -1) { + table.push(current); + new_idx[j] = table.length - 1; + } else { + new_idx[j] = existing; + } } } return { table, new_idx }; From 7cccff130585e6b4bf6509749c6c294b815a5cc9 Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Sun, 20 Sep 2020 18:22:44 +0200 Subject: [PATCH 12/25] remove hack --- src/compiler/preprocess/index.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index 357c63603cd4..5126fb1d13c9 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -214,17 +214,6 @@ export default async function preprocess( sourcemap_list.unshift(res.get_sourcemap()); } - // HACK - // remove `undefined` sources in first sourcemap - // otherwise remapper throws error: - // Error: Transformation map 0 must have exactly one source file. - // Did you specify these with the most recent transformation maps first? - // test: preprocess comments - (firstMap => { - if (firstMap && firstMap.sources) - firstMap.sources = firstMap.sources.filter(Boolean); - })(sourcemap_list[0] as any); - // https://github.com/ampproject/remapping#usage // https://github.com/mozilla/source-map#new-sourcemapconsumerrawsourcemap const map: ReturnType = From 6668f12148bf14ad8484fef5a600beac77417389 Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Sun, 20 Sep 2020 18:57:52 +0200 Subject: [PATCH 13/25] trigger test on travis ci From 459dd88b71813d4dcea58f88f81c7cdaeeaf2570 Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Mon, 21 Sep 2020 16:37:56 +0200 Subject: [PATCH 14/25] refactor --- src/compiler/preprocess/index.ts | 23 +- src/compiler/utils/string_with_sourcemap.ts | 230 ++++++------------ .../samples/binding-shorthand.skip/test.js | 2 +- 3 files changed, 86 insertions(+), 169 deletions(-) diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index 5126fb1d13c9..b521f5fd352c 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -95,22 +95,21 @@ function get_replacement( prefix: string, suffix: string ): StringWithSourcemap { - const generated_prefix = StringWithSourcemap.from_source( + const prefix_with_map = StringWithSourcemap.from_source( filename, prefix, get_location(offset)); - const generated_suffix = StringWithSourcemap.from_source( + const suffix_with_map = StringWithSourcemap.from_source( filename, suffix, get_location(offset + prefix.length + original.length)); - let generated; + let processed_map_shifted; if (processed.map) { - const full_map = typeof processed.map === "string" ? JSON.parse(processed.map) : processed.map; - const decoded_map = { ...full_map, mappings: sourcemap_decode(full_map.mappings) }; + const decoded_map = typeof processed.map === "string" ? JSON.parse(processed.map) : processed.map; + decoded_map.mappings = sourcemap_decode(decoded_map.mappings); const processed_offset = get_location(offset + prefix.length); - generated = StringWithSourcemap.from_generated(processed.code, sourcemap_add_offset(processed_offset, decoded_map)); - } else { - generated = StringWithSourcemap.from_generated(processed.code); + processed_map_shifted = sourcemap_add_offset(decoded_map, processed_offset); } - const map = generated_prefix.concat(generated).concat(generated_suffix); - return map; + const processed_with_map = StringWithSourcemap.from_processed(processed.code, processed_map_shifted); + + return prefix_with_map.concat(processed_with_map).concat(suffix_with_map); } export default async function preprocess( @@ -177,7 +176,7 @@ export default async function preprocess( : no_change(); } ); - source = res.generated; + source = res.string; sourcemap_list.unshift(res.get_sourcemap()); } @@ -210,7 +209,7 @@ export default async function preprocess( : no_change(); } ); - source = res.generated; + source = res.string; sourcemap_list.unshift(res.get_sourcemap()); } diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index fa384d18bda0..2c6df43e6a4b 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -16,145 +16,97 @@ type SourceLocation = { column: number; }; -function get_end_location(s: string): SourceLocation { - const parts = s.split("\n"); - return { - line: parts.length - 1, - column: parts[parts.length - 1].length - 1 - }; +function last_line_length(s: string) { + return s.length - s.lastIndexOf('\n') - 1; } export function sourcemap_add_offset( - offset: SourceLocation, - map: SourceMappings + map: SourceMappings, offset: SourceLocation ): SourceMappings { - const new_mappings = map.mappings.map((line) => - line.map((seg) => { - if (seg.length < 3) return seg; - const new_seg = seg.slice() as MappingSegment; - new_seg[2] = new_seg[2] + offset.line; - return new_seg; - }) - ); - - // column changed in first line - if (new_mappings.length > 0) { - new_mappings[0] = new_mappings[0].map((seg) => { - if (seg.length < 4) return seg; - const newSeg = seg.slice() as MappingSegment; - newSeg[3] = newSeg[3] + offset.column; - return newSeg; - }); - } - return { - sources: map.sources, - mappings: new_mappings + sources: map.sources.slice(), + mappings: map.mappings.map((line, line_idx) => + line.map(seg => { + const new_seg = seg.slice() as MappingSegment; + if (seg.length >= 4) { + new_seg[2] = new_seg[2] + offset.line; + if (line_idx == 0) + new_seg[3] = new_seg[3] + offset.column; + } + return new_seg; + }) + ) } as SourceMappings; } -function merge_tables( - original: T[], - extended: T[] -): { table: T[]; new_idx: number[] } { - const table = original.slice(); - const new_idx = []; - if (extended) { - for (let j = 0; j < extended.length; j++) { - const current = extended[j]; - const existing = table.indexOf(current); - if (existing == -1) { - table.push(current); - new_idx[j] = table.length - 1; - } else { - new_idx[j] = existing; - } +function merge_tables(this_table: T[], other_table): [T[], number[]] { + const new_table = this_table.slice(); + const idx_map = []; + other_table = other_table || []; + for (const [other_idx, other_val] of other_table.entries()) { + const this_idx = this_table.indexOf(other_val); + if (this_idx >= 0) { + idx_map[other_idx] = this_idx; + } else { + const new_idx = new_table.length; + new_table[new_idx] = other_val; + idx_map[other_idx] = new_idx; } } - return { table, new_idx }; + return [new_table, idx_map]; } export class StringWithSourcemap { - readonly generated: string; + readonly string: string; readonly map: SourceMappings; - constructor(generated: string, map: SourceMappings) { - this.generated = generated; + constructor(string: string, map: SourceMappings) { + this.string = string; this.map = map; } get_sourcemap() { return { version: 3, - sources: this.map.sources, + sources: this.map.sources.slice(), names: [], mappings: sourcemap_encode(this.map.mappings as any) }; } concat(other: StringWithSourcemap): StringWithSourcemap { - // if one is empty, return the other - if (this.generated.length == 0) return other; - if (other.generated.length == 0) return this; - - // combine sources - const { - table: new_sources, - new_idx: other_source_idx - } = merge_tables( - this.map.sources, - other.map.sources - ); + // noop: if one is empty, return the other + if (this.string == '') return other; + if (other.string == '') return this; - // combine names - const { - table: new_names, - new_idx: other_name_idx - } = merge_tables( - this.map.names, - other.map.names - ); + // combine sources and names + const [sources, new_source_idx] = merge_tables(this.map.sources, other.map.sources); + const [names, new_name_idx] = merge_tables(this.map.names, other.map.names); // update source refs and name refs in segments const other_mappings = other.map.mappings.map((line) => - line.map((seg) => { - // to reduce allocations, - // we only return a new segment if a value has changed - if ( - // new source idx - (seg.length > 1 && other_source_idx[seg[1]] != seg[1]) || - // new name idx - (seg.length == 5 && other_name_idx[seg[4]] != seg[4]) - ) { - const new_seg = seg.slice() as MappingSegment; - new_seg[1] = other_source_idx[seg[1]]; - if (seg.length == 5) { - new_seg[4] = other_name_idx[seg[4]]; - } - return new_seg; - } else { - return seg; - } + line.map(seg => { + const new_seg = seg.slice() as MappingSegment; + if (seg[1]) new_seg[1] = new_source_idx[seg[1]]; + if (seg[4]) new_seg[4] = new_name_idx[seg[4]]; + return new_seg; }) ); // combine the mappings - // this.map is read-only, so we copy - let new_mappings = this.map.mappings.slice(); - - // combine: + // combine // 1. last line of first map // 2. first line of second map // columns of 2 must be shifted - const end = get_end_location(this.generated); - const col_offset = end.column + 1; - const first_line = + const col_offset = last_line_length(this.string); + + const first_line: MappingSegment[] = other_mappings.length == 0 ? [] : col_offset == 0 - ? other_mappings[0] + ? other_mappings[0].slice() as MappingSegment[] : other_mappings[0].map((seg) => { // shift columns const new_seg = seg.slice() as MappingSegment; @@ -162,88 +114,54 @@ export class StringWithSourcemap { return new_seg; }); - // append segments to last line of first map - new_mappings[new_mappings.length - 1] = - new_mappings[new_mappings.length - 1].concat(first_line); - - // the other lines don't need modification and can just be appended - new_mappings = new_mappings.concat( - other_mappings.slice(1) as MappingSegment[][] - ); + const mappings: MappingSegment[][] = + this.map.mappings.slice(0, -1) + .concat([ + this.map.mappings.slice(-1)[0] // last line + .concat(first_line) + ]) + .concat(other_mappings.slice(1) as MappingSegment[][]); return new StringWithSourcemap( - this.generated + other.generated, { - sources: new_sources, - names: new_names, - mappings: new_mappings - }); + this.string + other.string, + { sources, names, mappings } + ); } - static from_generated( - generated: string, - map?: SourceMappings - ): StringWithSourcemap { - if (map) return new StringWithSourcemap(generated, map); - - const replacement_map: SourceMappings = { - names: [], - sources: [], - mappings: [] - }; - - if (generated.length == 0) - return new StringWithSourcemap(generated, replacement_map); - - // we generate a mapping - // where the source was overwritten by the generated - const end = get_end_location(generated); - for (let i = 0; i <= end.line; i++) { - replacement_map.mappings.push([]); // unmapped line - } - - return new StringWithSourcemap(generated, replacement_map); + static from_processed(string: string, map?: SourceMappings): StringWithSourcemap { + if (map) return new StringWithSourcemap(string, map); + map = { names: [], sources: [], mappings: [] }; + if (string == '') return new StringWithSourcemap(string, map); + // add empty MappingSegment[] for every line + const lineCount = string.split('\n').length; + map.mappings = Array.from({length: lineCount}).map(_ => []); + return new StringWithSourcemap(string, map); } static from_source( - source_file: string, - source: string, - offset_in_source?: SourceLocation + source_file: string, source: string, offset_in_source?: SourceLocation ): StringWithSourcemap { const offset = offset_in_source || { line: 0, column: 0 }; - const map: SourceMappings = { - names: [], - sources: [source_file], - mappings: [] - }; - + const map: SourceMappings = { names: [], sources: [source_file], mappings: [] }; if (source.length == 0) return new StringWithSourcemap(source, map); // we create a high resolution identity map here, // we know that it will eventually be merged with svelte's map, // at which stage the resolution will decrease. - const lines = source.split("\n"); - let pos = 0; - const identity_map = lines.map((line, line_idx) => { - const segs = line - .split(/([^\d\w\s]|\s+)/g) - .filter((s) => s !== "") - .map((s) => { + map.mappings = source.split("\n").map((line, line_idx) => { + let pos = 0; + const segs = line.split(/([^\d\w\s]|\s+)/g) + .filter(s => s !== "").map(s => { const seg: MappingSegment = [ - pos, - 0, + pos, 0, line_idx + offset.line, - // shift first line - pos + (line_idx == 0 ? offset.column : 0) + pos + (line_idx == 0 ? offset.column : 0) // shift first line ]; pos = pos + s.length; return seg; }); - pos = 0; return segs; }); - - map.mappings = identity_map; - return new StringWithSourcemap(source, map); } } diff --git a/test/sourcemaps/samples/binding-shorthand.skip/test.js b/test/sourcemaps/samples/binding-shorthand.skip/test.js index 591af0998782..13ecdbf88934 100644 --- a/test/sourcemaps/samples/binding-shorthand.skip/test.js +++ b/test/sourcemaps/samples/binding-shorthand.skip/test.js @@ -6,7 +6,7 @@ export function test({ assert, input, js }) { start = js.locate('potato'); start = js.locate('potato', start.character + 1); start = js.locate('potato', start.character + 1); - // we need the third instance of 'potato' + // we need the third instance of 'potato' const actual = js.mapConsumer.originalPositionFor({ line: start.line + 1, From 880f5567daa06d079e3d53e67796f7964e3f4138 Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Mon, 21 Sep 2020 17:11:26 +0200 Subject: [PATCH 15/25] ignore names in sourcemap --- src/compiler/utils/string_with_sourcemap.ts | 22 ++++++++++----------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index 2c6df43e6a4b..ad1930cd1eef 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -7,7 +7,7 @@ type MappingSegment = type SourceMappings = { sources: string[]; - names: string[]; + names: string[]; // names are ignored. might change in future mappings: MappingSegment[][]; }; @@ -79,16 +79,16 @@ export class StringWithSourcemap { if (this.string == '') return other; if (other.string == '') return this; - // combine sources and names + // combine sources const [sources, new_source_idx] = merge_tables(this.map.sources, other.map.sources); - const [names, new_name_idx] = merge_tables(this.map.names, other.map.names); + //const [names, new_name_idx] = merge_tables(this.map.names, other.map.names); - // update source refs and name refs in segments + // update source refs const other_mappings = other.map.mappings.map((line) => line.map(seg => { const new_seg = seg.slice() as MappingSegment; if (seg[1]) new_seg[1] = new_source_idx[seg[1]]; - if (seg[4]) new_seg[4] = new_name_idx[seg[4]]; + //if (seg[4]) new_seg[4] = new_name_idx[seg[4]]; return new_seg; }) ); @@ -107,12 +107,10 @@ export class StringWithSourcemap { ? [] : col_offset == 0 ? other_mappings[0].slice() as MappingSegment[] - : other_mappings[0].map((seg) => { - // shift columns - const new_seg = seg.slice() as MappingSegment; - new_seg[0] = seg[0] + col_offset; - return new_seg; - }); + : other_mappings[0].map(seg => ( + // shift column + [seg[0] + col_offset].concat(seg.slice(1)) as MappingSegment + )); const mappings: MappingSegment[][] = this.map.mappings.slice(0, -1) @@ -124,7 +122,7 @@ export class StringWithSourcemap { return new StringWithSourcemap( this.string + other.string, - { sources, names, mappings } + { sources, names: [], mappings } ); } From 2cf1ae6ad35ab44bf4c205cdca5dd8882ca6ce4e Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Tue, 22 Sep 2020 20:57:49 +0200 Subject: [PATCH 16/25] handle sourcemap.names --- src/compiler/utils/string_with_sourcemap.ts | 14 +++--- test/sourcemaps/index.js | 11 ++-- .../samples/sourcemap-names/_config.js | 50 +++++++++++++++++++ .../samples/sourcemap-names/input.svelte | 12 +++++ .../samples/sourcemap-names/test.js | 42 ++++++++++++++++ 5 files changed, 118 insertions(+), 11 deletions(-) create mode 100644 test/sourcemaps/samples/sourcemap-names/_config.js create mode 100644 test/sourcemaps/samples/sourcemap-names/input.svelte create mode 100644 test/sourcemaps/samples/sourcemap-names/test.js diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index ad1930cd1eef..4dbcc424263c 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -7,7 +7,7 @@ type MappingSegment = type SourceMappings = { sources: string[]; - names: string[]; // names are ignored. might change in future + names: string[]; mappings: MappingSegment[][]; }; @@ -69,7 +69,7 @@ export class StringWithSourcemap { return { version: 3, sources: this.map.sources.slice(), - names: [], + names: this.map.names.slice(), mappings: sourcemap_encode(this.map.mappings as any) }; } @@ -79,16 +79,16 @@ export class StringWithSourcemap { if (this.string == '') return other; if (other.string == '') return this; - // combine sources + // combine sources and names const [sources, new_source_idx] = merge_tables(this.map.sources, other.map.sources); - //const [names, new_name_idx] = merge_tables(this.map.names, other.map.names); + const [names, new_name_idx] = merge_tables(this.map.names, other.map.names); - // update source refs + // update source refs and name refs const other_mappings = other.map.mappings.map((line) => line.map(seg => { const new_seg = seg.slice() as MappingSegment; if (seg[1]) new_seg[1] = new_source_idx[seg[1]]; - //if (seg[4]) new_seg[4] = new_name_idx[seg[4]]; + if (seg[4]) new_seg[4] = new_name_idx[seg[4]]; return new_seg; }) ); @@ -122,7 +122,7 @@ export class StringWithSourcemap { return new StringWithSourcemap( this.string + other.string, - { sources, names: [], mappings } + { sources, names, mappings } ); } diff --git a/test/sourcemaps/index.js b/test/sourcemaps/index.js index adaac60ebf14..503f3c170714 100644 --- a/test/sourcemaps/index.js +++ b/test/sourcemaps/index.js @@ -49,9 +49,9 @@ describe("sourcemaps", () => { match => match.replace(/\d/g, "x") ); - fs.writeFileSync(`${outputBase}.html`, preprocessed.code); + fs.writeFileSync(`${outputBase}.svelte`, preprocessed.code); if (preprocessed.map) { - fs.writeFileSync(`${outputBase}.html.map`, JSON.stringify(preprocessed.map, null, 2)); + fs.writeFileSync(`${outputBase}.svelte.map`, JSON.stringify(preprocessed.map, null, 2)); } fs.writeFileSync( `${outputBase}.js`, @@ -77,13 +77,16 @@ describe("sourcemaps", () => { const { test } = require(`./samples/${dir}/test.js`); - js.mapConsumer = await new SourceMapConsumer(js.map); + preprocessed.mapConsumer = preprocessed.map && await new SourceMapConsumer(preprocessed.map); + preprocessed.locate = getLocator(preprocessed.code); + + js.mapConsumer = js.map && await new SourceMapConsumer(js.map); js.locate = getLocator(js.code); css.mapConsumer = css.map && await new SourceMapConsumer(css.map); css.locate = getLocator(css.code || ""); - test({ assert, input, js, css }); + test({ assert, input, preprocessed, js, css }); }); }); diff --git a/test/sourcemaps/samples/sourcemap-names/_config.js b/test/sourcemaps/samples/sourcemap-names/_config.js new file mode 100644 index 000000000000..96c969813272 --- /dev/null +++ b/test/sourcemaps/samples/sourcemap-names/_config.js @@ -0,0 +1,50 @@ +import MagicString from 'magic-string'; + +function replace(search, replace, content, src, options = { storeName: true }) { + let idx = -1; + while ((idx = content.indexOf(search, idx + 1)) != -1) { + src.overwrite(idx, idx + search.length, replace, options); + } +} + +function result(src, filename) { + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + hires: true, + includeContent: false + }) + }; +} + +export default { + preprocess: [ + { + markup: ({ content, filename }) => { + const src = new MagicString(content); + replace('baritone', 'bar', content, src); + replace('--bazitone', '--baz', content, src); + replace('old_name_1', 'temp_new_name_1', content, src); + replace('old_name_2', 'temp_new_name_2', content, src); + return result(src, filename); + } + }, + { + markup: ({ content, filename }) => { + const src = new MagicString(content); + replace('temp_new_name_1', 'temp_temp_new_name_1', content, src); + replace('temp_new_name_2', 'temp_temp_new_name_2', content, src); + return result(src, filename); + } + }, + { + markup: ({ content, filename }) => { + const src = new MagicString(content); + replace('temp_temp_new_name_1', 'new_name_1', content, src); + replace('temp_temp_new_name_2', 'new_name_2', content, src); + return result(src, filename); + } + } + ] +}; diff --git a/test/sourcemaps/samples/sourcemap-names/input.svelte b/test/sourcemaps/samples/sourcemap-names/input.svelte new file mode 100644 index 000000000000..b62715a85713 --- /dev/null +++ b/test/sourcemaps/samples/sourcemap-names/input.svelte @@ -0,0 +1,12 @@ + + +

use-names

+
{old_name_1.baritone}
+
{old_name_2}
diff --git a/test/sourcemaps/samples/sourcemap-names/test.js b/test/sourcemaps/samples/sourcemap-names/test.js new file mode 100644 index 000000000000..21cb4410e6cd --- /dev/null +++ b/test/sourcemaps/samples/sourcemap-names/test.js @@ -0,0 +1,42 @@ +// needed for workaround, TODO remove +import { getLocator } from 'locate-character'; + +export function test({ assert, input, preprocessed, js, css }) { + + assert.deepEqual( + preprocessed.map.names.sort(), + ['baritone', '--bazitone', 'old_name_1', 'old_name_2'].sort() + ); + + function test_name(old_name, new_name, where) { + + let loc = { character: -1 }; + while (loc = where.locate(new_name, loc.character + 1)) { + const actualMapping = where.mapConsumer.originalPositionFor({ + line: loc.line + 1, column: loc.column + }); + if (actualMapping.line === null) { + // location is not mapped - ignore + continue; + } + assert.equal(actualMapping.name, old_name); + } + if (loc === undefined) { + // workaround for bug in locate-character, TODO remove + // https://github.com/Rich-Harris/locate-character/pull/5 + where.locate = getLocator(where.code); + } + } + + test_name('baritone', 'bar', js); + test_name('baritone', 'bar', preprocessed); + + test_name('--bazitone', '--baz', css); + test_name('--bazitone', '--baz', preprocessed); + + test_name('old_name_1', 'new_name_1', js); + test_name('old_name_1', 'new_name_1', preprocessed); + + test_name('old_name_2', 'new_name_2', js); + test_name('old_name_2', 'new_name_2', preprocessed); +} From 1073120a16ca47d2c96b7ea60e2b312d25a02a5d Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Wed, 23 Sep 2020 13:53:58 +0200 Subject: [PATCH 17/25] remove unnecessary sourcemap encode --- src/compiler/preprocess/index.ts | 4 ++-- src/compiler/utils/string_with_sourcemap.ts | 11 ----------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index b521f5fd352c..87dada9fba76 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -177,7 +177,7 @@ export default async function preprocess( } ); source = res.string; - sourcemap_list.unshift(res.get_sourcemap()); + sourcemap_list.unshift(res.map); } for (const fn of style) { @@ -210,7 +210,7 @@ export default async function preprocess( } ); source = res.string; - sourcemap_list.unshift(res.get_sourcemap()); + sourcemap_list.unshift(res.map); } // https://github.com/ampproject/remapping#usage diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index 4dbcc424263c..cd7c2ea0a9e2 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -1,5 +1,3 @@ -import { encode as sourcemap_encode } from "sourcemap-codec"; - type MappingSegment = | [number] | [number, number, number, number] @@ -65,15 +63,6 @@ export class StringWithSourcemap { this.map = map; } - get_sourcemap() { - return { - version: 3, - sources: this.map.sources.slice(), - names: this.map.names.slice(), - mappings: sourcemap_encode(this.map.mappings as any) - }; - } - concat(other: StringWithSourcemap): StringWithSourcemap { // noop: if one is empty, return the other if (this.string == '') return other; From cf600f7a9f948d66f06e09a82e9ef26c0cb65181 Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Wed, 23 Sep 2020 16:51:08 +0200 Subject: [PATCH 18/25] add tests, fix empty map.sources, cleanup gitignore --- .gitignore | 4 -- src/compiler/preprocess/index.ts | 16 +++++-- test/sourcemaps/index.js | 19 ++++++-- .../samples/decoded-sourcemap/_config.js | 29 ++++++++++++ .../samples/decoded-sourcemap/input.svelte | 2 + .../samples/decoded-sourcemap/test.js | 19 ++++++++ .../samples/sourcemap-sources/_config.js | 46 +++++++++++++++++++ .../samples/sourcemap-sources/input.svelte | 4 ++ .../samples/sourcemap-sources/test.js | 12 +++++ 9 files changed, 138 insertions(+), 13 deletions(-) create mode 100644 test/sourcemaps/samples/decoded-sourcemap/_config.js create mode 100644 test/sourcemaps/samples/decoded-sourcemap/input.svelte create mode 100644 test/sourcemaps/samples/decoded-sourcemap/test.js create mode 100644 test/sourcemaps/samples/sourcemap-sources/_config.js create mode 100644 test/sourcemaps/samples/sourcemap-sources/input.svelte create mode 100644 test/sourcemaps/samples/sourcemap-sources/test.js diff --git a/.gitignore b/.gitignore index f7fac04ebaa9..a471c3aaa29c 100644 --- a/.gitignore +++ b/.gitignore @@ -18,10 +18,6 @@ node_modules /coverage/ /coverage.lcov /test/*/samples/_ -/test/sourcemaps/samples/*/output.js -/test/sourcemaps/samples/*/output.js.map -/test/sourcemaps/samples/*/output.css -/test/sourcemaps/samples/*/output.css.map /yarn-error.log _actual*.* _output diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index 87dada9fba76..ac7738f16f0a 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -132,7 +132,7 @@ export default async function preprocess( // sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1) // so we use sourcemap_list.unshift() to add new maps // https://github.com/ampproject/remapping#multiple-transformations-of-a-file - const sourcemap_list: Array = []; + let sourcemap_list: Array = []; for (const fn of markup) { @@ -213,14 +213,22 @@ export default async function preprocess( sourcemap_list.unshift(res.map); } - // https://github.com/ampproject/remapping#usage - // https://github.com/mozilla/source-map#new-sourcemapconsumerrawsourcemap + // remapper can throw error + // `Transformation map ${i} must have exactly one source file.` + sourcemap_list = sourcemap_list + .map(sourcemap => { + if ((sourcemap as any).sources.filter(Boolean).length == 0) + // fix missing source file + (sourcemap as any).sources = [filename]; + return sourcemap; + }); + const map: ReturnType = sourcemap_list.length == 0 ? null : remapper(sourcemap_list as any, () => null, true); // true: skip optional field `sourcesContent` - if (map) delete map.file; // skip optional field `file` + if (map && !map.file) delete map.file; // skip optional field `file` return { // TODO return separated output, in future version where svelte.compile supports it: diff --git a/test/sourcemaps/index.js b/test/sourcemaps/index.js index 503f3c170714..b823abb8eccd 100644 --- a/test/sourcemaps/index.js +++ b/test/sourcemaps/index.js @@ -29,11 +29,20 @@ describe("sourcemaps", () => { input.code = fs.readFileSync(inputFile, "utf-8"); input.locate = getLocator(input.code); - const preprocessed = await svelte.preprocess( - input.code, - config.preprocess, { - filename: "input.svelte" - }); + let preprocessed; + try { + preprocessed = await svelte.preprocess( + input.code, + config.preprocess, { + filename: "input.svelte" + }); + } catch (error) { + preprocessed = { + error, + code: '', + map: null + }; + } const { js, css } = svelte.compile( preprocessed.code, { diff --git a/test/sourcemaps/samples/decoded-sourcemap/_config.js b/test/sourcemaps/samples/decoded-sourcemap/_config.js new file mode 100644 index 000000000000..bc0db984b22c --- /dev/null +++ b/test/sourcemaps/samples/decoded-sourcemap/_config.js @@ -0,0 +1,29 @@ +import MagicString from 'magic-string'; + +function replace(search, replace, content, src, options = {}) { + let idx = -1; + while ((idx = content.indexOf(search, idx + 1)) != -1) { + src.overwrite(idx, idx + search.length, replace, options); + } +} + +function result(src, filename) { + return { + code: src.toString(), + map: src.generateDecodedMap({ // return decoded sourcemap + source: filename, + hires: true, + includeContent: false + }) + }; +} + +export default { + preprocess: { + markup: ({ content, filename }) => { + const src = new MagicString(content); + replace('replace me', 'success', content, src); + return result(src, filename); + } + } +}; diff --git a/test/sourcemaps/samples/decoded-sourcemap/input.svelte b/test/sourcemaps/samples/decoded-sourcemap/input.svelte new file mode 100644 index 000000000000..b233d7f670d7 --- /dev/null +++ b/test/sourcemaps/samples/decoded-sourcemap/input.svelte @@ -0,0 +1,2 @@ +

decoded-sourcemap

+
replace me
diff --git a/test/sourcemaps/samples/decoded-sourcemap/test.js b/test/sourcemaps/samples/decoded-sourcemap/test.js new file mode 100644 index 000000000000..54d930cb9714 --- /dev/null +++ b/test/sourcemaps/samples/decoded-sourcemap/test.js @@ -0,0 +1,19 @@ +export function test({ assert, input, preprocessed }) { + + const expected = input.locate('replace me'); + + const start = preprocessed.locate('success'); + + const actualbar = preprocessed.mapConsumer.originalPositionFor({ + line: start.line + 1, + column: start.column + }); + + assert.deepEqual(actualbar, { + source: 'input.svelte', + name: null, + line: expected.line + 1, + column: expected.column + }); + +} diff --git a/test/sourcemaps/samples/sourcemap-sources/_config.js b/test/sourcemaps/samples/sourcemap-sources/_config.js new file mode 100644 index 000000000000..df46e1348e77 --- /dev/null +++ b/test/sourcemaps/samples/sourcemap-sources/_config.js @@ -0,0 +1,46 @@ +import MagicString from 'magic-string'; + +function add(bundle, filename, source) { + bundle.addSource({ + filename: filename, + content: new MagicString(source) + }); +} + +function result(bundle, filename) { + return { + code: bundle.toString(), + map: bundle.generateMap({ + file: filename, + includeContent: true, + hires: true + }) + }; +} + +export default { + preprocess: [ + { + script: ({ content, filename }) => { + const bundle = new MagicString.Bundle(); + + add(bundle, filename, content); + add(bundle, 'foo.js', 'var answer = 42;'); + add(bundle, 'bar.js', 'console.log(answer);'); + + return result(bundle, filename); + } + }, + { + script: ({ content, filename }) => { + const bundle = new MagicString.Bundle(); + + add(bundle, filename, content); + add(bundle, 'foo2.js', 'var answer2 = 84;'); + add(bundle, 'bar2.js', 'console.log(answer2);'); + + return result(bundle, filename); + } + } + ] +}; diff --git a/test/sourcemaps/samples/sourcemap-sources/input.svelte b/test/sourcemaps/samples/sourcemap-sources/input.svelte new file mode 100644 index 000000000000..33c8a9d9a66b --- /dev/null +++ b/test/sourcemaps/samples/sourcemap-sources/input.svelte @@ -0,0 +1,4 @@ + +

sourcemap-sources

diff --git a/test/sourcemaps/samples/sourcemap-sources/test.js b/test/sourcemaps/samples/sourcemap-sources/test.js new file mode 100644 index 000000000000..0887afddd0b1 --- /dev/null +++ b/test/sourcemaps/samples/sourcemap-sources/test.js @@ -0,0 +1,12 @@ +export function test({ assert, preprocessed, js, css }) { + + const msg_expected = 'Transformation map 0 must have exactly one source file.'; + + assert.notEqual(preprocessed.error, undefined, 'expected preprocessed.error'); + + assert.equal( + preprocessed.error.message.slice(0, msg_expected.length), + msg_expected + ); + +} From 38f4ce48552a7c46f69633a937f6c9e6482308ee Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Thu, 24 Sep 2020 18:47:51 +0200 Subject: [PATCH 19/25] fix decode, dont fix missing map.sources --- src/compiler/preprocess/index.ts | 14 +++----- test/sourcemaps/index.js | 23 ++++++------- .../samples/sourcemap-names/test.js | 1 + .../samples/sourcemap-sources/_config.js | 33 +++++++++++-------- 4 files changed, 34 insertions(+), 37 deletions(-) diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index ac7738f16f0a..b4e956aab538 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -103,7 +103,8 @@ function get_replacement( let processed_map_shifted; if (processed.map) { const decoded_map = typeof processed.map === "string" ? JSON.parse(processed.map) : processed.map; - decoded_map.mappings = sourcemap_decode(decoded_map.mappings); + if (typeof(decoded_map.mappings) === 'string') + decoded_map.mappings = sourcemap_decode(decoded_map.mappings); const processed_offset = get_location(offset + prefix.length); processed_map_shifted = sourcemap_add_offset(decoded_map, processed_offset); } @@ -132,7 +133,7 @@ export default async function preprocess( // sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1) // so we use sourcemap_list.unshift() to add new maps // https://github.com/ampproject/remapping#multiple-transformations-of-a-file - let sourcemap_list: Array = []; + const sourcemap_list: Array = []; for (const fn of markup) { @@ -215,14 +216,7 @@ export default async function preprocess( // remapper can throw error // `Transformation map ${i} must have exactly one source file.` - sourcemap_list = sourcemap_list - .map(sourcemap => { - if ((sourcemap as any).sources.filter(Boolean).length == 0) - // fix missing source file - (sourcemap as any).sources = [filename]; - return sourcemap; - }); - + // for 0 <= i <= (sourcemap_list.length - 2) const map: ReturnType = sourcemap_list.length == 0 ? null diff --git a/test/sourcemaps/index.js b/test/sourcemaps/index.js index b823abb8eccd..7a2bf60982b4 100644 --- a/test/sourcemaps/index.js +++ b/test/sourcemaps/index.js @@ -22,8 +22,10 @@ describe("sourcemaps", () => { } (solo ? it.only : skip ? it.skip : it)(dir, async () => { + const { test } = require(`./samples/${dir}/test.js`); const inputFile = path.resolve(`${__dirname}/samples/${dir}/input.svelte`); - const outputBase = path.resolve(`${__dirname}/samples/${dir}/_actual`); + const outputName = '_actual'; + const outputBase = path.resolve(`${__dirname}/samples/${dir}/${outputName}`); const input = {}; input.code = fs.readFileSync(inputFile, "utf-8"); @@ -37,11 +39,9 @@ describe("sourcemaps", () => { filename: "input.svelte" }); } catch (error) { - preprocessed = { - error, - code: '', - map: null - }; + preprocessed = { error }; + // run test without js, css + return test({ assert, input, preprocessed }); } const { js, css } = svelte.compile( @@ -49,8 +49,8 @@ describe("sourcemaps", () => { filename: "input.svelte", sourcemap: preprocessed.map, // filenames for sourcemaps - outputFilename: "output.js", - cssOutputFilename: "output.css", + outputFilename: `${outputName}.js`, + cssOutputFilename: `${outputName}.css`, }); js.code = js.code.replace( @@ -64,7 +64,7 @@ describe("sourcemaps", () => { } fs.writeFileSync( `${outputBase}.js`, - `${js.code}\n//# sourceMappingURL=output.js.map` + `${js.code}\n//# sourceMappingURL=${outputName}.js.map` ); fs.writeFileSync( `${outputBase}.js.map`, @@ -73,7 +73,7 @@ describe("sourcemaps", () => { if (css.code) { fs.writeFileSync( `${outputBase}.css`, - `${css.code}\n/*# sourceMappingURL=output.css.map */` + `${css.code}\n/*# sourceMappingURL=${outputName}.css.map */` ); fs.writeFileSync( `${outputBase}.css.map`, @@ -84,8 +84,6 @@ describe("sourcemaps", () => { assert.deepEqual(js.map.sources, ["input.svelte"]); if (css.map) assert.deepEqual(css.map.sources, ["input.svelte"]); - const { test } = require(`./samples/${dir}/test.js`); - preprocessed.mapConsumer = preprocessed.map && await new SourceMapConsumer(preprocessed.map); preprocessed.locate = getLocator(preprocessed.code); @@ -96,7 +94,6 @@ describe("sourcemaps", () => { css.locate = getLocator(css.code || ""); test({ assert, input, preprocessed, js, css }); - }); }); }); diff --git a/test/sourcemaps/samples/sourcemap-names/test.js b/test/sourcemaps/samples/sourcemap-names/test.js index 21cb4410e6cd..85f4b1afdba5 100644 --- a/test/sourcemaps/samples/sourcemap-names/test.js +++ b/test/sourcemaps/samples/sourcemap-names/test.js @@ -8,6 +8,7 @@ export function test({ assert, input, preprocessed, js, css }) { ['baritone', '--bazitone', 'old_name_1', 'old_name_2'].sort() ); + // TODO move fn test_name to test/sourcemaps/index.js and use in samples/*/test.js function test_name(old_name, new_name, where) { let loc = { character: -1 }; diff --git a/test/sourcemaps/samples/sourcemap-sources/_config.js b/test/sourcemaps/samples/sourcemap-sources/_config.js index df46e1348e77..4fb923e1fc18 100644 --- a/test/sourcemaps/samples/sourcemap-sources/_config.js +++ b/test/sourcemaps/samples/sourcemap-sources/_config.js @@ -1,21 +1,26 @@ import MagicString from 'magic-string'; +// eslint warning: import/no-named-as-default-member + +//import MagicString, { Bundle } from 'magic-string'; +// TODO why does this break mocha? +// SyntaxError: Cannot use import statement outside a module function add(bundle, filename, source) { - bundle.addSource({ - filename: filename, - content: new MagicString(source) - }); + bundle.addSource({ + filename, + content: new MagicString(source) + }); } function result(bundle, filename) { - return { - code: bundle.toString(), - map: bundle.generateMap({ - file: filename, - includeContent: true, - hires: true - }) - }; + return { + code: bundle.toString(), + map: bundle.generateMap({ + file: filename, + includeContent: true, + hires: true + }) + }; } export default { @@ -28,7 +33,7 @@ export default { add(bundle, 'foo.js', 'var answer = 42;'); add(bundle, 'bar.js', 'console.log(answer);'); - return result(bundle, filename); + return result(bundle, filename); } }, { @@ -39,7 +44,7 @@ export default { add(bundle, 'foo2.js', 'var answer2 = 84;'); add(bundle, 'bar2.js', 'console.log(answer2);'); - return result(bundle, filename); + return result(bundle, filename); } } ] From 47ffc055fb22604f90d3b88fcb3f25a5e498105d Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Thu, 24 Sep 2020 19:39:55 +0200 Subject: [PATCH 20/25] optimize concat --- src/compiler/utils/string_with_sourcemap.ts | 39 ++++++++++++--------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index cd7c2ea0a9e2..1b2c72f0e653 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -37,10 +37,11 @@ export function sourcemap_add_offset( } as SourceMappings; } -function merge_tables(this_table: T[], other_table): [T[], number[]] { +function merge_tables(this_table: T[], other_table): [T[], number[], boolean] { const new_table = this_table.slice(); const idx_map = []; other_table = other_table || []; + let has_changed = false; for (const [other_idx, other_val] of other_table.entries()) { const this_idx = this_table.indexOf(other_val); if (this_idx >= 0) { @@ -49,9 +50,10 @@ function merge_tables(this_table: T[], other_table): [T[], number[]] { const new_idx = new_table.length; new_table[new_idx] = other_val; idx_map[other_idx] = new_idx; + has_changed = true; } } - return [new_table, idx_map]; + return [new_table, idx_map, has_changed]; } export class StringWithSourcemap { @@ -69,18 +71,20 @@ export class StringWithSourcemap { if (other.string == '') return this; // combine sources and names - const [sources, new_source_idx] = merge_tables(this.map.sources, other.map.sources); - const [names, new_name_idx] = merge_tables(this.map.names, other.map.names); + const [sources, new_source_idx, sources_changed] = merge_tables(this.map.sources, other.map.sources); + const [names, new_name_idx, names_changed] = merge_tables(this.map.names, other.map.names); // update source refs and name refs - const other_mappings = other.map.mappings.map((line) => - line.map(seg => { - const new_seg = seg.slice() as MappingSegment; - if (seg[1]) new_seg[1] = new_source_idx[seg[1]]; - if (seg[4]) new_seg[4] = new_name_idx[seg[4]]; - return new_seg; - }) - ); + const other_mappings = + (sources_changed || names_changed) + ? other.map.mappings.slice().map(line => + line.map(seg => { + if (seg[1]) seg[1] = new_source_idx[seg[1]]; + if (seg[4]) seg[4] = new_name_idx[seg[4]]; + return seg; + }) + ) + : other.map.mappings; // combine the mappings @@ -89,17 +93,18 @@ export class StringWithSourcemap { // 2. first line of second map // columns of 2 must be shifted - const col_offset = last_line_length(this.string); + const column_offset = last_line_length(this.string); const first_line: MappingSegment[] = other_mappings.length == 0 ? [] - : col_offset == 0 + : column_offset == 0 ? other_mappings[0].slice() as MappingSegment[] - : other_mappings[0].map(seg => ( + : other_mappings[0].slice().map(seg => { // shift column - [seg[0] + col_offset].concat(seg.slice(1)) as MappingSegment - )); + seg[0] += column_offset; + return seg; + }); const mappings: MappingSegment[][] = this.map.mappings.slice(0, -1) From b739bdbe192becb3fd3ea2d85d295df7e32d6cb1 Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Fri, 25 Sep 2020 09:44:34 +0200 Subject: [PATCH 21/25] optimize merge_tables, verbose remapper error --- src/compiler/preprocess/index.ts | 19 +++++++++++++++---- src/compiler/utils/string_with_sourcemap.ts | 6 ++++++ .../samples/sourcemap-sources/test.js | 8 ++++---- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index b4e956aab538..4a7578452dd7 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -217,10 +217,21 @@ export default async function preprocess( // remapper can throw error // `Transformation map ${i} must have exactly one source file.` // for 0 <= i <= (sourcemap_list.length - 2) - const map: ReturnType = - sourcemap_list.length == 0 - ? null - : remapper(sourcemap_list as any, () => null, true); // true: skip optional field `sourcesContent` + + let map: ReturnType; + try { + map = + sourcemap_list.length == 0 + ? null + : remapper(sourcemap_list as any, () => null, true); // true: skip optional field `sourcesContent` + } catch (error) { + throw { ...error, message: error.message + + '\n\ncould not combine sourcemaps:\n' + + JSON.stringify((sourcemap_list as any).map(m => { + return { ...m, mappings: JSON.stringify(m.mappings).slice(0, 100)+' ....'}; + }), null, 2) + }; + } if (map && !map.file) delete map.file; // skip optional field `file` diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index 1b2c72f0e653..a8a978acdba7 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -53,6 +53,12 @@ function merge_tables(this_table: T[], other_table): [T[], number[], boolean] has_changed = true; } } + if (has_changed) { + if (idx_map.find((val, idx) => val != idx) === undefined) { + // idx_map is identity map [0, 1, 2, 3, 4, ....] + has_changed = false; + } + } return [new_table, idx_map, has_changed]; } diff --git a/test/sourcemaps/samples/sourcemap-sources/test.js b/test/sourcemaps/samples/sourcemap-sources/test.js index 0887afddd0b1..e35618e18a04 100644 --- a/test/sourcemaps/samples/sourcemap-sources/test.js +++ b/test/sourcemaps/samples/sourcemap-sources/test.js @@ -1,12 +1,12 @@ export function test({ assert, preprocessed, js, css }) { - const msg_expected = 'Transformation map 0 must have exactly one source file.'; + assert.notEqual(preprocessed.error, undefined, 'expected preprocessed.error'); - assert.notEqual(preprocessed.error, undefined, 'expected preprocessed.error'); + const msg_expected_prefix = 'Transformation map 0 must have exactly one source file.'; assert.equal( - preprocessed.error.message.slice(0, msg_expected.length), - msg_expected + preprocessed.error.message.slice(0, msg_expected_prefix.length), + msg_expected_prefix ); } From 422cc0d96f4d1fd942e3835beb85d3acda904abf Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Fri, 25 Sep 2020 18:33:20 +0200 Subject: [PATCH 22/25] optimize: use mutable data, unswitch loops --- src/compiler/preprocess/index.ts | 17 +-- src/compiler/utils/string_with_sourcemap.ts | 157 ++++++++++++-------- 2 files changed, 99 insertions(+), 75 deletions(-) diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index 4a7578452dd7..d72eb73945bb 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -67,7 +67,7 @@ async function replace_async( ); return ''; }); - let out: StringWithSourcemap; + const out = new StringWithSourcemap(); let last_end = 0; for (const { offset, length, replacement } of await Promise.all( replacements @@ -75,15 +75,13 @@ async function replace_async( // content = source before replacement const content = StringWithSourcemap.from_source( filename, source.slice(last_end, offset), get_location(last_end)); - out = out ? out.concat(content) : content; - out = out.concat(replacement); + out.concat(content).concat(replacement); last_end = offset + length; } // final_content = source after last replacement const final_content = StringWithSourcemap.from_source( filename, source.slice(last_end), get_location(last_end)); - out = out.concat(final_content); - return out; + return out.concat(final_content); } function get_replacement( @@ -100,15 +98,14 @@ function get_replacement( const suffix_with_map = StringWithSourcemap.from_source( filename, suffix, get_location(offset + prefix.length + original.length)); - let processed_map_shifted; + let decoded_map; if (processed.map) { - const decoded_map = typeof processed.map === "string" ? JSON.parse(processed.map) : processed.map; + decoded_map = typeof processed.map === "string" ? JSON.parse(processed.map) : processed.map; if (typeof(decoded_map.mappings) === 'string') decoded_map.mappings = sourcemap_decode(decoded_map.mappings); - const processed_offset = get_location(offset + prefix.length); - processed_map_shifted = sourcemap_add_offset(decoded_map, processed_offset); + sourcemap_add_offset(decoded_map, get_location(offset + prefix.length)); } - const processed_with_map = StringWithSourcemap.from_processed(processed.code, processed_map_shifted); + const processed_with_map = StringWithSourcemap.from_processed(processed.code, decoded_map); return prefix_with_map.concat(processed_with_map).concat(suffix_with_map); } diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index a8a978acdba7..a5a89eea158d 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -4,6 +4,7 @@ type MappingSegment = | [number, number, number, number, number]; type SourceMappings = { + version: number; sources: string[]; names: string[]; mappings: MappingSegment[][]; @@ -18,30 +19,28 @@ function last_line_length(s: string) { return s.length - s.lastIndexOf('\n') - 1; } +// mutate map in-place export function sourcemap_add_offset( map: SourceMappings, offset: SourceLocation -): SourceMappings { - return { - sources: map.sources.slice(), - mappings: map.mappings.map((line, line_idx) => - line.map(seg => { - const new_seg = seg.slice() as MappingSegment; - if (seg.length >= 4) { - new_seg[2] = new_seg[2] + offset.line; - if (line_idx == 0) - new_seg[3] = new_seg[3] + offset.column; - } - return new_seg; - }) - ) - } as SourceMappings; +) { + // shift columns in first line + const m = map.mappings as any; + m[0].forEach(seg => { + if (seg[3]) seg[3] += offset.column; + }); + // shift lines + m.forEach(line => { + line.forEach(seg => { + if (seg[2]) seg[2] += offset.line; + }); + }); } -function merge_tables(this_table: T[], other_table): [T[], number[], boolean] { +function merge_tables(this_table: T[], other_table): [T[], number[], boolean, boolean] { const new_table = this_table.slice(); const idx_map = []; other_table = other_table || []; - let has_changed = false; + let val_changed = false; for (const [other_idx, other_val] of other_table.entries()) { const this_idx = this_table.indexOf(other_val); if (this_idx >= 0) { @@ -50,47 +49,85 @@ function merge_tables(this_table: T[], other_table): [T[], number[], boolean] const new_idx = new_table.length; new_table[new_idx] = other_val; idx_map[other_idx] = new_idx; - has_changed = true; + val_changed = true; } } - if (has_changed) { + let idx_changed = val_changed; + if (val_changed) { if (idx_map.find((val, idx) => val != idx) === undefined) { // idx_map is identity map [0, 1, 2, 3, 4, ....] - has_changed = false; + idx_changed = false; } } - return [new_table, idx_map, has_changed]; + return [new_table, idx_map, val_changed, idx_changed]; +} + +function pushArray(_this: T[], other: T[]) { + for (let i = 0; i < other.length; i++) + _this.push(other[i]); } export class StringWithSourcemap { - readonly string: string; - readonly map: SourceMappings; + string: string; + map: SourceMappings; - constructor(string: string, map: SourceMappings) { + constructor(string = '', map = null) { this.string = string; - this.map = map; + if (map) + this.map = map as SourceMappings; + else + this.map = { + version: 3, + mappings: [], + sources: [], + names: [] + }; } + // concat in-place (mutable), return this (chainable) + // will also mutate the `other` object concat(other: StringWithSourcemap): StringWithSourcemap { // noop: if one is empty, return the other - if (this.string == '') return other; if (other.string == '') return this; + if (this.string == '') { + this.string = other.string; + this.map = other.map; + return this; + } + + this.string += other.string; + + const m1 = this.map as any; + const m2 = other.map as any; // combine sources and names - const [sources, new_source_idx, sources_changed] = merge_tables(this.map.sources, other.map.sources); - const [names, new_name_idx, names_changed] = merge_tables(this.map.names, other.map.names); - - // update source refs and name refs - const other_mappings = - (sources_changed || names_changed) - ? other.map.mappings.slice().map(line => - line.map(seg => { - if (seg[1]) seg[1] = new_source_idx[seg[1]]; - if (seg[4]) seg[4] = new_name_idx[seg[4]]; - return seg; - }) - ) - : other.map.mappings; + const [sources, new_source_idx, sources_changed, sources_idx_changed] = merge_tables(m1.sources, m2.sources); + const [names, new_name_idx, names_changed, names_idx_changed] = merge_tables(m1.names, m2.names); + + if (sources_changed) m1.sources = sources; + if (names_changed) m1.names = names; + + // unswitched loops are faster + if (sources_idx_changed && names_idx_changed) { + m2.forEach(line => { + line.forEach(seg => { + if (seg[1]) seg[1] = new_source_idx[seg[1]]; + if (seg[4]) seg[4] = new_name_idx[seg[4]]; + }); + }); + } else if (sources_idx_changed) { + m2.forEach(line => { + line.forEach(seg => { + if (seg[1]) seg[1] = new_source_idx[seg[1]]; + }); + }); + } else if (names_idx_changed) { + m2.forEach(line => { + line.forEach(seg => { + if (seg[4]) seg[4] = new_name_idx[seg[4]]; + }); + }); + } // combine the mappings @@ -100,35 +137,25 @@ export class StringWithSourcemap { // columns of 2 must be shifted const column_offset = last_line_length(this.string); + if (m2.length > 0 && column_offset > 0) { + // shift columns in first line + m2[0].forEach(seg => { + seg[0] += column_offset; + }); + } + + // combine last line + first line + pushArray(m1.mappings[m1.mappings.length - 1], m2.mappings.shift()); + + // append other lines + pushArray(m1.mappings, m2.mappings); - const first_line: MappingSegment[] = - other_mappings.length == 0 - ? [] - : column_offset == 0 - ? other_mappings[0].slice() as MappingSegment[] - : other_mappings[0].slice().map(seg => { - // shift column - seg[0] += column_offset; - return seg; - }); - - const mappings: MappingSegment[][] = - this.map.mappings.slice(0, -1) - .concat([ - this.map.mappings.slice(-1)[0] // last line - .concat(first_line) - ]) - .concat(other_mappings.slice(1) as MappingSegment[][]); - - return new StringWithSourcemap( - this.string + other.string, - { sources, names, mappings } - ); + return this; } static from_processed(string: string, map?: SourceMappings): StringWithSourcemap { if (map) return new StringWithSourcemap(string, map); - map = { names: [], sources: [], mappings: [] }; + map = { version: 3, names: [], sources: [], mappings: [] }; if (string == '') return new StringWithSourcemap(string, map); // add empty MappingSegment[] for every line const lineCount = string.split('\n').length; @@ -140,7 +167,7 @@ export class StringWithSourcemap { source_file: string, source: string, offset_in_source?: SourceLocation ): StringWithSourcemap { const offset = offset_in_source || { line: 0, column: 0 }; - const map: SourceMappings = { names: [], sources: [source_file], mappings: [] }; + const map: SourceMappings = { version: 3, names: [], sources: [source_file], mappings: [] }; if (source.length == 0) return new StringWithSourcemap(source, map); // we create a high resolution identity map here, From 3d053d93956c8c31bc3f4b6ac56593b923e976d3 Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Sat, 26 Sep 2020 12:21:02 +0200 Subject: [PATCH 23/25] support default + named import --- test/setup.js | 22 ++++++++++++------- .../samples/sourcemap-sources/_config.js | 11 +++------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/test/setup.js b/test/setup.js index 7406a07dd9fb..14e0366387b1 100644 --- a/test/setup.js +++ b/test/setup.js @@ -8,22 +8,28 @@ require.extensions['.js'] = function(module, filename) { const exports = []; let code = fs.readFileSync(filename, 'utf-8') - .replace(/^import \* as (\w+) from ['"]([^'"]+)['"];?/gm, 'var $1 = require("$2");') - .replace(/^import (\w+) from ['"]([^'"]+)['"];?/gm, 'var {default: $1} = require("$2");') - .replace(/^import {([^}]+)} from ['"](.+)['"];?/gm, 'var {$1} = require("$2");') - .replace(/^export default /gm, 'exports.default = ') - .replace(/^export (const|let|var|class|function) (\w+)/gm, (match, type, name) => { + .replace(/^import\s+\*\s+as\s+(\w+)\s+from\s+(['"])(.+?)\2;?/gm, 'var $1 = require($2$3$2);') + .replace(/^import\s+(\w+)\s+from\s+(['"])(.+?)\2;?/gm, 'var {default: $1} = require($2$3$2);') + .replace(/^import\s+(?:(\w+)\s*,\s*)?{([^}]+)}(?:\s*,\s*(\w+))?\s+from\s+(['"])(.+?)\4;?/gm, + (match, default_name_1, names, default_name_2, quote, source) => { + names = names.replace(/\s+as\s+/g, ': '); + let default_name = default_name_1 || default_name_2; + default_name = default_name ? `default: ${default_name}, ` : ''; + return `var {${default_name}${names}} = require(${quote}${source}${quote});`; + }) + .replace(/^export\s+default\s+/gm, 'exports.default = ') + .replace(/^export\s+(const|let|var|class|function)\s+(\w+)/gm, (match, type, name) => { exports.push(name); return `${type} ${name}`; }) - .replace(/^export \{([^}]+)\}(?: from ['"]([^'"]+)['"];?)?/gm, (match, names, source) => { + .replace(/^export\s+\{([^}]+)\}(?:\s+from\s+(['"])(.+?)\2;?)?/gm, (match, names, quote, source) => { names.split(',').filter(Boolean).forEach(name => { exports.push(name); }); - return source ? `const { ${names} } = require("${source}");` : ''; + return source ? `const { ${names} } = require(${quote}${source}${quote});` : ''; }) - .replace(/^export function (\w+)/gm, 'exports.$1 = function $1'); + .replace(/^export\s+function\s+(\w+)/gm, 'exports.$1 = function $1'); exports.forEach(name => { code += `\nexports.${name} = ${name};`; diff --git a/test/sourcemaps/samples/sourcemap-sources/_config.js b/test/sourcemaps/samples/sourcemap-sources/_config.js index 4fb923e1fc18..0c413c93000f 100644 --- a/test/sourcemaps/samples/sourcemap-sources/_config.js +++ b/test/sourcemaps/samples/sourcemap-sources/_config.js @@ -1,9 +1,4 @@ -import MagicString from 'magic-string'; -// eslint warning: import/no-named-as-default-member - -//import MagicString, { Bundle } from 'magic-string'; -// TODO why does this break mocha? -// SyntaxError: Cannot use import statement outside a module +import MagicString, { Bundle } from 'magic-string'; function add(bundle, filename, source) { bundle.addSource({ @@ -27,7 +22,7 @@ export default { preprocess: [ { script: ({ content, filename }) => { - const bundle = new MagicString.Bundle(); + const bundle = new Bundle(); add(bundle, filename, content); add(bundle, 'foo.js', 'var answer = 42;'); @@ -38,7 +33,7 @@ export default { }, { script: ({ content, filename }) => { - const bundle = new MagicString.Bundle(); + const bundle = new Bundle(); add(bundle, filename, content); add(bundle, 'foo2.js', 'var answer2 = 84;'); From a0eb41f68d5809b84a4be5dfbe3aad33ecbffded Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Tue, 29 Sep 2020 22:13:07 +0200 Subject: [PATCH 24/25] support multiple source files, fix types --- src/compiler/compile/Component.ts | 4 +- src/compiler/preprocess/index.ts | 44 +++++++++++++------ src/compiler/utils/string_with_sourcemap.ts | 42 +++++++----------- test/sourcemaps/index.js | 36 +++++++++++---- .../samples/sourcemap-sources/_config.js | 15 +++++-- .../samples/sourcemap-sources/test.js | 31 ++++++++++--- 6 files changed, 112 insertions(+), 60 deletions(-) diff --git a/src/compiler/compile/Component.ts b/src/compiler/compile/Component.ts index f36790eb2080..caf0a42a8fe3 100644 --- a/src/compiler/compile/Component.ts +++ b/src/compiler/compile/Component.ts @@ -335,7 +335,7 @@ export default class Component { if (compile_options.sourcemap) { if (js.map) { const pre_remap_sources = js.map.sources; - js.map = remapping([js.map, compile_options.sourcemap], () => null); + js.map = remapping([js.map, compile_options.sourcemap], () => null, true); // remapper can remove our source if it isn't used (no segments map back to it). It is still handy to have a source // so we add it back if (js.map.sources && js.map.sources.length == 0) { @@ -357,7 +357,7 @@ export default class Component { }); } if (css.map) { - css.map = remapping([css.map, compile_options.sourcemap], () => null); + css.map = remapping([css.map, compile_options.sourcemap], () => null, true); } } } diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index d72eb73945bb..c27c65891904 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -1,12 +1,12 @@ -import remapper from '@ampproject/remapping'; -import { decode as sourcemap_decode } from 'sourcemap-codec'; +import remapping from '@ampproject/remapping'; +import { SourceMapInput, SourceMapLoader, RawSourceMap, DecodedSourceMap } from '@ampproject/remapping/dist/types/types'; +import { decode as decode_mappings } from 'sourcemap-codec'; import { getLocator } from 'locate-character'; import { StringWithSourcemap, sourcemap_add_offset } from '../utils/string_with_sourcemap'; - export interface Processed { code: string; - map?: object | string; + map?: SourceMapInput; dependencies?: string[]; } @@ -102,7 +102,7 @@ function get_replacement( if (processed.map) { decoded_map = typeof processed.map === "string" ? JSON.parse(processed.map) : processed.map; if (typeof(decoded_map.mappings) === 'string') - decoded_map.mappings = sourcemap_decode(decoded_map.mappings); + decoded_map.mappings = decode_mappings(decoded_map.mappings); sourcemap_add_offset(decoded_map, get_location(offset + prefix.length)); } const processed_with_map = StringWithSourcemap.from_processed(processed.code, decoded_map); @@ -130,7 +130,7 @@ export default async function preprocess( // sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1) // so we use sourcemap_list.unshift() to add new maps // https://github.com/ampproject/remapping#multiple-transformations-of-a-file - const sourcemap_list: Array = []; + const sourcemap_list: (DecodedSourceMap | RawSourceMap)[] = []; for (const fn of markup) { @@ -142,7 +142,12 @@ export default async function preprocess( if (processed && processed.dependencies) dependencies.push(...processed.dependencies); source = processed ? processed.code : source; - if (processed && processed.map) sourcemap_list.unshift(processed.map); + if (processed && processed.map) + sourcemap_list.unshift( + typeof(processed.map) === 'string' + ? JSON.parse(processed.map) as RawSourceMap + : processed.map as (RawSourceMap | DecodedSourceMap) + ); } for (const fn of script) { @@ -211,20 +216,31 @@ export default async function preprocess( sourcemap_list.unshift(res.map); } - // remapper can throw error - // `Transformation map ${i} must have exactly one source file.` - // for 0 <= i <= (sourcemap_list.length - 2) - - let map: ReturnType; + let map: RawSourceMap; + let map_idx = 0; try { map = sourcemap_list.length == 0 ? null - : remapper(sourcemap_list as any, () => null, true); // true: skip optional field `sourcesContent` + : sourcemap_list.slice(0, -1).find(m => m.sources.length !== 1) === undefined + ? remapping( // use array interface + sourcemap_list, + () => null, + true // skip optional field `sourcesContent` + ) + : remapping( // use loader interface + sourcemap_list[map_idx++], + function loader(sourcefile) { + if (sourcefile === filename) + return sourcemap_list[map_idx++] || null; + // bundle file = branch node + else return null; // source file = leaf node + } as SourceMapLoader + ); } catch (error) { throw { ...error, message: error.message + '\n\ncould not combine sourcemaps:\n' + - JSON.stringify((sourcemap_list as any).map(m => { + JSON.stringify(sourcemap_list.map(m => { return { ...m, mappings: JSON.stringify(m.mappings).slice(0, 100)+' ....'}; }), null, 2) }; diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index a5a89eea158d..9dcc51afe3be 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -1,14 +1,4 @@ -type MappingSegment = - | [number] - | [number, number, number, number] - | [number, number, number, number, number]; - -type SourceMappings = { - version: number; - sources: string[]; - names: string[]; - mappings: MappingSegment[][]; -}; +import { DecodedSourceMap, SourceMapSegment } from '@ampproject/remapping/dist/types/types'; type SourceLocation = { line: number; @@ -21,10 +11,10 @@ function last_line_length(s: string) { // mutate map in-place export function sourcemap_add_offset( - map: SourceMappings, offset: SourceLocation + map: DecodedSourceMap, offset: SourceLocation ) { // shift columns in first line - const m = map.mappings as any; + const m = map.mappings; m[0].forEach(seg => { if (seg[3]) seg[3] += offset.column; }); @@ -69,12 +59,12 @@ function pushArray(_this: T[], other: T[]) { export class StringWithSourcemap { string: string; - map: SourceMappings; + map: DecodedSourceMap; constructor(string = '', map = null) { this.string = string; if (map) - this.map = map as SourceMappings; + this.map = map as DecodedSourceMap; else this.map = { version: 3, @@ -97,8 +87,8 @@ export class StringWithSourcemap { this.string += other.string; - const m1 = this.map as any; - const m2 = other.map as any; + const m1 = this.map; + const m2 = other.map; // combine sources and names const [sources, new_source_idx, sources_changed, sources_idx_changed] = merge_tables(m1.sources, m2.sources); @@ -109,20 +99,20 @@ export class StringWithSourcemap { // unswitched loops are faster if (sources_idx_changed && names_idx_changed) { - m2.forEach(line => { + m2.mappings.forEach(line => { line.forEach(seg => { if (seg[1]) seg[1] = new_source_idx[seg[1]]; if (seg[4]) seg[4] = new_name_idx[seg[4]]; }); }); } else if (sources_idx_changed) { - m2.forEach(line => { + m2.mappings.forEach(line => { line.forEach(seg => { if (seg[1]) seg[1] = new_source_idx[seg[1]]; }); }); } else if (names_idx_changed) { - m2.forEach(line => { + m2.mappings.forEach(line => { line.forEach(seg => { if (seg[4]) seg[4] = new_name_idx[seg[4]]; }); @@ -137,9 +127,9 @@ export class StringWithSourcemap { // columns of 2 must be shifted const column_offset = last_line_length(this.string); - if (m2.length > 0 && column_offset > 0) { + if (m2.mappings.length > 0 && column_offset > 0) { // shift columns in first line - m2[0].forEach(seg => { + m2.mappings[0].forEach(seg => { seg[0] += column_offset; }); } @@ -153,11 +143,11 @@ export class StringWithSourcemap { return this; } - static from_processed(string: string, map?: SourceMappings): StringWithSourcemap { + static from_processed(string: string, map?: DecodedSourceMap): StringWithSourcemap { if (map) return new StringWithSourcemap(string, map); map = { version: 3, names: [], sources: [], mappings: [] }; if (string == '') return new StringWithSourcemap(string, map); - // add empty MappingSegment[] for every line + // add empty SourceMapSegment[] for every line const lineCount = string.split('\n').length; map.mappings = Array.from({length: lineCount}).map(_ => []); return new StringWithSourcemap(string, map); @@ -167,7 +157,7 @@ export class StringWithSourcemap { source_file: string, source: string, offset_in_source?: SourceLocation ): StringWithSourcemap { const offset = offset_in_source || { line: 0, column: 0 }; - const map: SourceMappings = { version: 3, names: [], sources: [source_file], mappings: [] }; + const map: DecodedSourceMap = { version: 3, names: [], sources: [source_file], mappings: [] }; if (source.length == 0) return new StringWithSourcemap(source, map); // we create a high resolution identity map here, @@ -177,7 +167,7 @@ export class StringWithSourcemap { let pos = 0; const segs = line.split(/([^\d\w\s]|\s+)/g) .filter(s => s !== "").map(s => { - const seg: MappingSegment = [ + const seg: SourceMapSegment = [ pos, 0, line_idx + offset.line, pos + (line_idx == 0 ? offset.column : 0) // shift first line diff --git a/test/sourcemaps/index.js b/test/sourcemaps/index.js index 7a2bf60982b4..ca0bdbfca31a 100644 --- a/test/sourcemaps/index.js +++ b/test/sourcemaps/index.js @@ -81,17 +81,37 @@ describe("sourcemaps", () => { ); } - assert.deepEqual(js.map.sources, ["input.svelte"]); - if (css.map) assert.deepEqual(css.map.sources, ["input.svelte"]); + assert.deepEqual( + js.map.sources.slice().sort(), + (config.js_map_sources || ["input.svelte"]).sort() + ); + if (css.map) { + assert.deepEqual( + css.map.sources.slice().sort(), + (config.css_map_sources || ["input.svelte"]).sort() + ); + }; - preprocessed.mapConsumer = preprocessed.map && await new SourceMapConsumer(preprocessed.map); - preprocessed.locate = getLocator(preprocessed.code); + // use locate_1 with mapConsumer: + // lines are one-based, columns are zero-based - js.mapConsumer = js.map && await new SourceMapConsumer(js.map); - js.locate = getLocator(js.code); + if (preprocessed.map) { + preprocessed.mapConsumer = await new SourceMapConsumer(preprocessed.map); + preprocessed.locate = getLocator(preprocessed.code); + preprocessed.locate_1 = getLocator(preprocessed.code, { offsetLine: 1 }); + } - css.mapConsumer = css.map && await new SourceMapConsumer(css.map); - css.locate = getLocator(css.code || ""); + if (js.map) { + js.mapConsumer = await new SourceMapConsumer(js.map); + js.locate = getLocator(js.code); + js.locate_1 = getLocator(js.code, { offsetLine: 1 }); + } + + if (css.map) { + css.mapConsumer = await new SourceMapConsumer(css.map); + css.locate = getLocator(css.code); + css.locate_1 = getLocator(css.code, { offsetLine: 1 }); + } test({ assert, input, preprocessed, js, css }); }); diff --git a/test/sourcemaps/samples/sourcemap-sources/_config.js b/test/sourcemaps/samples/sourcemap-sources/_config.js index 0c413c93000f..2c3e66c3b4b2 100644 --- a/test/sourcemaps/samples/sourcemap-sources/_config.js +++ b/test/sourcemaps/samples/sourcemap-sources/_config.js @@ -3,7 +3,9 @@ import MagicString, { Bundle } from 'magic-string'; function add(bundle, filename, source) { bundle.addSource({ filename, - content: new MagicString(source) + content: new MagicString(source), + separator: '\n', + //separator: '', // ERROR. probably a bug in magic-string }); } @@ -12,13 +14,20 @@ function result(bundle, filename) { code: bundle.toString(), map: bundle.generateMap({ file: filename, - includeContent: true, - hires: true + includeContent: false, + hires: false }) }; } export default { + js_map_sources: [ + 'input.svelte', + 'foo.js', + 'bar.js', + 'foo2.js', + 'bar2.js', + ], preprocess: [ { script: ({ content, filename }) => { diff --git a/test/sourcemaps/samples/sourcemap-sources/test.js b/test/sourcemaps/samples/sourcemap-sources/test.js index e35618e18a04..98fb64fc6ae3 100644 --- a/test/sourcemaps/samples/sourcemap-sources/test.js +++ b/test/sourcemaps/samples/sourcemap-sources/test.js @@ -1,12 +1,29 @@ -export function test({ assert, preprocessed, js, css }) { +export function test({ assert, preprocessed, js }) { - assert.notEqual(preprocessed.error, undefined, 'expected preprocessed.error'); + assert.equal(preprocessed.error, undefined); - const msg_expected_prefix = 'Transformation map 0 must have exactly one source file.'; + // sourcemap stores location only for 'answer = 42;' + // not for 'var answer = 42;' + [ + [js, 'foo.js', 'answer = 42;'], + [js, 'bar.js', 'console.log(answer);'], + [js, 'foo2.js', 'answer2 = 84;'], + [js, 'bar2.js', 'console.log(answer2);'], + ] + .forEach(([where, sourcefile, content]) => { - assert.equal( - preprocessed.error.message.slice(0, msg_expected_prefix.length), - msg_expected_prefix - ); + assert.deepEqual( + where.mapConsumer.originalPositionFor( + where.locate_1(content) + ), + { + source: sourcefile, + name: null, + line: 1, + column: 0 + }, + `failed to locate "${content}" from "${sourcefile}"` + ); + }); } From 18003d6487eb9490dfa7eca2924fa20829c8bd37 Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Sun, 4 Oct 2020 20:38:19 +0200 Subject: [PATCH 25/25] fix tests, use decoded mappings, show warnings --- src/compiler/compile/Component.ts | 81 +++++++--- src/compiler/compile/css/Stylesheet.ts | 2 +- src/compiler/compile/render_dom/index.ts | 11 +- src/compiler/compile/render_ssr/index.ts | 1 + src/compiler/interfaces.ts | 6 +- src/compiler/preprocess/index.ts | 82 ++++++---- src/compiler/utils/string_with_sourcemap.ts | 153 +++++++++++++++++- test/sourcemaps/index.js | 44 +++-- .../samples/decoded-sourcemap/_config.js | 3 + .../detect-lowres-sourcemaps/_config.js | 54 +++++++ .../detect-lowres-sourcemaps/input.svelte | 10 ++ .../samples/detect-lowres-sourcemaps/test.js | 10 ++ .../samples/preprocessed-markup/_config.js | 2 +- .../samples/preprocessed-multiple/_config.js | 6 +- .../samples/sourcemap-names/_config.js | 2 +- .../samples/sourcemap-sources/_config.js | 16 +- .../samples/sourcemap-sources/test.js | 12 +- .../warn-on-encoded-mappings/_config.js | 58 +++++++ .../warn-on-encoded-mappings/input.svelte | 2 + .../samples/warn-on-encoded-mappings/test.js | 13 ++ 20 files changed, 470 insertions(+), 98 deletions(-) create mode 100644 test/sourcemaps/samples/detect-lowres-sourcemaps/_config.js create mode 100644 test/sourcemaps/samples/detect-lowres-sourcemaps/input.svelte create mode 100644 test/sourcemaps/samples/detect-lowres-sourcemaps/test.js create mode 100644 test/sourcemaps/samples/warn-on-encoded-mappings/_config.js create mode 100644 test/sourcemaps/samples/warn-on-encoded-mappings/input.svelte create mode 100644 test/sourcemaps/samples/warn-on-encoded-mappings/test.js diff --git a/src/compiler/compile/Component.ts b/src/compiler/compile/Component.ts index caf0a42a8fe3..d3dd8d0a0bf2 100644 --- a/src/compiler/compile/Component.ts +++ b/src/compiler/compile/Component.ts @@ -29,8 +29,11 @@ import add_to_set from './utils/add_to_set'; import check_graph_for_cycles from './utils/check_graph_for_cycles'; import { print, x, b } from 'code-red'; import { is_reserved_keyword } from './utils/reserved_keywords'; -import remapping from '@ampproject/remapping'; +import { combine_sourcemaps, sourcemap_add_tostring_tourl, combine_sourcemaps_map_stats } from '../utils/string_with_sourcemap'; import Element from './nodes/Element'; +import { RawSourceMap } from '@ampproject/remapping/dist/types/types'; +import { encode as encode_mappings, decode as decode_mappings } from 'sourcemap-codec'; + interface ComponentOptions { namespace?: string; @@ -318,12 +321,19 @@ export default class Component { css = compile_options.customElement ? { code: null, map: null } - : result.css; + : result.css; // css.map.mappings are decoded js = print(program, { sourceMapSource: compile_options.filename }); + // TODO remove workaround + // js.map.mappings should be decoded + // https://github.com/Rich-Harris/code-red/issues/50 + if (js.map && typeof (js.map as any).mappings == 'string') { + (js.map as any).mappings = decode_mappings((js.map as any).mappings); + } + js.map.sources = [ compile_options.filename ? get_relative_path(compile_options.outputFilename || '', compile_options.filename) : null ]; @@ -332,34 +342,59 @@ export default class Component { this.source ]; + // combine sourcemaps + const map_stats: combine_sourcemaps_map_stats = { + sourcemapWarnLoss: 0, // segment loss is usually high, so we ignore + sourcemapEncodedWarn: true // TODO config + // property `result` is set by combine_sourcemaps + }; + if (compile_options.sourcemap) { if (js.map) { - const pre_remap_sources = js.map.sources; - js.map = remapping([js.map, compile_options.sourcemap], () => null, true); - // remapper can remove our source if it isn't used (no segments map back to it). It is still handy to have a source - // so we add it back - if (js.map.sources && js.map.sources.length == 0) { - js.map.sources = pre_remap_sources; + js.map = combine_sourcemaps( + this.file, + [ + js.map, // idx 1: internal + compile_options.sourcemap // idx 0: external: svelte.preprocess, etc + ], + map_stats + ) as RawSourceMap; + sourcemap_add_tostring_tourl(js.map); + if (map_stats.result && map_stats.result.maps_encoded && map_stats.result.maps_encoded.length > 0) { + console.log('warning. svelte.compile received encoded script sourcemaps (index '+ + map_stats.result.maps_encoded.join(', ')+'). '+ + 'this is slow. make your sourcemap-generators return decoded mappings '+ + 'or disable this warning with svelte.compile(_, _, { sourcemapEncodedWarn: false })' + ); } - Object.defineProperties(js.map, { - toString: { - enumerable: false, - value: function toString() { - return JSON.stringify(this); - } - }, - toUrl: { - enumerable: false, - value: function toUrl() { - return 'data:application/json;charset=utf-8;base64,' + btoa(this.toString()); - } - } - }); } if (css.map) { - css.map = remapping([css.map, compile_options.sourcemap], () => null, true); + css.map = combine_sourcemaps( + this.file, + [ + css.map, // idx 1: internal + compile_options.sourcemap // idx 0: external: svelte.preprocess, etc + ] + ) as RawSourceMap; + sourcemap_add_tostring_tourl(css.map); + if (map_stats.result && map_stats.result.maps_encoded && map_stats.result.maps_encoded.length > 0) { + console.log('warning. svelte.compile received encoded style sourcemaps (index '+ + map_stats.result.maps_encoded.join(', ')+'). '+ + 'this is slow. make your sourcemap-generators return decoded mappings '+ + 'or disable this warning with svelte.compile(_, _, { sourcemapEncodedWarn: false })' + ); + } } } + + // encode mappings only once, after all sourcemaps are combined + if (js.map && typeof(js.map.mappings) == 'object') { + (js.map as RawSourceMap).mappings = encode_mappings(js.map.mappings); + } + if (css.map && typeof(css.map.mappings) == 'object') { + (css.map as RawSourceMap).mappings = encode_mappings(css.map.mappings); + } + } return { diff --git a/src/compiler/compile/css/Stylesheet.ts b/src/compiler/compile/css/Stylesheet.ts index dc464d7df806..9d18d642749e 100644 --- a/src/compiler/compile/css/Stylesheet.ts +++ b/src/compiler/compile/css/Stylesheet.ts @@ -410,7 +410,7 @@ export default class Stylesheet { return { code: code.toString(), - map: code.generateMap({ + map: code.generateDecodedMap({ includeContent: true, source: this.filename, file diff --git a/src/compiler/compile/render_dom/index.ts b/src/compiler/compile/render_dom/index.ts index ab4ee904e8a8..523141b57b6b 100644 --- a/src/compiler/compile/render_dom/index.ts +++ b/src/compiler/compile/render_dom/index.ts @@ -30,8 +30,12 @@ export default function dom( } const css = component.stylesheet.render(options.filename, !options.customElement); + + // TODO fix css.map.toUrl - stylesheet.render returns decoded mappings, map.toUrl needs encoded mappings + // TODO use combined css.map? see compile/Component.ts const styles = component.stylesheet.has_styles && options.dev - ? `${css.code}\n/*# sourceMappingURL=${css.map.toUrl()} */` + //? `${css.code}\n/*# sourceMappingURL=${css.map.toUrl()} */` + ? `${css.code}\n/*# sourceMappingURL=TODO_FIXME */` : css.code; const add_css = component.get_unique_name('add_css'); @@ -467,12 +471,15 @@ export default function dom( } if (options.customElement) { + // TODO use combined css.map? see compile/Component.ts + // TODO css.map.toUrl needs encoded mappings + // ${css.code && b`this.shadowRoot.innerHTML = \`\`;`} const declaration = b` class ${name} extends @SvelteElement { constructor(options) { super(); - ${css.code && b`this.shadowRoot.innerHTML = \`\`;`} + ${css.code && b`this.shadowRoot.innerHTML = \`\`;`} @init(this, { target: this.shadowRoot }, ${definition}, ${has_create_fragment ? 'create_fragment': 'null'}, ${not_equal}, ${prop_indexes}, ${dirty}); diff --git a/src/compiler/compile/render_ssr/index.ts b/src/compiler/compile/render_ssr/index.ts index ff45abd78b05..62e2d1831476 100644 --- a/src/compiler/compile/render_ssr/index.ts +++ b/src/compiler/compile/render_ssr/index.ts @@ -137,6 +137,7 @@ export default function ssr( main ].filter(Boolean); + // TODO use combined css.map? see compile/Component.ts const js = b` ${css.code ? b` const #css = { diff --git a/src/compiler/interfaces.ts b/src/compiler/interfaces.ts index 7843cf8938db..263f25d7d759 100644 --- a/src/compiler/interfaces.ts +++ b/src/compiler/interfaces.ts @@ -1,5 +1,7 @@ import { Node, Program } from "estree"; -import { SourceMap } from 'magic-string'; + +// eslint-disable-next-line import/named +import { DecodedSourceMap } from 'magic-string'; interface BaseNode { start: number; @@ -166,5 +168,5 @@ export interface Var { export interface CssResult { code: string; - map: SourceMap; + map: DecodedSourceMap; } diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index c27c65891904..30f4d08f1b23 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -1,8 +1,7 @@ -import remapping from '@ampproject/remapping'; -import { SourceMapInput, SourceMapLoader, RawSourceMap, DecodedSourceMap } from '@ampproject/remapping/dist/types/types'; +import { SourceMapInput, RawSourceMap, DecodedSourceMap } from '@ampproject/remapping/dist/types/types'; import { decode as decode_mappings } from 'sourcemap-codec'; import { getLocator } from 'locate-character'; -import { StringWithSourcemap, sourcemap_add_offset } from '../utils/string_with_sourcemap'; +import { StringWithSourcemap, sourcemap_add_offset, combine_sourcemaps, combine_sourcemaps_map_stats } from '../utils/string_with_sourcemap'; export interface Processed { code: string; @@ -113,10 +112,17 @@ function get_replacement( export default async function preprocess( source: string, preprocessor: PreprocessorGroup | PreprocessorGroup[], - options?: { filename?: string } + options?: { + filename?: string, + sourcemapWarnLoss?: number, // default 0.5 + sourcemapEncodedWarn?: boolean // default true + } ) { // @ts-ignore todo: doublecheck const filename = (options && options.filename) || preprocessor.filename; // legacy + const sourcemapWarnLoss = (options && options.sourcemapWarnLoss != undefined) ? options.sourcemapWarnLoss : 0.5; + const sourcemapEncodedWarn = (options && options.sourcemapEncodedWarn != undefined) ? options.sourcemapEncodedWarn : true; + const dependencies = []; const preprocessors = preprocessor @@ -130,7 +136,9 @@ export default async function preprocess( // sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1) // so we use sourcemap_list.unshift() to add new maps // https://github.com/ampproject/remapping#multiple-transformations-of-a-file - const sourcemap_list: (DecodedSourceMap | RawSourceMap)[] = []; + const sourcemap_list: Array = []; + + // TODO keep track: what preprocessor generated what sourcemap? to make debugging easier = detect low-resolution sourcemaps in fn combine_mappings for (const fn of markup) { @@ -150,6 +158,8 @@ export default async function preprocess( ); } + // TODO run script and style in parallel + for (const fn of script) { const get_location = getLocator(source); const res = await replace_async( @@ -216,37 +226,41 @@ export default async function preprocess( sourcemap_list.unshift(res.map); } - let map: RawSourceMap; - let map_idx = 0; - try { - map = - sourcemap_list.length == 0 - ? null - : sourcemap_list.slice(0, -1).find(m => m.sources.length !== 1) === undefined - ? remapping( // use array interface - sourcemap_list, - () => null, - true // skip optional field `sourcesContent` - ) - : remapping( // use loader interface - sourcemap_list[map_idx++], - function loader(sourcefile) { - if (sourcefile === filename) - return sourcemap_list[map_idx++] || null; - // bundle file = branch node - else return null; // source file = leaf node - } as SourceMapLoader - ); - } catch (error) { - throw { ...error, message: error.message + - '\n\ncould not combine sourcemaps:\n' + - JSON.stringify(sourcemap_list.map(m => { - return { ...m, mappings: JSON.stringify(m.mappings).slice(0, 100)+' ....'}; - }), null, 2) - }; + const map_stats: combine_sourcemaps_map_stats = { + sourcemapWarnLoss, + sourcemapEncodedWarn + // property `result` is set by combine_sourcemaps + }; + + const map: DecodedSourceMap = combine_sourcemaps( + filename, + sourcemap_list, + map_stats, + true // explicitly decode mappings + // TODO remove this, when `remapping` allows to return decoded mappings, so we skip the unnecessary encode + decode steps + ) as DecodedSourceMap; + + // TODO better than console.log? + + if (map_stats.result && map_stats.result.segments_lost) { + const { segment_loss_per_map, segments_per_map } = map_stats.result; + console.log('warning. svelte.preprocess seems to receive low-resolution sourcemaps. '+ + 'relative segment loss per combine_sourcemaps step: '+ + segment_loss_per_map.map(f => f.toFixed(2)).join(' -> ')+ + '. absolute number of segments per sourcemap: '+ + segments_per_map.join(' -> ')+ + '. make your preprocessors return high-resolution sourcemaps '+ + 'or increase the tolerated loss with svelte.preprocess(_, _, { sourcemapWarnLoss: 0.8 })' + ); } - if (map && !map.file) delete map.file; // skip optional field `file` + if (map_stats.result && map_stats.result.maps_encoded && map_stats.result.maps_encoded.length > 0) { + console.log('warning. svelte.preprocess received encoded sourcemaps (index '+ + map_stats.result.maps_encoded.join(', ')+'). '+ + 'this is slow. make your sourcemap-generators return decoded mappings '+ + 'or disable this warning with svelte.preprocess(_, _, { sourcemapEncodedWarn: false })' + ); + } return { // TODO return separated output, in future version where svelte.compile supports it: diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index 9dcc51afe3be..0e0843609241 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -1,4 +1,6 @@ -import { DecodedSourceMap, SourceMapSegment } from '@ampproject/remapping/dist/types/types'; +import { DecodedSourceMap, RawSourceMap, SourceMapSegment, SourceMapLoader } from '@ampproject/remapping/dist/types/types'; +import remapping from '@ampproject/remapping'; +import { decode as decode_mappings } from 'sourcemap-codec'; type SourceLocation = { line: number; @@ -180,3 +182,152 @@ export class StringWithSourcemap { return new StringWithSourcemap(source, map); } } + +export type combine_sourcemaps_map_stats = { + sourcemapEncodedWarn?: boolean, + sourcemapWarnLoss?: number, + result?: { + maps_encoded?: number[], + segments_lost?: boolean + segment_loss_per_map?: number[], + segments_per_map?: number[], + } +}; + +export function combine_sourcemaps( + filename: string, + sourcemap_list: Array, + map_stats?: combine_sourcemaps_map_stats, + do_decode_mappings?: boolean +): (RawSourceMap | DecodedSourceMap) { + if (sourcemap_list.length == 0) return null; + + if (map_stats) { + map_stats.result = {}; + const { result } = map_stats; + + const last_map_idx = sourcemap_list.length - 1; + + // TODO allow to set options per preprocessor -> extend preprocessor config object + // some sourcemap-generators produce ultra-high-resolution sourcemaps (1 token = 1 character), so a high segment loss can be tolerable + + // sourcemapEncodedWarn: show warning + // if preprocessors return sourcemaps with encoded mappings + // we need decoded mappings, so that is a waste of time + + if (map_stats.sourcemapEncodedWarn) { + result.maps_encoded = []; + + for (let map_idx = last_map_idx; map_idx >= 0; map_idx--) { + const map = sourcemap_list[map_idx]; + if (typeof(map) == 'string') { + sourcemap_list[map_idx] = JSON.parse(map); + } + if (typeof(map.mappings) == 'string') { + result.maps_encoded.push(last_map_idx - map_idx); // chronological index + } + } + } + + // sourcemapWarnLoss: show warning if source files were lost + // disable warning with `svelte.preprocess(_, _, { sourcemapWarnLoss: false })` + // value 1 : never warn + // value 0.8: seldom warn + // value 0.5: average warn + // value 0.2: often warn + // value 0 : nonsense -> never warn + // -Infinity <= loss <= 1 and 0 < sourcemapWarnLoss <= 1 + + if (map_stats.sourcemapWarnLoss) { + + // guess if segments were lost because of lowres sourcemaps + // assert: typeof(result) == 'object' + result.segments_per_map = []; + result.segment_loss_per_map = []; + result.segments_lost = false; + + let last_num_segments; + + for (let map_idx = last_map_idx; map_idx >= 0; map_idx--) { + + const map = sourcemap_list[map_idx]; + if (typeof(map) == 'string') { + sourcemap_list[map_idx] = JSON.parse(map); + } + if (typeof(map.mappings) == 'string') { + // do this before remapping to avoid double decoding + // remapping does not mutate its input data + map.mappings = decode_mappings(map.mappings); + } + let num_segments = 0; + for (const line of map.mappings) { + num_segments += line.length; + } + // get relative loss, compared to last map + const loss = map_idx == last_map_idx + ? 0 : (last_num_segments - num_segments) / last_num_segments; + if (loss > map_stats.sourcemapWarnLoss) { + result.segments_lost = true; + } + + // chronological index + result.segment_loss_per_map.push(loss); + result.segments_per_map.push(num_segments); + + last_num_segments = num_segments; + } + } + + } + + let map_idx = 1; + const map: RawSourceMap = + sourcemap_list.slice(0, -1) + .find(m => m.sources.length !== 1) === undefined + + ? remapping( // use array interface + // only the oldest sourcemap can have multiple sources + sourcemap_list, + () => null, + true // skip optional field `sourcesContent` + ) + + : remapping( // use loader interface + sourcemap_list[0], // last map + function loader(sourcefile) { + if (sourcefile === filename && sourcemap_list[map_idx]) { + return sourcemap_list[map_idx++]; // idx 1, 2, ... + // bundle file = branch node + } + else return null; // source file = leaf node + } as SourceMapLoader, + true + ); + + if (!map.file) delete map.file; // skip optional field `file` + + if (do_decode_mappings) { + // explicitly decode mappings + // TODO remove this, when `remapping` allows to return decoded mappings, so we skip the unnecessary encode + decode steps + (map as unknown as DecodedSourceMap).mappings = decode_mappings(map.mappings); + } + + return map; +} + +export function sourcemap_add_tostring_tourl(map) { + Object.defineProperties(map, { + toString: { + enumerable: false, + value: function toString() { + return JSON.stringify(this); + } + }, + toUrl: { + enumerable: false, + value: function toUrl() { + return 'data:application/json;charset=utf-8;base64,' + btoa(this.toString()); + } + } + }); +} diff --git a/test/sourcemaps/index.js b/test/sourcemaps/index.js index ca0bdbfca31a..b8ba4aaa2a39 100644 --- a/test/sourcemaps/index.js +++ b/test/sourcemaps/index.js @@ -6,6 +6,7 @@ import { loadConfig, svelte } from "../helpers.js"; // https://github.com/mozilla/source-map/issues/400 import { SourceMapConsumer } from "source-map"; import { getLocator } from "locate-character"; +import { encode as encode_mappings, decode as decode_mappings } from 'sourcemap-codec'; describe("sourcemaps", () => { fs.readdirSync(`${__dirname}/samples`).forEach(dir => { @@ -44,6 +45,13 @@ describe("sourcemaps", () => { return test({ assert, input, preprocessed }); } + // preprocessed.map.mappings should be decoded + // to avoid unnecessary encode + decode steps + if (preprocessed.map) { + assert.equal(typeof preprocessed.map.mappings, 'object', 'preprocessed.map.mappings should be decoded'); + assert.equal(Array.isArray(preprocessed.map.mappings), true, 'preprocessed.map.mappings should be decoded'); + } + const { js, css } = svelte.compile( preprocessed.code, { filename: "input.svelte", @@ -60,7 +68,11 @@ describe("sourcemaps", () => { fs.writeFileSync(`${outputBase}.svelte`, preprocessed.code); if (preprocessed.map) { - fs.writeFileSync(`${outputBase}.svelte.map`, JSON.stringify(preprocessed.map, null, 2)); + fs.writeFileSync( + `${outputBase}.svelte.map`, + // TODO encode mappings for output - svelte.preprocess returns decoded mappings + JSON.stringify(preprocessed.map, null, 2) + ); } fs.writeFileSync( `${outputBase}.js`, @@ -92,26 +104,26 @@ describe("sourcemaps", () => { ); }; + // stupid workaround (unnecessary encode + decode steps) + // TODO find a SourceMapConsumer who also consumes decoded mappings + if (preprocessed.map) { + preprocessed.map.mappings = encode_mappings(preprocessed.map.mappings); + } + // use locate_1 with mapConsumer: // lines are one-based, columns are zero-based - if (preprocessed.map) { - preprocessed.mapConsumer = await new SourceMapConsumer(preprocessed.map); - preprocessed.locate = getLocator(preprocessed.code); - preprocessed.locate_1 = getLocator(preprocessed.code, { offsetLine: 1 }); - } + preprocessed.mapConsumer = preprocessed.map && await new SourceMapConsumer(preprocessed.map); + preprocessed.locate = getLocator(preprocessed.code); + preprocessed.locate_1 = getLocator(preprocessed.code, { offsetLine: 1 }); - if (js.map) { - js.mapConsumer = await new SourceMapConsumer(js.map); - js.locate = getLocator(js.code); - js.locate_1 = getLocator(js.code, { offsetLine: 1 }); - } + js.mapConsumer = js.map && await new SourceMapConsumer(js.map); + js.locate = getLocator(js.code); + js.locate_1 = getLocator(js.code, { offsetLine: 1 }); - if (css.map) { - css.mapConsumer = await new SourceMapConsumer(css.map); - css.locate = getLocator(css.code); - css.locate_1 = getLocator(css.code, { offsetLine: 1 }); - } + css.mapConsumer = css.map && await new SourceMapConsumer(css.map); + css.locate = getLocator(css.code || ''); + css.locate_1 = getLocator(css.code || '', { offsetLine: 1 }); test({ assert, input, preprocessed, js, css }); }); diff --git a/test/sourcemaps/samples/decoded-sourcemap/_config.js b/test/sourcemaps/samples/decoded-sourcemap/_config.js index bc0db984b22c..fc4d2a03c90e 100644 --- a/test/sourcemaps/samples/decoded-sourcemap/_config.js +++ b/test/sourcemaps/samples/decoded-sourcemap/_config.js @@ -19,6 +19,9 @@ function result(src, filename) { } export default { + + js_map_sources: [], // test component has no scripts + preprocess: { markup: ({ content, filename }) => { const src = new MagicString(content); diff --git a/test/sourcemaps/samples/detect-lowres-sourcemaps/_config.js b/test/sourcemaps/samples/detect-lowres-sourcemaps/_config.js new file mode 100644 index 000000000000..b11594dd855f --- /dev/null +++ b/test/sourcemaps/samples/detect-lowres-sourcemaps/_config.js @@ -0,0 +1,54 @@ +import MagicString from 'magic-string'; + +// TODO move util fns to test index.js + +function result(filename, src, extraOptions = {}) { + return { + code: src.toString(), + map: src.generateDecodedMap({ + source: filename, + hires: true, + includeContent: false, + ...extraOptions + }) + }; +} + +function replace_all(src, search, replace) { + let idx = src.original.indexOf(search); + if (idx == -1) throw new Error('search not found in src'); + do { + src.overwrite(idx, idx + search.length, replace); + } while ((idx = src.original.indexOf(search, idx + 1)) != -1); +} + +function replace_first(src, search, replace) { + const idx = src.original.indexOf(search); + if (idx == -1) throw new Error('search not found in src'); + src.overwrite(idx, idx + search.length, replace); +} + +export default { + + preprocess_options: { + sourcemapLossWarn: 0.9 // warn often + }, + + js_map_sources: [], // test component has no scripts + + preprocess: [ + { markup: ({ content, filename }) => { + const src = new MagicString(content); + replace_all(src, 'replace_me', 'done_replace'); + return result(filename, src, { hires: true }); + } }, + { markup: ({ content, filename }) => { + const src = new MagicString(content); + replace_first(src, 'done_replace', 'version_3'); + // return low-resolution sourcemap + // this should make previous mappings unreachable + return result(filename, src, { hires: false }); + } } + ] + +}; diff --git a/test/sourcemaps/samples/detect-lowres-sourcemaps/input.svelte b/test/sourcemaps/samples/detect-lowres-sourcemaps/input.svelte new file mode 100644 index 000000000000..2b3afd881b3f --- /dev/null +++ b/test/sourcemaps/samples/detect-lowres-sourcemaps/input.svelte @@ -0,0 +1,10 @@ +replace_me +replace_me +replace_me +replace_me +replace_me +replace_me +replace_me +replace_me +replace_me +replace_me diff --git a/test/sourcemaps/samples/detect-lowres-sourcemaps/test.js b/test/sourcemaps/samples/detect-lowres-sourcemaps/test.js new file mode 100644 index 000000000000..0f63efb358d0 --- /dev/null +++ b/test/sourcemaps/samples/detect-lowres-sourcemaps/test.js @@ -0,0 +1,10 @@ +export function test({ assert, preprocessed, js }) { + + assert.equal(preprocessed.error, undefined); + + // TODO can we automate this test? + // we need the output of console.log + // to test the warning message. + // or use a different method for warnings? + +} diff --git a/test/sourcemaps/samples/preprocessed-markup/_config.js b/test/sourcemaps/samples/preprocessed-markup/_config.js index f515cc6d1256..cb3eb90e01c6 100644 --- a/test/sourcemaps/samples/preprocessed-markup/_config.js +++ b/test/sourcemaps/samples/preprocessed-markup/_config.js @@ -8,7 +8,7 @@ export default { src.overwrite(idx, idx+"baritone".length, "bar"); return { code: src.toString(), - map: src.generateMap({ + map: src.generateDecodedMap({ source: filename, includeContent: false }) diff --git a/test/sourcemaps/samples/preprocessed-multiple/_config.js b/test/sourcemaps/samples/preprocessed-multiple/_config.js index c587a71bbd99..d4c1e6cdbeb5 100644 --- a/test/sourcemaps/samples/preprocessed-multiple/_config.js +++ b/test/sourcemaps/samples/preprocessed-multiple/_config.js @@ -11,7 +11,7 @@ export default { src.overwrite(css_idx, css_idx + "--bazitone".length, "--baz"); return { code: src.toString(), - map: src.generateMap({ + map: src.generateDecodedMap({ source: filename, hires: true, includeContent: false @@ -24,7 +24,7 @@ export default { src.prependLeft(idx, " "); return { code: src.toString(), - map: src.generateMap({ + map: src.generateDecodedMap({ source: filename, hires: true, includeContent: false @@ -37,7 +37,7 @@ export default { src.prependLeft(idx, " "); return { code: src.toString(), - map: src.generateMap({ + map: src.generateDecodedMap({ source: filename, hires: true, includeContent: false diff --git a/test/sourcemaps/samples/sourcemap-names/_config.js b/test/sourcemaps/samples/sourcemap-names/_config.js index 96c969813272..35c7badb296a 100644 --- a/test/sourcemaps/samples/sourcemap-names/_config.js +++ b/test/sourcemaps/samples/sourcemap-names/_config.js @@ -10,7 +10,7 @@ function replace(search, replace, content, src, options = { storeName: true }) { function result(src, filename) { return { code: src.toString(), - map: src.generateMap({ + map: src.generateDecodedMap({ source: filename, hires: true, includeContent: false diff --git a/test/sourcemaps/samples/sourcemap-sources/_config.js b/test/sourcemaps/samples/sourcemap-sources/_config.js index 2c3e66c3b4b2..999fb20dfeff 100644 --- a/test/sourcemaps/samples/sourcemap-sources/_config.js +++ b/test/sourcemaps/samples/sourcemap-sources/_config.js @@ -4,8 +4,8 @@ function add(bundle, filename, source) { bundle.addSource({ filename, content: new MagicString(source), - separator: '\n', - //separator: '', // ERROR. probably a bug in magic-string + separator: '\n' + //separator: '' // ERROR. probably a bug in magic-string }); } @@ -15,7 +15,7 @@ function result(bundle, filename) { map: bundle.generateMap({ file: filename, includeContent: false, - hires: false + hires: true // required for remapping }) }; } @@ -26,7 +26,7 @@ export default { 'foo.js', 'bar.js', 'foo2.js', - 'bar2.js', + 'bar2.js' ], preprocess: [ { @@ -34,8 +34,8 @@ export default { const bundle = new Bundle(); add(bundle, filename, content); - add(bundle, 'foo.js', 'var answer = 42;'); - add(bundle, 'bar.js', 'console.log(answer);'); + add(bundle, 'foo.js', 'var answer = 42; // foo.js\n'); + add(bundle, 'bar.js', 'console.log(answer); // bar.js\n'); return result(bundle, filename); } @@ -45,8 +45,8 @@ export default { const bundle = new Bundle(); add(bundle, filename, content); - add(bundle, 'foo2.js', 'var answer2 = 84;'); - add(bundle, 'bar2.js', 'console.log(answer2);'); + add(bundle, 'foo2.js', 'var answer2 = 84; // foo2.js\n'); + add(bundle, 'bar2.js', 'console.log(answer2); // bar2.js\n'); return result(bundle, filename); } diff --git a/test/sourcemaps/samples/sourcemap-sources/test.js b/test/sourcemaps/samples/sourcemap-sources/test.js index 98fb64fc6ae3..78a4c80a1748 100644 --- a/test/sourcemaps/samples/sourcemap-sources/test.js +++ b/test/sourcemaps/samples/sourcemap-sources/test.js @@ -5,12 +5,12 @@ export function test({ assert, preprocessed, js }) { // sourcemap stores location only for 'answer = 42;' // not for 'var answer = 42;' [ - [js, 'foo.js', 'answer = 42;'], - [js, 'bar.js', 'console.log(answer);'], - [js, 'foo2.js', 'answer2 = 84;'], - [js, 'bar2.js', 'console.log(answer2);'], + [js, 'foo.js', 'answer = 42;', 4], + [js, 'bar.js', 'console.log(answer);', 0], + [js, 'foo2.js', 'answer2 = 84;', 4], + [js, 'bar2.js', 'console.log(answer2);', 0] ] - .forEach(([where, sourcefile, content]) => { + .forEach(([where, sourcefile, content, column]) => { assert.deepEqual( where.mapConsumer.originalPositionFor( @@ -20,7 +20,7 @@ export function test({ assert, preprocessed, js }) { source: sourcefile, name: null, line: 1, - column: 0 + column }, `failed to locate "${content}" from "${sourcefile}"` ); diff --git a/test/sourcemaps/samples/warn-on-encoded-mappings/_config.js b/test/sourcemaps/samples/warn-on-encoded-mappings/_config.js new file mode 100644 index 000000000000..d33573e27d72 --- /dev/null +++ b/test/sourcemaps/samples/warn-on-encoded-mappings/_config.js @@ -0,0 +1,58 @@ +import MagicString from 'magic-string'; + +// TODO move util fns to test index.js + +function result(filename, src, options = {}) { + const map_fn = options.encodeMappings ? src.generateMap : src.generateDecodedMap; + delete options.encodeMappings; + return { + code: src.toString(), + map: map_fn.apply(src, [{ + source: filename, + hires: true, + includeContent: false, + ...options + }]) + }; +} + +function replace_all(src, search, replace) { + let idx = src.original.indexOf(search); + if (idx == -1) throw new Error('search not found in src'); + do { + src.overwrite(idx, idx + search.length, replace); + } while ((idx = src.original.indexOf(search, idx + 1)) != -1); +} + +export default { + + js_map_sources: [], // test component has no scripts + + preprocess: [ + // preprocessor 0 + { markup: ({ content, filename }) => { + const src = new MagicString(content); + replace_all(src, 'replace_me', 'version_1'); + return result(filename, src, { encodeMappings: true }); + } }, + // 1 + { markup: ({ content, filename }) => { + const src = new MagicString(content); + replace_all(src, 'version_1', 'version_2'); + return result(filename, src); + } }, + // 2 + { markup: ({ content, filename }) => { + const src = new MagicString(content); + replace_all(src, 'version_2', 'version_3'); + return result(filename, src, { encodeMappings: true }); + } }, + // 3 + { markup: ({ content, filename }) => { + const src = new MagicString(content); + replace_all(src, 'version_3', 'version_4'); + return result(filename, src); + } } + ] + +}; diff --git a/test/sourcemaps/samples/warn-on-encoded-mappings/input.svelte b/test/sourcemaps/samples/warn-on-encoded-mappings/input.svelte new file mode 100644 index 000000000000..ac8f9b30ab75 --- /dev/null +++ b/test/sourcemaps/samples/warn-on-encoded-mappings/input.svelte @@ -0,0 +1,2 @@ +replace_me +replace_me diff --git a/test/sourcemaps/samples/warn-on-encoded-mappings/test.js b/test/sourcemaps/samples/warn-on-encoded-mappings/test.js new file mode 100644 index 000000000000..9e25f21e55c7 --- /dev/null +++ b/test/sourcemaps/samples/warn-on-encoded-mappings/test.js @@ -0,0 +1,13 @@ +export function test({ assert, preprocessed, js }) { + + assert.equal(preprocessed.error, undefined); + + // TODO can we automate this test? + // we need the output of console.log + // to test the warning message. + // or use a different method for warnings? + + // expected warning message: + // warning. svelte.preprocess received encoded sourcemaps (index 0, 2). [....] + +}