Merge pull request #698 from mrkishi/gh-689

Fix export queue

src/api/export.ts

@@ -132,26 +132,32 @@ async function _export({
 		if (seen.has(pathname)) return;
 		seen.add(pathname);
 
-		const timeout_deferred = new Deferred();
-		const the_timeout = setTimeout(() => {
-			timeout_deferred.reject(new Error(`Timed out waiting for ${url.href}`));
-		}, timeout);
+		const r = await q.add(async () => {
+			const timeout_deferred = new Deferred();
+			const the_timeout = setTimeout(() => {
+				timeout_deferred.reject(new Error(`Timed out waiting for ${url.href}`));
+			}, timeout);
 
-		const r = await Promise.race([
-			q.add(() => fetch(url.href, {
-				redirect: 'manual'
-			})),
-			timeout_deferred.promise
-		]);
+			const r = await Promise.race([
+				fetch(url.href, {
+					redirect: 'manual'
+				}),
+				timeout_deferred.promise
+			]);
 
-		clearTimeout(the_timeout); // prevent it hanging at the end
+			clearTimeout(the_timeout); // prevent it hanging at the end
+
+			return r;
+		}) as Response;
 
 		let type = r.headers.get('Content-Type');
 
 		let body = await r.text();
 
 		const range = ~~(r.status / 100);
 
+		let tasks = [];
+
 		if (range === 2) {
 			if (type === 'text/html') {
 				// parse link rel=preload headers and embed them in the HTML
@@ -173,8 +179,6 @@ async function _export({
 					let match;
 					let pattern = /<a ([\s\S]+?)>/gm;
 
-					let promise;
-
 					while (match = pattern.exec(cleaned)) {
 						const attrs = match[1];
 						const href = get_href(attrs);
@@ -183,12 +187,10 @@ async function _export({
 							const url = resolve(base.href, href);
 
 							if (url.protocol === protocol && url.host === host) {
-								promise = handle(url);
+								tasks.push(handle(url));
 							}
 						}
 					}
-
-					await promise;
 				}
 			}
 		}
@@ -199,10 +201,12 @@ async function _export({
 			type = 'text/html';
 			body = `<script>window.location.href = "${location.replace(origin, '')}"</script>`;
 
-			await handle(resolve(root.href, location));
+			tasks.push(handle(resolve(root.href, location)));
 		}
 
 		save(pathname, r.status, type, body);
+
+		await Promise.all(tasks);
 	}
 
 	try {

test/apps/export-queue/rollup.config.js

@@ -0,0 +1,58 @@
+import resolve from 'rollup-plugin-node-resolve';
+import replace from 'rollup-plugin-replace';
+import svelte from 'rollup-plugin-svelte';
+
+const mode = process.env.NODE_ENV;
+const dev = mode === 'development';
+
+const config = require('../../../config/rollup.js');
+
+export default {
+	client: {
+		input: config.client.input(),
+		output: config.client.output(),
+		plugins: [
+			replace({
+				'process.browser': true,
+				'process.env.NODE_ENV': JSON.stringify(mode)
+			}),
+			svelte({
+				dev,
+				hydratable: true,
+				emitCss: true
+			}),
+			resolve()
+		]
+	},
+
+	server: {
+		input: config.server.input(),
+		output: config.server.output(),
+		plugins: [
+			replace({
+				'process.browser': false,
+				'process.env.NODE_ENV': JSON.stringify(mode)
+			}),
+			svelte({
+				generate: 'ssr',
+				dev
+			}),
+			resolve({
+				preferBuiltins: true
+			})
+		],
+		external: ['sirv', 'polka']
+	},
+
+	serviceworker: {
+		input: config.serviceworker.input(),
+		output: config.serviceworker.output(),
+		plugins: [
+			resolve(),
+			replace({
+				'process.browser': true,
+				'process.env.NODE_ENV': JSON.stringify(mode)
+			})
+		]
+	}
+};

test/apps/export-queue/src/client.js

@@ -0,0 +1,9 @@
+import * as sapper from '@sapper/app';
+
+window.start = () => sapper.start({
+	target: document.querySelector('#sapper')
+});
+
+window.prefetchRoutes = () => sapper.prefetchRoutes();
+window.prefetch = href => sapper.prefetch(href);
+window.goto = href => sapper.goto(href);

test/apps/export-queue/src/routes/_error.svelte

@@ -0,0 +1,3 @@
+<h1>{status}</h1>
+
+<p>{error.message}</p>

test/apps/export-queue/src/routes/a-[x].svelte

@@ -0,0 +1,15 @@
+<script context="module">
+	export function preload({ params }) {
+		if (params.x === 'a') {
+			return new Promise(resolve => setTimeout(resolve, 100));
+		}
+
+		return params;
+	}
+</script>
+
+<script>
+	export let x;
+</script>
+
+<a href="b-{x}">b-{x}</a>

test/apps/export-queue/src/routes/b-[x].svelte

@@ -0,0 +1,5 @@
+<script>
+	export let x;
+</script>
+
+<p>b-{x}</p>

test/apps/export-queue/src/routes/index.svelte

@@ -0,0 +1,3 @@
+
+<a href="a-a">1</a>
+<a href="a-b">2</a>

test/apps/export-queue/src/server.js

@@ -0,0 +1,13 @@
+import sirv from 'sirv';
+import polka from 'polka';
+import * as sapper from '@sapper/server';
+
+import { start, dev } from '../../common.js';
+
+const app = polka()
+	.use(
+		sirv('static', { dev }),
+		sapper.middleware()
+	);
+
+start(app);

test/apps/export-queue/src/service-worker.js

@@ -0,0 +1,82 @@
+import * as sapper from '@sapper/service-worker';
+
+const ASSETS = `cache${sapper.timestamp}`;
+
+// `shell` is an array of all the files generated by webpack,
+// `files` is an array of everything in the `static` directory
+const to_cache = sapper.shell.concat(sapper.files);
+const cached = new Set(to_cache);
+
+self.addEventListener('install', event => {
+	event.waitUntil(
+		caches
+			.open(ASSETS)
+			.then(cache => cache.addAll(to_cache))
+			.then(() => {
+				self.skipWaiting();
+			})
+	);
+});
+
+self.addEventListener('activate', event => {
+	event.waitUntil(
+		caches.keys().then(async keys => {
+			// delete old caches
+			for (const key of keys) {
+				if (key !== ASSETS) await caches.delete(key);
+			}
+
+			self.clients.claim();
+		})
+	);
+});
+
+self.addEventListener('fetch', event => {
+	if (event.request.method !== 'GET') return;
+
+	const url = new URL(event.request.url);
+
+	// don't try to handle e.g. data: URIs
+	if (!url.protocol.startsWith('http')) return;
+
+	// ignore dev server requests
+	if (url.hostname === self.location.hostname && url.port !== self.location.port) return;
+
+	// always serve assets and webpack-generated files from cache
+	if (url.host === self.location.host && cached.has(url.pathname)) {
+		event.respondWith(caches.match(event.request));
+		return;
+	}
+
+	// for pages, you might want to serve a shell `index.html` file,
+	// which Sapper has generated for you. It's not right for every
+	// app, but if it's right for yours then uncomment this section
+	/*
+	if (url.origin === self.origin && routes.find(route => route.pattern.test(url.pathname))) {
+		event.respondWith(caches.match('/index.html'));
+		return;
+	}
+	*/
+
+	if (event.request.cache === 'only-if-cached') return;
+
+	// for everything else, try the network first, falling back to
+	// cache if the user is offline. (If the pages never change, you
+	// might prefer a cache-first approach to a network-first one.)
+	event.respondWith(
+		caches
+			.open(`offline${sapper.timestamp}`)
+			.then(async cache => {
+				try {
+					const response = await fetch(event.request);
+					cache.put(event.request, response.clone());
+					return response;
+				} catch(err) {
+					const response = await cache.match(event.request);
+					if (response) return response;
+
+					throw err;
+				}
+			})
+	);
+});

test/apps/export-queue/src/template.html

@@ -0,0 +1,14 @@
+<!doctype html>
+<html lang="en">
+<head>
+	<meta charset='utf-8'>
+
+	%sapper.base%
+	%sapper.styles%
+	%sapper.head%
+</head>
+<body>
+	<div id='sapper'>%sapper.html%</div>
+	%sapper.scripts%
+</body>
+</html>

test/apps/export-queue/static/global.css

@@ -0,0 +1,3 @@
+body {
+	font-family: 'Comic Sans MS';
+}

test/apps/export-queue/test.ts

@@ -0,0 +1,13 @@
+import * as api from '../../../api';
+
+describe('export-queue', function() {
+	this.timeout(10000);
+
+	// hooks
+	before('build app', () => api.build({ cwd: __dirname }));
+
+	// tests
+	it('exports a site with inconsistent load time', async () => {
+		await api.export({ cwd: __dirname });
+	});
+});