Explorations: Stream API

packages/docs/site/docs/01-start-here/01-index.md

@@ -5,6 +5,12 @@ slug: /
 
 # WordPress Playground
 
+:::info **Looking for the official Playground website?**
+
+WordPress Playground website was moved to [wp.org/playground](wp.org/playground). The site you're at right now is now a home for the documentation.
+
+:::
+
 👋 Hi! Welcome to WordPress Playground documentation. Playground is an online tool to experiment and learn about WordPress – learn more in the [overview section](./02-overview.md).
 
 The documentation consists of two major sections:

packages/php-wasm/universal/src/lib/index.ts

@@ -17,6 +17,9 @@ export type {
 	SpawnHandler,
 } from './universal-php';
 
+export { iterateFiles, readAllBytes, writeToPath } from './iterate-files';
+export type { FileEntry, IterateFilesOptions } from './iterate-files';
+
 export { UnhandledRejectionsTarget } from './wasm-error-reporting';
 
 export { PHPResponse } from './php-response';

packages/php-wasm/universal/src/lib/iterate-files.ts

@@ -0,0 +1,121 @@
+import { dirname, joinPaths, normalizePath } from '@php-wasm/util';
+import { UniversalPHP } from './universal-php';
+
+export type FileEntry = {
+	path: string;
+	isDirectory?: boolean;
+	bytes: () => Promise<Uint8Array>;
+};
+
+export type IterateFilesOptions = {
+	/**
+	 * Should yield paths relative to the root directory?
+	 * If false, all paths will be absolute.
+	 */
+	relativePaths?: boolean;
+
+	/**
+	 * A prefix to add to all paths.
+	 * Only used if `relativePaths` is true.
+	 */
+	pathPrefix?: string;
+
+	/**
+	 * A list of paths to exclude from the results.
+	 */
+	exceptPaths?: string[];
+};
+
+/**
+ * Iterates over all files in a php directory and its subdirectories.
+ *
+ * @param php - The PHP instance.
+ * @param root - The root directory to start iterating from.
+ * @param options - Optional configuration.
+ * @returns All files found in the tree.
+ */
+export async function* iterateFiles(
+	php: UniversalPHP,
+	root: string,
+	{
+		relativePaths = true,
+		pathPrefix,
+		exceptPaths = [],
+	}: IterateFilesOptions = {}
+): AsyncGenerator<FileEntry> {
+	root = normalizePath(root);
+	const stack: string[] = [root];
+	while (stack.length) {
+		const currentParent = stack.pop();
+		if (!currentParent) {
+			return;
+		}
+		const files = await php.listFiles(currentParent);
+		for (const file of files) {
+			const absPath = `${currentParent}/${file}`;
+			if (exceptPaths.includes(absPath.substring(root.length + 1))) {
+				continue;
+			}
+			const isDir = await php.isDir(absPath);
+			if (isDir) {
+				stack.push(absPath);
+			} else {
+				yield {
+					path: relativePaths
+						? joinPaths(
+								pathPrefix || '',
+								absPath.substring(root.length + 1)
+						  )
+						: absPath,
+					bytes: async () => await php.readFileAsBuffer(absPath),
+				};
+			}
+		}
+	}
+}
+
+export async function readAllBytes(
+	reader: ReadableStreamDefaultReader<Uint8Array>
+): Promise<Uint8Array> {
+	let size = 0;
+	const chunks: Uint8Array[] = [];
+	while (true) {
+		const { done, value } = await reader.read();
+		if (done) {
+			break;
+		}
+		chunks.push(value);
+		size += value.length;
+	}
+	const result = new Uint8Array(size);
+	let offset = 0;
+	for (const chunk of chunks) {
+		result.set(chunk, offset);
+		offset += chunk.length;
+	}
+	return result;
+}
+
+/**
+ * Replaces the contents of a Playground directory with the given files.
+ *
+ * @param client
+ * @param root
+ * @param newFiles
+ */
+export async function writeToPath(
+	client: UniversalPHP,
+	root: string,
+	files: AsyncIterable<FileEntry>
+) {
+	await client.mkdir(root);
+	for await (const file of files) {
+		const filePath = joinPaths(root, file.path);
+		if (file.isDirectory) {
+			await client.mkdir(filePath);
+		} else {
+			await client.mkdir(dirname(filePath));
+			await client.writeFile(filePath, await file.bytes());
+		}
+	}
+}

packages/php-wasm/util/tsconfig.lib.json

@@ -5,6 +5,6 @@
 		"declaration": true,
 		"types": ["node"]
 	},
-	"include": ["src/**/*.ts"],
+	"include": ["src/**/*.ts", "../universal/src/lib/iterate-files.ts"],
 	"exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"]
 }

packages/playground/blueprints/src/index.ts

@@ -1,5 +1,6 @@
 export * from './lib/steps';
 export * from './lib/steps/handlers';
+export * from './lib/zip';
 export { runBlueprintSteps, compileBlueprint } from './lib/compile';
 export type { Blueprint } from './lib/blueprint';
 export type {

packages/playground/blueprints/src/lib/resources.ts

@@ -2,7 +2,7 @@ import {
 	cloneResponseMonitorProgress,
 	ProgressTracker,
 } from '@php-wasm/progress';
-import { UniversalPHP } from '@php-wasm/universal';
+import { readAllBytes, UniversalPHP } from '@php-wasm/universal';
 import { Semaphore } from '@php-wasm/util';
 import { File, zipNameToHumanName } from './steps/common';
 
@@ -220,7 +220,12 @@ export abstract class FetchResource extends Resource {
 		if (response.status !== 200) {
 			throw new Error(`Could not download "${url}"`);
 		}
-		return new File([await response.blob()], this.name);
+
+		return new StreamedFile(
+			response.body!,
+			this.name,
+			response.headers.get('content-type') ?? undefined
+		);
 	}
 
 	/**
@@ -410,3 +415,28 @@ export class SemaphoreResource<
 		return this.semaphore.run(() => super.resolve());
 	}
 }
+
+export class StreamedFile extends File {
+	constructor(
+		private readableStream: ReadableStream<Uint8Array>,
+		name: string,
+		type?: string
+	) {
+		super([], name, { type });
+	}
+
+	override slice(): Blob {
+		throw new Error('Not implemented');
+	}
+
+	override stream() {
+		return this.readableStream;
+	}
+
+	override async text() {
+		return new TextDecoder().decode(await this.arrayBuffer());
+	}
+	override async arrayBuffer() {
+		return await readAllBytes(this.readableStream.getReader());
+	}
+}

packages/playground/blueprints/src/lib/steps/import-wp-content.ts

@@ -2,7 +2,7 @@ import { StepHandler } from '.';
 import { unzip } from './unzip';
 import { dirname, joinPaths, phpVar } from '@php-wasm/util';
 import { wpContentFilesExcludedFromExport } from './common';
-import { UniversalPHP } from '@php-wasm/universal';
+import { FileEntry, UniversalPHP } from '@php-wasm/universal';
 
 /**
  * @inheritDoc importWordPressFiles
@@ -24,7 +24,8 @@ export interface ImportWordPressFilesStep<ResourceType> {
 	 * The zip file containing the top-level WordPress files and
 	 * directories.
 	 */
-	wordPressFilesZip: ResourceType;
+	wordPressFilesZip?: ResourceType;
+	files?: AsyncIterable<FileEntry>;
 	/**
 	 * The path inside the zip file where the WordPress files are.
 	 */
@@ -46,11 +47,11 @@ export interface ImportWordPressFilesStep<ResourceType> {
  */
 export const importWordPressFiles: StepHandler<
 	ImportWordPressFilesStep<File>
-> = async (playground, { wordPressFilesZip, pathInZip = '' }) => {
+> = async (playground, { wordPressFilesZip, files, pathInZip = '' }) => {
 	const zipPath = '/import.zip';
 	await playground.writeFile(
 		zipPath,
-		new Uint8Array(await wordPressFilesZip.arrayBuffer())
+		new Uint8Array(await wordPressFilesZip!.arrayBuffer())
 	);
 
 	const documentRoot = await playground.documentRoot;

packages/playground/blueprints/src/lib/steps/install-asset.ts

@@ -1,12 +1,16 @@
-import type { UniversalPHP } from '@php-wasm/universal';
-import { writeFile } from './write-file';
-import { unzip } from './unzip';
+import { writeToPath, type UniversalPHP, FileEntry } from '@php-wasm/universal';
+import { dirname, joinPaths } from '@php-wasm/util';
 
 export interface InstallAssetOptions {
 	/**
-	 * The zip file to install.
+	 * The files to install.
 	 */
-	zipFile: File;
+	files: AsyncIterable<FileEntry>;
+	/**
+	 * The default name of the asset.
+	 * Used if the zip file contains more than one folder.
+	 */
+	defaultAssetName: string;
 	/**
 	 * Target path to extract the main folder.
 	 * @example
@@ -23,75 +27,33 @@ export interface InstallAssetOptions {
  */
 export async function installAsset(
 	playground: UniversalPHP,
-	{ targetPath, zipFile }: InstallAssetOptions
-): Promise<{
-	assetFolderPath: string;
-	assetFolderName: string;
-}> {
-	// Extract to temporary folder so we can find asset folder name
-
-	const zipFileName = zipFile.name;
-	const assetNameGuess = zipFileName.replace(/\.zip$/, '');
-
-	const tmpUnzippedFilesPath = `/tmp/assets/${assetNameGuess}`;
-	const tmpZipPath = `/tmp/${zipFileName}`;
-
-	const removeTmpFolder = () =>
-		playground.rmdir(tmpUnzippedFilesPath, {
-			recursive: true,
-		});
-
-	if (await playground.fileExists(tmpUnzippedFilesPath)) {
-		await removeTmpFolder();
-	}
-
-	await writeFile(playground, {
-		path: tmpZipPath,
-		data: zipFile,
-	});
-
-	const cleanup = () =>
-		Promise.all([removeTmpFolder, () => playground.unlink(tmpZipPath)]);
-
-	try {
-		await unzip(playground, {
-			zipPath: tmpZipPath,
-			extractToPath: tmpUnzippedFilesPath,
-		});
-
-		// Find the path asset folder name
-		const files = await playground.listFiles(tmpUnzippedFilesPath, {
-			prependPath: true,
-		});
+	{ targetPath, files, defaultAssetName }: InstallAssetOptions
+): Promise<string> {
+	const extractionPath = joinPaths(targetPath, crypto.randomUUID());
+	await writeToPath(playground, extractionPath, files);
+	return await flattenDirectory(playground, extractionPath, defaultAssetName);
+}
 
-		/**
-		 * If the zip only contains a single entry that is directory,
-		 * we assume that's the asset folder. Otherwise, the zip
-		 * probably contains the plugin files without an intermediate folder.
-		 */
-		const zipHasRootFolder =
-			files.length === 1 && (await playground.isDir(files[0]));
-		let assetFolderName;
-		let tmpAssetPath = '';
-		if (zipHasRootFolder) {
-			tmpAssetPath = files[0];
-			assetFolderName = files[0].split('/').pop()!;
-		} else {
-			tmpAssetPath = tmpUnzippedFilesPath;
-			assetFolderName = assetNameGuess;
+export async function flattenDirectory(
+	php: UniversalPHP,
+	directoryPath: string,
+	defaultName: string
+) {
+	const parentPath = dirname(directoryPath);
+
+	const filesInside = await php.listFiles(directoryPath);
+	if (filesInside.length === 1) {
+		const onlyFilePath = joinPaths(directoryPath, filesInside[0]);
+		const isDir = await php.isDir(onlyFilePath);
+		if (isDir) {
+			const finalPath = joinPaths(parentPath, filesInside[0]);
+			await php.mv(onlyFilePath, finalPath);
+			await php.rmdir(directoryPath, { recursive: true });
+			return finalPath;
 		}
-
-		// Move asset folder to target path
-		const assetFolderPath = `${targetPath}/${assetFolderName}`;
-		await playground.mv(tmpAssetPath, assetFolderPath);
-		await cleanup();
-
-		return {
-			assetFolderPath,
-			assetFolderName,
-		};
-	} catch (error) {
-		await cleanup();
-		throw error;
 	}
+
+	const finalPath = joinPaths(parentPath, defaultName);
+	await php.mv(directoryPath, finalPath);
+	return finalPath;
 }