Add --omit-paths flag to templates apply

src/spec-configuration/containerCollectionsOCI.ts

@@ -504,7 +504,7 @@ export async function getPublishedTags(params: CommonParams, ref: OCIRef): Promi
 	}
 }
 
-export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, expectedDigest: string, ignoredFilesDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> {
+export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, expectedDigest: string, omitDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> {
 	// TODO: Parallelize if multiple layers (not likely).
 	// TODO: Seeking might be needed if the size is too large.
 
@@ -543,24 +543,37 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st
 		await mkdirpLocal(destCachePath);
 		await writeLocalFile(tempTarballPath, resBody);
 
+		// https://github.com/devcontainers/spec/blob/main/docs/specs/devcontainer-templates.md#the-optionalpaths-property
+		const directoriesToOmit = omitDuringExtraction.filter(f => f.endsWith('/*')).map(f => f.slice(0, -1));
+		const filesToOmit = omitDuringExtraction.filter(f => !f.endsWith('/*'));
+
+		output.write(`omitDuringExtraction: '${omitDuringExtraction.join(', ')}`, LogLevel.Trace);
+		output.write(`Files to omit: '${filesToOmit.join(', ')}'`, LogLevel.Info);
+		if (directoriesToOmit.length) {
+			output.write(`Dirs to omit : '${directoriesToOmit.join(', ')}'`, LogLevel.Info);
+		}
+
 		const files: string[] = [];
 		await tar.x(
 			{
 				file: tempTarballPath,
 				cwd: destCachePath,
-				filter: (path: string, stat: tar.FileStat) => {
-					// Skip files that are in the ignore list
-					if (ignoredFilesDuringExtraction.some(f => path.indexOf(f) !== -1)) {
-						// Skip.
-						output.write(`Skipping file '${path}' during blob extraction`, LogLevel.Trace);
-						return false;
+				filter: (tPath: string, stat: tar.FileStat) => {
+					output.write(`Testing '${tPath}'(${stat.type})`, LogLevel.Trace);
+					const cleanedPath = tPath
+						.replace(/\\/g, '/')
+						.replace(/^\.\//, '');
+
+					if (filesToOmit.includes(cleanedPath) || directoriesToOmit.some(d => cleanedPath.startsWith(d))) {
+						output.write(`  Omitting '${tPath}'`, LogLevel.Trace);
+						return false; // Skip
 					}
-					// Keep track of all files extracted, in case the caller is interested.
-					output.write(`${path} : ${stat.type}`, LogLevel.Trace);
-					if ((stat.type.toString() === 'File')) {
-						files.push(path);
+
+					if (stat.type.toString() === 'File') {
+						files.push(tPath);
 					}
-					return true;
+
+					return true; // Keep
 				}
 			}
 		);
@@ -576,8 +589,8 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st
 			{
 				file: tempTarballPath,
 				cwd: ociCacheDir,
-				filter: (path: string, _: tar.FileStat) => {
-					return path === `./${metadataFile}`;
+				filter: (tPath: string, _: tar.FileStat) => {
+					return tPath === `./${metadataFile}`;
 				}
 			});
 		const pathToMetadataFile = path.join(ociCacheDir, metadataFile);

src/spec-configuration/containerTemplatesOCI.ts

@@ -19,12 +19,13 @@ export interface SelectedTemplate {
 	id: string;
 	options: TemplateOptions;
 	features: TemplateFeatureOption[];
+	omitPaths: string[];
 }
 
 export async function fetchTemplate(params: CommonParams, selectedTemplate: SelectedTemplate, templateDestPath: string, userProvidedTmpDir?: string): Promise<string[] | undefined> {
 	const { output } = params;
 
-	let { id: userSelectedId, options: userSelectedOptions } = selectedTemplate;
+	let { id: userSelectedId, options: userSelectedOptions, omitPaths } = selectedTemplate;
 	const templateRef = getRef(output, userSelectedId);
 	if (!templateRef) {
 		output.write(`Failed to parse template ref for ${userSelectedId}`, LogLevel.Error);
@@ -46,7 +47,7 @@ export async function fetchTemplate(params: CommonParams, selectedTemplate: Sele
 	output.write(`blob url: ${blobUrl}`, LogLevel.Trace);
 
 	const tmpDir = userProvidedTmpDir || path.join(os.tmpdir(), 'vsch-template-temp', `${Date.now()}`);
-	const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, ['devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json');
+	const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, [...omitPaths, 'devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json');
 
 	if (!blobResult) {
 		throw new Error(`Failed to download package for ${templateRef.resource}`);

src/spec-node/templatesCLI/apply.ts

@@ -15,6 +15,7 @@ export function templateApplyOptions(y: Argv) {
 			'features': { type: 'string', alias: 'f', default: '[]', description: 'Features to add to the provided Template, provided as JSON.' },
 			'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' },
 			'tmp-dir': { type: 'string', description: 'Directory to use for temporary files. If not provided, the system default will be inferred.' },
+			'omit-paths': { type: 'string', default: '[]', description: 'List of paths within the Template to omit applying, provided as JSON.  To ignore a directory append \'/*\'. Eg: \'[".github/*", "dir/a/*", "file.ts"]\'' },
 		})
 		.check(_argv => {
 			return true;
@@ -34,6 +35,7 @@ async function templateApply({
 	'features': featuresArgs,
 	'log-level': inputLogLevel,
 	'tmp-dir': userProvidedTmpDir,
+	'omit-paths': omitPathsArg,
 }: TemplateApplyArgs) {
 	const disposables: (() => Promise<unknown> | undefined)[] = [];
 	const dispose = async () => {
@@ -65,13 +67,23 @@ async function templateApply({
 		process.exit(1);
 	}
 
+	let omitPaths: string[] = [];
+	if (omitPathsArg) {
+		let omitPathsErrors: jsonc.ParseError[] = [];
+		omitPaths = jsonc.parse(omitPathsArg, omitPathsErrors);
+		if (!Array.isArray(omitPaths)) {
+			output.write('Invalid \'--omitPaths\' argument provided. Provide as a JSON array, eg: \'[".github/*", "dir/a/*", "file.ts"]\'', LogLevel.Error);
+			process.exit(1);
+		}
+	}
+
 	const selectedTemplate: SelectedTemplate = {
 		id: templateId,
 		options,
-		features
+		features,
+		omitPaths,
 	};
 
-
 	const files = await fetchTemplate({ output, env: process.env }, selectedTemplate, workspaceFolder, userProvidedTmpDir);
 	if (!files) {
 		output.write(`Failed to fetch template '${id}'.`, LogLevel.Error);

src/test/container-templates/containerTemplatesOCI.test.ts

@@ -1,8 +1,9 @@
-import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log';
 import * as assert from 'assert';
+import * as os from 'os';
+import * as path from 'path';
+import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log';
 export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace));
 import { fetchTemplate, SelectedTemplate } from '../../spec-configuration/containerTemplatesOCI';
-import * as path from 'path';
 import { readLocalFile } from '../../spec-utils/pfs';
 
 describe('fetchTemplate', async function () {
@@ -14,7 +15,8 @@ describe('fetchTemplate', async function () {
 		const selectedTemplate: SelectedTemplate = {
 			id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest',
 			options: { 'installZsh': 'false', 'upgradePackages': 'true', 'dockerVersion': '20.10', 'moby': 'true' },
-			features: []
+			features: [],
+			omitPaths: [],
 		};
 
 		const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp1'));
@@ -43,7 +45,8 @@ describe('fetchTemplate', async function () {
 		const selectedTemplate: SelectedTemplate = {
 			id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest',
 			options: {},
-			features: []
+			features: [],
+			omitPaths: [],
 		};
 
 		const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp2'));
@@ -72,7 +75,8 @@ describe('fetchTemplate', async function () {
 		const selectedTemplate: SelectedTemplate = {
 			id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest',
 			options: { 'installZsh': 'false', 'upgradePackages': 'true', 'dockerVersion': '20.10', 'moby': 'true', 'enableNonRootDocker': 'true' },
-			features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }]
+			features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }],
+			omitPaths: [],
 		};
 
 		const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp3'));
@@ -104,7 +108,8 @@ describe('fetchTemplate', async function () {
 		const selectedTemplate: SelectedTemplate = {
 			id: 'ghcr.io/devcontainers/templates/anaconda-postgres:latest',
 			options: { 'nodeVersion': 'lts/*' },
-			features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }, { id: 'ghcr.io/devcontainers/features/git:1', options: { 'version': 'latest', ppa: true } }]
+			features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }, { id: 'ghcr.io/devcontainers/features/git:1', options: { 'version': 'latest', ppa: true } }],
+			omitPaths: [],
 		};
 
 		const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp4'));
@@ -123,4 +128,109 @@ describe('fetchTemplate', async function () {
 		assert.match(devcontainer, /"ghcr.io\/devcontainers\/features\/azure-cli:1": {}/);
 		assert.match(devcontainer, /"ghcr.io\/devcontainers\/features\/git:1": {\n\t\t\t"version": "latest",\n\t\t\t"ppa": true/);
 	});
-});
+
+	describe('omit-path', async function () {
+		this.timeout('120s');
+
+		// https://github.com/codspace/templates/pkgs/container/templates%2Fmytemplate/255979159?tag=1.0.4
+		const id = 'ghcr.io/codspace/templates/mytemplate@sha256:57cbf968907c74c106b7b2446063d114743ab3f63345f7c108c577915c535185';
+		const templateFiles = [
+			'./c1.ts',
+			'./c2.ts',
+			'./c3.ts',
+			'./.devcontainer/devcontainer.json',
+			'./.github/dependabot.yml',
+			'./assets/hello.md',
+			'./assets/hi.md',
+			'./example-projects/exampleA/a1.ts',
+			'./example-projects/exampleA/.github/dependabot.yml',
+			'./example-projects/exampleA/subFolderA/a2.ts',
+			'./example-projects/exampleB/b1.ts',
+			'./example-projects/exampleB/.github/dependabot.yml',
+			'./example-projects/exampleB/subFolderB/b2.ts',
+		];
+
+		// NOTE: Certain files, like the 'devcontainer-template.json', are always filtered
+		//	     out as they are not part of the Template.
+		it('Omit nothing', async () => {
+			const selectedTemplate: SelectedTemplate = {
+				id,
+				options: {},
+				features: [],
+				omitPaths: [],
+			};
+
+			const files = await fetchTemplate(
+				{ output, env: process.env },
+				selectedTemplate,
+				path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`)
+			);
+
+			assert.ok(files);
+			assert.strictEqual(files.length, templateFiles.length);
+			for (const file of templateFiles) {
+				assert.ok(files.includes(file));
+			}
+		});
+
+		it('Omit nested folder', async () => {
+			const selectedTemplate: SelectedTemplate = {
+				id,
+				options: {},
+				features: [],
+				omitPaths: ['example-projects/exampleB/*'],
+			};
+
+			const files = await fetchTemplate(
+				{ output, env: process.env },
+				selectedTemplate,
+				path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`)
+			);
+
+			const expectedRemovedFiles = [
+				'./example-projects/exampleB/b1.ts',
+				'./example-projects/example/.github/dependabot.yml',
+				'./example-projects/exampleB/subFolderB/b2.ts',
+			];
+
+			assert.ok(files);
+			assert.strictEqual(files.length, templateFiles.length - 3);
+			for (const file of expectedRemovedFiles) {
+				assert.ok(!files.includes(file));
+			}
+		});
+
+		it('Omit single file, root folder, and nested folder', async () => {
+			const selectedTemplate: SelectedTemplate = {
+				id,
+				options: {},
+				features: [],
+				omitPaths: ['.github/*', 'example-projects/exampleA/*', 'c1.ts'],
+			};
+
+			const files = await fetchTemplate(
+				{ output, env: process.env },
+				selectedTemplate,
+				path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`)
+			);
+
+			const expectedRemovedFiles = [
+				'./c1.ts',
+				'./.github/dependabot.yml',
+				'./example-projects/exampleA/a1.ts',
+				'./example-projects/exampleA/.github/dependabot.yml',
+				'./example-projects/exampleA/subFolderA/a2.ts',
+			];
+
+			assert.ok(files);
+			assert.strictEqual(files.length, templateFiles.length - 5);
+			for (const file of expectedRemovedFiles) {
+				assert.ok(!files.includes(file));
+			}
+		});
+	});
+
+
+});
+
+