Add --omit-paths flag to templates apply

src/spec-configuration/containerCollectionsOCI.ts

@@ -543,22 +543,35 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st
 		await mkdirpLocal(destCachePath);
 		await writeLocalFile(tempTarballPath, resBody);
 
+		output.write(`Filtering out the following paths from the blob: '${ignoredFilesDuringExtraction.join(', ')}`, LogLevel.Trace);
 		const files: string[] = [];
 		await tar.x(
 			{
 				file: tempTarballPath,
 				cwd: destCachePath,
-				filter: (path: string, stat: tar.FileStat) => {
-					// Skip files that are in the ignore list
-					if (ignoredFilesDuringExtraction.some(f => path.indexOf(f) !== -1)) {
-						// Skip.
-						output.write(`Skipping file '${path}' during blob extraction`, LogLevel.Trace);
+				filter: (tPath: string, stat: tar.FileStat) => {
+					output.write(`Testing '${tPath}'(${stat.type})`, LogLevel.Trace);
+					// Skip files or folders that are in the ignore list
+					// Contents of ignoredFilesDuringExtraction should be resolved relative to the root of the tarball.
+					// Examples:
+					//    - 'devcontainer-template.json' (single file at the root)
+					//    - '.github/' 					 (entire folder at the root)
+					//    - 'my-project/index.ts' 		 (file in a subfolder)
+					if (ignoredFilesDuringExtraction.some(ignoredFileOrFolder => {
+						const fileOrFolder =
+							tPath.replace(/\\/g, '/')
+								.replace(/^\.\//, '');
+						const ignored =
+							ignoredFileOrFolder.replace(/\\/g, '/')
+								.replace(/^\.\//, '');
+						return ignored.endsWith('/') ? fileOrFolder.startsWith(ignored) : fileOrFolder === ignored;
+					})) {
+						output.write(`   ** Ignoring '${tPath}' during blob extraction`, LogLevel.Trace);
 						return false;
 					}
 					// Keep track of all files extracted, in case the caller is interested.
-					output.write(`${path} : ${stat.type}`, LogLevel.Trace);
 					if ((stat.type.toString() === 'File')) {
-						files.push(path);
+						files.push(tPath);
 					}
 					return true;
 				}
@@ -576,8 +589,8 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st
 			{
 				file: tempTarballPath,
 				cwd: ociCacheDir,
-				filter: (path: string, _: tar.FileStat) => {
-					return path === `./${metadataFile}`;
+				filter: (tPath: string, _: tar.FileStat) => {
+					return tPath === `./${metadataFile}`;
 				}
 			});
 		const pathToMetadataFile = path.join(ociCacheDir, metadataFile);

src/spec-configuration/containerTemplatesOCI.ts

@@ -19,12 +19,13 @@ export interface SelectedTemplate {
 	id: string;
 	options: TemplateOptions;
 	features: TemplateFeatureOption[];
+	omitPaths: string[];
 }
 
 export async function fetchTemplate(params: CommonParams, selectedTemplate: SelectedTemplate, templateDestPath: string, userProvidedTmpDir?: string): Promise<string[] | undefined> {
 	const { output } = params;
 
-	let { id: userSelectedId, options: userSelectedOptions } = selectedTemplate;
+	let { id: userSelectedId, options: userSelectedOptions, omitPaths } = selectedTemplate;
 	const templateRef = getRef(output, userSelectedId);
 	if (!templateRef) {
 		output.write(`Failed to parse template ref for ${userSelectedId}`, LogLevel.Error);
@@ -46,7 +47,7 @@ export async function fetchTemplate(params: CommonParams, selectedTemplate: Sele
 	output.write(`blob url: ${blobUrl}`, LogLevel.Trace);
 
 	const tmpDir = userProvidedTmpDir || path.join(os.tmpdir(), 'vsch-template-temp', `${Date.now()}`);
-	const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, ['devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json');
+	const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, [...omitPaths, 'devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json');
 
 	if (!blobResult) {
 		throw new Error(`Failed to download package for ${templateRef.resource}`);

src/spec-node/templatesCLI/apply.ts

@@ -15,6 +15,7 @@ export function templateApplyOptions(y: Argv) {
 			'features': { type: 'string', alias: 'f', default: '[]', description: 'Features to add to the provided Template, provided as JSON.' },
 			'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' },
 			'tmp-dir': { type: 'string', description: 'Directory to use for temporary files. If not provided, the system default will be inferred.' },
+			'omit-paths': { type: 'string', default: '[]', description: 'List of paths within the Template to omit applying, provided as JSON.  Likely resolved from the Template\'s \'optionalPaths\' property.' },
 		})
 		.check(_argv => {
 			return true;
@@ -34,6 +35,7 @@ async function templateApply({
 	'features': featuresArgs,
 	'log-level': inputLogLevel,
 	'tmp-dir': userProvidedTmpDir,
+	'omit-paths': omitPathsArg,
 }: TemplateApplyArgs) {
 	const disposables: (() => Promise<unknown> | undefined)[] = [];
 	const dispose = async () => {
@@ -65,13 +67,23 @@ async function templateApply({
 		process.exit(1);
 	}
 
+	let omitPaths: string[] = [];
+	if (omitPathsArg) {
+		let omitPathsErrors: jsonc.ParseError[] = [];
+		omitPaths = jsonc.parse(omitPathsArg, omitPathsErrors);
+		if (!Array.isArray(omitPaths)) {
+			output.write('Invalid \'--omitPaths\' argument provided. Provide as a JSON array, eg: \'[".github", "project/"]\'', LogLevel.Error);
+			process.exit(1);
+		}
+	}
+
 	const selectedTemplate: SelectedTemplate = {
 		id: templateId,
 		options,
-		features
+		features,
+		omitPaths,
 	};
 
-
 	const files = await fetchTemplate({ output, env: process.env }, selectedTemplate, workspaceFolder, userProvidedTmpDir);
 	if (!files) {
 		output.write(`Failed to fetch template '${id}'.`, LogLevel.Error);

src/test/container-templates/containerTemplatesOCI.test.ts

@@ -1,8 +1,9 @@
-import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log';
 import * as assert from 'assert';
+import * as os from 'os';
+import * as path from 'path';
+import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log';
 export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace));
 import { fetchTemplate, SelectedTemplate } from '../../spec-configuration/containerTemplatesOCI';
-import * as path from 'path';
 import { readLocalFile } from '../../spec-utils/pfs';
 
 describe('fetchTemplate', async function () {
@@ -14,7 +15,8 @@ describe('fetchTemplate', async function () {
 		const selectedTemplate: SelectedTemplate = {
 			id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest',
 			options: { 'installZsh': 'false', 'upgradePackages': 'true', 'dockerVersion': '20.10', 'moby': 'true' },
-			features: []
+			features: [],
+			omitPaths: [],
 		};
 
 		const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp1'));
@@ -43,7 +45,8 @@ describe('fetchTemplate', async function () {
 		const selectedTemplate: SelectedTemplate = {
 			id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest',
 			options: {},
-			features: []
+			features: [],
+			omitPaths: [],
 		};
 
 		const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp2'));
@@ -72,7 +75,8 @@ describe('fetchTemplate', async function () {
 		const selectedTemplate: SelectedTemplate = {
 			id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest',
 			options: { 'installZsh': 'false', 'upgradePackages': 'true', 'dockerVersion': '20.10', 'moby': 'true', 'enableNonRootDocker': 'true' },
-			features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }]
+			features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }],
+			omitPaths: [],
 		};
 
 		const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp3'));
@@ -104,7 +108,8 @@ describe('fetchTemplate', async function () {
 		const selectedTemplate: SelectedTemplate = {
 			id: 'ghcr.io/devcontainers/templates/anaconda-postgres:latest',
 			options: { 'nodeVersion': 'lts/*' },
-			features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }, { id: 'ghcr.io/devcontainers/features/git:1', options: { 'version': 'latest', ppa: true } }]
+			features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }, { id: 'ghcr.io/devcontainers/features/git:1', options: { 'version': 'latest', ppa: true } }],
+			omitPaths: [],
 		};
 
 		const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp4'));
@@ -123,4 +128,109 @@ describe('fetchTemplate', async function () {
 		assert.match(devcontainer, /"ghcr.io\/devcontainers\/features\/azure-cli:1": {}/);
 		assert.match(devcontainer, /"ghcr.io\/devcontainers\/features\/git:1": {\n\t\t\t"version": "latest",\n\t\t\t"ppa": true/);
 	});
-});
+
+	describe('omit-path', async function () {
+		this.timeout('120s');
+
+		// https://github.com/codspace/templates/pkgs/container/templates%2Fmytemplate/252099017?tag=1.0.3
+		const id = 'ghcr.io/codspace/templates/mytemplate@sha256:c44cb27efa68ee87a71838a59d1d2892b3c2de24be6f94c136652e45a19f017e';
+		const templateFiles = [
+			'./c1.ts',
+			'./c2.ts',
+			'./c3.ts',
+			'./.devcontainer/devcontainer.json',
+			'./.github/dependabot.yml',
+			'./assets/hello.md',
+			'./assets/hi.md',
+			'./example-projects/exampleA/a1.ts',
+			'./example-projects/exampleA/.github/dependabot.yml',
+			'./example-projects/exampleA/subFolderA/a2.ts',
+			'./example-projects/exampleB/b1.ts',
+			'./example-projects/exampleB/.github/dependabot.yml',
+			'./example-projects/exampleB/subFolderB/b2.ts',
+		];
+
+		// NOTE: Certain files, like the 'devcontainer-template.json', are always filtered
+		//	     out as they are not part of the Template.
+		it('Omit nothing', async () => {
+			const selectedTemplate: SelectedTemplate = {
+				id,
+				options: {},
+				features: [],
+				omitPaths: [],
+			};
+
+			const files = await fetchTemplate(
+				{ output, env: process.env },
+				selectedTemplate,
+				path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`)
+			);
+
+			assert.ok(files);
+			assert.strictEqual(files.length, templateFiles.length);
+			for (const file of templateFiles) {
+				assert.ok(files.includes(file));
+			}
+		});
+
+		it('Omit nested folder', async () => {
+			const selectedTemplate: SelectedTemplate = {
+				id,
+				options: {},
+				features: [],
+				omitPaths: ['example-projects/exampleB/'],
+			};
+
+			const files = await fetchTemplate(
+				{ output, env: process.env },
+				selectedTemplate,
+				path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`)
+			);
+
+			const expectedRemovedFiles = [
+				'./example-projects/exampleB/b1.ts',
+				'./example-projects/example/.github/dependabot.yml',
+				'./example-projects/exampleB/subFolderB/b2.ts',
+			];
+
+			assert.ok(files);
+			assert.strictEqual(files.length, templateFiles.length - 3);
+			for (const file of expectedRemovedFiles) {
+				assert.ok(!files.includes(file));
+			}
+		});
+
+		it('Omit single file, root folder, and nested folder', async () => {
+			const selectedTemplate: SelectedTemplate = {
+				id,
+				options: {},
+				features: [],
+				omitPaths: ['.github/', 'example-projects/exampleA/', 'c1.ts'],
+			};
+
+			const files = await fetchTemplate(
+				{ output, env: process.env },
+				selectedTemplate,
+				path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`)
+			);
+
+			const expectedRemovedFiles = [
+				'./c1.ts',
+				'./.github/dependabot.yml',
+				'./example-projects/exampleA/a1.ts',
+				'./example-projects/exampleA/.github/dependabot.yml',
+				'./example-projects/exampleA/subFolderA/a2.ts',
+			];
+
+			assert.ok(files);
+			assert.strictEqual(files.length, templateFiles.length - 5);
+			for (const file of expectedRemovedFiles) {
+				assert.ok(!files.includes(file));
+			}
+		});
+	});
+
+
+});
+
+