From 68d50ca975e4a64d9b5dfae3f650d3d4e2b7c3b5 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 2 Dec 2022 00:02:16 +0000 Subject: [PATCH 01/18] setup reference implementation registry for testing --- .gitignore | 1 + .../containerCollectionsOCI.ts | 158 +++++++++++------- .../containerCollectionsOCIPush.ts | 70 ++++---- src/spec-node/utils.ts | 4 +- src/spec-utils/httpRequest.ts | 15 +- .../containerFeaturesOCI.test.ts | 2 +- .../containerFeaturesOCIPush.test.ts | 63 ++++++- 7 files changed, 209 insertions(+), 104 deletions(-) diff --git a/.gitignore b/.gitignore index 55f76ad58..b954d43f1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ logs *.tgz tmp tmp[0-9] +tmp/* build-tmp .DS_Store .env diff --git a/src/spec-configuration/containerCollectionsOCI.ts b/src/spec-configuration/containerCollectionsOCI.ts index 302a27c20..430274a29 100644 --- a/src/spec-configuration/containerCollectionsOCI.ts +++ b/src/spec-configuration/containerCollectionsOCI.ts @@ -11,7 +11,7 @@ export const DEVCONTAINER_MANIFEST_MEDIATYPE = 'application/vnd.devcontainers'; export const DEVCONTAINER_TAR_LAYER_MEDIATYPE = 'application/vnd.devcontainers.layer.v1+tar'; export const DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE = 'application/vnd.devcontainers.collection.layer.v1+json'; -export type HEADERS = { 'authorization'?: string; 'user-agent': string; 'content-type'?: string; 'accept'?: string }; +export type HEADERS = { 'authorization'?: string; 'user-agent': string; 'content-type'?: string; 'accept'?: string; 'content-length'?: string }; // Represents the unique OCI identifier for a Feature or Template. // eg: ghcr.io/devcontainers/features/go:1.0.0 @@ -201,9 +201,9 @@ export async function getManifest(output: Log, env: NodeJS.ProcessEnv, url: stri 'accept': mimeType || 'application/vnd.oci.image.manifest.v1+json', }; - const auth = authToken ?? await fetchRegistryAuthToken(output, ref.registry, ref.path, env, 'pull'); - if (auth) { - headers['authorization'] = `Bearer ${auth}`; + const authorization = authToken ?? await fetchAuthorization(output, ref.registry, ref.path, env, 'pull'); + if (authorization) { + headers['authorization'] = authorization; } const options = { @@ -221,72 +221,108 @@ export async function getManifest(output: Log, env: NodeJS.ProcessEnv, url: stri } } -// https://github.com/oras-project/oras-go/blob/97a9c43c52f9d89ecf5475bc59bd1f96c8cc61f6/registry/remote/auth/scope.go#L60-L74 -export async function fetchRegistryAuthToken(output: Log, registry: string, ociRepoPath: string, env: NodeJS.ProcessEnv, operationScopes: string): Promise { - if (registry === 'mcr.microsoft.com') { - return undefined; - } - - const headers: HEADERS = { - 'user-agent': 'devcontainer' - }; +async function getBasicAuthCredential(output: Log, registry: string, env: NodeJS.ProcessEnv): Promise { + // TODO: Also read OS keychain/docker config for auth in various registries! - // TODO: Read OS keychain/docker config for auth in various registries! - - let userToken = ''; + let userToken: string | undefined = undefined; if (!!env['GITHUB_TOKEN'] && registry === 'ghcr.io') { userToken = env['GITHUB_TOKEN']; } else if (!!env['DEVCONTAINERS_OCI_AUTH']) { - // eg: DEVCONTAINERS_OCI_AUTH=domain1:token1,domain2:token2 + // eg: DEVCONTAINERS_OCI_AUTH=domain1|user1|token1,domain2|user2|token2 const authContexts = env['DEVCONTAINERS_OCI_AUTH'].split(','); - const authContext = authContexts.find(a => a.split(':')[0] === registry); - if (authContext && authContext.length === 2) { - userToken = authContext.split(':')[1]; + const authContext = authContexts.find(a => a.split('|')[0] === registry); + + if (authContext) { + output.write(`Using match from DEVCONTAINERS_OCI_AUTH for registry '${registry}'`, LogLevel.Trace); + const split = authContext.split('|'); + userToken = `${split[1]}:${split[2]}`; } - } else { - output.write('No oauth authentication credentials found.', LogLevel.Trace); } if (userToken) { - const base64Encoded = Buffer.from(`USERNAME:${userToken}`).toString('base64'); - headers['authorization'] = `Basic ${base64Encoded}`; + return Buffer.from(userToken).toString('base64'); } - const authServer = registry === 'docker.io' ? 'auth.docker.io' : registry; - const registryServer = registry === 'docker.io' ? 'registry.docker.io' : registry; - const url = `https://${authServer}/token?scope=repository:${ociRepoPath}:${operationScopes}&service=${registryServer}`; - output.write(`url: ${url}`, LogLevel.Trace); - - const options = { - type: 'GET', - url: url, - headers: headers - }; + // Error + output.write(`No authentication credentials found for registry '${registry}'.`, LogLevel.Error); + return undefined; +} - let authReq: Buffer; - try { - authReq = await request(options, output); - } catch (e: any) { - output.write(`Failed to get registry auth token with error: ${e}`, LogLevel.Error); - return undefined; +// https://github.com/oras-project/oras-go/blob/97a9c43c52f9d89ecf5475bc59bd1f96c8cc61f6/registry/remote/auth/scope.go#L60-L74 +// Some registries (eg: ghcr.io) expect a scoped token to target resources. + +// async function generateScopeTokenCredential(output: Log, registry: string, ociRepoPath: string, env: NodeJS.ProcessEnv, operationScopes: string, basicAuthTokenBase64: string | undefined = undefined): Promise { +// if (registry === 'mcr.microsoft.com') { +// return undefined; +// } + +// const headers: HEADERS = { +// 'user-agent': 'devcontainer' +// }; + +// if (!basicAuthTokenBase64) { +// basicAuthTokenBase64 = await getBasicAuthCredential(output, registry, env); +// } + +// headers['authorization'] = `Basic ${basicAuthTokenBase64}`; + +// const authServer = registry === 'docker.io' ? 'auth.docker.io' : registry; +// const registryServer = registry === 'docker.io' ? 'registry.docker.io' : registry; +// const url = `https://${authServer}/token?scope=repository:${ociRepoPath}:${operationScopes}&service=${registryServer}`; +// output.write(`url: ${url}`, LogLevel.Trace); + +// const options = { +// type: 'GET', +// url: url, +// headers: headers +// }; + +// let authReq: Buffer; +// try { +// authReq = await request(options, output); +// } catch (e: any) { +// output.write(`Unable to request scope token from registry ${registry}: ${e}`, LogLevel.Warning); +// return; +// } + +// if (!authReq) { +// output.write('Failed to get registry auth token', LogLevel.Error); +// return undefined; +// } + +// let scopeToken: string | undefined; +// try { +// scopeToken = JSON.parse(authReq.toString())?.token; +// } catch { +// // not JSON +// } +// if (!scopeToken) { +// output.write('Failed to parse registry auth token response', LogLevel.Error); +// return undefined; +// } + +// return scopeToken; +// } + + +// Exported Function +// Will attempt to generate/fetch the correct authorization header for subsequent requests (Bearer or Basic) +export async function fetchAuthorization(output: Log, registry: string, _ociRepoPath: string, env: NodeJS.ProcessEnv, _operationScopes: string): Promise { + const basicAuthTokenBase64 = await getBasicAuthCredential(output, registry, env); + // const scopeToken = generateScopeTokenCredential(output, registry, ociRepoPath, env, operationScopes, basicAuthTokenBase64); + + // if (scopeToken) { + // output.write(`Using scope token for registry '${registry}'`, LogLevel.Trace); + // return `Bearer ${scopeToken}`; + // } + + if (basicAuthTokenBase64) { + output.write(`Using basic auth token for registry '${registry}'`, LogLevel.Trace); + return `Basic ${basicAuthTokenBase64}`; } - if (!authReq) { - output.write('Failed to get registry auth token', LogLevel.Error); - return undefined; - } + return undefined; - let token: string | undefined; - try { - token = JSON.parse(authReq.toString())?.token; - } catch { - // not JSON - } - if (!token) { - output.write('Failed to parse registry auth token response', LogLevel.Error); - return undefined; - } - return token; } // Lists published versions/tags of a feature/template @@ -295,9 +331,9 @@ export async function getPublishedVersions(ref: OCIRef, output: Log, sorted: boo try { const url = `https://${ref.registry}/v2/${ref.namespace}/${ref.id}/tags/list`; - let authToken = await fetchRegistryAuthToken(output, ref.registry, ref.path, process.env, 'pull'); + let authorization = await fetchAuthorization(output, ref.registry, ref.path, process.env, 'pull'); - if (!authToken) { + if (!authorization) { output.write(`(!) ERR: Failed to get published versions for ${collectionType}: ${ref.resource}`, LogLevel.Error); return undefined; } @@ -305,7 +341,7 @@ export async function getPublishedVersions(ref: OCIRef, output: Log, sorted: boo const headers: HEADERS = { 'user-agent': 'devcontainer', 'accept': 'application/json', - 'authorization': `Bearer ${authToken}` + 'authorization': authorization }; const options = { @@ -352,9 +388,9 @@ export async function getBlob(output: Log, env: NodeJS.ProcessEnv, url: string, 'accept': 'application/vnd.oci.image.manifest.v1+json', }; - const auth = authToken ?? await fetchRegistryAuthToken(output, ociRef.registry, ociRef.path, env, 'pull'); - if (auth) { - headers['authorization'] = `Bearer ${auth}`; + const authorization = authToken ?? await fetchAuthorization(output, ociRef.registry, ociRef.path, env, 'pull'); + if (authorization) { + headers['authorization'] = authorization; } const options = { diff --git a/src/spec-configuration/containerCollectionsOCIPush.ts b/src/spec-configuration/containerCollectionsOCIPush.ts index b97f555bb..2385c3aee 100644 --- a/src/spec-configuration/containerCollectionsOCIPush.ts +++ b/src/spec-configuration/containerCollectionsOCIPush.ts @@ -5,7 +5,8 @@ import { delay } from '../spec-common/async'; import { headRequest, requestResolveHeaders } from '../spec-utils/httpRequest'; import { Log, LogLevel } from '../spec-utils/log'; import { isLocalFile, readLocalFile } from '../spec-utils/pfs'; -import { DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE, DEVCONTAINER_TAR_LAYER_MEDIATYPE, fetchOCIManifestIfExists, fetchRegistryAuthToken, HEADERS, OCICollectionRef, OCILayer, OCIManifest, OCIRef } from './containerCollectionsOCI'; +import { DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE, DEVCONTAINER_TAR_LAYER_MEDIATYPE, fetchOCIManifestIfExists, fetchAuthorization, HEADERS, OCICollectionRef, OCILayer, OCIManifest, OCIRef } from './containerCollectionsOCI'; +import { promisify } from 'util'; // (!) Entrypoint function to push a single feature/template to a registry. // Devcontainer Spec (features) : https://containers.dev/implementors/features-distribution/#oci-registry @@ -17,8 +18,8 @@ export async function pushOCIFeatureOrTemplate(output: Log, ociRef: OCIRef, path const env = process.env; // Generate registry auth token with `pull,push` scopes. - const registryAuthToken = await fetchRegistryAuthToken(output, ociRef.registry, ociRef.path, env, 'pull,push'); - if (!registryAuthToken) { + const authorization = await fetchAuthorization(output, ociRef.registry, ociRef.path, env, 'pull,push'); + if (!authorization) { output.write(`Failed to get registry auth token`, LogLevel.Error); return false; } @@ -32,10 +33,10 @@ export async function pushOCIFeatureOrTemplate(output: Log, ociRef: OCIRef, path output.write(`Generated manifest: \n${JSON.stringify(manifest?.manifestObj, undefined, 4)}`, LogLevel.Trace); // If the exact manifest digest already exists in the registry, we don't need to push individual blobs (it's already there!) - const existingManifest = await fetchOCIManifestIfExists(output, env, ociRef, manifest.digest, registryAuthToken); + const existingManifest = await fetchOCIManifestIfExists(output, env, ociRef, manifest.digest, authorization); if (manifest.digest && existingManifest) { output.write(`Not reuploading blobs, digest already exists.`, LogLevel.Trace); - return await putManifestWithTags(output, manifest.manifestStr, ociRef, tags, registryAuthToken); + return await putManifestWithTags(output, manifest.manifestStr, ociRef, tags, authorization); } const blobsToPush = [ @@ -50,7 +51,7 @@ export async function pushOCIFeatureOrTemplate(output: Log, ociRef: OCIRef, path ]; // Obtain session ID with `/v2//blobs/uploads/` - const blobPutLocationUriPath = await postUploadSessionId(output, ociRef, registryAuthToken); + const blobPutLocationUriPath = await postUploadSessionId(output, ociRef, authorization); if (!blobPutLocationUriPath) { output.write(`Failed to get upload session ID`, LogLevel.Error); return false; @@ -58,12 +59,12 @@ export async function pushOCIFeatureOrTemplate(output: Log, ociRef: OCIRef, path for await (const blob of blobsToPush) { const { name, digest } = blob; - const blobExistsConfigLayer = await checkIfBlobExists(output, ociRef, digest, registryAuthToken); + const blobExistsConfigLayer = await checkIfBlobExists(output, ociRef, digest, authorization); output.write(`blob: '${name}' with digest '${digest}' ${blobExistsConfigLayer ? 'already exists' : 'does not exist'} in registry.`, LogLevel.Trace); // PUT blobs if (!blobExistsConfigLayer) { - if (!(await putBlob(output, pathToTgz, blobPutLocationUriPath, ociRef, digest, registryAuthToken))) { + if (!(await putBlob(output, pathToTgz, blobPutLocationUriPath, ociRef, digest, authorization))) { output.write(`Failed to PUT blob '${name}' with digest '${digest}'`, LogLevel.Error); return false; } @@ -71,7 +72,7 @@ export async function pushOCIFeatureOrTemplate(output: Log, ociRef: OCIRef, path } // Send a final PUT to combine blobs and tag manifest properly. - return await putManifestWithTags(output, manifest.manifestStr, ociRef, tags, registryAuthToken); + return await putManifestWithTags(output, manifest.manifestStr, ociRef, tags, authorization); } // (!) Entrypoint function to push a collection metadata/overview file for a set of features/templates to a registry. @@ -83,8 +84,8 @@ export async function pushCollectionMetadata(output: Log, collectionRef: OCIColl output.write(`${JSON.stringify(collectionRef, null, 2)}`, LogLevel.Trace); const env = process.env; - const registryAuthToken = await fetchRegistryAuthToken(output, collectionRef.registry, collectionRef.path, env, 'pull,push'); - if (!registryAuthToken) { + const authorization = await fetchAuthorization(output, collectionRef.registry, collectionRef.path, env, 'pull,push'); + if (!authorization) { output.write(`Failed to get registry auth token`, LogLevel.Error); return false; } @@ -98,14 +99,14 @@ export async function pushCollectionMetadata(output: Log, collectionRef: OCIColl output.write(`Generated manifest: \n${JSON.stringify(manifest?.manifestObj, undefined, 4)}`, LogLevel.Trace); // If the exact manifest digest already exists in the registry, we don't need to push individual blobs (it's already there!) - const existingManifest = await fetchOCIManifestIfExists(output, env, collectionRef, manifest.digest, registryAuthToken); + const existingManifest = await fetchOCIManifestIfExists(output, env, collectionRef, manifest.digest, authorization); if (manifest.digest && existingManifest) { output.write(`Not reuploading blobs, digest already exists.`, LogLevel.Trace); - return await putManifestWithTags(output, manifest.manifestStr, collectionRef, ['latest'], registryAuthToken); + return await putManifestWithTags(output, manifest.manifestStr, collectionRef, ['latest'], authorization); } // Obtain session ID with `/v2//blobs/uploads/` - const blobPutLocationUriPath = await postUploadSessionId(output, collectionRef, registryAuthToken); + const blobPutLocationUriPath = await postUploadSessionId(output, collectionRef, authorization); if (!blobPutLocationUriPath) { output.write(`Failed to get upload session ID`, LogLevel.Error); return false; @@ -124,12 +125,12 @@ export async function pushCollectionMetadata(output: Log, collectionRef: OCIColl for await (const blob of blobsToPush) { const { name, digest } = blob; - const blobExistsConfigLayer = await checkIfBlobExists(output, collectionRef, digest, registryAuthToken); + const blobExistsConfigLayer = await checkIfBlobExists(output, collectionRef, digest, authorization); output.write(`blob: '${name}' with digest '${digest}' ${blobExistsConfigLayer ? 'already exists' : 'does not exist'} in registry.`, LogLevel.Trace); // PUT blobs if (!blobExistsConfigLayer) { - if (!(await putBlob(output, pathToCollectionJson, blobPutLocationUriPath, collectionRef, digest, registryAuthToken))) { + if (!(await putBlob(output, pathToCollectionJson, blobPutLocationUriPath, collectionRef, digest, authorization))) { output.write(`Failed to PUT blob '${name}' with digest '${digest}'`, LogLevel.Error); return false; } @@ -138,13 +139,13 @@ export async function pushCollectionMetadata(output: Log, collectionRef: OCIColl // Send a final PUT to combine blobs and tag manifest properly. // Collections are always tagged 'latest' - return await putManifestWithTags(output, manifest.manifestStr, collectionRef, ['latest'], registryAuthToken); + return await putManifestWithTags(output, manifest.manifestStr, collectionRef, ['latest'], authorization); } // --- Helper Functions // Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests (PUT /manifests/) -async function putManifestWithTags(output: Log, manifestStr: string, ociRef: OCIRef | OCICollectionRef, tags: string[], registryAuthToken: string): Promise { +async function putManifestWithTags(output: Log, manifestStr: string, ociRef: OCIRef | OCICollectionRef, tags: string[], authorization: string): Promise { output.write(`Tagging manifest with tags: ${tags.join(', ')}`, LogLevel.Trace); for await (const tag of tags) { @@ -155,7 +156,7 @@ async function putManifestWithTags(output: Log, manifestStr: string, ociRef: OCI type: 'PUT', url, headers: { - 'Authorization': `Bearer ${registryAuthToken}`, + 'Authorization': authorization, // Eg: 'Bearer ' or 'Basic ' 'Content-Type': 'application/vnd.oci.image.manifest.v1+json', }, data: Buffer.from(manifestStr), @@ -187,34 +188,43 @@ async function putManifestWithTags(output: Log, manifestStr: string, ociRef: OCI } // Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#post-then-put (PUT ?digest=) -async function putBlob(output: Log, pathToBlob: string, blobPutLocationUriPath: string, ociRef: OCIRef | OCICollectionRef, digest: string, registryAuthToken: string): Promise { - output.write(`PUT new blob -> '${digest}'`, LogLevel.Info); +async function putBlob(output: Log, pathToBlob: string, blobPutLocationUriPath: string, ociRef: OCIRef | OCICollectionRef, digest: string, authorization: string): Promise { if (!(await isLocalFile(pathToBlob))) { output.write(`Blob ${pathToBlob} does not exist`, LogLevel.Error); return false; } + const blobSize = (await promisify(fs.stat)(pathToBlob)).size; + + output.write(`PUT new blob -> '${digest}' (size=${blobSize})`, LogLevel.Info); + const headers: HEADERS = { 'user-agent': 'devcontainer', - 'authorization': `Bearer ${registryAuthToken}`, + 'authorization': authorization, 'content-type': 'application/octet-stream', + 'content-length': `${blobSize}` }; // OCI distribution spec is ambiguous on whether we get back an absolute or relative path. let url = ''; - if (blobPutLocationUriPath.startsWith('https://')) { + if (blobPutLocationUriPath.startsWith('https://') || blobPutLocationUriPath.startsWith('http://')) { url = blobPutLocationUriPath; } else { url = `https://${ociRef.registry}${blobPutLocationUriPath}`; } - url += `?digest=${digest}`; + + if (url.indexOf('?') === -1) { + url += `?digest=${digest}`; + } else { + url += `&digest=${digest}`; + } output.write(`Crafted blob url: ${url}`, LogLevel.Trace); - const { statusCode } = await requestResolveHeaders({ type: 'PUT', url, headers, data: await readLocalFile(pathToBlob) }, output); + const { statusCode, resBody } = await requestResolveHeaders({ type: 'PUT', url, headers, data: await readLocalFile(pathToBlob) }, output); if (statusCode !== 201) { - output.write(`${statusCode}: Failed to upload blob '${pathToBlob}' to '${url}'`, LogLevel.Error); + output.write(`${statusCode}: Failed to upload blob '${pathToBlob}' to '${url}' -> ${resBody}`, LogLevel.Error); return false; } @@ -288,10 +298,10 @@ export async function calculateDataLayer(output: Log, pathToData: string, mediaT // Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#checking-if-content-exists-in-the-registry // Requires registry auth token. -export async function checkIfBlobExists(output: Log, ociRef: OCIRef | OCICollectionRef, digest: string, authToken: string): Promise { +export async function checkIfBlobExists(output: Log, ociRef: OCIRef | OCICollectionRef, digest: string, authorization: string): Promise { const headers: HEADERS = { 'user-agent': 'devcontainer', - 'authorization': `Bearer ${authToken}`, + 'authorization': authorization, }; const url = `https://${ociRef.registry}/v2/${ociRef.path}/blobs/${digest}`; @@ -303,10 +313,10 @@ export async function checkIfBlobExists(output: Log, ociRef: OCIRef | OCICollect // Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#post-then-put // Requires registry auth token. -async function postUploadSessionId(output: Log, ociRef: OCIRef | OCICollectionRef, authToken: string): Promise { +async function postUploadSessionId(output: Log, ociRef: OCIRef | OCICollectionRef, authorization: string): Promise { const headers: HEADERS = { 'user-agent': 'devcontainer', - 'authorization': `Bearer ${authToken}`, + 'authorization': authorization }; const url = `https://${ociRef.registry}/v2/${ociRef.path}/blobs/uploads/`; diff --git a/src/spec-node/utils.ts b/src/spec-node/utils.ts index d0814884b..8133c4df8 100644 --- a/src/spec-node/utils.ts +++ b/src/spec-node/utils.ts @@ -25,7 +25,7 @@ import { Event } from '../spec-utils/event'; import { Mount } from '../spec-configuration/containerFeaturesConfiguration'; import { PackageConfiguration } from '../spec-utils/product'; import { ImageMetadataEntry } from './imageMetadata'; -import { fetchRegistryAuthToken, getManifest, getRef, HEADERS } from '../spec-configuration/containerCollectionsOCI'; +import { fetchAuthorization, getManifest, getRef, HEADERS } from '../spec-configuration/containerCollectionsOCI'; import { request } from '../spec-utils/httpRequest'; export { getConfigFilePath, getDockerfilePath, isDockerFileConfig, resolveConfigFilePath } from '../spec-configuration/configuration'; @@ -218,7 +218,7 @@ export async function inspectImageInRegistry(output: Log, name: string, authToke if (!ref) { throw new Error(`Could not parse image name '${name}'`); } - const auth = authToken ?? await fetchRegistryAuthToken(output, ref.registry, ref.path, process.env, 'pull'); + const auth = authToken ?? await fetchAuthorization(output, ref.registry, ref.path, process.env, 'pull'); const registryServer = ref.registry === 'docker.io' ? 'registry-1.docker.io' : ref.registry; const manifestUrl = `https://${registryServer}/v2/${ref.path}/manifests/${ref.version}`; diff --git a/src/spec-utils/httpRequest.ts b/src/spec-utils/httpRequest.ts index 44058ae44..b50c3f940 100644 --- a/src/spec-utils/httpRequest.ts +++ b/src/spec-utils/httpRequest.ts @@ -4,12 +4,12 @@ *--------------------------------------------------------------------------------------------*/ import type { RequestOptions } from 'https'; -import { https } from 'follow-redirects'; +import { https, http } from 'follow-redirects'; import ProxyAgent from 'proxy-agent'; import * as url from 'url'; import { Log, LogLevel } from './log'; -export function request(options: { type: string; url: string; headers: Record; data?: Buffer }, output?: Log) { +export function request(options: { type: string; url: string; headers: Record; data?: Buffer }, output?: Log, plainHTTP = true) { return new Promise((resolve, reject) => { const parsed = new url.URL(options.url); const reqOptions: RequestOptions = { @@ -20,7 +20,7 @@ export function request(options: { type: string; url: string; headers: Record { + const req = (plainHTTP ? http : https).request(reqOptions, res => { if (res.statusCode! < 200 || res.statusCode! > 299) { reject(new Error(`HTTP ${res.statusCode}: ${res.statusMessage}`)); if (output) { @@ -42,7 +42,7 @@ export function request(options: { type: string; url: string; headers: Record }, output?: Log) { +export function headRequest(options: { url: string; headers: Record }, output?: Log, plainHTTP = true) { return new Promise((resolve, reject) => { const parsed = new url.URL(options.url); const reqOptions: RequestOptions = { @@ -53,7 +53,8 @@ export function headRequest(options: { url: string; headers: Record { + + const req = (plainHTTP ? http : https).request(reqOptions, res => { res.on('error', reject); if (output) { output.write(`HEAD ${options.url} -> ${res.statusCode}`, LogLevel.Trace); @@ -67,7 +68,7 @@ export function headRequest(options: { url: string; headers: Record; data?: Buffer }, _output?: Log) { +export function requestResolveHeaders(options: { type: string; url: string; headers: Record; data?: Buffer }, _output?: Log, plainHTTP = true) { return new Promise<{ statusCode: number; resHeaders: Record; resBody: Buffer }>((resolve, reject) => { const parsed = new url.URL(options.url); const reqOptions: RequestOptions = { @@ -78,7 +79,7 @@ export function requestResolveHeaders(options: { type: string; url: string; head headers: options.headers, agent: new ProxyAgent(), }; - const req = https.request(reqOptions, res => { + const req = (plainHTTP ? http : https).request(reqOptions, res => { res.on('error', reject); // Resolve response body diff --git a/src/test/container-features/containerFeaturesOCI.test.ts b/src/test/container-features/containerFeaturesOCI.test.ts index b3db8ecbf..26c3a7806 100644 --- a/src/test/container-features/containerFeaturesOCI.test.ts +++ b/src/test/container-features/containerFeaturesOCI.test.ts @@ -5,7 +5,7 @@ import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace)); describe('getCollectionRef()', async function () { - this.timeout('120s'); + this.timeout('240s'); it('valid getCollectionRef()', async () => { diff --git a/src/test/container-features/containerFeaturesOCIPush.test.ts b/src/test/container-features/containerFeaturesOCIPush.test.ts index 2ccfdec7e..3d6550a09 100644 --- a/src/test/container-features/containerFeaturesOCIPush.test.ts +++ b/src/test/container-features/containerFeaturesOCIPush.test.ts @@ -1,16 +1,73 @@ import { assert } from 'chai'; -import { fetchRegistryAuthToken, DEVCONTAINER_TAR_LAYER_MEDIATYPE, getRef } from '../../spec-configuration/containerCollectionsOCI'; +import { fetchAuthorization, DEVCONTAINER_TAR_LAYER_MEDIATYPE, getRef } from '../../spec-configuration/containerCollectionsOCI'; import { fetchOCIFeatureManifestIfExistsFromUserIdentifier } from '../../spec-configuration/containerFeaturesOCI'; import { calculateDataLayer, checkIfBlobExists, calculateManifestAndContentDigest } from '../../spec-configuration/containerCollectionsOCIPush'; import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; +import { ExecResult, shellExec } from '../testUtils'; +import * as path from 'path'; +import { writeLocalFile } from '../../spec-utils/pfs'; +const pkg = require('../../../package.json'); export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace)); const testAssetsDir = `${__dirname}/assets`; +describe('Test OCI Push against reference registry', async function () { + this.timeout('240s'); + + const tmp = path.relative(process.cwd(), path.join(__dirname, 'tmp', Date.now().toString())); + const cli = `npx --prefix ${tmp} devcontainer`; + + before('Install CLI and Start reference implementation registry', async () => { + // Clean up any potential previous runs + await shellExec(`docker rm registry -f`, {}, false, true); + + // Install CLI + await shellExec(`rm -rf ${tmp}/node_modules`); + await shellExec(`mkdir -p ${tmp}`); + await shellExec(`npm --prefix ${tmp} install devcontainers-cli-${pkg.version}.tgz`); + + // Write htpasswd file to simulate basic auth. + // Generated from 'htpasswd -cB -b auth.htpasswd myuser mypass' + writeLocalFile(path.join(tmp, 'auth.htpasswd'), 'myuser:$2y$05$xmGlPoyYqECe3AY8GhO2ve1XvpxbSqe3yvPT2agOClbIeDIRAVPLC'); + + const resolvedTmpPath = path.resolve(tmp); + const startRegistryCmd = `docker run -d -p 5000:5000 \ +-v ${resolvedTmpPath}/certs:/certs \ +-v ${resolvedTmpPath}/auth.htpasswd:/etc/docker/registry/auth.htpasswd \ +-e REGISTRY_AUTH="{htpasswd: {realm: localhost, path: /etc/docker/registry/auth.htpasswd}}" \ +--name registry \ +registry`; + + await shellExec(startRegistryCmd, { cwd: tmp }); + + // Wait for registry to start + await shellExec(`docker exec registry sh -c "while ! nc -z localhost 5000; do sleep 3; done"`, { cwd: tmp }); + + // Login with basic auth creds + await shellExec('docker login -u myuser -p mypass localhost:5000'); + }); + + it('Publish Features to registry', async () => { + const collectionFolder = `${__dirname}/example-v2-features-sets/simple`; + let success = false; + let result: ExecResult | undefined = undefined; + try { + result = await shellExec(`${cli} features publish --log-level trace -r localhost:5000 -n octocat/features ${collectionFolder}/src`, { env: { ...process.env, 'DEVCONTAINERS_OCI_AUTH': 'localhost:5000|myuser|mypass' } }); + success = true; + + } catch (error) { + assert.fail('features publish sub-command should not throw'); + } + + assert.isTrue(success); + assert.isDefined(result); + }); +}); + // NOTE: // Test depends on https://github.com/codspace/features/pkgs/container/features%2Fgo/29819216?tag=1 -describe('Test OCI Push', () => { +describe('Test OCI Push Helper Functions', () => { it('Generates the correct tgz manifest layer', async () => { // Calculate the tgz layer and digest @@ -50,7 +107,7 @@ describe('Test OCI Push', () => { assert.fail('getRef() for the Feature should not be undefined'); } const { registry, resource } = ociFeatureRef; - const sessionAuth = await fetchRegistryAuthToken(output, registry, resource, process.env, 'pull'); + const sessionAuth = await fetchAuthorization(output, registry, resource, process.env, 'pull'); if (!sessionAuth) { assert.fail('Could not get registry auth token'); } From ebfc4c3829e3d694e82209013d50b928a6790864 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 2 Dec 2022 18:30:40 +0000 Subject: [PATCH 02/18] uncomment --- .../containerCollectionsOCI.ts | 117 +++++++++--------- .../containerCollectionsOCIPush.ts | 18 +-- 2 files changed, 68 insertions(+), 67 deletions(-) diff --git a/src/spec-configuration/containerCollectionsOCI.ts b/src/spec-configuration/containerCollectionsOCI.ts index 430274a29..7e4e5be72 100644 --- a/src/spec-configuration/containerCollectionsOCI.ts +++ b/src/spec-configuration/containerCollectionsOCI.ts @@ -250,71 +250,70 @@ async function getBasicAuthCredential(output: Log, registry: string, env: NodeJS // https://github.com/oras-project/oras-go/blob/97a9c43c52f9d89ecf5475bc59bd1f96c8cc61f6/registry/remote/auth/scope.go#L60-L74 // Some registries (eg: ghcr.io) expect a scoped token to target resources. +async function generateScopeTokenCredential(output: Log, registry: string, ociRepoPath: string, env: NodeJS.ProcessEnv, operationScopes: string, basicAuthTokenBase64: string | undefined = undefined): Promise { + if (registry === 'mcr.microsoft.com') { + return undefined; + } -// async function generateScopeTokenCredential(output: Log, registry: string, ociRepoPath: string, env: NodeJS.ProcessEnv, operationScopes: string, basicAuthTokenBase64: string | undefined = undefined): Promise { -// if (registry === 'mcr.microsoft.com') { -// return undefined; -// } - -// const headers: HEADERS = { -// 'user-agent': 'devcontainer' -// }; - -// if (!basicAuthTokenBase64) { -// basicAuthTokenBase64 = await getBasicAuthCredential(output, registry, env); -// } - -// headers['authorization'] = `Basic ${basicAuthTokenBase64}`; - -// const authServer = registry === 'docker.io' ? 'auth.docker.io' : registry; -// const registryServer = registry === 'docker.io' ? 'registry.docker.io' : registry; -// const url = `https://${authServer}/token?scope=repository:${ociRepoPath}:${operationScopes}&service=${registryServer}`; -// output.write(`url: ${url}`, LogLevel.Trace); - -// const options = { -// type: 'GET', -// url: url, -// headers: headers -// }; - -// let authReq: Buffer; -// try { -// authReq = await request(options, output); -// } catch (e: any) { -// output.write(`Unable to request scope token from registry ${registry}: ${e}`, LogLevel.Warning); -// return; -// } - -// if (!authReq) { -// output.write('Failed to get registry auth token', LogLevel.Error); -// return undefined; -// } - -// let scopeToken: string | undefined; -// try { -// scopeToken = JSON.parse(authReq.toString())?.token; -// } catch { -// // not JSON -// } -// if (!scopeToken) { -// output.write('Failed to parse registry auth token response', LogLevel.Error); -// return undefined; -// } - -// return scopeToken; -// } + const headers: HEADERS = { + 'user-agent': 'devcontainer' + }; + + if (!basicAuthTokenBase64) { + basicAuthTokenBase64 = await getBasicAuthCredential(output, registry, env); + } + + headers['authorization'] = `Basic ${basicAuthTokenBase64}`; + + const authServer = registry === 'docker.io' ? 'auth.docker.io' : registry; + const registryServer = registry === 'docker.io' ? 'registry.docker.io' : registry; + const url = `https://${authServer}/token?scope=repository:${ociRepoPath}:${operationScopes}&service=${registryServer}`; + output.write(`url: ${url}`, LogLevel.Trace); + + const options = { + type: 'GET', + url: url, + headers: headers + }; + + let authReq: Buffer; + try { + authReq = await request(options, output); + } catch (e: any) { + output.write(`Unable to request scope token from registry ${registry}: ${e}`, LogLevel.Warning); + return; + } + + if (!authReq) { + output.write('Failed to get registry auth token', LogLevel.Error); + return undefined; + } + + let scopeToken: string | undefined; + try { + scopeToken = JSON.parse(authReq.toString())?.token; + } catch { + // not JSON + } + if (!scopeToken) { + output.write('Failed to parse registry auth token response', LogLevel.Error); + return undefined; + } + + return scopeToken; +} // Exported Function // Will attempt to generate/fetch the correct authorization header for subsequent requests (Bearer or Basic) -export async function fetchAuthorization(output: Log, registry: string, _ociRepoPath: string, env: NodeJS.ProcessEnv, _operationScopes: string): Promise { +export async function fetchAuthorization(output: Log, registry: string, ociRepoPath: string, env: NodeJS.ProcessEnv, operationScopes: string): Promise { const basicAuthTokenBase64 = await getBasicAuthCredential(output, registry, env); - // const scopeToken = generateScopeTokenCredential(output, registry, ociRepoPath, env, operationScopes, basicAuthTokenBase64); + const scopeToken = generateScopeTokenCredential(output, registry, ociRepoPath, env, operationScopes, basicAuthTokenBase64); - // if (scopeToken) { - // output.write(`Using scope token for registry '${registry}'`, LogLevel.Trace); - // return `Bearer ${scopeToken}`; - // } + if (scopeToken) { + output.write(`Using scope token for registry '${registry}'`, LogLevel.Trace); + return `Bearer ${scopeToken}`; + } if (basicAuthTokenBase64) { output.write(`Using basic auth token for registry '${registry}'`, LogLevel.Trace); @@ -455,4 +454,4 @@ export async function getBlob(output: Log, env: NodeJS.ProcessEnv, url: string, output.write(`error: ${e}`, LogLevel.Error); return undefined; } -} +} \ No newline at end of file diff --git a/src/spec-configuration/containerCollectionsOCIPush.ts b/src/spec-configuration/containerCollectionsOCIPush.ts index 2385c3aee..f180388df 100644 --- a/src/spec-configuration/containerCollectionsOCIPush.ts +++ b/src/spec-configuration/containerCollectionsOCIPush.ts @@ -162,7 +162,7 @@ async function putManifestWithTags(output: Log, manifestStr: string, ociRef: OCI data: Buffer.from(manifestStr), }; - let { statusCode, resHeaders } = await requestResolveHeaders(options, output); + let { statusCode, resHeaders, resBody } = await requestResolveHeaders(options, output); // Retry logic: when request fails with HTTP 429: too many requests if (statusCode === 429) { @@ -175,7 +175,7 @@ async function putManifestWithTags(output: Log, manifestStr: string, ociRef: OCI } if (statusCode !== 201) { - output.write(`Failed to PUT manifest for tag ${tag}`, LogLevel.Error); + output.write(`Failed to PUT manifest for tag ${tag}\n${JSON.stringify(resBody, undefined, 4)}`, LogLevel.Error); return false; } @@ -196,8 +196,7 @@ async function putBlob(output: Log, pathToBlob: string, blobPutLocationUriPath: } const blobSize = (await promisify(fs.stat)(pathToBlob)).size; - - output.write(`PUT new blob -> '${digest}' (size=${blobSize})`, LogLevel.Info); + output.write(`Starting PUT of blob '${digest}' (size=${blobSize})`, LogLevel.Info); const headers: HEADERS = { 'user-agent': 'devcontainer', @@ -214,17 +213,20 @@ async function putBlob(output: Log, pathToBlob: string, blobPutLocationUriPath: url = `https://${ociRef.registry}${blobPutLocationUriPath}`; } + // The MAY contain critical query parameters. + // Additionally, it SHOULD match exactly the obtained from the POST request. + // It SHOULD NOT be assembled manually by clients except where absolute/relative conversion is necessary. if (url.indexOf('?') === -1) { url += `?digest=${digest}`; } else { url += `&digest=${digest}`; } - output.write(`Crafted blob url: ${url}`, LogLevel.Trace); + output.write(`PUT blob to -> ${url}`, LogLevel.Trace); const { statusCode, resBody } = await requestResolveHeaders({ type: 'PUT', url, headers, data: await readLocalFile(pathToBlob) }, output); if (statusCode !== 201) { - output.write(`${statusCode}: Failed to upload blob '${pathToBlob}' to '${url}' -> ${resBody}`, LogLevel.Error); + output.write(`${statusCode}: Failed to upload blob '${pathToBlob}' to '${url}' \n${JSON.stringify(resBody, undefined, 4)}`, LogLevel.Error); return false; } @@ -329,12 +331,12 @@ async function postUploadSessionId(output: Log, ociRef: OCIRef | OCICollectionRe output.write(`${url}: Got 202 status code, but no location header found.`, LogLevel.Error); return undefined; } + output.write(`Generated Upload URL: ${locationHeader}`, LogLevel.Trace); return locationHeader; } else { // Any other statusCode besides 202 is unexpected // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#error-codes - const displayResBody = resBody ? ` -> ${resBody}` : ''; - output.write(`${url}: Unexpected status code '${statusCode}'${displayResBody}`, LogLevel.Error); + output.write(`${url}: Unexpected status code '${statusCode}' \n${JSON.stringify(resBody, undefined, 4)}`, LogLevel.Error); return undefined; } } From 68ae246902e54c486ae1e960592d92bc132640d4 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 2 Dec 2022 18:48:52 +0000 Subject: [PATCH 03/18] revert plainHTTP defaulting to true --- src/spec-configuration/containerCollectionsOCI.ts | 7 +++---- src/spec-utils/httpRequest.ts | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/spec-configuration/containerCollectionsOCI.ts b/src/spec-configuration/containerCollectionsOCI.ts index 7e4e5be72..e7d642cc6 100644 --- a/src/spec-configuration/containerCollectionsOCI.ts +++ b/src/spec-configuration/containerCollectionsOCI.ts @@ -226,7 +226,7 @@ async function getBasicAuthCredential(output: Log, registry: string, env: NodeJS let userToken: string | undefined = undefined; if (!!env['GITHUB_TOKEN'] && registry === 'ghcr.io') { - userToken = env['GITHUB_TOKEN']; + userToken = `USERNAME:${env['GITHUB_TOKEN']}`; } else if (!!env['DEVCONTAINERS_OCI_AUTH']) { // eg: DEVCONTAINERS_OCI_AUTH=domain1|user1|token1,domain2|user2|token2 const authContexts = env['DEVCONTAINERS_OCI_AUTH'].split(','); @@ -268,7 +268,7 @@ async function generateScopeTokenCredential(output: Log, registry: string, ociRe const authServer = registry === 'docker.io' ? 'auth.docker.io' : registry; const registryServer = registry === 'docker.io' ? 'registry.docker.io' : registry; const url = `https://${authServer}/token?scope=repository:${ociRepoPath}:${operationScopes}&service=${registryServer}`; - output.write(`url: ${url}`, LogLevel.Trace); + output.write(`Fetching scope token from: ${url}`, LogLevel.Trace); const options = { type: 'GET', @@ -299,7 +299,6 @@ async function generateScopeTokenCredential(output: Log, registry: string, ociRe output.write('Failed to parse registry auth token response', LogLevel.Error); return undefined; } - return scopeToken; } @@ -308,7 +307,7 @@ async function generateScopeTokenCredential(output: Log, registry: string, ociRe // Will attempt to generate/fetch the correct authorization header for subsequent requests (Bearer or Basic) export async function fetchAuthorization(output: Log, registry: string, ociRepoPath: string, env: NodeJS.ProcessEnv, operationScopes: string): Promise { const basicAuthTokenBase64 = await getBasicAuthCredential(output, registry, env); - const scopeToken = generateScopeTokenCredential(output, registry, ociRepoPath, env, operationScopes, basicAuthTokenBase64); + const scopeToken = await generateScopeTokenCredential(output, registry, ociRepoPath, env, operationScopes, basicAuthTokenBase64); if (scopeToken) { output.write(`Using scope token for registry '${registry}'`, LogLevel.Trace); diff --git a/src/spec-utils/httpRequest.ts b/src/spec-utils/httpRequest.ts index b50c3f940..53c9c8a21 100644 --- a/src/spec-utils/httpRequest.ts +++ b/src/spec-utils/httpRequest.ts @@ -9,7 +9,7 @@ import ProxyAgent from 'proxy-agent'; import * as url from 'url'; import { Log, LogLevel } from './log'; -export function request(options: { type: string; url: string; headers: Record; data?: Buffer }, output?: Log, plainHTTP = true) { +export function request(options: { type: string; url: string; headers: Record; data?: Buffer }, output?: Log, plainHTTP = false) { return new Promise((resolve, reject) => { const parsed = new url.URL(options.url); const reqOptions: RequestOptions = { @@ -42,7 +42,7 @@ export function request(options: { type: string; url: string; headers: Record }, output?: Log, plainHTTP = true) { +export function headRequest(options: { url: string; headers: Record }, output?: Log, plainHTTP = false) { return new Promise((resolve, reject) => { const parsed = new url.URL(options.url); const reqOptions: RequestOptions = { @@ -68,7 +68,7 @@ export function headRequest(options: { url: string; headers: Record; data?: Buffer }, _output?: Log, plainHTTP = true) { +export function requestResolveHeaders(options: { type: string; url: string; headers: Record; data?: Buffer }, _output?: Log, plainHTTP = false) { return new Promise<{ statusCode: number; resHeaders: Record; resBody: Buffer }>((resolve, reject) => { const parsed = new url.URL(options.url); const reqOptions: RequestOptions = { From 19458168542cf6e07c6698eb1172cf5a6ca3836a Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 2 Dec 2022 19:03:32 +0000 Subject: [PATCH 04/18] mode for plain http when targetting localhost --- src/spec-utils/httpRequest.ts | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/spec-utils/httpRequest.ts b/src/spec-utils/httpRequest.ts index 53c9c8a21..373f6493b 100644 --- a/src/spec-utils/httpRequest.ts +++ b/src/spec-utils/httpRequest.ts @@ -9,7 +9,7 @@ import ProxyAgent from 'proxy-agent'; import * as url from 'url'; import { Log, LogLevel } from './log'; -export function request(options: { type: string; url: string; headers: Record; data?: Buffer }, output?: Log, plainHTTP = false) { +export function request(options: { type: string; url: string; headers: Record; data?: Buffer }, output?: Log) { return new Promise((resolve, reject) => { const parsed = new url.URL(options.url); const reqOptions: RequestOptions = { @@ -20,6 +20,12 @@ export function request(options: { type: string; url: string; headers: Record { if (res.statusCode! < 200 || res.statusCode! > 299) { reject(new Error(`HTTP ${res.statusCode}: ${res.statusMessage}`)); @@ -42,7 +48,7 @@ export function request(options: { type: string; url: string; headers: Record }, output?: Log, plainHTTP = false) { +export function headRequest(options: { url: string; headers: Record }, output?: Log) { return new Promise((resolve, reject) => { const parsed = new url.URL(options.url); const reqOptions: RequestOptions = { @@ -54,6 +60,11 @@ export function headRequest(options: { url: string; headers: Record { res.on('error', reject); if (output) { @@ -68,7 +79,7 @@ export function headRequest(options: { url: string; headers: Record; data?: Buffer }, _output?: Log, plainHTTP = false) { +export function requestResolveHeaders(options: { type: string; url: string; headers: Record; data?: Buffer }, output?: Log) { return new Promise<{ statusCode: number; resHeaders: Record; resBody: Buffer }>((resolve, reject) => { const parsed = new url.URL(options.url); const reqOptions: RequestOptions = { @@ -79,6 +90,12 @@ export function requestResolveHeaders(options: { type: string; url: string; head headers: options.headers, agent: new ProxyAgent(), }; + + const plainHTTP = parsed.protocol === 'http:' || parsed.hostname === 'localhost'; + if (output) { + output.write('Sending as plain HTTP request', LogLevel.Warning); + } + const req = (plainHTTP ? http : https).request(reqOptions, res => { res.on('error', reject); From 2701867688b92ac126423956ac97abd1b9d833c0 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 2 Dec 2022 19:04:24 +0000 Subject: [PATCH 05/18] print only when plain HTTP --- src/spec-utils/httpRequest.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/spec-utils/httpRequest.ts b/src/spec-utils/httpRequest.ts index 373f6493b..f62d428ce 100644 --- a/src/spec-utils/httpRequest.ts +++ b/src/spec-utils/httpRequest.ts @@ -22,7 +22,7 @@ export function request(options: { type: string; url: string; headers: Record Date: Fri, 2 Dec 2022 19:18:24 +0000 Subject: [PATCH 06/18] surface JSON errors better --- src/spec-configuration/containerCollectionsOCIPush.ts | 9 ++++++--- src/spec-utils/httpRequest.ts | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/spec-configuration/containerCollectionsOCIPush.ts b/src/spec-configuration/containerCollectionsOCIPush.ts index f180388df..3fe405526 100644 --- a/src/spec-configuration/containerCollectionsOCIPush.ts +++ b/src/spec-configuration/containerCollectionsOCIPush.ts @@ -175,7 +175,8 @@ async function putManifestWithTags(output: Log, manifestStr: string, ociRef: OCI } if (statusCode !== 201) { - output.write(`Failed to PUT manifest for tag ${tag}\n${JSON.stringify(resBody, undefined, 4)}`, LogLevel.Error); + const parsed = JSON.parse(resBody?.toString() || '{}'); + output.write(`Failed to PUT manifest for tag ${tag}\n${JSON.stringify(parsed, undefined, 4)}`, LogLevel.Error); return false; } @@ -226,7 +227,8 @@ async function putBlob(output: Log, pathToBlob: string, blobPutLocationUriPath: const { statusCode, resBody } = await requestResolveHeaders({ type: 'PUT', url, headers, data: await readLocalFile(pathToBlob) }, output); if (statusCode !== 201) { - output.write(`${statusCode}: Failed to upload blob '${pathToBlob}' to '${url}' \n${JSON.stringify(resBody, undefined, 4)}`, LogLevel.Error); + const parsed = JSON.parse(resBody?.toString() || '{}'); + output.write(`${statusCode}: Failed to upload blob '${pathToBlob}' to '${url}' \n${JSON.stringify(parsed, undefined, 4)}`, LogLevel.Error); return false; } @@ -336,7 +338,8 @@ async function postUploadSessionId(output: Log, ociRef: OCIRef | OCICollectionRe } else { // Any other statusCode besides 202 is unexpected // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#error-codes - output.write(`${url}: Unexpected status code '${statusCode}' \n${JSON.stringify(resBody, undefined, 4)}`, LogLevel.Error); + const parsed = JSON.parse(resBody?.toString() || '{}'); + output.write(`${url}: Unexpected status code '${statusCode}' \n${JSON.stringify(parsed, undefined, 4)}`, LogLevel.Error); return undefined; } } diff --git a/src/spec-utils/httpRequest.ts b/src/spec-utils/httpRequest.ts index f62d428ce..c2a8c4749 100644 --- a/src/spec-utils/httpRequest.ts +++ b/src/spec-utils/httpRequest.ts @@ -30,7 +30,7 @@ export function request(options: { type: string; url: string; headers: Record 299) { reject(new Error(`HTTP ${res.statusCode}: ${res.statusMessage}`)); if (output) { - output.write(`HTTP request failed with status code ${res.statusCode}: : ${res.statusMessage}`, LogLevel.Error); + output.write(`[-] HTTP request failed with status code ${res.statusCode}: : ${res.statusMessage}`, LogLevel.Error); } } else { res.on('error', reject); From ec68d10a154fad3d948f34fad2f74eaa792b9df8 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 2 Dec 2022 22:32:30 +0000 Subject: [PATCH 07/18] refactor out reading in tgz Buffer to only do that work once --- .../containerCollectionsOCIPush.ts | 85 +++++++++++-------- .../containerFeaturesOCIPush.test.ts | 7 +- 2 files changed, 53 insertions(+), 39 deletions(-) diff --git a/src/spec-configuration/containerCollectionsOCIPush.ts b/src/spec-configuration/containerCollectionsOCIPush.ts index 3fe405526..a6a5b0d32 100644 --- a/src/spec-configuration/containerCollectionsOCIPush.ts +++ b/src/spec-configuration/containerCollectionsOCIPush.ts @@ -4,19 +4,25 @@ import * as crypto from 'crypto'; import { delay } from '../spec-common/async'; import { headRequest, requestResolveHeaders } from '../spec-utils/httpRequest'; import { Log, LogLevel } from '../spec-utils/log'; -import { isLocalFile, readLocalFile } from '../spec-utils/pfs'; +import { isLocalFile } from '../spec-utils/pfs'; import { DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE, DEVCONTAINER_TAR_LAYER_MEDIATYPE, fetchOCIManifestIfExists, fetchAuthorization, HEADERS, OCICollectionRef, OCILayer, OCIManifest, OCIRef } from './containerCollectionsOCI'; -import { promisify } from 'util'; // (!) Entrypoint function to push a single feature/template to a registry. // Devcontainer Spec (features) : https://containers.dev/implementors/features-distribution/#oci-registry // Devcontainer Spec (templates): https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-templates-distribution.md#oci-registry // OCI Spec : https://github.com/opencontainers/distribution-spec/blob/main/spec.md#push export async function pushOCIFeatureOrTemplate(output: Log, ociRef: OCIRef, pathToTgz: string, tags: string[], collectionType: string): Promise { - output.write(`Starting push of ${collectionType} '${ociRef.id}' to '${ociRef.resource}' with tags '${tags.join(', ')}'`); + output.write(`-- Starting push of ${collectionType} '${ociRef.id}' to '${ociRef.resource}' with tags '${tags.join(', ')}'`); output.write(`${JSON.stringify(ociRef, null, 2)}`, LogLevel.Trace); const env = process.env; + if (!(await isLocalFile(pathToTgz))) { + output.write(`Blob ${pathToTgz} does not exist.`, LogLevel.Error); + return false; + } + + const dataBytes = fs.readFileSync(pathToTgz); + // Generate registry auth token with `pull,push` scopes. const authorization = await fetchAuthorization(output, ociRef.registry, ociRef.path, env, 'pull,push'); if (!authorization) { @@ -25,7 +31,7 @@ export async function pushOCIFeatureOrTemplate(output: Log, ociRef: OCIRef, path } // Generate Manifest for given feature/template artifact. - const manifest = await generateCompleteManifestForIndividualFeatureOrTemplate(output, pathToTgz, ociRef, collectionType); + const manifest = await generateCompleteManifestForIndividualFeatureOrTemplate(output, dataBytes, pathToTgz, ociRef, collectionType); if (!manifest) { output.write(`Failed to generate manifest for ${ociRef.id}`, LogLevel.Error); return false; @@ -43,10 +49,14 @@ export async function pushOCIFeatureOrTemplate(output: Log, ociRef: OCIRef, path { name: 'configLayer', digest: manifest.manifestObj.config.digest, + size: manifest.manifestObj.config.size, + contents: Buffer.alloc(0), }, { name: 'tgzLayer', digest: manifest.manifestObj.layers[0].digest, + size: manifest.manifestObj.layers[0].size, + contents: dataBytes, } ]; @@ -60,11 +70,11 @@ export async function pushOCIFeatureOrTemplate(output: Log, ociRef: OCIRef, path for await (const blob of blobsToPush) { const { name, digest } = blob; const blobExistsConfigLayer = await checkIfBlobExists(output, ociRef, digest, authorization); - output.write(`blob: '${name}' with digest '${digest}' ${blobExistsConfigLayer ? 'already exists' : 'does not exist'} in registry.`, LogLevel.Trace); + output.write(`blob: '${name}' ${blobExistsConfigLayer ? 'DOES exists' : 'DOES NOT exist'} in registry.`, LogLevel.Trace); // PUT blobs if (!blobExistsConfigLayer) { - if (!(await putBlob(output, pathToTgz, blobPutLocationUriPath, ociRef, digest, authorization))) { + if (!(await putBlob(output, blobPutLocationUriPath, ociRef, blob, authorization))) { output.write(`Failed to PUT blob '${name}' with digest '${digest}'`, LogLevel.Error); return false; } @@ -90,8 +100,15 @@ export async function pushCollectionMetadata(output: Log, collectionRef: OCIColl return false; } + if (!(await isLocalFile(pathToCollectionJson))) { + output.write(`Collection Metadata was not found at expected location: ${pathToCollectionJson}`, LogLevel.Error); + return false; + } + + const dataBytes = fs.readFileSync(pathToCollectionJson); + // Generate Manifest for collection artifact. - const manifest = await generateCompleteManifestForCollectionFile(output, pathToCollectionJson, collectionRef); + const manifest = await generateCompleteManifestForCollectionFile(output, dataBytes, collectionRef); if (!manifest) { output.write(`Failed to generate manifest for ${collectionRef.path}`, LogLevel.Error); return false; @@ -116,10 +133,14 @@ export async function pushCollectionMetadata(output: Log, collectionRef: OCIColl { name: 'configLayer', digest: manifest.manifestObj.config.digest, + size: manifest.manifestObj.config.size, + contents: Buffer.alloc(0), }, { name: 'collectionLayer', digest: manifest.manifestObj.layers[0].digest, + size: manifest.manifestObj.layers[0].size, + contents: dataBytes, } ]; @@ -130,7 +151,7 @@ export async function pushCollectionMetadata(output: Log, collectionRef: OCIColl // PUT blobs if (!blobExistsConfigLayer) { - if (!(await putBlob(output, pathToCollectionJson, blobPutLocationUriPath, collectionRef, digest, authorization))) { + if (!(await putBlob(output, blobPutLocationUriPath, collectionRef, blob, authorization))) { output.write(`Failed to PUT blob '${name}' with digest '${digest}'`, LogLevel.Error); return false; } @@ -189,21 +210,17 @@ async function putManifestWithTags(output: Log, manifestStr: string, ociRef: OCI } // Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#post-then-put (PUT ?digest=) -async function putBlob(output: Log, pathToBlob: string, blobPutLocationUriPath: string, ociRef: OCIRef | OCICollectionRef, digest: string, authorization: string): Promise { +async function putBlob(output: Log, blobPutLocationUriPath: string, ociRef: OCIRef | OCICollectionRef, blob: { name: string; digest: string; size: number; contents: Buffer }, authorization: string): Promise { - if (!(await isLocalFile(pathToBlob))) { - output.write(`Blob ${pathToBlob} does not exist`, LogLevel.Error); - return false; - } + const { name, digest, size, contents } = blob; - const blobSize = (await promisify(fs.stat)(pathToBlob)).size; - output.write(`Starting PUT of blob '${digest}' (size=${blobSize})`, LogLevel.Info); + output.write(`Starting PUT of ${name} blob '${digest}' (size=${size})`, LogLevel.Info); const headers: HEADERS = { 'user-agent': 'devcontainer', 'authorization': authorization, 'content-type': 'application/octet-stream', - 'content-length': `${blobSize}` + 'content-length': `${size}` }; // OCI distribution spec is ambiguous on whether we get back an absolute or relative path. @@ -217,18 +234,20 @@ async function putBlob(output: Log, pathToBlob: string, blobPutLocationUriPath: // The MAY contain critical query parameters. // Additionally, it SHOULD match exactly the obtained from the POST request. // It SHOULD NOT be assembled manually by clients except where absolute/relative conversion is necessary. - if (url.indexOf('?') === -1) { + const queryParamsStart = url.indexOf('?'); + if (queryParamsStart === -1) { + // Just append digest to the end. url += `?digest=${digest}`; } else { - url += `&digest=${digest}`; + url = url.substring(0, queryParamsStart) + `?digest=${digest}` + '&' + url.substring(queryParamsStart + 1); } output.write(`PUT blob to -> ${url}`, LogLevel.Trace); - const { statusCode, resBody } = await requestResolveHeaders({ type: 'PUT', url, headers, data: await readLocalFile(pathToBlob) }, output); + const { statusCode, resBody } = await requestResolveHeaders({ type: 'PUT', url, headers, data: contents }, output); if (statusCode !== 201) { const parsed = JSON.parse(resBody?.toString() || '{}'); - output.write(`${statusCode}: Failed to upload blob '${pathToBlob}' to '${url}' \n${JSON.stringify(parsed, undefined, 4)}`, LogLevel.Error); + output.write(`${statusCode}: Failed to upload blob '${digest}' to '${url}' \n${JSON.stringify(parsed, undefined, 4)}`, LogLevel.Error); return false; } @@ -238,8 +257,8 @@ async function putBlob(output: Log, pathToBlob: string, blobPutLocationUriPath: // Generate a layer that follows the `application/vnd.devcontainers.layer.v1+tar` mediaType as defined in // Devcontainer Spec (features) : https://containers.dev/implementors/features-distribution/#oci-registry // Devcontainer Spec (templates): https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-templates-distribution.md#oci-registry -async function generateCompleteManifestForIndividualFeatureOrTemplate(output: Log, pathToTgz: string, ociRef: OCIRef, collectionType: string): Promise<{ manifestObj: OCIManifest; manifestStr: string; digest: string } | undefined> { - const tgzLayer = await calculateDataLayer(output, pathToTgz, DEVCONTAINER_TAR_LAYER_MEDIATYPE); +async function generateCompleteManifestForIndividualFeatureOrTemplate(output: Log, dataBytes: Buffer, pathToTgz: string, ociRef: OCIRef, collectionType: string): Promise<{ manifestObj: OCIManifest; manifestStr: string; digest: string } | undefined> { + const tgzLayer = await calculateDataLayer(output, dataBytes, path.basename(pathToTgz), DEVCONTAINER_TAR_LAYER_MEDIATYPE); if (!tgzLayer) { output.write(`Failed to calculate tgz layer.`, LogLevel.Error); return undefined; @@ -259,8 +278,8 @@ async function generateCompleteManifestForIndividualFeatureOrTemplate(output: Lo // Generate a layer that follows the `application/vnd.devcontainers.collection.layer.v1+json` mediaType as defined in // Devcontainer Spec (features) : https://containers.dev/implementors/features-distribution/#oci-registry // Devcontainer Spec (templates): https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-templates-distribution.md#oci-registry -async function generateCompleteManifestForCollectionFile(output: Log, pathToCollectionFile: string, collectionRef: OCICollectionRef): Promise<{ manifestObj: OCIManifest; manifestStr: string; digest: string } | undefined> { - const collectionMetadataLayer = await calculateDataLayer(output, pathToCollectionFile, DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE); +async function generateCompleteManifestForCollectionFile(output: Log, dataBytes: Buffer, collectionRef: OCICollectionRef): Promise<{ manifestObj: OCIManifest; manifestStr: string; digest: string } | undefined> { + const collectionMetadataLayer = await calculateDataLayer(output, dataBytes, 'devcontainer-collection.json', DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE); if (!collectionMetadataLayer) { output.write(`Failed to calculate collection file layer.`, LogLevel.Error); return undefined; @@ -278,24 +297,18 @@ async function generateCompleteManifestForCollectionFile(output: Log, pathToColl } // Generic construction of a layer in the manifest and digest for the generated layer. -export async function calculateDataLayer(output: Log, pathToData: string, mediaType: string): Promise { - output.write(`Creating manifest from ${pathToData}`, LogLevel.Trace); - if (!(await isLocalFile(pathToData))) { - output.write(`${pathToData} does not exist.`, LogLevel.Error); - return undefined; - } - - const dataBytes = fs.readFileSync(pathToData); +export async function calculateDataLayer(output: Log, data: Buffer, basename: string, mediaType: string): Promise { + output.write(`Creating manifest from data`, LogLevel.Trace); - const tarSha256 = crypto.createHash('sha256').update(dataBytes).digest('hex'); - output.write(`${pathToData}: sha256:${tarSha256} (size: ${dataBytes.byteLength})`, LogLevel.Info); + const tarSha256 = crypto.createHash('sha256').update(data).digest('hex'); + output.write(`sha256:${tarSha256} (size: ${data.byteLength})`, LogLevel.Info); return { mediaType, digest: `sha256:${tarSha256}`, - size: dataBytes.byteLength, + size: data.byteLength, annotations: { - 'org.opencontainers.image.title': path.basename(pathToData), + 'org.opencontainers.image.title': basename, } }; } diff --git a/src/test/container-features/containerFeaturesOCIPush.test.ts b/src/test/container-features/containerFeaturesOCIPush.test.ts index 3d6550a09..dc4786599 100644 --- a/src/test/container-features/containerFeaturesOCIPush.test.ts +++ b/src/test/container-features/containerFeaturesOCIPush.test.ts @@ -5,6 +5,7 @@ import { calculateDataLayer, checkIfBlobExists, calculateManifestAndContentDiges import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; import { ExecResult, shellExec } from '../testUtils'; import * as path from 'path'; +import * as fs from 'fs'; import { writeLocalFile } from '../../spec-utils/pfs'; const pkg = require('../../../package.json'); @@ -33,7 +34,6 @@ describe('Test OCI Push against reference registry', async function () { const resolvedTmpPath = path.resolve(tmp); const startRegistryCmd = `docker run -d -p 5000:5000 \ --v ${resolvedTmpPath}/certs:/certs \ -v ${resolvedTmpPath}/auth.htpasswd:/etc/docker/registry/auth.htpasswd \ -e REGISTRY_AUTH="{htpasswd: {realm: localhost, path: /etc/docker/registry/auth.htpasswd}}" \ --name registry \ @@ -43,7 +43,6 @@ registry`; // Wait for registry to start await shellExec(`docker exec registry sh -c "while ! nc -z localhost 5000; do sleep 3; done"`, { cwd: tmp }); - // Login with basic auth creds await shellExec('docker login -u myuser -p mypass localhost:5000'); }); @@ -70,8 +69,10 @@ registry`; describe('Test OCI Push Helper Functions', () => { it('Generates the correct tgz manifest layer', async () => { + const dataBytes = fs.readFileSync(`${testAssetsDir}/go.tgz`); + // Calculate the tgz layer and digest - const res = await calculateDataLayer(output, `${testAssetsDir}/go.tgz`, DEVCONTAINER_TAR_LAYER_MEDIATYPE); + const res = await calculateDataLayer(output, dataBytes, 'go.tgz', DEVCONTAINER_TAR_LAYER_MEDIATYPE); const expected = { digest: 'sha256:b2006e7647191f7b47222ae48df049c6e21a4c5a04acfad0c4ef614d819de4c5', mediaType: 'application/vnd.devcontainers.layer.v1+tar', From 1d5ed62e9785e6ea5bb6a037202803fe5ba2b4e1 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 2 Dec 2022 22:39:44 +0000 Subject: [PATCH 08/18] generate a postUploadSessionId for each blob that needs to be uploaded --- .../containerCollectionsOCIPush.ts | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/spec-configuration/containerCollectionsOCIPush.ts b/src/spec-configuration/containerCollectionsOCIPush.ts index a6a5b0d32..a3a61bf17 100644 --- a/src/spec-configuration/containerCollectionsOCIPush.ts +++ b/src/spec-configuration/containerCollectionsOCIPush.ts @@ -60,12 +60,6 @@ export async function pushOCIFeatureOrTemplate(output: Log, ociRef: OCIRef, path } ]; - // Obtain session ID with `/v2//blobs/uploads/` - const blobPutLocationUriPath = await postUploadSessionId(output, ociRef, authorization); - if (!blobPutLocationUriPath) { - output.write(`Failed to get upload session ID`, LogLevel.Error); - return false; - } for await (const blob of blobsToPush) { const { name, digest } = blob; @@ -74,6 +68,14 @@ export async function pushOCIFeatureOrTemplate(output: Log, ociRef: OCIRef, path // PUT blobs if (!blobExistsConfigLayer) { + + // Obtain session ID with `/v2//blobs/uploads/` + const blobPutLocationUriPath = await postUploadSessionId(output, ociRef, authorization); + if (!blobPutLocationUriPath) { + output.write(`Failed to get upload session ID`, LogLevel.Error); + return false; + } + if (!(await putBlob(output, blobPutLocationUriPath, ociRef, blob, authorization))) { output.write(`Failed to PUT blob '${name}' with digest '${digest}'`, LogLevel.Error); return false; @@ -122,13 +124,6 @@ export async function pushCollectionMetadata(output: Log, collectionRef: OCIColl return await putManifestWithTags(output, manifest.manifestStr, collectionRef, ['latest'], authorization); } - // Obtain session ID with `/v2//blobs/uploads/` - const blobPutLocationUriPath = await postUploadSessionId(output, collectionRef, authorization); - if (!blobPutLocationUriPath) { - output.write(`Failed to get upload session ID`, LogLevel.Error); - return false; - } - const blobsToPush = [ { name: 'configLayer', @@ -151,6 +146,14 @@ export async function pushCollectionMetadata(output: Log, collectionRef: OCIColl // PUT blobs if (!blobExistsConfigLayer) { + + // Obtain session ID with `/v2//blobs/uploads/` + const blobPutLocationUriPath = await postUploadSessionId(output, collectionRef, authorization); + if (!blobPutLocationUriPath) { + output.write(`Failed to get upload session ID`, LogLevel.Error); + return false; + } + if (!(await putBlob(output, blobPutLocationUriPath, collectionRef, blob, authorization))) { output.write(`Failed to PUT blob '${name}' with digest '${digest}'`, LogLevel.Error); return false; From 20d77741209874e2fc92de95f7dfc57e55ffbccd Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 2 Dec 2022 23:11:29 +0000 Subject: [PATCH 09/18] update auth contract change in utils file --- src/spec-configuration/containerCollectionsOCI.ts | 9 ++++++--- src/spec-node/utils.ts | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/spec-configuration/containerCollectionsOCI.ts b/src/spec-configuration/containerCollectionsOCI.ts index e7d642cc6..883b04182 100644 --- a/src/spec-configuration/containerCollectionsOCI.ts +++ b/src/spec-configuration/containerCollectionsOCI.ts @@ -217,6 +217,7 @@ export async function getManifest(output: Log, env: NodeJS.ProcessEnv, url: stri return manifest; } catch (e) { + output.write(`(!) Failed to fetch manifest: ${e}`, LogLevel.Error); return undefined; } } @@ -243,8 +244,7 @@ async function getBasicAuthCredential(output: Log, registry: string, env: NodeJS return Buffer.from(userToken).toString('base64'); } - // Error - output.write(`No authentication credentials found for registry '${registry}'.`, LogLevel.Error); + output.write(`No authentication credentials found for registry '${registry}'.`, LogLevel.Warning); return undefined; } @@ -263,7 +263,10 @@ async function generateScopeTokenCredential(output: Log, registry: string, ociRe basicAuthTokenBase64 = await getBasicAuthCredential(output, registry, env); } - headers['authorization'] = `Basic ${basicAuthTokenBase64}`; + if (basicAuthTokenBase64) { + headers['authorization'] = `Basic ${basicAuthTokenBase64}`; + } + const authServer = registry === 'docker.io' ? 'auth.docker.io' : registry; const registryServer = registry === 'docker.io' ? 'registry.docker.io' : registry; diff --git a/src/spec-node/utils.ts b/src/spec-node/utils.ts index 8133c4df8..f7ca8cc71 100644 --- a/src/spec-node/utils.ts +++ b/src/spec-node/utils.ts @@ -236,7 +236,7 @@ export async function inspectImageInRegistry(output: Log, name: string, authToke }; if (auth) { - headers['authorization'] = `Bearer ${auth}`; + headers['authorization'] = auth; } const options = { From 28dd1634863f71f9d2541f56d446be4630411568 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 2 Dec 2022 23:11:49 +0000 Subject: [PATCH 10/18] factor out containerFeaturesOCIPush test to its own matrix run --- .github/workflows/dev-containers.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev-containers.yml b/.github/workflows/dev-containers.yml index 8803f9c2c..7ff110f56 100644 --- a/.github/workflows/dev-containers.yml +++ b/.github/workflows/dev-containers.yml @@ -54,8 +54,9 @@ jobs: "src/test/cli.exec.nonBuildKit.2.test.ts", "src/test/cli.test.ts", "src/test/cli.up.test.ts", + "src/test/container-features/containerFeaturesOCIPush.test.ts", # Run all except the above: - "--exclude src/test/container-features/e2e.test.ts --exclude src/test/container-features/featuresCLICommands.test.ts --exclude src/test/container-features/containerFeaturesOrder.test.ts --exclude src/test/cli.build.test.ts --exclude src/test/cli.exec.buildKit.1.test.ts --exclude src/test/cli.exec.buildKit.2.test.ts --exclude src/test/cli.exec.nonBuildKit.1.test.ts --exclude src/test/cli.exec.nonBuildKit.2.test.ts --exclude src/test/cli.test.ts --exclude src/test/cli.up.test.ts 'src/test/**/*.test.ts'", + "--exclude src/test/container-features/containerFeaturesOCIPush.test.ts --exclude src/test/container-features/e2e.test.ts --exclude src/test/container-features/featuresCLICommands.test.ts --exclude src/test/container-features/containerFeaturesOrder.test.ts --exclude src/test/cli.build.test.ts --exclude src/test/cli.exec.buildKit.1.test.ts --exclude src/test/cli.exec.buildKit.2.test.ts --exclude src/test/cli.exec.nonBuildKit.1.test.ts --exclude src/test/cli.exec.nonBuildKit.2.test.ts --exclude src/test/cli.test.ts --exclude src/test/cli.up.test.ts 'src/test/**/*.test.ts'", ] steps: - name: Checkout From de93dad1775f5fe2d620e0894657cb8c75f8087c Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 2 Dec 2022 23:13:10 +0000 Subject: [PATCH 11/18] clarifying comment --- src/spec-configuration/containerCollectionsOCI.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/spec-configuration/containerCollectionsOCI.ts b/src/spec-configuration/containerCollectionsOCI.ts index 883b04182..5cecc6fac 100644 --- a/src/spec-configuration/containerCollectionsOCI.ts +++ b/src/spec-configuration/containerCollectionsOCI.ts @@ -244,6 +244,7 @@ async function getBasicAuthCredential(output: Log, registry: string, env: NodeJS return Buffer.from(userToken).toString('base64'); } + // Represents anonymous access. output.write(`No authentication credentials found for registry '${registry}'.`, LogLevel.Warning); return undefined; } From b27f9fc163740f7508c50af05cdf9a831c43c58d Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 2 Dec 2022 23:24:42 +0000 Subject: [PATCH 12/18] add comments clarifying auth --- .../containerCollectionsOCI.ts | 58 +++++++++++-------- src/spec-utils/httpRequest.ts | 2 +- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/spec-configuration/containerCollectionsOCI.ts b/src/spec-configuration/containerCollectionsOCI.ts index 5cecc6fac..5a9e5d6db 100644 --- a/src/spec-configuration/containerCollectionsOCI.ts +++ b/src/spec-configuration/containerCollectionsOCI.ts @@ -222,6 +222,34 @@ export async function getManifest(output: Log, env: NodeJS.ProcessEnv, url: stri } } +// Exported Function +// Will attempt to generate/fetch the correct authorization header for subsequent requests (Bearer or Basic) +export async function fetchAuthorization(output: Log, registry: string, ociRepoPath: string, env: NodeJS.ProcessEnv, operationScopes: string): Promise { + const basicAuthTokenBase64 = await getBasicAuthCredential(output, registry, env); + const scopeToken = await generateScopeTokenCredential(output, registry, ociRepoPath, env, operationScopes, basicAuthTokenBase64); + + // Prefer returned a Bearer token retreived from the /token endpoint. + if (scopeToken) { + output.write(`Using scope token for registry '${registry}'`, LogLevel.Trace); + return `Bearer ${scopeToken}`; + } + + // If all we have are Basic auth credentials, return those for the caller to use. + if (basicAuthTokenBase64) { + output.write(`Using basic auth token for registry '${registry}'`, LogLevel.Trace); + return `Basic ${basicAuthTokenBase64}`; + } + + // If we have no credentials, and we weren't able to get a scope token anonymously, return undefined. + return undefined; +} + + +// * Internal helper for 'fetchAuthorization(...)' +// Attempts to get the Basic auth credentials for the provided registry. +// These may be programatically crafted via environment variables (GITHUB_TOKEN), +// parsed out of a special DEVCONTAINERS_OCI_AUTH environment variable, +// TODO: or directly read out of the local docker config file/credential helper. async function getBasicAuthCredential(output: Log, registry: string, env: NodeJS.ProcessEnv): Promise { // TODO: Also read OS keychain/docker config for auth in various registries! @@ -249,8 +277,11 @@ async function getBasicAuthCredential(output: Log, registry: string, env: NodeJS return undefined; } +// * Internal helper for 'fetchAuthorization(...)' // https://github.com/oras-project/oras-go/blob/97a9c43c52f9d89ecf5475bc59bd1f96c8cc61f6/registry/remote/auth/scope.go#L60-L74 -// Some registries (eg: ghcr.io) expect a scoped token to target resources. +// Using the provided Basic auth credentials, (or if none, anonymously), to ask the registry's '/token' endpoint for a token. +// Some registries (eg: ghcr.io) expect a scoped token to target resources and will not operate with just Basic Auth. +// Other registries (eg: the OCI Reference Implementation) will not return a valid token from '/token' async function generateScopeTokenCredential(output: Log, registry: string, ociRepoPath: string, env: NodeJS.ProcessEnv, operationScopes: string, basicAuthTokenBase64: string | undefined = undefined): Promise { if (registry === 'mcr.microsoft.com') { return undefined; @@ -268,7 +299,6 @@ async function generateScopeTokenCredential(output: Log, registry: string, ociRe headers['authorization'] = `Basic ${basicAuthTokenBase64}`; } - const authServer = registry === 'docker.io' ? 'auth.docker.io' : registry; const registryServer = registry === 'docker.io' ? 'registry.docker.io' : registry; const url = `https://${authServer}/token?scope=repository:${ociRepoPath}:${operationScopes}&service=${registryServer}`; @@ -284,7 +314,8 @@ async function generateScopeTokenCredential(output: Log, registry: string, ociRe try { authReq = await request(options, output); } catch (e: any) { - output.write(`Unable to request scope token from registry ${registry}: ${e}`, LogLevel.Warning); + // This is ok if the registry is trying to speak Basic Auth with us. + output.write(`Not used a scoped token for ${registry}: ${e}`, LogLevel.Trace); return; } @@ -306,27 +337,6 @@ async function generateScopeTokenCredential(output: Log, registry: string, ociRe return scopeToken; } - -// Exported Function -// Will attempt to generate/fetch the correct authorization header for subsequent requests (Bearer or Basic) -export async function fetchAuthorization(output: Log, registry: string, ociRepoPath: string, env: NodeJS.ProcessEnv, operationScopes: string): Promise { - const basicAuthTokenBase64 = await getBasicAuthCredential(output, registry, env); - const scopeToken = await generateScopeTokenCredential(output, registry, ociRepoPath, env, operationScopes, basicAuthTokenBase64); - - if (scopeToken) { - output.write(`Using scope token for registry '${registry}'`, LogLevel.Trace); - return `Bearer ${scopeToken}`; - } - - if (basicAuthTokenBase64) { - output.write(`Using basic auth token for registry '${registry}'`, LogLevel.Trace); - return `Basic ${basicAuthTokenBase64}`; - } - - return undefined; - -} - // Lists published versions/tags of a feature/template // Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#content-discovery export async function getPublishedVersions(ref: OCIRef, output: Log, sorted: boolean = false, collectionType: string = 'feature'): Promise { diff --git a/src/spec-utils/httpRequest.ts b/src/spec-utils/httpRequest.ts index c2a8c4749..9c5b30a51 100644 --- a/src/spec-utils/httpRequest.ts +++ b/src/spec-utils/httpRequest.ts @@ -30,7 +30,7 @@ export function request(options: { type: string; url: string; headers: Record 299) { reject(new Error(`HTTP ${res.statusCode}: ${res.statusMessage}`)); if (output) { - output.write(`[-] HTTP request failed with status code ${res.statusCode}: : ${res.statusMessage}`, LogLevel.Error); + output.write(`[-] HTTP request failed with status code ${res.statusCode}: : ${res.statusMessage}`, LogLevel.Trace); } } else { res.on('error', reject); From 56ceefc85dddd71030578499c8cf506902a5a496 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 2 Dec 2022 23:50:56 +0000 Subject: [PATCH 13/18] allow localhost past the domain name check --- src/spec-configuration/containerCollectionsOCI.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/spec-configuration/containerCollectionsOCI.ts b/src/spec-configuration/containerCollectionsOCI.ts index 5a9e5d6db..4a0accef4 100644 --- a/src/spec-configuration/containerCollectionsOCI.ts +++ b/src/spec-configuration/containerCollectionsOCI.ts @@ -168,7 +168,8 @@ export function getCollectionRef(output: Log, registry: string, namespace: strin export async function fetchOCIManifestIfExists(output: Log, env: NodeJS.ProcessEnv, ref: OCIRef | OCICollectionRef, manifestDigest?: string, authToken?: string): Promise { // Simple mechanism to avoid making a DNS request for // something that is not a domain name. - if (ref.registry.indexOf('.') < 0) { + if (ref.registry.indexOf('.') < 0 && !ref.registry.startsWith('localhost')) { + output.write(`ERR: Registry '${ref.registry}' is not a valid domain name or IP address.`, LogLevel.Error); return undefined; } @@ -255,6 +256,7 @@ async function getBasicAuthCredential(output: Log, registry: string, env: NodeJS let userToken: string | undefined = undefined; if (!!env['GITHUB_TOKEN'] && registry === 'ghcr.io') { + output.write('Using environment GITHUB_TOKEN for auth', LogLevel.Trace); userToken = `USERNAME:${env['GITHUB_TOKEN']}`; } else if (!!env['DEVCONTAINERS_OCI_AUTH']) { // eg: DEVCONTAINERS_OCI_AUTH=domain1|user1|token1,domain2|user2|token2 From 866ce3e9dcfbe8c88e761ef9b70f929b687e5ddd Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Fri, 2 Dec 2022 23:51:28 +0000 Subject: [PATCH 14/18] add 'info tags' and 'info manifest' commands' --- src/spec-node/devContainersSpecCLI.ts | 14 +++-- src/spec-node/featuresCLI/infoManifest.ts | 53 +++++++++++++++++++ .../featuresCLI/{info.ts => infoTags.ts} | 34 ++++++------ 3 files changed, 79 insertions(+), 22 deletions(-) create mode 100644 src/spec-node/featuresCLI/infoManifest.ts rename src/spec-node/featuresCLI/{info.ts => infoTags.ts} (63%) diff --git a/src/spec-node/devContainersSpecCLI.ts b/src/spec-node/devContainersSpecCLI.ts index 2a5bfa677..60313b582 100644 --- a/src/spec-node/devContainersSpecCLI.ts +++ b/src/spec-node/devContainersSpecCLI.ts @@ -29,12 +29,13 @@ import { FeaturesConfig, generateFeaturesConfig, getContainerFeaturesFolder } fr import { featuresTestOptions, featuresTestHandler } from './featuresCLI/test'; import { featuresPackageHandler, featuresPackageOptions } from './featuresCLI/package'; import { featuresPublishHandler, featuresPublishOptions } from './featuresCLI/publish'; -import { featuresInfoHandler, featuresInfoOptions } from './featuresCLI/info'; +import { featureInfoTagsHandler, featuresInfoTagsOptions } from './featuresCLI/infoTags'; import { beforeContainerSubstitute, containerSubstitute } from '../spec-common/variableSubstitution'; import { getPackageConfig, PackageConfiguration } from '../spec-utils/product'; import { getDevcontainerMetadata, getImageBuildInfo, getImageMetadataFromContainer, ImageMetadataEntry, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata'; import { templatesPublishHandler, templatesPublishOptions } from './templatesCLI/publish'; import { templateApplyHandler, templateApplyOptions } from './templatesCLI/apply'; +import { featuresInfoManifestHandler, featuresInfoManifestOptions } from './featuresCLI/infoManifest'; const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell'; @@ -63,10 +64,13 @@ const mountRegex = /^type=(bind|volume),source=([^,]+),target=([^,]+)(?:,externa y.command('run-user-commands', 'Run user commands', runUserCommandsOptions, runUserCommandsHandler); y.command('read-configuration', 'Read configuration', readConfigurationOptions, readConfigurationHandler); y.command('features', 'Features commands', (y: Argv) => { - y.command('test [target]', 'Test features', featuresTestOptions, featuresTestHandler); - y.command('package ', 'Package features', featuresPackageOptions, featuresPackageHandler); - y.command('publish ', 'Package and publish features', featuresPublishOptions, featuresPublishHandler); - y.command('info ', 'Fetch info on a feature', featuresInfoOptions, featuresInfoHandler); + y.command('test [target]', 'Test Features', featuresTestOptions, featuresTestHandler); + y.command('package ', 'Package Features', featuresPackageOptions, featuresPackageHandler); + y.command('publish ', 'Package and publish Features', featuresPublishOptions, featuresPublishHandler); + y.command('info', 'Fetch metadata on published Features', (y: Argv) => { + y.command('tags ', 'Fetch tags for a specific Feature', featuresInfoTagsOptions, featureInfoTagsHandler); + y.command('manifest ', 'Fetch the manifest for a specific Feature', featuresInfoManifestOptions, featuresInfoManifestHandler); + }); }); y.command('templates', 'Templates commands', (y: Argv) => { y.command('apply', 'Apply a template to the project', templateApplyOptions, templateApplyHandler); diff --git a/src/spec-node/featuresCLI/infoManifest.ts b/src/spec-node/featuresCLI/infoManifest.ts new file mode 100644 index 000000000..13d73edfc --- /dev/null +++ b/src/spec-node/featuresCLI/infoManifest.ts @@ -0,0 +1,53 @@ +import { Argv } from 'yargs'; +import { fetchAuthorization, fetchOCIManifestIfExists, getPublishedVersions, getRef } from '../../spec-configuration/containerCollectionsOCI'; +import { fetchOCIFeatureManifestIfExistsFromUserIdentifier } from '../../spec-configuration/containerFeaturesOCI'; +import { Log, LogLevel, mapLogLevel } from '../../spec-utils/log'; +import { getPackageConfig } from '../../spec-utils/product'; +import { createLog } from '../devContainers'; +import { UnpackArgv } from '../devContainersSpecCLI'; + +export function featuresInfoManifestOptions(y: Argv) { + return y + .options({ + 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' }, + 'output-format': { choices: ['text' as 'text', 'json' as 'json'], default: 'text', description: 'Output format.' }, + }) + .positional('feature', { type: 'string', demandOption: true, description: 'Feature Id' }); +} + +export type FeaturesInfoManifestArgs = UnpackArgv>; + +export function featuresInfoManifestHandler(args: FeaturesInfoManifestArgs) { + (async () => await featuresInfoManifest(args))().catch(console.error); +} + +async function featuresInfoManifest({ + 'feature': featureId, + 'log-level': inputLogLevel, + 'output-format': outputFormat, +}: FeaturesInfoManifestArgs) { + const disposables: (() => Promise | undefined)[] = []; + const dispose = async () => { + await Promise.all(disposables.map(d => d())); + }; + + const pkg = getPackageConfig(); + + const output = createLog({ + logLevel: mapLogLevel(inputLogLevel), + logFormat: 'text', + log: (str) => process.stderr.write(str), + terminalDimensions: undefined, + }, pkg, new Date(), disposables, true); + + + const featureRef = getRef(output, featureId); + if (!featureRef) { + return undefined; + } + const authorization = await fetchAuthorization(output, featureRef.registry, featureRef.path, process.env, 'pull'); + const manifest = await fetchOCIManifestIfExists(output, process.env, featureRef, undefined, authorization); + + console.log(JSON.stringify(manifest, undefined, 4)); + process.exit(); +} diff --git a/src/spec-node/featuresCLI/info.ts b/src/spec-node/featuresCLI/infoTags.ts similarity index 63% rename from src/spec-node/featuresCLI/info.ts rename to src/spec-node/featuresCLI/infoTags.ts index 49681e2db..f736d0d86 100644 --- a/src/spec-node/featuresCLI/info.ts +++ b/src/spec-node/featuresCLI/infoTags.ts @@ -5,26 +5,26 @@ import { getPackageConfig } from '../../spec-utils/product'; import { createLog } from '../devContainers'; import { UnpackArgv } from '../devContainersSpecCLI'; -export function featuresInfoOptions(y: Argv) { +export function featuresInfoTagsOptions(y: Argv) { return y .options({ 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' }, 'output-format': { choices: ['text' as 'text', 'json' as 'json'], default: 'text', description: 'Output format.' }, }) - .positional('featureId', { type: 'string', demandOption: true, description: 'Feature Id' }); + .positional('feature', { type: 'string', demandOption: true, description: 'Feature Id' }); } -export type FeaturesInfoArgs = UnpackArgv>; +export type FeaturesInfoTagsArgs = UnpackArgv>; -export function featuresInfoHandler(args: FeaturesInfoArgs) { - (async () => await featuresInfo(args))().catch(console.error); +export function featureInfoTagsHandler(args: FeaturesInfoTagsArgs) { + (async () => await featuresInfoTags(args))().catch(console.error); } -async function featuresInfo({ - 'featureId': featureId, +async function featuresInfoTags({ + 'feature': featureId, 'log-level': inputLogLevel, 'output-format': outputFormat, -}: FeaturesInfoArgs) { +}: FeaturesInfoTagsArgs) { const disposables: (() => Promise | undefined)[] = []; const dispose = async () => { await Promise.all(disposables.map(d => d())); @@ -35,7 +35,7 @@ async function featuresInfo({ const output = createLog({ logLevel: mapLogLevel(inputLogLevel), logFormat: 'text', - log: (str) => process.stdout.write(str), + log: (str) => process.stderr.write(str), terminalDimensions: undefined, }, pkg, new Date(), disposables, true); @@ -52,9 +52,9 @@ async function featuresInfo({ const publishedVersions = await getPublishedVersions(featureOciRef, output, true); if (!publishedVersions || publishedVersions.length === 0) { if (outputFormat === 'json') { - output.raw(JSON.stringify({}), LogLevel.Info); + console.log(JSON.stringify({}), LogLevel.Info); } else { - output.raw(`No published versions found for feature '${featureId}'\n`, LogLevel.Error); + console.log(`No published versions found for feature '${featureId}'\n`, LogLevel.Error); } process.exit(1); } @@ -64,19 +64,19 @@ async function featuresInfo({ }; if (outputFormat === 'json') { - printAsJson(output, data); + printAsJson(data); } else { - printAsPlainText(output, data); + printAsPlainText(data); } await dispose(); process.exit(0); } -function printAsJson(output: Log, data: { publishedVersions: string[] }) { - output.raw(JSON.stringify(data, null, 2), LogLevel.Info); +function printAsJson(data: { publishedVersions: string[] }) { + console.log(JSON.stringify(data, null, 2), LogLevel.Info); } -function printAsPlainText(output: Log, data: { publishedVersions: string[] }) { - output.raw(`Published Versions: \n ${data.publishedVersions.join('\n ')}\n`, LogLevel.Info); +function printAsPlainText(data: { publishedVersions: string[] }) { + console.log(`Published Versions: \n ${data.publishedVersions.join('\n ')}\n`, LogLevel.Info); } From 227a85ee8a799e8358b8f4df6b02916f0c2d3f6a Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Sat, 3 Dec 2022 00:10:10 +0000 Subject: [PATCH 15/18] use new info commands in test --- src/spec-node/featuresCLI/infoTags.ts | 8 ++-- .../containerFeaturesOCIPush.test.ts | 44 +++++++++++++++++-- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/spec-node/featuresCLI/infoTags.ts b/src/spec-node/featuresCLI/infoTags.ts index f736d0d86..b87b7cdf8 100644 --- a/src/spec-node/featuresCLI/infoTags.ts +++ b/src/spec-node/featuresCLI/infoTags.ts @@ -42,9 +42,9 @@ async function featuresInfoTags({ const featureOciRef = getRef(output, featureId); if (!featureOciRef) { if (outputFormat === 'json') { - output.raw(JSON.stringify({}), LogLevel.Info); + console.log(JSON.stringify({}), LogLevel.Info); } else { - output.raw(`Failed to parse Feature identifier '${featureId}'\n`, LogLevel.Error); + console.log(`Failed to parse Feature identifier '${featureId}'\n`, LogLevel.Error); } process.exit(1); } @@ -74,9 +74,9 @@ async function featuresInfoTags({ } function printAsJson(data: { publishedVersions: string[] }) { - console.log(JSON.stringify(data, null, 2), LogLevel.Info); + console.log(JSON.stringify(data, null, 2)); } function printAsPlainText(data: { publishedVersions: string[] }) { - console.log(`Published Versions: \n ${data.publishedVersions.join('\n ')}\n`, LogLevel.Info); + console.log(`Published Versions: \n ${data.publishedVersions.join('\n ')}\n`); } diff --git a/src/test/container-features/containerFeaturesOCIPush.test.ts b/src/test/container-features/containerFeaturesOCIPush.test.ts index dc4786599..d357a9a32 100644 --- a/src/test/container-features/containerFeaturesOCIPush.test.ts +++ b/src/test/container-features/containerFeaturesOCIPush.test.ts @@ -50,9 +50,13 @@ registry`; it('Publish Features to registry', async () => { const collectionFolder = `${__dirname}/example-v2-features-sets/simple`; let success = false; - let result: ExecResult | undefined = undefined; + + let publishResult: ExecResult | undefined = undefined; + let infoTagsResult: ExecResult | undefined = undefined; + let infoManifestResult: ExecResult | undefined = undefined; + try { - result = await shellExec(`${cli} features publish --log-level trace -r localhost:5000 -n octocat/features ${collectionFolder}/src`, { env: { ...process.env, 'DEVCONTAINERS_OCI_AUTH': 'localhost:5000|myuser|mypass' } }); + publishResult = await shellExec(`${cli} features publish --log-level trace -r localhost:5000 -n octocat/features ${collectionFolder}/src`, { env: { ...process.env, 'DEVCONTAINERS_OCI_AUTH': 'localhost:5000|myuser|mypass' } }); success = true; } catch (error) { @@ -60,7 +64,41 @@ registry`; } assert.isTrue(success); - assert.isDefined(result); + assert.isDefined(publishResult); + + // --- See that the Features can be queried from the Dev Container CLI. + + success = false; // Reset success flag. + try { + infoTagsResult = await shellExec(`${cli} features info tags localhost:5000/octocat/features/hello --output-format json --log-level trace`, { env: { ...process.env, 'DEVCONTAINERS_OCI_AUTH': 'localhost:5000|myuser|mypass' } }); + success = true; + + } catch (error) { + assert.fail('features info tags sub-command should not throw'); + } + + assert.isTrue(success); + assert.isDefined(infoTagsResult); + const tags = JSON.parse(infoTagsResult.stdout); + const publishedVersions: string[] = tags['publishedVersions']; + assert.equal(publishedVersions.length, 4); + + success = false; // Reset success flag. + try { + infoManifestResult = await shellExec(`${cli} features info manifest localhost:5000/octocat/features/hello --log-level trace`, { env: { ...process.env, 'DEVCONTAINERS_OCI_AUTH': 'localhost:5000|myuser|mypass' } }); + success = true; + + } catch (error) { + assert.fail('features info tags sub-command should not throw'); + } + + assert.isTrue(success); + assert.isDefined(infoManifestResult); + const manifest = infoManifestResult.stdout; + const regex = /application\/vnd\.devcontainers\.layer\.v1\+tar/; + assert.match(manifest, regex); + + success = false; // Reset success flag. }); }); From f80ba5842f671b2d2324044135b7b68c6ba2003f Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Sat, 3 Dec 2022 00:14:30 +0000 Subject: [PATCH 16/18] typecheck --- src/spec-node/featuresCLI/infoManifest.ts | 7 +++---- src/spec-node/featuresCLI/infoTags.ts | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/spec-node/featuresCLI/infoManifest.ts b/src/spec-node/featuresCLI/infoManifest.ts index 13d73edfc..e1c140b46 100644 --- a/src/spec-node/featuresCLI/infoManifest.ts +++ b/src/spec-node/featuresCLI/infoManifest.ts @@ -1,7 +1,6 @@ import { Argv } from 'yargs'; -import { fetchAuthorization, fetchOCIManifestIfExists, getPublishedVersions, getRef } from '../../spec-configuration/containerCollectionsOCI'; -import { fetchOCIFeatureManifestIfExistsFromUserIdentifier } from '../../spec-configuration/containerFeaturesOCI'; -import { Log, LogLevel, mapLogLevel } from '../../spec-utils/log'; +import { fetchAuthorization, fetchOCIManifestIfExists, getRef } from '../../spec-configuration/containerCollectionsOCI'; +import { mapLogLevel } from '../../spec-utils/log'; import { getPackageConfig } from '../../spec-utils/product'; import { createLog } from '../devContainers'; import { UnpackArgv } from '../devContainersSpecCLI'; @@ -24,7 +23,6 @@ export function featuresInfoManifestHandler(args: FeaturesInfoManifestArgs) { async function featuresInfoManifest({ 'feature': featureId, 'log-level': inputLogLevel, - 'output-format': outputFormat, }: FeaturesInfoManifestArgs) { const disposables: (() => Promise | undefined)[] = []; const dispose = async () => { @@ -49,5 +47,6 @@ async function featuresInfoManifest({ const manifest = await fetchOCIManifestIfExists(output, process.env, featureRef, undefined, authorization); console.log(JSON.stringify(manifest, undefined, 4)); + await dispose(); process.exit(); } diff --git a/src/spec-node/featuresCLI/infoTags.ts b/src/spec-node/featuresCLI/infoTags.ts index b87b7cdf8..01901fdfb 100644 --- a/src/spec-node/featuresCLI/infoTags.ts +++ b/src/spec-node/featuresCLI/infoTags.ts @@ -1,6 +1,6 @@ import { Argv } from 'yargs'; import { getPublishedVersions, getRef } from '../../spec-configuration/containerCollectionsOCI'; -import { Log, LogLevel, mapLogLevel } from '../../spec-utils/log'; +import { LogLevel, mapLogLevel } from '../../spec-utils/log'; import { getPackageConfig } from '../../spec-utils/product'; import { createLog } from '../devContainers'; import { UnpackArgv } from '../devContainersSpecCLI'; From cdd92f581db4a158b234833436520565dafc1d40 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Sat, 3 Dec 2022 00:24:08 +0000 Subject: [PATCH 17/18] clarify error case of getManifest() --- src/spec-configuration/containerCollectionsOCI.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/spec-configuration/containerCollectionsOCI.ts b/src/spec-configuration/containerCollectionsOCI.ts index 4a0accef4..c3e039715 100644 --- a/src/spec-configuration/containerCollectionsOCI.ts +++ b/src/spec-configuration/containerCollectionsOCI.ts @@ -218,7 +218,8 @@ export async function getManifest(output: Log, env: NodeJS.ProcessEnv, url: stri return manifest; } catch (e) { - output.write(`(!) Failed to fetch manifest: ${e}`, LogLevel.Error); + // A 404 is expected here if the manifest does not exist on the remote. + output.write(`Did not fetch manifest: ${e}`, LogLevel.Trace); return undefined; } } From fd4de3fc50fdf299c1d9dce64a6d0c4f29dfe935 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Tue, 6 Dec 2022 09:24:58 -0800 Subject: [PATCH 18/18] Apply suggestions from code review Co-authored-by: Samruddhi Khandale --- src/spec-configuration/containerCollectionsOCI.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spec-configuration/containerCollectionsOCI.ts b/src/spec-configuration/containerCollectionsOCI.ts index c3e039715..b2883ef83 100644 --- a/src/spec-configuration/containerCollectionsOCI.ts +++ b/src/spec-configuration/containerCollectionsOCI.ts @@ -230,7 +230,7 @@ export async function fetchAuthorization(output: Log, registry: string, ociRepoP const basicAuthTokenBase64 = await getBasicAuthCredential(output, registry, env); const scopeToken = await generateScopeTokenCredential(output, registry, ociRepoPath, env, operationScopes, basicAuthTokenBase64); - // Prefer returned a Bearer token retreived from the /token endpoint. + // Prefer returned a Bearer token retrieved from the /token endpoint. if (scopeToken) { output.write(`Using scope token for registry '${registry}'`, LogLevel.Trace); return `Bearer ${scopeToken}`;