Skip to content

Commit

Permalink
Follow Docker Token Authentication Specification (#341)
Browse files Browse the repository at this point in the history 
* refactor oci http auth code into a new file, and implement using wwwAuthenticate

* clean up nesting

* log statements

* progress adopting new function

* refactor to send all HTTP requests through httpOciRegistry

* fix scope

* fix inspectImageInRegistry

* read in auth from .docker/config.json and assert ACR can anonymously pull features

* add src/test/container-features/registryCompatibilityOCI.test.ts to matrix

* improve performance by caching last successfully used token

* whitespace

* codereview: harden using auth headers to per registry

* remove outdated comments
  • Loading branch information
@joshspicer
joshspicer authored Jan 3, 2023
1 parent a77d855 commit 0583b9c
Show file tree
Hide file tree
Showing 12 changed files with 467 additions and 260 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/dev-containers.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,9 @@ jobs:
"src/test/cli.test.ts",
"src/test/cli.up.test.ts",
"src/test/container-features/containerFeaturesOCIPush.test.ts",
"src/test/container-features/registryCompatibilityOCI.test.ts",
# Run all except the above:
"--exclude src/test/container-features/containerFeaturesOCIPush.test.ts --exclude src/test/container-features/e2e.test.ts --exclude src/test/container-features/featuresCLICommands.test.ts --exclude src/test/container-features/containerFeaturesOrder.test.ts --exclude src/test/cli.build.test.ts --exclude src/test/cli.exec.buildKit.1.test.ts --exclude src/test/cli.exec.buildKit.2.test.ts --exclude src/test/cli.exec.nonBuildKit.1.test.ts --exclude src/test/cli.exec.nonBuildKit.2.test.ts --exclude src/test/cli.test.ts --exclude src/test/cli.up.test.ts 'src/test/**/*.test.ts'",
"--exclude src/test/container-features/registryCompatibilityOCI.test.ts --exclude src/test/container-features/containerFeaturesOCIPush.test.ts --exclude src/test/container-features/e2e.test.ts --exclude src/test/container-features/featuresCLICommands.test.ts --exclude src/test/container-features/containerFeaturesOrder.test.ts --exclude src/test/cli.build.test.ts --exclude src/test/cli.exec.buildKit.1.test.ts --exclude src/test/cli.exec.buildKit.2.test.ts --exclude src/test/cli.exec.nonBuildKit.1.test.ts --exclude src/test/cli.exec.nonBuildKit.2.test.ts --exclude src/test/cli.test.ts --exclude src/test/cli.up.test.ts 'src/test/**/*.test.ts'",
]
steps:
- name: Checkout
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/test-windows.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ jobs:
# "src/test/dockerUtils.test.ts",
# "src/test/imageMetadata.test.ts",
# "src/test/variableSubstitution.test.ts",
# "src/test/container-features/registryCompatibilityOCI.test.ts",
# # Run all except the above:
# "--exclude .....",
]
Expand Down
234 changes: 64 additions & 170 deletions src/spec-configuration/containerCollectionsOCI.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,19 @@ import * as semver from 'semver';
import * as tar from 'tar';
import * as jsonc from 'jsonc-parser';

import { request } from '../spec-utils/httpRequest';
import { Log, LogLevel } from '../spec-utils/log';
import { isLocalFile, mkdirpLocal, readLocalFile, writeLocalFile } from '../spec-utils/pfs';
import { requestEnsureAuthenticated } from './httpOCIRegistry';

export const DEVCONTAINER_MANIFEST_MEDIATYPE = 'application/vnd.devcontainers';
export const DEVCONTAINER_TAR_LAYER_MEDIATYPE = 'application/vnd.devcontainers.layer.v1+tar';
export const DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE = 'application/vnd.devcontainers.collection.layer.v1+json';

export type HEADERS = { 'authorization'?: string; 'user-agent': string; 'content-type'?: string; 'accept'?: string; 'content-length'?: string };

export interface CommonParams {
env: NodeJS.ProcessEnv;
output: Log;
cachedAuthHeader?: Record<string, string>; // <registry, authHeader>
}

// Represents the unique OCI identifier for a Feature or Template.
Expand Down Expand Up @@ -170,7 +170,7 @@ export function getCollectionRef(output: Log, registry: string, namespace: strin

// Validate if a manifest exists and is reachable about the declared feature/template.
// Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests
export async function fetchOCIManifestIfExists(params: CommonParams, ref: OCIRef | OCICollectionRef, manifestDigest?: string, authToken?: string): Promise<OCIManifest | undefined> {
export async function fetchOCIManifestIfExists(params: CommonParams, ref: OCIRef | OCICollectionRef, manifestDigest?: string): Promise<OCIManifest | undefined> {
const { output } = params;

// Simple mechanism to avoid making a DNS request for
Expand All @@ -188,7 +188,7 @@ export async function fetchOCIManifestIfExists(params: CommonParams, ref: OCIRef
}
const manifestUrl = `https://${ref.registry}/v2/${ref.path}/manifests/${reference}`;
output.write(`manifest url: ${manifestUrl}`, LogLevel.Trace);
const manifest = await getManifest(params, manifestUrl, ref, authToken);
const manifest = await getManifest(params, manifestUrl, ref);

if (!manifest) {
return;
Expand All @@ -202,184 +202,79 @@ export async function fetchOCIManifestIfExists(params: CommonParams, ref: OCIRef
return manifest;
}

export async function getManifest(params: CommonParams, url: string, ref: OCIRef | OCICollectionRef, authToken?: string | false, mimeType?: string): Promise<OCIManifest | undefined> {
export async function getManifest(params: CommonParams, url: string, ref: OCIRef | OCICollectionRef, mimeType?: string): Promise<OCIManifest | undefined> {
const { output } = params;
let body: string = '';
try {
const headers: HEADERS = {
const headers = {
'user-agent': 'devcontainer',
'accept': mimeType || 'application/vnd.oci.image.manifest.v1+json',
};

const authorization = authToken ?? await fetchAuthorizationHeader(params, ref.registry, ref.path, 'pull');
if (authorization) {
headers['authorization'] = authorization;
}

const options = {
type: 'GET',
const httpOptions = {
type: 'GET',
url: url,
headers: headers
};

const response = await request(options);
const manifest: OCIManifest = JSON.parse(response.toString());

return manifest;
} catch (e) {
// A 404 is expected here if the manifest does not exist on the remote.
output.write(`Did not fetch manifest: ${e}`, LogLevel.Trace);
return undefined;
}
}

// Exported Function
// Will attempt to generate/fetch the correct authorization header for subsequent requests (Bearer or Basic)
export async function fetchAuthorizationHeader(params: CommonParams, registry: string, ociRepoPath: string, operationScopes: string): Promise<string | undefined> {
const { output } = params;

const basicAuthTokenBase64 = await getBasicAuthCredential(params, registry);
const scopeToken = await fetchRegistryBearerToken(params, registry, ociRepoPath, operationScopes, basicAuthTokenBase64);

// Prefer returned a Bearer token retrieved from the /token endpoint.
if (scopeToken) {
output.write(`Using scope token for registry '${registry}'`, LogLevel.Trace);
return `Bearer ${scopeToken}`;
}

// If all we have are Basic auth credentials, return those for the caller to use.
if (basicAuthTokenBase64) {
output.write(`Using basic auth token for registry '${registry}'`, LogLevel.Trace);
return `Basic ${basicAuthTokenBase64}`;
}

// If we have no credentials, and we weren't able to get a scope token anonymously, return undefined.
return undefined;
}


// * Internal helper for 'fetchAuthorizationHeader(...)'
// Attempts to get the Basic auth credentials for the provided registry.
// These may be programatically crafted via environment variables (GITHUB_TOKEN),
// parsed out of a special DEVCONTAINERS_OCI_AUTH environment variable,
// TODO: or directly read out of the local docker config file/credential helper.
async function getBasicAuthCredential(params: CommonParams, registry: string): Promise<string | undefined> {
const { output, env } = params;

let userToken: string | undefined = undefined;
if (!!env['GITHUB_TOKEN'] && registry === 'ghcr.io') {
output.write('Using environment GITHUB_TOKEN for auth', LogLevel.Trace);
userToken = `USERNAME:${env['GITHUB_TOKEN']}`;
} else if (!!env['DEVCONTAINERS_OCI_AUTH']) {
// eg: DEVCONTAINERS_OCI_AUTH=realm1|user1|token1,realm2|user2|token2
const authContexts = env['DEVCONTAINERS_OCI_AUTH'].split(',');
const authContext = authContexts.find(a => a.split('|')[0] === registry);

if (authContext) {
output.write(`Using match from DEVCONTAINERS_OCI_AUTH for registry '${registry}'`, LogLevel.Trace);
const split = authContext.split('|');
userToken = `${split[1]}:${split[2]}`;
const res = await requestEnsureAuthenticated(params, httpOptions, ref);
if (!res) {
output.write('Request failed', LogLevel.Error);
return;
}
}

if (userToken) {
return Buffer.from(userToken).toString('base64');
}

// Represents anonymous access.
output.write(`No authentication credentials found for registry '${registry}'.`, LogLevel.Warning);
return undefined;
}

// * Internal helper for 'fetchAuthorizationHeader(...)'
// https://github.com/oras-project/oras-go/blob/97a9c43c52f9d89ecf5475bc59bd1f96c8cc61f6/registry/remote/auth/scope.go#L60-L74
// Using the provided Basic auth credentials, (or if none, anonymously), to ask the registry's '/token' endpoint for a token.
// Some registries (eg: ghcr.io) expect a scoped token to target resources and will not operate with just Basic Auth.
// Other registries (eg: the OCI Reference Implementation) will not return a valid token from '/token'
async function fetchRegistryBearerToken(params: CommonParams, registry: string, ociRepoPath: string, operationScopes: string, basicAuthTokenBase64: string | undefined = undefined): Promise<string | undefined> {
const { output } = params;

if (registry === 'mcr.microsoft.com') {
return undefined;
}

const headers: HEADERS = {
'user-agent': 'devcontainer'
};

if (!basicAuthTokenBase64) {
basicAuthTokenBase64 = await getBasicAuthCredential(params, registry);
}

if (basicAuthTokenBase64) {
headers['authorization'] = `Basic ${basicAuthTokenBase64}`;
}

const authServer = registry === 'docker.io' ? 'auth.docker.io' : registry;
const registryServer = registry === 'docker.io' ? 'registry.docker.io' : registry;
const url = `https://${authServer}/token?scope=repository:${ociRepoPath}:${operationScopes}&service=${registryServer}`;
output.write(`Fetching scope token from: ${url}`, LogLevel.Trace);
const { resBody, statusCode } = res;
body = resBody.toString();

const options = {
type: 'GET',
url: url,
headers: headers
};
// NOTE: A 404 is expected here if the manifest does not exist on the remote.
if (statusCode > 299) {
output.write(`Did not fetch manifest: ${body}`, LogLevel.Trace);
return;
}

let authReq: Buffer;
try {
authReq = await request(options, output);
} catch (e: any) {
// This is ok if the registry is trying to speak Basic Auth with us.
output.write(`Not used a scoped token for ${registry}: ${e}`, LogLevel.Trace);
return JSON.parse(body);
} catch (e) {
output.write(`Failed to parse manifest: ${body}`, LogLevel.Error);
return;
}

if (!authReq) {
output.write('Failed to get registry auth token', LogLevel.Error);
return undefined;
}

let scopeToken: string | undefined;
try {
scopeToken = JSON.parse(authReq.toString())?.token;
} catch {
// not JSON
}
if (!scopeToken) {
output.write('Failed to parse registry auth token response', LogLevel.Error);
return undefined;
}
return scopeToken;
}

// Lists published versions/tags of a feature/template
// Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#content-discovery
export async function getPublishedVersions(params: CommonParams, ref: OCIRef, sorted: boolean = false, collectionType: string = 'feature'): Promise<string[] | undefined> {
export async function getPublishedVersions(params: CommonParams, ref: OCIRef, sorted: boolean = false): Promise<string[] | undefined> {
const { output } = params;
try {
const url = `https://${ref.registry}/v2/${ref.namespace}/${ref.id}/tags/list`;

let authorization = await fetchAuthorizationHeader(params, ref.registry, ref.path, 'pull');

if (!authorization) {
output.write(`(!) ERR: Failed to get published versions for ${collectionType}: ${ref.resource}`, LogLevel.Error);
return undefined;
}

const headers: HEADERS = {
'user-agent': 'devcontainer',
const headers = {
'accept': 'application/json',
'authorization': authorization
};

const options = {
const httpOptions = {
type: 'GET',
url: url,
headers: headers
};

const response = await request(options);
const publishedVersionsResponse: OCITagList = JSON.parse(response.toString());
const res = await requestEnsureAuthenticated(params, httpOptions, ref);
if (!res) {
output.write('Request failed', LogLevel.Error);
return;
}

const { statusCode, resBody } = res;
const body = resBody.toString();

// Expected when publishing for the first time
if (statusCode === 404) {
return [];
// Unexpected Error
} else if (statusCode > 299) {
output.write(`(!) ERR: Could not fetch published tags for '${ref.namespace}/${ref.id}' : ${resBody ?? ''} `, LogLevel.Error);
return;
}

const publishedVersionsResponse: OCITagList = JSON.parse(body);

if (!sorted) {
return publishedVersionsResponse.tags;
Expand All @@ -394,17 +289,12 @@ export async function getPublishedVersions(params: CommonParams, ref: OCIRef, so

return hasLatest ? ['latest', ...sortedVersions] : sortedVersions;
} catch (e) {
// Publishing for the first time
if (e?.message.includes('HTTP 404: Not Found')) {
return [];
}

output.write(`(!) ERR: Could not fetch published tags for '${ref.namespace}/${ref.id}' : ${e?.message ?? ''} `, LogLevel.Error);
return undefined;
output.write(`Failed to parse published versions: ${e}`, LogLevel.Error);
return;
}
}

export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, authToken?: string, ignoredFilesDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> {
export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, ignoredFilesDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> {
// TODO: Parallelize if multiple layers (not likely).
// TODO: Seeking might be needed if the size is too large.

Expand All @@ -413,26 +303,30 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st
await mkdirpLocal(ociCacheDir);
const tempTarballPath = path.join(ociCacheDir, 'blob.tar');

const headers: HEADERS = {
'user-agent': 'devcontainer',
const headers = {
'accept': 'application/vnd.oci.image.manifest.v1+json',
};

const authorization = authToken ?? await fetchAuthorizationHeader(params, ociRef.registry, ociRef.path, 'pull');
if (authorization) {
headers['authorization'] = authorization;
}

const options = {
const httpOptions = {
type: 'GET',
url: url,
headers: headers
};

const blob = await request(options, output);
const res = await requestEnsureAuthenticated(params, httpOptions, ociRef);
if (!res) {
output.write('Request failed', LogLevel.Error);
return;
}

const { statusCode, resBody } = res;
if (statusCode > 299) {
output.write(`Failed to fetch blob (${url}): ${resBody}`, LogLevel.Error);
return;
}

await mkdirpLocal(destCachePath);
await writeLocalFile(tempTarballPath, blob);
await writeLocalFile(tempTarballPath, resBody);

const files: string[] = [];
await tar.x(
Expand Down Expand Up @@ -482,7 +376,7 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st
files, metadata
};
} catch (e) {
output.write(`error: ${e}`, LogLevel.Error);
return undefined;
output.write(`Error getting blob: ${e}`, LogLevel.Error);
return;
}
}
Loading

0 comments on commit 0583b9c

Please sign in to comment.