Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fetch Features/Templates by digest #480

Merged
merged 5 commits into from
Apr 14, 2023
Merged

Fetch Features/Templates by digest #480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
77afa58
fetch a Feature or Template by its sha256 digest
joshspicer Apr 4, 2023
1161266
update test
joshspicer Apr 4, 2023
db00742
update tests in featureHelpers.test.ts
joshspicer Apr 5, 2023
4e97da9
code review cleanup
joshspicer Apr 7, 2023
8d02d5d
update an E2E test for Feature by SHA
joshspicer Apr 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 71 additions & 29 deletions src/spec-configuration/containerCollectionsOCI.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ export interface CommonParams {

// Represents the unique OCI identifier for a Feature or Template.
// eg: ghcr.io/devcontainers/features/go:1.0.0
// eg: ghcr.io/devcontainers/features/go@sha256:fe73f123927bd9ed1abda190d3009c4d51d0e17499154423c5913cf344af15a3
// Constructed by 'getRef()'
export interface OCIRef {
registry: string; // 'ghcr.io'
Expand All @@ -28,7 +29,10 @@ export interface OCIRef {
path: string; // 'devcontainers/features/go'
resource: string; // 'ghcr.io/devcontainers/features/go'
id: string; // 'go'
version?: string; // '1.0.0'

version: string; // (Either the contents of 'tag' or 'digest')
joshspicer marked this conversation as resolved.
Show resolved Hide resolved
tag?: string; // '1.0.0'
digest?: string; // 'sha256:fe73f123927bd9ed1abda190d3009c4d51d0e17499154423c5913cf344af15a3'
}

// Represents the unique OCI identifier for a Collection's Metadata artifact.
Expand All @@ -38,6 +42,7 @@ export interface OCICollectionRef {
registry: string; // 'ghcr.io'
path: string; // 'devcontainers/features'
resource: string; // 'ghcr.io/devcontainers/features'
tag: 'latest'; // 'latest' (always)
version: 'latest'; // 'latest' (always)
}

Expand Down Expand Up @@ -88,12 +93,14 @@ interface OCIImageIndex {
// Following Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests
// Alternative Spec: https://docs.docker.com/registry/spec/api/#overview
//
// Entire path ('namespace' in spec terminology) for the given repository
// The path:
// 'namespace' in spec terminology for the given repository
// (eg: devcontainers/features/go)
const regexForPath = /^[a-z0-9]+([._-][a-z0-9]+)*(\/[a-z0-9]+([._-][a-z0-9]+)*)*$/;
// The reference:
// MUST be either (a) the digest of the manifest or (b) a tag
// MUST be at most 128 characters in length and MUST match the following regular expression:
const regexForReference = /^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$/;
const regexForVersionOrDigest = /^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$/;

// https://go.dev/doc/install/source#environment
// Expected by OCI Spec as seen here: https://github.com/opencontainers/image-spec/blob/main/image-index.md#image-index-property-descriptions
Expand Down Expand Up @@ -124,20 +131,54 @@ export function getRef(output: Log, input: string): OCIRef | undefined {
input = input.toLowerCase();

const indexOfLastColon = input.lastIndexOf(':');
const indexOfLastAtCharacter = input.lastIndexOf('@');

let resource = '';
let version = ''; // TODO: Support parsing out manifest digest (...@sha256:...)

// 'If' condition is true in the following cases:
// 1. The final colon is before the first slash (a port) : eg: ghcr.io:8081/codspace/features/ruby
// 2. There is no version : eg: ghcr.io/codspace/features/ruby
// In both cases, assume 'latest' tag.
if (indexOfLastColon === -1 || indexOfLastColon < input.indexOf('/')) {
resource = input;
version = 'latest';
let tag: string | undefined = undefined;
let digest: string | undefined = undefined;

if (indexOfLastAtCharacter !== -1) {
// The version is specified by digest
// eg: ghcr.io/codspace/features/ruby@sha256:abcdefgh
resource = input.substring(0, indexOfLastAtCharacter);
const digestWithHashingAlgorithm = input.substring(indexOfLastAtCharacter + 1);
const splitOnColon = digestWithHashingAlgorithm.split(':');
if (splitOnColon.length !== 2) {
output.write(`Failed to parse digest '${digestWithHashingAlgorithm}'. Expected format: 'sha256:abcdefghijk'`, LogLevel.Error);
return;
}

if (splitOnColon[0] !== 'sha256') {
output.write(`Digest algorithm for input '${input}' failed validation. Expected hashing algorithm to be 'sha256'.`, LogLevel.Error);
joshspicer marked this conversation as resolved.
Show resolved Hide resolved
return;
}

if (!regexForVersionOrDigest.test(splitOnColon[1])) {
output.write(`Digest for input '${input}' failed validation. Expected digest to match regex '${regexForVersionOrDigest}'.`, LogLevel.Error);
}

digest = digestWithHashingAlgorithm;
} else {
resource = input.substring(0, indexOfLastColon);
version = input.substring(indexOfLastColon + 1);
// In both cases, assume 'latest' tag.
if (indexOfLastColon === -1 || indexOfLastColon < input.lastIndexOf('/')) {
// 1. The final colon is before the first slash (a port)
// eg: ghcr.io:8081/codspace/features/ruby
// 2. There is no tag at all
// eg: ghcr.io/codspace/features/ruby
// In both cases, assume the 'latest' tag
resource = input;
tag = 'latest';
} else {
// The version is specified by tag
// eg: ghcr.io/codspace/features/ruby:1.0.0
resource = input.substring(0, indexOfLastColon);
tag = input.substring(indexOfLastColon + 1);
}
}

if (tag && !regexForVersionOrDigest.test(tag)) {
output.write(`Tag '${tag}' for input '${input}' failed validation. Expected digest to match regex '${regexForVersionOrDigest}'.`, LogLevel.Error);
return;
}

const splitOnSlash = resource.split('/');
Expand All @@ -149,36 +190,36 @@ export function getRef(output: Log, input: string): OCIRef | undefined {

const path = `${namespace}/${id}`;

if (!regexForPath.exec(path)) {
output.write(`Path '${path}' for input '${input}' failed validation. Expected path to match regex '${regexForPath}'.`, LogLevel.Error);
return;
}

const version = digest || tag || 'latest'; // The most specific version.

output.write(`> input: ${input}`, LogLevel.Trace);
output.write(`>`, LogLevel.Trace);
output.write(`> resource: ${resource}`, LogLevel.Trace);
output.write(`> id: ${id}`, LogLevel.Trace);
output.write(`> version: ${version}`, LogLevel.Trace);
output.write(`> owner: ${owner}`, LogLevel.Trace);
output.write(`> namespace: ${namespace}`, LogLevel.Trace); // TODO: We assume 'namespace' includes at least one slash (eg: 'devcontainers/features')
output.write(`> registry: ${registry}`, LogLevel.Trace);
output.write(`> path: ${path}`, LogLevel.Trace);

// Validate results of parse.

if (!regexForPath.exec(path)) {
output.write(`Parsed path '${path}' for input '${input}' failed validation.`, LogLevel.Error);
return undefined;
}

if (!regexForReference.test(version)) {
output.write(`Parsed version '${version}' for input '${input}' failed validation.`, LogLevel.Error);
return undefined;
}
output.write(`>`, LogLevel.Trace);
output.write(`> version: ${version}`, LogLevel.Trace);
output.write(`> tag?: ${tag}`, LogLevel.Trace);
output.write(`> digest?: ${digest}`, LogLevel.Trace);

return {
id,
version,
owner,
namespace,
registry,
resource,
path,
version,
tag,
digest,
};
}

Expand All @@ -203,7 +244,8 @@ export function getCollectionRef(output: Log, registry: string, namespace: strin
registry,
path,
resource,
version: 'latest'
version: 'latest',
tag: 'latest',
};
}

Expand Down
10 changes: 3 additions & 7 deletions src/spec-configuration/containerFeaturesConfiguration.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -724,18 +724,14 @@ export async function getFeatureIdType(params: CommonParams, userFeatureId: stri
return { type: 'file-path', manifest: undefined };
}

// version identifier for a github release feature.
// DEPRECATED: This is a legacy feature-set ID
if (userFeatureId.includes('@')) {
return { type: 'github-repo', manifest: undefined };
}

const manifest = await fetchOCIFeatureManifestIfExistsFromUserIdentifier(params, userFeatureId);
if (manifest) {
return { type: 'oci', manifest: manifest };
} else {
output.write(`Could not resolve Feature manifest for '${userFeatureId}'. If necessary, provide registry credentials with 'docker login <registry>'.`, LogLevel.Warning);
output.write(`Falling back to legacy GitHub Releases mode to acquire Feature.`, LogLevel.Trace);

// DEPRECATED: This is a legacy feature-set ID
output.write('(!) WARNING: Falling back to deprecated GitHub Release syntax. See https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-features.md#referencing-a-feature for updated specification.', LogLevel.Warning);
return { type: 'github-repo', manifest: undefined };
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/test/container-features/configs/dockerfile-with-v2-oci-features/.devcontainer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
},
"features": {
"terraform": "latest",
"ghcr.io/devcontainers/features/docker-in-docker:1": {},
"ghcr.io/devcontainers/features/docker-in-docker@sha256:e32e8937c87345ff7a937d22cacb7f395d41deffde9943291ef3cc0ac91a8ac6": {},
"node": "16"
}
}
48 changes: 42 additions & 6 deletions src/test/container-features/containerFeaturesOCI.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ describe('getCollectionRef()', async function () {
assert.equal(collectionRef.path, 'devcontainers/templates');
assert.equal(collectionRef.resource, 'ghcr.io/devcontainers/templates');
assert.equal(collectionRef.version, 'latest');
assert.equal(collectionRef.tag, collectionRef.version);
});

it('valid getCollectionRef() that was originally uppercase', async () => {
Expand All @@ -30,6 +31,7 @@ describe('getCollectionRef()', async function () {
assert.equal(collectionRef.path, 'devcontainers/templates');
assert.equal(collectionRef.resource, 'ghcr.io/devcontainers/templates');
assert.equal(collectionRef.version, 'latest');
assert.equal(collectionRef.tag, collectionRef.version);
});

it('valid getCollectionRef() with port in registry', async () => {
Expand All @@ -42,6 +44,7 @@ describe('getCollectionRef()', async function () {
assert.equal(collectionRef.path, 'devcontainers/templates');
assert.equal(collectionRef.resource, 'ghcr.io:8001/devcontainers/templates');
assert.equal(collectionRef.version, 'latest');
assert.equal(collectionRef.tag, collectionRef.version);
});

it('invalid getCollectionRef() with an invalid character in path', async () => {
Expand Down Expand Up @@ -70,10 +73,28 @@ describe('getRef()', async function () {
assert.equal(feat.owner, 'devcontainers');
assert.equal(feat.registry, 'ghcr.io');
assert.equal(feat.resource, 'ghcr.io/devcontainers/templates/docker-from-docker');
assert.equal(feat.version, 'latest');
assert.equal(feat.tag, 'latest');
assert.equal(feat.tag, feat.version);
assert.equal(feat.path, 'devcontainers/templates/docker-from-docker');
});

it('valid getRef() with a digest', async () => {
const feat = getRef(output, 'ghcr.io/my-org/my-features/my-feat@sha256:1234567890123456789012345678901234567890123456789012345678901234');
if (!feat) {
assert.fail('featureRef should not be undefined');
}
assert.ok(feat);
assert.equal(feat.id, 'my-feat');
assert.equal(feat.namespace, 'my-org/my-features');
assert.equal(feat.owner, 'my-org');
assert.equal(feat.registry, 'ghcr.io');
assert.equal(feat.resource, 'ghcr.io/my-org/my-features/my-feat');
assert.equal(feat.path, 'my-org/my-features/my-feat');
assert.isUndefined(feat.tag);
assert.equal(feat.digest, 'sha256:1234567890123456789012345678901234567890123456789012345678901234');
assert.equal(feat.digest, feat.version);
});

it('valid getRef() without a version tag', async () => {
const feat = getRef(output, 'ghcr.io/devcontainers/templates/docker-from-docker');
if (!feat) {
Expand All @@ -86,7 +107,9 @@ describe('getRef()', async function () {
assert.equal(feat.registry, 'ghcr.io');
assert.equal(feat.resource, 'ghcr.io/devcontainers/templates/docker-from-docker');
assert.equal(feat.path, 'devcontainers/templates/docker-from-docker');
assert.equal(feat.version, 'latest'); // Defaults to 'latest' if not version supplied.
assert.equal(feat.tag, 'latest'); // Defaults to 'latest' if not version supplied.
assert.isUndefined(feat.digest);
assert.equal(feat.tag, feat.version);
});

it('valid getRef() automatically downcases', async () => {
Expand All @@ -101,7 +124,8 @@ describe('getRef()', async function () {
assert.equal(feat.registry, 'ghcr.io');
assert.equal(feat.resource, 'ghcr.io/devcontainers/templates/docker-from-docker');
assert.equal(feat.path, 'devcontainers/templates/docker-from-docker');
assert.equal(feat.version, 'latest'); // Defaults to 'latest' if not version supplied.
assert.equal(feat.tag, 'latest'); // Defaults to 'latest' if not version supplied.
assert.equal(feat.tag, feat.version);
});

it('valid getRef() with a registry that contains a port.', async () => {
Expand All @@ -116,7 +140,8 @@ describe('getRef()', async function () {
assert.equal(feat.registry, 'docker.io:8001');
assert.equal(feat.resource, 'docker.io:8001/devcontainers/templates/docker-from-docker');
assert.equal(feat.path, 'devcontainers/templates/docker-from-docker');
assert.equal(feat.version, 'latest'); // Defaults to 'latest' if not version supplied.
assert.equal(feat.tag, 'latest'); // Defaults to 'latest' if not version supplied.
assert.equal(feat.tag, feat.version);
});

it('valid getRef() really short path and no version', async () => {
Expand All @@ -131,7 +156,8 @@ describe('getRef()', async function () {
assert.equal(feat.registry, 'docker.io:8001');
assert.equal(feat.resource, 'docker.io:8001/a/b/c');
assert.equal(feat.path, 'a/b/c');
assert.equal(feat.version, 'latest'); // Defaults to 'latest' if not version supplied.
assert.equal(feat.tag, 'latest'); // Defaults to 'latest' if not version supplied.
assert.equal(feat.tag, feat.version);
});

it('invalid getRef() with duplicate version tags', async () => {
Expand Down Expand Up @@ -164,6 +190,16 @@ describe('getRef()', async function () {
assert.isUndefined(feat);
});

it('invalid getRef() with unsupported digest hashing algorithm', async () => {
const feat = getRef(output, 'ghcr.io/devcontainers//templates/docker-from-docker@sha100:1234567890123456789012345678901234567890123456789012345678901234');
assert.isUndefined(feat);
});

it('invalid getRef() with mis-shaped digest', async () => {
const feat = getRef(output, 'ghcr.io/devcontainers//templates/docker-from-docker@1234567890123456789012345678901234567890123456789012345678901234');
assert.isUndefined(feat);
});

});

describe('Test OCI Pull', () => {
Expand All @@ -179,7 +215,7 @@ describe('Test OCI Pull', () => {
assert.equal(feat.owner, 'codspace');
assert.equal(feat.registry, 'ghcr.io');
assert.equal(feat.resource, 'ghcr.io/codspace/features/ruby');
assert.equal(feat.version, '1');
assert.equal(feat.tag, '1');
assert.equal(feat.path, 'codspace/features/ruby');
});

Expand Down
1 change: 1 addition & 0 deletions src/test/container-features/containerFeaturesOrder.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ describe('Container features install order', function () {
owner: spiltOnSlash[1],
registry: spiltOnSlash[0],
resource: splitOnColon[0],
tag: splitOnColon[1],
version: splitOnColon[1],
path: `${spiltOnSlash[1]}/${spiltOnSlash[2]}/spiltOnSlash[3]`
},
Expand Down
Loading