From e9c2cb0a1951fd2a6057ce843da108128263ce97 Mon Sep 17 00:00:00 2001 From: Miroslav Shubernetskiy Date: Mon, 25 Nov 2024 12:20:45 -0500 Subject: [PATCH] feat: DOCKER_ANNOTATIONS and _IMAGE_ANNOTATIONS keys Docker supports adding OCI-style annotations which are stored per manifest, not per image config. During `docker build`, annotations can be provided via `--annotation` flag. Chalk now reports: * DOCKER_ANNOTATIONS - all `--annotation` flags used in CLI during build * _IMAGE_ANNOTATIONS - all found annotations in the registry for existing image --- src/chalk_common.nim | 1 + src/configs/base_keyspecs.c4m | 21 +++++++++++++++++++++ src/configs/base_plugins.c4m | 4 ++-- src/configs/base_report_templates.c4m | 10 ++++++++++ src/configs/crashoverride.c4m | 2 ++ src/configs/dockercmd.c4m | 1 + src/docker/build.nim | 1 + src/docker/cmdline.nim | 9 +++++++++ src/docker/collect.nim | 4 ++++ src/docker/manifest.nim | 20 +++++++++++++++----- src/util.nim | 8 ++++++++ tests/functional/chalk/runner.py | 6 ++++++ tests/functional/test_docker.py | 25 ++++++++++++++++++++----- tests/functional/utils/docker.py | 16 +++++++++++++++- 14 files changed, 115 insertions(+), 13 deletions(-) diff --git a/src/chalk_common.nim b/src/chalk_common.nim index 4a88c13d..41aa6ae7 100644 --- a/src/chalk_common.nim +++ b/src/chalk_common.nim @@ -376,6 +376,7 @@ type foundFileArg*: string foundContext*: string foundLabels*: OrderedTableRef[string, string] + foundAnnotations*: OrderedTableRef[string, string] foundTags*: seq[DockerImage] foundBuildArgs*: TableRef[string, string] foundPlatforms*: seq[DockerPlatform] diff --git a/src/configs/base_keyspecs.c4m b/src/configs/base_keyspecs.c4m index fd0d17fd..58be76bc 100644 --- a/src/configs/base_keyspecs.c4m +++ b/src/configs/base_keyspecs.c4m @@ -1692,6 +1692,18 @@ Labels added to a docker image during the build process, if any. """ } +keyspec DOCKER_ANNOTATIONS { + kind: ChalkTimeArtifact + type: dict[string, string] + standard: true + system: false + since: "0.4.15" + shortdoc: "Annotations added to docker image during build process" + doc: """ +Labels added to a docker image during the build process, if any. +""" +} + keyspec DOCKER_TAGS { kind: ChalkTimeArtifact type: list[string] @@ -2854,6 +2866,15 @@ Key-value pairs adding metadata to images """ } +keyspec _IMAGE_ANNOTATIONS { + kind: RunTimeArtifact + type: dict[string, string] + standard: true + doc: """ +Key-value pairs adding metadata to image manifests +""" +} + keyspec _IMAGE_STOP_SIGNAL { kind: RunTimeArtifact type: string diff --git a/src/configs/base_plugins.c4m b/src/configs/base_plugins.c4m index 2a50e2d7..5b4dd6e5 100644 --- a/src/configs/base_plugins.c4m +++ b/src/configs/base_plugins.c4m @@ -534,7 +534,7 @@ plugin docker { pre_chalk_keys: ["ARTIFACT_TYPE", "DOCKER_FILE", "DOCKER_FILE_CHALKED", "DOCKERFILE_PATH", "DOCKERFILE_PATH_WITHIN_VCTL", "DOCKER_PLATFORM", - "DOCKER_PLATFORMS", "DOCKER_LABELS", "DOCKER_TAGS", + "DOCKER_PLATFORMS", "DOCKER_LABELS", "DOCKER_ANNOTATIONS", "DOCKER_TAGS", "DOCKER_BASE_IMAGE", "DOCKER_BASE_IMAGE_REPO", "DOCKER_BASE_IMAGE_TAG", "DOCKER_BASE_IMAGE_DIGEST", "DOCKER_CONTEXT", "DOCKER_ADDITIONAL_CONTEXTS", @@ -553,7 +553,7 @@ plugin docker { "_IMAGE_HEALTHCHECK_START_PERIOD", "_IMAGE_HEALTHCHECK_START_INTERVAL", "_IMAGE_HEALTHCHECK_RETRIES", "_IMAGE_MOUNTS", "_IMAGE_WORKINGDIR", "_IMAGE_ENTRYPOINT", "_IMAGE_NETWORK_DISABLED", "_IMAGE_MAC_ADDR", - "_IMAGE_ONBUILD", "_IMAGE_LABELS", "_IMAGE_STOP_SIGNAL", + "_IMAGE_ONBUILD", "_IMAGE_LABELS", "_IMAGE_ANNOTATIONS", "_IMAGE_STOP_SIGNAL", "_IMAGE_STOP_TIMEOUT", "_IMAGE_SHELL", "_INSTANCE_CONTAINER_ID","_INSTANCE_CREATION_DATETIME", "_INSTANCE_ENTRYPOINT_PATH", "_INSTANCE_ENTRYPOINT_ARGS", diff --git a/src/configs/base_report_templates.c4m b/src/configs/base_report_templates.c4m index b6b130c3..16aa1905 100644 --- a/src/configs/base_report_templates.c4m +++ b/src/configs/base_report_templates.c4m @@ -120,6 +120,7 @@ report and subtract from it. key.DOCKER_PLATFORM.use = true key.DOCKER_PLATFORMS.use = true key.DOCKER_LABELS.use = true + key.DOCKER_ANNOTATIONS.use = true key.DOCKER_TAGS.use = true key.DOCKER_TARGET.use = true key.DOCKER_CONTEXT.use = true @@ -196,6 +197,7 @@ report and subtract from it. key._IMAGE_MAC_ADDR.use = true key._IMAGE_ONBUILD.use = true key._IMAGE_LABELS.use = true + key._IMAGE_ANNOTATIONS.use = true key._IMAGE_STOP_SIGNAL.use = true key._IMAGE_STOP_TIMEOUT.use = true key._IMAGE_SHELL.use = true @@ -701,6 +703,7 @@ doc: """ key.DOCKER_PLATFORM.use = true key.DOCKER_PLATFORMS.use = true key.DOCKER_LABELS.use = true + key.DOCKER_ANNOTATIONS.use = true key.DOCKER_TAGS.use = true key.DOCKER_TARGET.use = true key.DOCKER_CONTEXT.use = true @@ -776,6 +779,7 @@ doc: """ key._IMAGE_MAC_ADDR.use = true key._IMAGE_ONBUILD.use = true key._IMAGE_LABELS.use = true + key._IMAGE_ANNOTATIONS.use = true key._IMAGE_STOP_SIGNAL.use = true key._IMAGE_STOP_TIMEOUT.use = true key._IMAGE_SHELL.use = true @@ -1166,6 +1170,7 @@ container. key.DOCKER_PLATFORM.use = true key.DOCKER_PLATFORMS.use = true key.DOCKER_LABELS.use = true + key.DOCKER_ANNOTATIONS.use = true key.DOCKER_TAGS.use = true key.DOCKER_TARGET.use = true key.DOCKER_CONTEXT.use = true @@ -1238,6 +1243,7 @@ container. key._IMAGE_MAC_ADDR.use = true key._IMAGE_ONBUILD.use = true key._IMAGE_LABELS.use = true + key._IMAGE_ANNOTATIONS.use = true key._IMAGE_STOP_SIGNAL.use = true key._IMAGE_STOP_TIMEOUT.use = true key._IMAGE_SHELL.use = true @@ -1631,6 +1637,7 @@ and keep the run-time key. key.DOCKER_PLATFORM.use = true key.DOCKER_PLATFORMS.use = true key.DOCKER_LABELS.use = true + key.DOCKER_ANNOTATIONS.use = true key.DOCKER_TAGS.use = true key.DOCKER_CONTEXT.use = true key.DOCKER_TARGET.use = true @@ -1703,6 +1710,7 @@ and keep the run-time key. key._IMAGE_MAC_ADDR.use = true key._IMAGE_ONBUILD.use = true key._IMAGE_LABELS.use = true + key._IMAGE_ANNOTATIONS.use = true key._IMAGE_STOP_SIGNAL.use = true key._IMAGE_STOP_TIMEOUT.use = true key._IMAGE_SHELL.use = true @@ -1991,6 +1999,7 @@ running insert commands. key.DOCKER_PLATFORM.use = true key.DOCKER_PLATFORMS.use = true key.DOCKER_LABELS.use = true + key.DOCKER_ANNOTATIONS.use = true key.DOCKER_TAGS.use = true key.DOCKER_BASE_IMAGE.use = true key.DOCKER_BASE_IMAGE_REPO.use = false @@ -2075,6 +2084,7 @@ running commands that do NOT insert chalk marks. key.DOCKER_PLATFORM.use = true key.DOCKER_PLATFORMS.use = true key.DOCKER_LABELS.use = true + key.DOCKER_ANNOTATIONS.use = true key.DOCKER_TAGS.use = true key.DOCKER_BASE_IMAGE.use = true key.DOCKER_BASE_IMAGE_REPO.use = true diff --git a/src/configs/crashoverride.c4m b/src/configs/crashoverride.c4m index 8ba0ffff..75844db2 100644 --- a/src/configs/crashoverride.c4m +++ b/src/configs/crashoverride.c4m @@ -235,6 +235,7 @@ This is mostly a copy of insert template however all keys are immutable. ~key.DOCKER_PLATFORM.use = true ~key.DOCKER_PLATFORMS.use = true ~key.DOCKER_LABELS.use = true + ~key.DOCKER_ANNOTATIONS.use = true ~key.DOCKER_TAGS.use = true ~key.DOCKER_TARGET.use = true ~key.DOCKER_CONTEXT.use = true @@ -307,6 +308,7 @@ This is mostly a copy of insert template however all keys are immutable. ~key._IMAGE_MAC_ADDR.use = true ~key._IMAGE_ONBUILD.use = true ~key._IMAGE_LABELS.use = true + ~key._IMAGE_ANNOTATIONS.use = true ~key._IMAGE_STOP_SIGNAL.use = true ~key._IMAGE_STOP_TIMEOUT.use = true ~key._IMAGE_SHELL.use = true diff --git a/src/configs/dockercmd.c4m b/src/configs/dockercmd.c4m index 1933defe..b2353820 100644 --- a/src/configs/dockercmd.c4m +++ b/src/configs/dockercmd.c4m @@ -230,6 +230,7 @@ docker { } flag_multi_arg iidfile { } flag_multi_arg label { } + flag_multi_arg annotation { } flag_yn load { yes_aliases: [] no_aliases: [] diff --git a/src/docker/build.nim b/src/docker/build.nim index 4d2accc6..3041a7a8 100644 --- a/src/docker/build.nim +++ b/src/docker/build.nim @@ -347,6 +347,7 @@ proc collectBeforeChalkTime*(chalk: ChalkObj, ctx: DockerInvocation) = dict.setIfNeeded("DOCKER_PLATFORM", $(chalk.platform.normalize())) dict.setIfNeeded("DOCKER_PLATFORMS", $(ctx.platforms.normalize())) dict.setIfNeeded("DOCKER_LABELS", ctx.foundLabels) + dict.setIfNeeded("DOCKER_ANNOTATIONS", ctx.foundAnnotations) dict.setIfNeeded("DOCKER_TAGS", ctx.foundTags.asRepoTag()) dict.setIfNeeded("DOCKER_BASE_IMAGE", $(base.image)) dict.setIfNeeded("DOCKER_BASE_IMAGE_REPO", base.image.repo) diff --git a/src/docker/cmdline.nim b/src/docker/cmdline.nim index 6182a417..76ea0d2d 100644 --- a/src/docker/cmdline.nim +++ b/src/docker/cmdline.nim @@ -83,6 +83,14 @@ proc extractLabels(ctx: DockerInvocation) = let arr = item.split("=") ctx.foundLabels[arr[0]] = arr[^1] +proc extractAnnotations(ctx: DockerInvocation) = + ctx.foundAnnotations = newOrderedTable[string, string]() + if "annotation" in ctx.processedFlags: + let rawAnnotations = unpack[seq[string]](ctx.processedFlags["annotation"].getValue()) + for item in rawAnnotations: + let arr = item.split("=") + ctx.foundAnnotations[arr[0]] = arr[^1] + proc extractExtraContexts(ctx: DockerInvocation) = ctx.foundExtraContexts = newOrderedTable[string, string]() if "build-contexts" in ctx.processedFlags: @@ -249,6 +257,7 @@ proc extractDockerCommand*(self: DockerInvocation): DockerCmd = self.extractBuildArgs() self.extractTarget() self.extractLabels() + self.extractAnnotations() self.extractExtraContexts() self.extractPlatforms() self.extractTags() diff --git a/src/docker/collect.nim b/src/docker/collect.nim index c6d4b312..738edc46 100644 --- a/src/docker/collect.nim +++ b/src/docker/collect.nim @@ -54,6 +54,7 @@ let dockerImageAutoMap: JsonToChalkKeysMapping = { "Config.MacAddress": ("_IMAGE_MAC_ADDR", identity), "Config.OnBuild": ("_IMAGE_ONBUILD", identity), "Config.Labels": ("_IMAGE_LABELS", identity), + "Config.Annotations": ("_IMAGE_ANNOTATIONS", identity), "Config.StopSignal": ("_IMAGE_STOP_SIGNAL", identity), "Config.StopTimeout": ("_IMAGE_STOP_TIMEOUT", identity), }.toOrderedTable() @@ -200,6 +201,7 @@ proc collectImageFrom(chalk: ChalkObj, arch = caseless{"architecture"}.getStr() variant = caseless{"variant"}.getStr() platform = DockerPlatform(os: os, architecture: arch, variant: variant) + annotations = newJObject() if chalk.name == "": chalk.name = name if chalk.cachedHash == "": @@ -219,6 +221,7 @@ proc collectImageFrom(chalk: ChalkObj, let manifest = fetchImageManifest(repo, platform) imageRepo = manifest.asImageRepo() + annotations.update(manifest.annotations) chalk.repos[repo.repo] = imageRepo + chalk.repos.getOrDefault(repo.repo) except: trace("docker: " & getCurrentExceptionMsg()) @@ -262,6 +265,7 @@ proc collectImageFrom(chalk: ChalkObj, chalk.setIfNeeded("_REPO_DIGESTS", repoDigests) chalk.setIfNeeded("_REPO_LIST_DIGESTS", repoListDigests) chalk.setIfNeeded("_REPO_TAGS", repoTags) + chalk.setIfNeeded("_IMAGE_ANNOTATIONS", annotations.nimJsonToBox()) proc collectProvenance(chalk: ChalkObj) = if not isSubscribedKey("_IMAGE_PROVENANCE"): diff --git a/src/docker/manifest.nim b/src/docker/manifest.nim index 450025d2..68bea2d6 100644 --- a/src/docker/manifest.nim +++ b/src/docker/manifest.nim @@ -8,7 +8,7 @@ ## module for interacting with remote registry docker manifests import std/[httpclient] -import ".."/[chalk_common, config, www_authenticate, semver] +import ".."/[chalk_common, config, www_authenticate, semver, util] import "."/[exe, json, ids, registry] # cache is by repo ref as its normalized in buildx imagetools command @@ -129,6 +129,10 @@ proc mimickLocalConfig(self: DockerManifest) = # config object does not contain size so we add compressed size # for easier metadata collection self.json["compressedSize"] = %(self.image.getCompressedSize()) + if self.image.annotations != nil: + if "config" notin self.json: + self.json["config"] = newJObject() + self.json["config"]["annotations"] = self.image.annotations proc setImageConfig(self: DockerManifest, data: DigestedJson) = if self.kind != DockerManifestType.image: @@ -146,6 +150,10 @@ proc setImageConfig(self: DockerManifest, data: DigestedJson) = ) self.config = config +proc setAnnotations(self: DockerManifest, data: JsonNode): DockerManifest {.discardable.} = + self.annotations = self.annotations.update(data{"annotations"}) + return self + proc setImagePlatform(self: DockerManifest, platform: DockerPlatform) = if self.kind != DockerManifestType.image: raise newException(AssertionDefect, "can only set image platform on image manifests") @@ -168,8 +176,7 @@ proc setImageLayers(self: DockerManifest, data: DigestedJson) = mediaType: layer{"mediaType"}.getStr(), digest: layer{"digest"}.getStr(), size: layer{"size"}.getInt(), - annotations: layer{"annotations"}, - )) + ).setAnnotations(data.json)) proc fetch(self: DockerManifest, fetchConfig = true): DockerManifest {.discardable.} = result = self @@ -186,6 +193,7 @@ proc fetch(self: DockerManifest, fetchConfig = true): DockerManifest {.discardab error("docker: " & getCurrentExceptionMsg()) requestManifestJson(self.asImage()) self.setJson(data) + self.setAnnotations(data.json) self.setImageConfig(data) self.setImageLayers(data) if fetchConfig: @@ -204,6 +212,7 @@ proc fetch(self: DockerManifest, fetchConfig = true): DockerManifest {.discardab error("docker: " & getCurrentExceptionMsg()) requestManifestJson(self.asImage()) self.setJson(data) + self.setAnnotations(data.json) self.mimickLocalConfig() self.configPlatform = DockerPlatform( os: data.json{"os"}.getStr(), @@ -227,6 +236,7 @@ proc newManifest(name: DockerImage, data: DigestedJson, otherNames: seq[DockerIm manifests: @[], ) list.setJson(data) + list.setAnnotations(json) for item in json["manifests"].items(): let platform = item{"platform"} list.manifests.add(DockerManifest( @@ -237,13 +247,12 @@ proc newManifest(name: DockerImage, data: DigestedJson, otherNames: seq[DockerIm mediaType: item{"mediaType"}.getStr(), digest: item{"digest"}.getStr(), size: item{"size"}.getInt(), - annotations: item{"annotations"}, platform: DockerPlatform( os: platform{"os"}.getStr(), architecture: platform{"architecture"}.getStr(), variant: platform{"variant"}.getStr(), ) - )) + ).setAnnotations(item)) return list elif "config" in json and "layers" in json: @@ -255,6 +264,7 @@ proc newManifest(name: DockerImage, data: DigestedJson, otherNames: seq[DockerIm mediaType: json{"mediaType"}.getStr(), ) image.setJson(data) + image.setAnnotations(json) image.setImageConfig(data) image.setImageLayers(data) return image diff --git a/src/util.nim b/src/util.nim index c9df7db3..1783bc35 100644 --- a/src/util.nim +++ b/src/util.nim @@ -579,6 +579,14 @@ proc update*(self: ChalkDict, other: ChalkDict): ChalkDict {.discardable.} = for k, v in other: self[k] = v +proc update*(self: JsonNode, other: JsonNode): JsonNode {.discardable.} = + if self == nil: + return other + if other != nil: + for k, v in other.pairs(): + self[k] = v + return self + proc merge*(self: ChalkDict, other: ChalkDict): ChalkDict {.discardable.} = result = self for k, v in other: diff --git a/tests/functional/chalk/runner.py b/tests/functional/chalk/runner.py index d6b9cd3a..b4d12a30 100644 --- a/tests/functional/chalk/runner.py +++ b/tests/functional/chalk/runner.py @@ -563,6 +563,8 @@ def docker_build( sbom: bool = False, run_docker: bool = True, show_config: bool = False, + labels: Optional[dict[str, str]] = None, + annotations: Optional[dict[str, str]] = None, ) -> tuple[str, ChalkProgram]: cwd = cwd or Path(os.getcwd()) context = context or getattr(dockerfile, "parent", cwd) @@ -585,6 +587,8 @@ def docker_build( secrets=secrets, provenance=provenance, sbom=sbom, + labels=labels, + annotations=annotations, ) with Docker.build_cmd( @@ -601,6 +605,8 @@ def docker_build( buildkit=buildkit, provenance=provenance, sbom=sbom, + labels=labels, + annotations=annotations, ) as (params, stdin): image_hash, result = Docker.with_image_id( self.run( diff --git a/tests/functional/test_docker.py b/tests/functional/test_docker.py index 904952b5..2f91e279 100644 --- a/tests/functional/test_docker.py +++ b/tests/functional/test_docker.py @@ -37,8 +37,6 @@ logger = get_logger() -TEST_LABEL = "CRASH_OVERRIDE_TEST_LABEL" - @pytest.fixture(scope="session", autouse=True) def do_docker_cleanup() -> Iterator[None]: @@ -935,13 +933,30 @@ def test_docker_heartbeat(chalk_copy: Chalk, random_hex: str): def test_docker_labels(chalk: Chalk, random_hex: str): - tag = f"test_image_{random_hex}" + tag = f"{REGISTRY}/test_image_{random_hex}" # build container with env vars - chalk.docker_build( + _, build = chalk.docker_build( + buildx=True, dockerfile=DOCKERFILES / "valid" / "sample_1" / "Dockerfile", tag=tag, config=CONFIGS / "docker_heartbeat.c4m", + labels={"foo": "bar"}, + annotations={"hello": "there"}, + push=True, + ) + + assert build.mark.has( + DOCKER_LABELS={ + "foo": "bar", + "run.crashoverride.hello": MISSING, # only known on host-keys + }, + _IMAGE_LABELS={ + "foo": "bar", + "run.crashoverride.hello": "CRASH_OVERRIDE_TEST_LABEL", + }, + DOCKER_ANNOTATIONS={"hello": "there"}, + _IMAGE_ANNOTATIONS={"hello": "there"}, ) inspected = Docker.inspect(tag) @@ -950,7 +965,7 @@ def test_docker_labels(chalk: Chalk, random_hex: str): docker_configs = inspected[0]["Config"] assert "Labels" in docker_configs labels = docker_configs["Labels"] - assert TEST_LABEL in labels.values() + assert "CRASH_OVERRIDE_TEST_LABEL" in labels.values() @pytest.mark.parametrize( diff --git a/tests/functional/utils/docker.py b/tests/functional/utils/docker.py index 9d2a83c2..2f03bc83 100644 --- a/tests/functional/utils/docker.py +++ b/tests/functional/utils/docker.py @@ -41,6 +41,8 @@ def build_cmd( buildkit: bool = True, provenance: bool = False, sbom: bool = False, + labels: Optional[dict[str, str]] = None, + annotations: Optional[dict[str, str]] = None, ): stdin = b"" tags = tags or [] @@ -51,7 +53,7 @@ def build_cmd( cmd += ["buildx"] buildx = True cmd += ["build"] - if buildx and not platforms: + if buildx and not platforms and not provenance and not sbom: cmd += ["--load"] for t in tags: cmd += ["-t", t] @@ -75,6 +77,14 @@ def build_cmd( cmd += ["--provenance=false"] if sbom: cmd += ["--sbom=true"] + if labels: + for k, v in labels.items(): + cmd += [f"--label={k}={v}"] + if annotations: + if not buildx: + raise ValueError("--annotation only works with buildx") + for k, v in annotations.items(): + cmd += [f"--annotation={k}={v}"] cmd += [str(context or ".")] yield cmd, stdin @@ -97,6 +107,8 @@ def build( provenance: bool = False, sbom: bool = False, env: Optional[dict[str, str]] = None, + labels: Optional[dict[str, str]] = None, + annotations: Optional[dict[str, str]] = None, ) -> tuple[str, Program]: """ run docker build with parameters @@ -115,6 +127,8 @@ def build( buildkit=buildkit, provenance=provenance, sbom=sbom, + labels=labels, + annotations=annotations, ) as (params, stdin): return Docker.with_image_id( run(