From 4f2dbe19de1a9674b0bc73dc21949f4487e9c2fa Mon Sep 17 00:00:00 2001 From: Miroslav Shubernetskiy Date: Wed, 4 Dec 2024 15:34:20 -0500 Subject: [PATCH] feat: adding _IMAGE_LAYERS Collecting raw layer digests from the registry. This will allow to match base images by the base layers on the backend. --- CHANGELOG.md | 5 +++++ src/configs/base_keyspecs.c4m | 10 ++++++++++ src/configs/base_plugins.c4m | 2 +- src/configs/base_report_templates.c4m | 4 ++++ src/configs/crashoverride.c4m | 1 + src/docker/collect.nim | 5 +++++ tests/functional/test_docker.py | 7 ++++++- 7 files changed, 32 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f627ea0..2c80f374 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -155,6 +155,11 @@ ([#453](https://github.com/crashappsec/chalk/pull/453)) +- `_IMAGE_LAYERS` key which collects image layer digests as it is stored + in the registry. This should allow to correlate base images by matching + layer combinations from other images. + ([#456](https://github.com/crashappsec/chalk/pull/456)) + ## 0.4.14 **Nov 11, 2024** diff --git a/src/configs/base_keyspecs.c4m b/src/configs/base_keyspecs.c4m index 3f4c19e7..c1b47e2e 100644 --- a/src/configs/base_keyspecs.c4m +++ b/src/configs/base_keyspecs.c4m @@ -2750,6 +2750,16 @@ The layer IDs of the image's root filesystem """ } +keyspec _IMAGE_LAYERS { + kind: RunTimeArtifact + type: list[string] + standard: true + since: "0.4.15" + doc: """ +Layer digests of the image as they are stored in the registry. +""" +} + keyspec _IMAGE_HOSTNAME { kind: RunTimeArtifact type: string diff --git a/src/configs/base_plugins.c4m b/src/configs/base_plugins.c4m index bc35bd8e..8dfabfb3 100644 --- a/src/configs/base_plugins.c4m +++ b/src/configs/base_plugins.c4m @@ -547,7 +547,7 @@ plugin docker { "_IMAGE_PROVENANCE", "_IMAGE_SBOM", "_IMAGE_DOCKER_VERSION", "_IMAGE_AUTHOR", "_IMAGE_ARCHITECTURE", "_IMAGE_VARIANT", "_IMAGE_OS", "_IMAGE_OS_VERSION", "_IMAGE_SIZE", - "_IMAGE_ROOT_FS_TYPE", "_IMAGE_ROOT_FS_LAYERS", "_IMAGE_HOSTNAME", + "_IMAGE_ROOT_FS_TYPE", "_IMAGE_ROOT_FS_LAYERS", "_IMAGE_LAYERS", "_IMAGE_HOSTNAME", "_IMAGE_DOMAINNAME", "_IMAGE_USER", "_IMAGE_EXPOSED_PORTS", "_IMAGE_ENV", "_IMAGE_CMD", "_IMAGE_NAME", "_IMAGE_HEALTHCHECK_TEST", "_IMAGE_HEALTHCHECK_INTERVAL", "_IMAGE_HEALTHCHECK_TIMEOUT", diff --git a/src/configs/base_report_templates.c4m b/src/configs/base_report_templates.c4m index e1cc48f2..e6cce3b9 100644 --- a/src/configs/base_report_templates.c4m +++ b/src/configs/base_report_templates.c4m @@ -182,6 +182,7 @@ report and subtract from it. key._IMAGE_SIZE.use = true key._IMAGE_ROOT_FS_TYPE.use = true key._IMAGE_ROOT_FS_LAYERS.use = true + key._IMAGE_LAYERS.use = true key._IMAGE_HOSTNAME.use = true key._IMAGE_DOMAINNAME.use = true key._IMAGE_USER.use = true @@ -771,6 +772,7 @@ doc: """ key._IMAGE_SIZE.use = true key._IMAGE_ROOT_FS_TYPE.use = true key._IMAGE_ROOT_FS_LAYERS.use = true + key._IMAGE_LAYERS.use = true key._IMAGE_HOSTNAME.use = true key._IMAGE_DOMAINNAME.use = true key._IMAGE_USER.use = true @@ -1241,6 +1243,7 @@ container. key._IMAGE_SIZE.use = true key._IMAGE_ROOT_FS_TYPE.use = true key._IMAGE_ROOT_FS_LAYERS.use = true + key._IMAGE_LAYERS.use = true key._IMAGE_HOSTNAME.use = true key._IMAGE_DOMAINNAME.use = true key._IMAGE_USER.use = true @@ -1714,6 +1717,7 @@ and keep the run-time key. key._IMAGE_SIZE.use = true key._IMAGE_ROOT_FS_TYPE.use = true key._IMAGE_ROOT_FS_LAYERS.use = true + key._IMAGE_LAYERS.use = true key._IMAGE_HOSTNAME.use = true key._IMAGE_DOMAINNAME.use = true key._IMAGE_USER.use = true diff --git a/src/configs/crashoverride.c4m b/src/configs/crashoverride.c4m index 91504459..2fff27de 100644 --- a/src/configs/crashoverride.c4m +++ b/src/configs/crashoverride.c4m @@ -294,6 +294,7 @@ This is mostly a copy of insert template however all keys are immutable. ~key._IMAGE_SIZE.use = true ~key._IMAGE_ROOT_FS_TYPE.use = true ~key._IMAGE_ROOT_FS_LAYERS.use = true + ~key._IMAGE_LAYERS.use = true ~key._IMAGE_HOSTNAME.use = true ~key._IMAGE_DOMAINNAME.use = true ~key._IMAGE_USER.use = true diff --git a/src/docker/collect.nim b/src/docker/collect.nim index a6ee8bdf..cef27bfd 100644 --- a/src/docker/collect.nim +++ b/src/docker/collect.nim @@ -202,6 +202,8 @@ proc collectImageFrom(chalk: ChalkObj, variant = caseless{"variant"}.getStr() platform = DockerPlatform(os: os, architecture: arch, variant: variant) annotations = newJObject() + var + layers = newSeq[string]() if chalk.platform != nil and chalk.platform != platform: raise newException(ValueError, "platform does not match chalk platform") if chalk.name == "": @@ -225,6 +227,8 @@ proc collectImageFrom(chalk: ChalkObj, imageRepo = manifest.asImageRepo(tag = repo.tag) annotations.update(manifest.annotations) chalk.repos[repo.repo] = imageRepo + chalk.repos.getOrDefault(repo.repo) + for layer in manifest.layers: + layers.add(layer.digest) except: trace("docker: " & getCurrentExceptionMsg()) continue @@ -270,6 +274,7 @@ proc collectImageFrom(chalk: ChalkObj, chalk.setIfNeeded("_REPO_DIGESTS", repoDigests) chalk.setIfNeeded("_REPO_LIST_DIGESTS", repoListDigests) chalk.setIfNeeded("_REPO_TAGS", repoTags) + chalk.setIfNeeded("_IMAGE_LAYERS", layers) chalk.setIfNeeded("_IMAGE_ANNOTATIONS", annotations.nimJsonToBox()) chalk.setIfNeeded("COMMIT_ID", annotations{"org.opencontainers.image.revision"}.getStr()) let source = annotations{"org.opencontainers.image.source"}.getStr() diff --git a/tests/functional/test_docker.py b/tests/functional/test_docker.py index eb51634e..04498947 100644 --- a/tests/functional/test_docker.py +++ b/tests/functional/test_docker.py @@ -3,6 +3,7 @@ # This file is part of Chalk # (see https://crashoverride.com/docs/chalk) import itertools +import operator import platform import re import shutil @@ -29,7 +30,7 @@ REGISTRY_TLS_INSECURE, ROOT, ) -from .utils.dict import ANY, MISSING, Contains, IfExists +from .utils.dict import ANY, MISSING, Contains, IfExists, Length from .utils.docker import Docker from .utils.git import Git from .utils.log import get_logger @@ -1080,6 +1081,9 @@ def test_build_and_push( push_result = build_result # if without --push at build time, explicitly push to registry if not push: + assert build_result.mark.has( + _IMAGE_LAYERS=MISSING, + ) push_result = chalk.docker_push(tag, buildkit=buildkit) image_digest, _ = Docker.with_image_digest(build_result) @@ -1125,6 +1129,7 @@ def test_build_and_push( name: [image_digest], } }, + _IMAGE_LAYERS=Length(1, operator.ge), ) pull = chalk.docker_pull(tag)