feat: adding _IMAGE_LAYERS

Collecting raw layer digests from the registry.
This will allow to match base images by the base layers on the backend.

CHANGELOG.md

@@ -155,6 +155,11 @@
 
   ([#453](https://github.com/crashappsec/chalk/pull/453))
 
+- `_IMAGE_LAYERS` key which collects image layer digests as it is stored
+  in the registry. This should allow to correlate base images by matching
+  layer combinations from other images.
+  ([#456](https://github.com/crashappsec/chalk/pull/456))
+
 ## 0.4.14
 
 **Nov 11, 2024**

src/configs/base_keyspecs.c4m

@@ -2750,6 +2750,16 @@ The layer IDs of the image's root filesystem
 """
 }
 
+keyspec _IMAGE_LAYERS {
+    kind:     RunTimeArtifact
+    type:     list[string]
+    standard: true
+    since:    "0.4.15"
+    doc: """
+Layer digests of the image as they are stored in the registry.
+"""
+}
+
 keyspec _IMAGE_HOSTNAME {
     kind:     RunTimeArtifact
     type:     string

src/configs/base_plugins.c4m

@@ -547,7 +547,7 @@ plugin docker {
         "_IMAGE_PROVENANCE", "_IMAGE_SBOM",
         "_IMAGE_DOCKER_VERSION", "_IMAGE_AUTHOR", "_IMAGE_ARCHITECTURE",
         "_IMAGE_VARIANT", "_IMAGE_OS", "_IMAGE_OS_VERSION", "_IMAGE_SIZE",
-        "_IMAGE_ROOT_FS_TYPE", "_IMAGE_ROOT_FS_LAYERS", "_IMAGE_HOSTNAME",
+        "_IMAGE_ROOT_FS_TYPE", "_IMAGE_ROOT_FS_LAYERS", "_IMAGE_LAYERS", "_IMAGE_HOSTNAME",
         "_IMAGE_DOMAINNAME", "_IMAGE_USER", "_IMAGE_EXPOSED_PORTS",
         "_IMAGE_ENV", "_IMAGE_CMD", "_IMAGE_NAME", "_IMAGE_HEALTHCHECK_TEST",
         "_IMAGE_HEALTHCHECK_INTERVAL", "_IMAGE_HEALTHCHECK_TIMEOUT",

src/configs/base_report_templates.c4m

@@ -182,6 +182,7 @@ report and subtract from it.
   key._IMAGE_SIZE.use                         = true
   key._IMAGE_ROOT_FS_TYPE.use                 = true
   key._IMAGE_ROOT_FS_LAYERS.use               = true
+  key._IMAGE_LAYERS.use                       = true
   key._IMAGE_HOSTNAME.use                     = true
   key._IMAGE_DOMAINNAME.use                   = true
   key._IMAGE_USER.use                         = true
@@ -771,6 +772,7 @@ doc: """
   key._IMAGE_SIZE.use                         = true
   key._IMAGE_ROOT_FS_TYPE.use                 = true
   key._IMAGE_ROOT_FS_LAYERS.use               = true
+  key._IMAGE_LAYERS.use                       = true
   key._IMAGE_HOSTNAME.use                     = true
   key._IMAGE_DOMAINNAME.use                   = true
   key._IMAGE_USER.use                         = true
@@ -1241,6 +1243,7 @@ container.
   key._IMAGE_SIZE.use                         = true
   key._IMAGE_ROOT_FS_TYPE.use                 = true
   key._IMAGE_ROOT_FS_LAYERS.use               = true
+  key._IMAGE_LAYERS.use                       = true
   key._IMAGE_HOSTNAME.use                     = true
   key._IMAGE_DOMAINNAME.use                   = true
   key._IMAGE_USER.use                         = true
@@ -1714,6 +1717,7 @@ and keep the run-time key.
   key._IMAGE_SIZE.use                         = true
   key._IMAGE_ROOT_FS_TYPE.use                 = true
   key._IMAGE_ROOT_FS_LAYERS.use               = true
+  key._IMAGE_LAYERS.use                       = true
   key._IMAGE_HOSTNAME.use                     = true
   key._IMAGE_DOMAINNAME.use                   = true
   key._IMAGE_USER.use                         = true

src/configs/crashoverride.c4m

@@ -294,6 +294,7 @@ This is mostly a copy of insert template however all keys are immutable.
   ~key._IMAGE_SIZE.use                         = true
   ~key._IMAGE_ROOT_FS_TYPE.use                 = true
   ~key._IMAGE_ROOT_FS_LAYERS.use               = true
+  ~key._IMAGE_LAYERS.use                       = true
   ~key._IMAGE_HOSTNAME.use                     = true
   ~key._IMAGE_DOMAINNAME.use                   = true
   ~key._IMAGE_USER.use                         = true

src/docker/collect.nim

@@ -202,6 +202,8 @@ proc collectImageFrom(chalk:    ChalkObj,
     variant            = caseless{"variant"}.getStr()
     platform           = DockerPlatform(os: os, architecture: arch, variant: variant)
     annotations        = newJObject()
+  var
+    layers             = newSeq[string]()
   if chalk.platform != nil and chalk.platform != platform:
     raise newException(ValueError, "platform does not match chalk platform")
   if chalk.name == "":
@@ -225,6 +227,8 @@ proc collectImageFrom(chalk:    ChalkObj,
         imageRepo = manifest.asImageRepo(tag = repo.tag)
       annotations.update(manifest.annotations)
       chalk.repos[repo.repo] = imageRepo + chalk.repos.getOrDefault(repo.repo)
+      for layer in manifest.layers:
+        layers.add(layer.digest)
     except:
       trace("docker: " & getCurrentExceptionMsg())
       continue
@@ -270,6 +274,7 @@ proc collectImageFrom(chalk:    ChalkObj,
     chalk.setIfNeeded("_REPO_DIGESTS",      repoDigests)
     chalk.setIfNeeded("_REPO_LIST_DIGESTS", repoListDigests)
     chalk.setIfNeeded("_REPO_TAGS",         repoTags)
+    chalk.setIfNeeded("_IMAGE_LAYERS",      layers)
     chalk.setIfNeeded("_IMAGE_ANNOTATIONS", annotations.nimJsonToBox())
     chalk.setIfNeeded("COMMIT_ID",          annotations{"org.opencontainers.image.revision"}.getStr())
     let source = annotations{"org.opencontainers.image.source"}.getStr()

tests/functional/test_docker.py

@@ -3,6 +3,7 @@
 # This file is part of Chalk
 # (see https://crashoverride.com/docs/chalk)
 import itertools
+import operator
 import platform
 import re
 import shutil
@@ -29,7 +30,7 @@
     REGISTRY_TLS_INSECURE,
     ROOT,
 )
-from .utils.dict import ANY, MISSING, Contains, IfExists
+from .utils.dict import ANY, MISSING, Contains, IfExists, Length
 from .utils.docker import Docker
 from .utils.git import Git
 from .utils.log import get_logger
@@ -1080,6 +1081,9 @@ def test_build_and_push(
     push_result = build_result
     # if without --push at build time, explicitly push to registry
     if not push:
+        assert build_result.mark.has(
+            _IMAGE_LAYERS=MISSING,
+        )
         push_result = chalk.docker_push(tag, buildkit=buildkit)
 
     image_digest, _ = Docker.with_image_digest(build_result)
@@ -1125,6 +1129,7 @@ def test_build_and_push(
                 name: [image_digest],
             }
         },
+        _IMAGE_LAYERS=Length(1, operator.ge),
     )
 
     pull = chalk.docker_pull(tag)