feat: reporting base image as part of _ARTIFACTS

instead of reporting the chalkmakr nested within the build image
chalkmark, all base image information is now reported as part of
_ARTIFACTS key which will eventually include more artifacts
about which chalk collects metadata for.

CHANGELOG.md

@@ -140,15 +140,20 @@
   ([#452](https://github.com/crashappsec/chalk/pull/452))
 
 - Docker base image keys:
+
+  - `_OP_ARTIFACT_CONTEXT` - what is the context of the artifact.
+    For `docker build` its either `build` or `base`.
   - `DOCKER_BASE_IMAGE_REGISTRY` - just registry of the base image
   - `DOCKER_BASE_IMAGE_NAME` - repo name within the registry
-  - `DOCKER_BASE_IMAGE_CHALK` - base image chalkmark
+  - `DOCKER_BASE_IMAGE_ID` - image id (config digest) of the base image
   - `DOCKER_BASE_IMAGE_METADATA_ID` - id of the base image chalkmark
-  - `DOCKER_BASE_IMAGE_ORIGIN_URI` - origin uri of the base image,
-    either from chalkmark or from OCI annotations
-  - `DOCKER_BASE_IMAGE_COMMIT_ID` - commit id of the base image,
-    either from chalkmark or from OCI annotations
-    ([#453](https://github.com/crashappsec/chalk/pull/453))
+  - `_ARTIFACTS` - similar to `_CHALKS` but reports potentially collected
+    information about non-chalked artifacts such as the base image.
+    If the base image is chalked it can be correlated with the build
+    chalkmark via `METADATA_ID`. Otherwise both artifacts can be linked
+    via the digest or the image id.
+
+  ([#453](https://github.com/crashappsec/chalk/pull/453))
 
 ## 0.4.14
 

src/chalk_common.nim

@@ -120,6 +120,7 @@ type
   CollectionCtx* = ref object
     currentErrorObject*:       Option[ChalkObj]
     allChalks*:                seq[ChalkObj]
+    allArtifacts*:             seq[ChalkObj]
     unmarked*:                 seq[string]
     report*:                   Box
     args*:                     seq[string]

src/collect.nim

@@ -257,7 +257,6 @@ proc collectRunTimeHostInfo*() =
             getCurrentExceptionMsg())
       dumpExOnDebug()
 
-
 # The two below functions are helpers for the artifacts() iterator
 # and the self-extractor (in the case of findChalk anyway).
 proc ignoreArtifact(path: string, regexps: seq[Regex]): bool {.inline.} =

src/configs/base_keyspecs.c4m

@@ -1799,32 +1799,6 @@ If the base image is chalked, its METADATA_ID.
 """
 }
 
-keyspec DOCKER_BASE_IMAGE_ORIGIN_URI {
-    kind:             ChalkTimeArtifact
-    type:             `x
-    standard:         true
-    system:           false
-    since:            "0.4.15"
-    doc: """
-If the base image is chalked, its `ORIGIN_URI`.
-Otherwise origin is attempted to be fetched from
-OCI image annotation `org.opencontainers.image.source`.
-"""
-}
-
-keyspec DOCKER_BASE_IMAGE_COMMIT_ID {
-    kind:             ChalkTimeArtifact
-    type:             `x
-    standard:         true
-    system:           false
-    since:            "0.4.15"
-    doc: """
-If the base image is chalked, its `COMMIT_ID`.
-Otherwise origin is attempted to be fetched from
-OCI image annotation `org.opencontainers.image.revision`.
-"""
-}
-
 keyspec DOCKER_BASE_IMAGE {
     kind:             ChalkTimeArtifact
     type:             string
@@ -1897,6 +1871,18 @@ Note this only specifies the base image of the final build target.
 """
 }
 
+keyspec DOCKER_BASE_IMAGE_ID {
+    kind:             ChalkTimeArtifact
+    type:             string
+    standard:         true
+    system:           false
+    since:            "0.4.15"
+    doc: """
+Docker image id (config digest) of the base image.
+Note this only specifies the base image of the final build target.
+"""
+}
+
 keyspec DOCKER_BASE_IMAGES {
     kind:             ChalkTimeArtifact
     type:             dict[string, `x]
@@ -2123,6 +2109,22 @@ these at insertion time.
 """
 }
 
+keyspec _OP_ARTIFACT_CONTEXT {
+    kind:             RunTimeArtifact
+    type:             string
+    standard:         true
+    codec:            true
+    since:            "0.4.15"
+    shortdoc:         "Context of the artifact in the report"
+    doc : """
+Context of the artifact how it is referenced in the report.
+For example during a docker build, one chalkmark can be the final
+built image however there can also be a base image chalkmark.
+This key allows to differentiate in what context the same artifact
+is being referenced.
+"""
+}
+
 keyspec _OP_ARTIFACT_PATH {
     kind:             RunTimeArtifact
     type:             string
@@ -4380,6 +4382,19 @@ report will NOT output chalks.
 """
 }
 
+keyspec _ARTIFACTS {
+    kind:             RunTimeHost
+    type:             string
+    standard:         true
+    system:           true
+    since:            "0.4.15"
+    normalized_order: 7
+    shortdoc:         "Contains software artifact information for the current operation"
+    doc:              """
+Similar to _CHALKS but can report metadata non-chalked artifacts.
+"""
+}
+
 keyspec _OP_CHALK_COUNT {
     kind:             RunTimeHost
     type:             int

src/configs/base_report_templates.c4m

@@ -127,19 +127,18 @@ report and subtract from it.
   key.DOCKER_ADDITIONAL_CONTEXTS.use          = true
   key.DOCKER_CHALK_ADDED_LABELS.use           = true
   key.DOCKER_CHALK_ADDED_TO_DOCKERFILE.use    = true
-  key.DOCKER_BASE_IMAGE_CHALK.use             = true
   key.DOCKER_BASE_IMAGE_METADATA_ID.use       = true
-  key.DOCKER_BASE_IMAGE_COMMIT_ID.use         = true
-  key.DOCKER_BASE_IMAGE_ORIGIN_URI.use        = true
   key.DOCKER_BASE_IMAGE.use                   = true
   key.DOCKER_BASE_IMAGE_REPO.use              = true
   key.DOCKER_BASE_IMAGE_REGISTRY.use          = true
   key.DOCKER_BASE_IMAGE_NAME.use              = true
   key.DOCKER_BASE_IMAGE_TAG.use               = true
   key.DOCKER_BASE_IMAGE_DIGEST.use            = true
+  key.DOCKER_BASE_IMAGE_ID.use                = true
   key.DOCKER_BASE_IMAGES.use                  = true
   key.DOCKER_COPY_IMAGES.use                  = true
   key._OP_ARTIFACT_TYPE.use                   = true
+  key._OP_ARTIFACT_CONTEXT.use                = true
   key._OP_ARTIFACT_PATH.use                   = true
   key._CURRENT_HASH.use                       = true
   key._VALIDATED_METADATA.use                 = true
@@ -351,6 +350,7 @@ report and subtract from it.
   key._TZ_OFFSET.use                          = true
   key._DATETIME.use                           = true
   key._CHALKS.use                             = true
+  key._ARTIFACTS.use                          = true
   key._OP_CHALK_COUNT.use                     = true
   key._OP_UNMARKED_COUNT.use                  = true
   key._OP_CMD_FLAGS.use                       = true
@@ -506,6 +506,7 @@ doc: """
   key._TZ_OFFSET.use                          = true
   key._DATETIME.use                           = true
   key._CHALKS.use                             = true
+  key._ARTIFACTS.use                          = true
   key._OP_CHALK_COUNT.use                     = true
   key._OP_UNMARKED_COUNT.use                  = true
   key._OP_CMD_FLAGS.use                       = true
@@ -716,19 +717,18 @@ doc: """
   key.DOCKER_ADDITIONAL_CONTEXTS.use          = true
   key.DOCKER_CHALK_ADDED_LABELS.use           = true
   key.DOCKER_CHALK_ADDED_TO_DOCKERFILE.use    = true
-  key.DOCKER_BASE_IMAGE_CHALK.use             = true
   key.DOCKER_BASE_IMAGE_METADATA_ID.use       = true
-  key.DOCKER_BASE_IMAGE_COMMIT_ID.use         = true
-  key.DOCKER_BASE_IMAGE_ORIGIN_URI.use        = true
   key.DOCKER_BASE_IMAGE.use                   = true
   key.DOCKER_BASE_IMAGE_REPO.use              = true
   key.DOCKER_BASE_IMAGE_REGISTRY.use          = true
   key.DOCKER_BASE_IMAGE_NAME.use              = true
   key.DOCKER_BASE_IMAGE_TAG.use               = true
   key.DOCKER_BASE_IMAGE_DIGEST.use            = true
+  key.DOCKER_BASE_IMAGE_ID.use                = true
   key.DOCKER_BASE_IMAGES.use                  = true
   key.DOCKER_COPY_IMAGES.use                  = true
   key._OP_ARTIFACT_TYPE.use                   = true
+  key._OP_ARTIFACT_CONTEXT.use                = true
   key._OP_ARTIFACT_PATH.use                   = true
   key._CURRENT_HASH.use                       = true
   key._VIRTUAL.use                            = true
@@ -1003,6 +1003,7 @@ container.
   key._TZ_OFFSET.use                          = false
   key._DATETIME.use                           = true
   key._CHALKS.use                             = true
+  key._ARTIFACTS.use                          = true
   key._OP_CHALK_COUNT.use                     = true
   key._OP_UNMARKED_COUNT.use                  = true
   key._OP_CMD_FLAGS.use                       = true
@@ -1189,19 +1190,18 @@ container.
   key.DOCKER_ADDITIONAL_CONTEXTS.use          = true
   key.DOCKER_CHALK_ADDED_LABELS.use           = true
   key.DOCKER_CHALK_ADDED_TO_DOCKERFILE.use    = true
-  key.DOCKER_BASE_IMAGE_CHALK.use             = true
   key.DOCKER_BASE_IMAGE_METADATA_ID.use       = true
-  key.DOCKER_BASE_IMAGE_COMMIT_ID.use         = true
-  key.DOCKER_BASE_IMAGE_ORIGIN_URI.use        = true
   key.DOCKER_BASE_IMAGE.use                   = true
   key.DOCKER_BASE_IMAGE_REPO.use              = true
   key.DOCKER_BASE_IMAGE_REGISTRY.use          = true
   key.DOCKER_BASE_IMAGE_NAME.use              = true
   key.DOCKER_BASE_IMAGE_TAG.use               = true
   key.DOCKER_BASE_IMAGE_DIGEST.use            = true
+  key.DOCKER_BASE_IMAGE_ID.use                = true
   key.DOCKER_BASE_IMAGES.use                  = true
   key.DOCKER_COPY_IMAGES.use                  = true
   key._OP_ARTIFACT_TYPE.use                   = true
+  key._OP_ARTIFACT_CONTEXT.use                = true
   key._OP_ARTIFACT_PATH.use                   = true
   key._CURRENT_HASH.use                       = true
   key._VIRTUAL.use                            = true
@@ -1476,6 +1476,7 @@ and keep the run-time key.
   key._TZ_OFFSET.use                          = false
   key._DATETIME.use                           = true
   key._CHALKS.use                             = true
+  key._ARTIFACTS.use                          = true
   key._OP_CHALK_COUNT.use                     = true
   key._OP_UNMARKED_COUNT.use                  = true
   key._OP_CMD_FLAGS.use                       = true
@@ -1662,19 +1663,18 @@ and keep the run-time key.
   key.DOCKER_ADDITIONAL_CONTEXTS.use          = true
   key.DOCKER_CHALK_ADDED_LABELS.use           = true
   key.DOCKER_CHALK_ADDED_TO_DOCKERFILE.use    = true
-  key.DOCKER_BASE_IMAGE_CHALK.use             = true
   key.DOCKER_BASE_IMAGE_METADATA_ID.use       = true
-  key.DOCKER_BASE_IMAGE_COMMIT_ID.use         = true
-  key.DOCKER_BASE_IMAGE_ORIGIN_URI.use        = true
   key.DOCKER_BASE_IMAGE.use                   = true
   key.DOCKER_BASE_IMAGE_REPO.use              = true
   key.DOCKER_BASE_IMAGE_REGISTRY.use          = true
   key.DOCKER_BASE_IMAGE_NAME.use              = true
   key.DOCKER_BASE_IMAGE_TAG.use               = true
   key.DOCKER_BASE_IMAGE_DIGEST.use            = true
+  key.DOCKER_BASE_IMAGE_ID.use                = true
   key.DOCKER_BASE_IMAGES.use                  = true
   key.DOCKER_COPY_IMAGES.use                  = true
   key._OP_ARTIFACT_TYPE.use                   = true
+  key._OP_ARTIFACT_CONTEXT.use                = true
   key._OP_ARTIFACT_PATH.use                   = true
   key._CURRENT_HASH.use                       = true
   key._VIRTUAL.use                            = true
@@ -1952,6 +1952,7 @@ minimal.
   ~key.METADATA_ID.use                        = true
   ~key.CHALK_ID.use                           = true
   ~key._CHALKS.use                            = true
+  key._ARTIFACTS.use                          = true
   ~key._OPERATION.use                         = true
   ~key._TIMESTAMP.use                         = true
   ~key._DATETIME.use                          = true
@@ -1969,6 +1970,7 @@ running insert commands.
   key._OPERATION.use                          = true
   key._DATETIME.use                           = true
   key._CHALKS.use                             = true
+  key._ARTIFACTS.use                          = true
   key._OP_ARGV.use                            = true
   key._OP_CHALKER_VERSION.use                 = true
   key._OP_CHALK_COUNT.use                     = true
@@ -2025,16 +2027,14 @@ running insert commands.
   key.DOCKER_LABELS.use                       = true
   key.DOCKER_ANNOTATIONS.use                  = true
   key.DOCKER_TAGS.use                         = true
-  key.DOCKER_BASE_IMAGE_CHALK.use             = true
   key.DOCKER_BASE_IMAGE_METADATA_ID.use       = true
-  key.DOCKER_BASE_IMAGE_COMMIT_ID.use         = true
-  key.DOCKER_BASE_IMAGE_ORIGIN_URI.use        = true
   key.DOCKER_BASE_IMAGE.use                   = true
   key.DOCKER_BASE_IMAGE_REPO.use              = false
   key.DOCKER_BASE_IMAGE_REGISTRY.use          = true
   key.DOCKER_BASE_IMAGE_NAME.use              = true
   key.DOCKER_BASE_IMAGE_TAG.use               = false
   key.DOCKER_BASE_IMAGE_DIGEST.use            = false
+  key.DOCKER_BASE_IMAGE_ID.use                = false
   key.DOCKER_BASE_IMAGES.use                  = false
   key.DOCKER_COPY_IMAGES.use                  = false
   key.INFERRED_TECH_STACKS.use                = true
@@ -2061,6 +2061,7 @@ running commands that do NOT insert chalk marks.
   key._OPERATION.use                          = true
   key._DATETIME.use                           = true
   key._CHALKS.use                             = true
+  key._ARTIFACTS.use                          = true
   key._OP_ARGV.use                            = true
   key._OP_CHALKER_VERSION.use                 = true
   key._OP_CHALK_COUNT.use                     = true
@@ -2116,16 +2117,14 @@ running commands that do NOT insert chalk marks.
   key.DOCKER_LABELS.use                       = true
   key.DOCKER_ANNOTATIONS.use                  = true
   key.DOCKER_TAGS.use                         = true
-  key.DOCKER_BASE_IMAGE_CHALK.use             = true
   key.DOCKER_BASE_IMAGE_METADATA_ID.use       = true
-  key.DOCKER_BASE_IMAGE_COMMIT_ID.use         = true
-  key.DOCKER_BASE_IMAGE_ORIGIN_URI.use        = true
   key.DOCKER_BASE_IMAGE.use                   = true
   key.DOCKER_BASE_IMAGE_REPO.use              = true
   key.DOCKER_BASE_IMAGE_REGISTRY.use          = true
   key.DOCKER_BASE_IMAGE_NAME.use              = true
   key.DOCKER_BASE_IMAGE_TAG.use               = true
   key.DOCKER_BASE_IMAGE_DIGEST.use            = true
+  key.DOCKER_BASE_IMAGE_ID.use                = true
   key.DOCKER_BASE_IMAGES.use                  = true
   key.DOCKER_COPY_IMAGES.use                  = true
   key.METADATA_ID.use                         = true
@@ -2134,6 +2133,7 @@ running commands that do NOT insert chalk marks.
   key._VALIDATED_SIGNATURE.use                = true
   key._CURRENT_HASH.use                       = true
   key._OP_ARTIFACT_TYPE.use                   = true
+  key._OP_ARTIFACT_CONTEXT.use                = true
   key._OP_ARTIFACT_PATH.use                   = true
   key._CURRENT_HASH.use                       = true
   key._IMAGE_ID.use                           = true

src/configs/crashoverride.c4m

@@ -56,6 +56,7 @@ This is mostly a copy of insert template however all keys are immutable.
   ~key._TZ_OFFSET.use                                 = false
   ~key._DATETIME.use                                  = true
   ~key._CHALKS.use                                    = true
+  ~key._ARTIFACTS.use                                 = true
   ~key._OP_CHALK_COUNT.use                            = true
   ~key._OP_UNMARKED_COUNT.use                         = true
   ~key._OP_CMD_FLAGS.use                              = true
@@ -242,19 +243,18 @@ This is mostly a copy of insert template however all keys are immutable.
   ~key.DOCKER_ADDITIONAL_CONTEXTS.use          = true
   ~key.DOCKER_CHALK_ADDED_LABELS.use           = true
   ~key.DOCKER_CHALK_ADDED_TO_DOCKERFILE.use    = true
-  ~key.DOCKER_BASE_IMAGE_CHALK.use             = true
   ~key.DOCKER_BASE_IMAGE_METADATA_ID.use       = true
-  ~key.DOCKER_BASE_IMAGE_COMMIT_ID.use         = true
-  ~key.DOCKER_BASE_IMAGE_ORIGIN_URI.use        = true
   ~key.DOCKER_BASE_IMAGE.use                   = true
   ~key.DOCKER_BASE_IMAGE_REPO.use              = true
   ~key.DOCKER_BASE_IMAGE_REGISTRY.use          = true
   ~key.DOCKER_BASE_IMAGE_NAME.use              = true
   ~key.DOCKER_BASE_IMAGE_TAG.use               = true
   ~key.DOCKER_BASE_IMAGE_DIGEST.use            = true
+  ~key.DOCKER_BASE_IMAGE_ID.use                = true
   ~key.DOCKER_BASE_IMAGES.use                  = true
   ~key.DOCKER_COPY_IMAGES.use                  = true
   ~key._OP_ARTIFACT_TYPE.use                   = true
+  ~key._OP_ARTIFACT_CONTEXT.use                = true
   ~key._OP_ARTIFACT_PATH.use                   = true
   ~key._CURRENT_HASH.use                       = true
   ~key._VIRTUAL.use                            = true

src/docker/build.nim

@@ -349,16 +349,15 @@ proc collectBaseImage(chalk: ChalkObj, baseSection: DockerFileSection) =
           baseChalk.collectedData
         else:
           ChalkDict()
+    baseChalk.addToAllArtifacts()
+    baseChalk.collectedData["_OP_ARTIFACT_CONTEXT"] = pack("base")
     chalk.baseChalk = baseChalk
-    if baseChalk.marked:
-      dict.setIfNeeded("DOCKER_BASE_IMAGE_CHALK", baseChalk.extract)
+    if baseChalk.isMarked():
+      dict.setIfNeeded("DOCKER_BASE_IMAGE_METADATA_ID", baseChalk.extract["METADATA_ID"])
     else:
       trace("docker: base image is not chalked " & $baseSection.image)
-    for k, v in collected:
-      let suffix = k.strip(chars = {'_'})
-      dict.setIfNeeded("DOCKER_BASE_IMAGE_" & suffix, baseChalk.collectedData[k])
-      # some keys already start with IMAGE such as _IMAGE_ANNOTATIONS
-      dict.setIfNeeded("DOCKER_BASE_" & suffix, baseChalk.collectedData[k])
+    if "_IMAGE_ID" in baseChalk.collectedData:
+      dict.setIfNeeded("DOCKER_BASE_IMAGE_ID", baseChalk.collectedData["_IMAGE_ID"])
   except:
     trace("docker: unable to scan base image due to: " & getCurrentExceptionMsg())
 
@@ -581,6 +580,7 @@ proc dockerBuild*(ctx: DockerInvocation): int =
   trace("docker: collecting post-build runtime data")
   for _, chalk in chalksByPlatform:
     chalk.addToAllChalks()
+    chalk.collectedData["_OP_ARTIFACT_CONTEXT"] = pack("build")
     chalk.collectRunTimeArtifactInfo()
     chalk.marked = true
   collectRunTimeHostInfo()

src/docker/push.nim

@@ -33,6 +33,7 @@ proc dockerPush*(ctx: DockerInvocation): int =
 
   initCollection()
   chalk.addToAllChalks()
+  chalk.collectedData["_OP_ARTIFACT_CONTEXT"] = pack("push")
   chalk.collectChalkTimeArtifactInfo()
 
   result = setExitCode(ctx.runMungedDockerInvocation())