test: removing validate.py in favor of in-line runner validations (#441)

* test: removing validate.py in favor of in-line runner validations

Moving most of the functionality into the runner utils (e.g. insert,
extract). It checks common keys are present such as `_OPERATION` for
each wrapped command and other bits such as if its not virtual chalking,
virtual chalk json should be absent. As this is validated in each
invocation of the runner, we dont need to separately call additional
validation utilities in tests. Only thing each test needs to assert is
specific keys (if any) from either report/chalkmark. To make that
simpler, added some utility properties like `marks_by_path` which
group all chalkmarks by the path of the artifacts hence allowing
to assert that all expected files were chalked as expected.

* build: fixing buildx 0.18 compatibility in compose file

Makefile

@@ -131,11 +131,15 @@ ifneq "$(shell which systemctl 2> /dev/null)" ""
 		|| echo Please restart docker daemon after changing docker config
 endif
 
+$(HOME)/.pdbrc.py:
+	touch $@
+
 .PHONY: docker-setup
 docker-setup: /etc/docker/daemon.json
 
 .PHONY: tests
 tests: DOCKER=$(_DOCKER) # force rebuilds to use docker to match tests
+tests: $(HOME)/.pdbrc.py
 tests: docker-setup
 tests: $(BINARY) # note this will rebuild chalk if necessary
 	docker compose run --rm tests $(make_args) $(args)

docker-compose.yml

@@ -27,7 +27,7 @@ services:
   # --------------------------------------------------------------------------
   # SERVER
 
-  server: &server
+  server:
     build:
       context: ./server
       target: deps
@@ -50,8 +50,13 @@ services:
       interval: 1s
 
   server-tls:
-    <<: *server
+    build:
+      context: ./server
+      target: deps
     command: run -r -p 5858 --domain=tls.chalk.local --keyfile=cert.key --certfile=cert.pem --use-existing-cert
+    working_dir: /chalk/server
+    volumes:
+      - .:/chalk
     ports:
       - 5858:5858
     networks:
@@ -122,6 +127,7 @@ services:
       - seccomp=unconfined # for gdb
     volumes:
       - $PWD:$PWD
+      - $HOME/.pdbrc.py:/root/.pdbrc.py
       - /var/run/docker.sock:/var/run/docker.sock
       - /etc/buildkit:/etc/buildkit
       - /etc/docker:/etc/docker

tests/functional/chalk/runner.py

@@ -11,7 +11,7 @@
 
 from ..conf import MAGIC
 from ..utils.bin import sha256
-from ..utils.dict import ContainsMixin
+from ..utils.dict import ContainsMixin, MISSING, ANY, IfExists
 from ..utils.docker import Docker
 from ..utils.log import get_logger
 from ..utils.os import CalledProcessError, Program, run
@@ -37,17 +37,58 @@
 logger = get_logger()
 
 
+def artifact_type(path: Path) -> str:
+    if path.suffix == ".py":
+        return "python"
+    elif path.suffix == ".zip":
+        return "ZIP"
+    else:
+        return "ELF"
+
+
 class ChalkReport(ContainsMixin, dict):
     name = "report"
 
     def __init__(self, report: dict[str, Any]):
         super().__init__(**report)
 
+    def deterministic(self, ignore: Optional[set[str]] = None):
+        return self.__class__(
+            {
+                k: v
+                for k, v in self.items()
+                if k
+                not in {
+                    "_TIMESTAMP",
+                    "_DATETIME",
+                    "_ACTION_ID",
+                    "_ARGV",
+                    "_OP_ARGV",
+                    "_EXEC_ID",
+                    # docker does not have deterministic output
+                    # insecure registries are not consistently ordered
+                    "_DOCKER_INFO",
+                }
+                | (ignore or set())
+            }
+        )
+
     @property
     def marks(self):
         assert len(self["_CHALKS"]) > 0
         return [ChalkMark(i, report=self) for i in self["_CHALKS"]]
 
+    @property
+    def marks_by_path(self):
+        return ContainsMixin(
+            {
+                i.get("PATH_WHEN_CHALKED", i.get("_OP_ARTIFACT_PATH")): i
+                for i in self.marks
+                # paths can be missing for example in minimum report profile
+                if "PATH_WHEN_CHALKED" in i or "_OP_ARTIFACT_PATH" in i
+            }
+        )
+
     @property
     def mark(self):
         assert len(self.marks) == 1
@@ -179,6 +220,11 @@ def report(self):
         assert len(self.reports) == 1
         return self.reports[0]
 
+    @property
+    def first_report(self):
+        assert len(self.reports) > 0
+        return self.reports[0]
+
     @property
     def mark(self):
         return self.report.mark
@@ -187,6 +233,27 @@ def mark(self):
     def marks(self):
         return self.report.marks
 
+    @property
+    def marks_by_path(self):
+        return self.report.marks_by_path
+
+    @property
+    def virtual_path(self):
+        return Path.cwd() / "virtual-chalk.json"
+
+    @property
+    def vmarks(self):
+        assert self.virtual_path.exists()
+        return [
+            ChalkMark.from_json(i) for i in self.virtual_path.read_text().splitlines()
+        ]
+
+    @property
+    def vmark(self):
+        marks = self.vmarks
+        assert len(marks) == 1
+        return marks[0]
+
 
 class Chalk:
     def __init__(
@@ -284,22 +351,22 @@ def run(
 
         # if chalk outputs report, sanity check its operation matches chalk_cmd
         if expecting_report:
-            try:
-                report = result.report
-            except Exception:
-                pass
-            else:
-                # report could be silenced on the profile level
-                if report:
-                    operation = cast(str, command)
-                    # when calling docker, the arg after docker is the operation
-                    if not operation and "docker" in params:
-                        try:
-                            operation = params[params.index("buildx") + 1]
-                        except ValueError:
-                            operation = params[params.index("docker") + 1]
-                    if operation:
-                        assert report.has(_OPERATION=operation)
+            report = result.first_report
+            operation = cast(str, command)
+            # when calling docker, the arg after docker is the operation
+            if not operation and "docker" in params:
+                try:
+                    operation = params[params.index("buildx") + 1]
+                except ValueError:
+                    operation = params[params.index("docker") + 1]
+            if operation:
+                assert report.has(_OPERATION=IfExists(operation))
+                if "_CHALKS" in report:
+                    for mark in report.marks:
+                        assert mark.has_if(
+                            operation in {"insert", "build"},
+                            _VIRTUAL=IfExists(virtual),
+                        )
 
         return result
 
@@ -313,16 +380,38 @@ def insert(
         log_level: ChalkLogLevel = "trace",
         env: Optional[dict[str, str]] = None,
         ignore_errors: bool = False,
+        expecting_report: bool = True,
+        expecting_chalkmarks: bool = True,
     ) -> ChalkProgram:
-        return self.run(
+        result = self.run(
             command="insert",
             target=artifact,
             config=config,
             virtual=virtual,
             log_level=log_level,
             env=env,
             ignore_errors=ignore_errors,
+            expecting_report=expecting_report,
         )
+        if expecting_report:
+            if expecting_chalkmarks:
+                for chalk in result.marks:
+                    assert chalk.has(_VIRTUAL=IfExists(virtual))
+                if virtual:
+                    assert result.virtual_path.exists()
+                    for mark in result.vmarks:
+                        assert mark.has(
+                            CHALK_ID=ANY,
+                            MAGIC=MAGIC,
+                        )
+            else:
+                assert result.report.has(
+                    _CHALKS=MISSING,
+                    _UNMARKED=IfExists(ANY),
+                )
+        if not virtual:
+            assert not result.virtual_path.exists()
+        return result
 
     def extract(
         self,
@@ -332,8 +421,10 @@ def extract(
         config: Optional[Path] = None,
         log_level: ChalkLogLevel = "trace",
         env: Optional[dict[str, str]] = None,
+        virtual: bool = False,
+        expecting_chalkmarks: bool = True,
     ) -> ChalkProgram:
-        return self.run(
+        result = self.run(
             command="extract",
             target=artifact,
             log_level=log_level,
@@ -342,6 +433,22 @@ def extract(
             config=config,
             env=env,
         )
+        if virtual:
+            assert result.report.has(
+                _CHALKS=MISSING,
+                _UNMARKED=IfExists(ANY),
+            )
+        else:
+            if Path(artifact).exists() and expecting_chalkmarks:
+                for path, chalk in result.marks_by_path.items():
+                    assert chalk.has(
+                        ARTIFACT_TYPE=artifact_type(Path(path)),
+                        PLATFORM_WHEN_CHALKED=result.report["_OP_PLATFORM"],
+                        INJECTOR_COMMIT_ID=result.report["_OP_CHALKER_COMMIT_ID"],
+                    )
+            if not expecting_chalkmarks:
+                assert "_CHALKS" not in result.report
+        return result
 
     def exec(
         self,
@@ -373,7 +480,7 @@ def dump(self, path: Optional[Path] = None) -> ChalkProgram:
         if path is not None:
             assert not path.is_file()
             args = [str(path)]
-        result = self.run(command="dump", params=args)
+        result = self.run(command="dump", params=args, expecting_report=False)
         if path is not None:
             assert path.is_file()
         return result
@@ -487,10 +594,13 @@ def docker_build(
                 )
             )
         if expecting_report and expected_success and image_hash:
+            assert result.report.has(_VIRTUAL=IfExists(virtual))
             if platforms:
                 assert len(result.marks) == len(platforms)
             else:
                 assert len(result.marks) == 1
+            for chalk in result.marks:
+                assert chalk.has(_OP_ARTIFACT_TYPE="Docker Image")
             # sanity check that chalk mark includes basic chalk keys
             assert image_hash in [i["_CURRENT_HASH"] for i in result.marks]
             assert image_hash in [i["_IMAGE_ID"] for i in result.marks]
@@ -517,4 +627,5 @@ def docker_push(self, image: str, buildkit: bool = True):
     def docker_pull(self, image: str):
         return self.run(
             params=["docker", "pull", image],
+            expecting_report=False,
         )