Bugfix/leakreplay output setting (#790)

* add test for general hitlog writing code path

* bump copyright

* detector uses all_outputs, have hitlog use all_outputs also

* amend attempt output manipulation to edit underlying structure

* check hitlog file presence

* xplatform compat

* reset plugin cache singleton

* reset cache after each plugin cache test
* remove fixture not needed by hitlog test

Signed-off-by: Jeffrey Martin <jemartin@nvidia.com>

---------

Signed-off-by: Jeffrey Martin <jemartin@nvidia.com>
Co-authored-by: Jeffrey Martin <jemartin@nvidia.com>

garak/attempt.py

@@ -1,6 +1,5 @@
 """Defines the Attempt class, which encapsulates a prompt with metadata and results"""
 
-from collections.abc import Iterable
 from types import GeneratorType
 from typing import Any, List
 import uuid

garak/evaluators/base.py

@@ -102,7 +102,7 @@ def evaluate(self, attempts: Iterable[garak.attempt.Attempt]) -> None:
                                 {
                                     "goal": attempt.goal,
                                     "prompt": attempt.prompt,
-                                    "output": attempt.outputs[idx],
+                                    "output": attempt.all_outputs[idx],
                                     "trigger": trigger,
                                     "score": score,
                                     "run_id": str(_config.transient.run_id),

garak/probes/leakreplay.py

@@ -64,7 +64,10 @@ def _attempt_prestore_hook(self, attempt: Attempt, seq: int) -> Attempt:
         return attempt
 
     def _postprocess_hook(self, attempt: Attempt) -> Attempt:
-        attempt.outputs = [re.sub("</?name>", "", o) for o in attempt.outputs]
+        for idx, thread in enumerate(attempt.messages):
+            attempt.messages[idx][-1]["content"] = re.sub(
+                "</?name>", "", thread[-1]["content"]
+            )
         return attempt
 
 

tests/plugins/test_plugin_cache.py

@@ -5,6 +5,14 @@
 from garak._plugins import PluginCache
 
 
+@pytest.fixture(autouse=True)
+def reset_cache(request) -> None:
+    def reset_plugin_cache():
+        PluginCache._plugin_cache_dict = None
+
+    request.addfinalizer(reset_plugin_cache)
+
+
 @pytest.fixture
 def temp_cache_location(request) -> None:
     # override the cache file with a tmp location

tests/probes/test_probes_leakreplay.py

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: Portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+
+import garak
+import garak._config
+import garak._plugins
+import garak.attempt
+import garak.cli
+
+
+def test_leakreplay_hitlog():
+
+    args = "-m test.Blank -p leakreplay -d always.Fail".split()
+    garak.cli.main(args)
+
+
+def test_leakreplay_output_count():
+    generations = 1
+    garak._config.load_base_config()
+    garak._config.transient.reportfile = open(os.devnull, "w+")
+    a = garak.attempt.Attempt(prompt="test")
+    p = garak._plugins.load_plugin(
+        "probes.leakreplay.LiteratureCloze80", config_root=garak._config
+    )
+    g = garak._plugins.load_plugin("generators.test.Blank", config_root=garak._config)
+    g.generations = generations
+    p.generator = g
+    results = p._execute_all([a])
+    assert len(a.all_outputs) == generations

tests/test_hitlog.py

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: Portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import contextlib
+import os
+import pytest
+
+import garak
+import garak.cli
+
+
+CODEPATH_PREFIX = "_garak_test_hitlog_codepath"
+
+
+def test_hitlog_codepath():
+
+    args = f"-m test.Blank --report_prefix {CODEPATH_PREFIX} -p test.Test -d always.Fail".split()
+    garak.cli.main(args)
+    assert os.path.isfile(f"{CODEPATH_PREFIX}.hitlog.jsonl")