diff --git a/garak/analyze/report_digest.py b/garak/analyze/report_digest.py index a655a4e2..c0911e47 100644 --- a/garak/analyze/report_digest.py +++ b/garak/analyze/report_digest.py @@ -7,6 +7,7 @@ import json import markdown import os +import pprint import re import sys @@ -63,6 +64,7 @@ def plugin_docstring_to_description(docstring): def compile_digest(report_path, taxonomy=_config.reporting.taxonomy): evals = [] + payloads = [] setup = defaultdict(str) with open(report_path, "r", encoding="utf-8") as reportfile: for line in reportfile: @@ -75,6 +77,12 @@ def compile_digest(report_path, taxonomy=_config.reporting.taxonomy): run_uuid = record["run"] elif record["entry_type"] == "start_run setup": setup = record + elif record["entry_type"] == "payload_init": + payloads.append( + record["payload_name"] + + " " + + pprint.pformat(record, sort_dicts=True, width=60) + ) calibration = garak.analyze.calibration.Calibration() calibration_used = False @@ -85,10 +93,11 @@ def compile_digest(report_path, taxonomy=_config.reporting.taxonomy): "garak_version": garak_version, "start_time": start_time, "run_uuid": run_uuid, - "setup": repr(setup), + "setup": pprint.pformat(setup, sort_dicts=True, width=60), "probespec": setup["plugins.probe_spec"], "model_type": setup["plugins.model_type"], "model_name": setup["plugins.model_name"], + "payloads": payloads, } ) diff --git a/garak/analyze/templates/digest_header.jinja b/garak/analyze/templates/digest_header.jinja index 00d73492..7ab1e0a2 100644 --- a/garak/analyze/templates/digest_header.jinja +++ b/garak/analyze/templates/digest_header.jinja @@ -71,13 +71,25 @@ h2,h3,h4 {padding-top: 10px; padding-bottom: 10px}
filename: {{reportfile}}
-garak version: {{garak_version}}
-generator: {{model_type}}.{{model_name}}
-started at: {{start_time}}
-run config: {{setup}}
-probe spec: {{probespec}}
++filename: {{reportfile}} + +garak version: {{garak_version}} + +generator: {{model_type}}.{{model_name}} + +probe spec: {{probespec}} + +run started at: {{start_time}} + +run config: {{setup}} + +{% for payload in payloads %} +payload: {{payload}} + +{% endfor %} +