diff --git a/garak/analyze/report_digest.py b/garak/analyze/report_digest.py index a655a4e2..c0911e47 100644 --- a/garak/analyze/report_digest.py +++ b/garak/analyze/report_digest.py @@ -7,6 +7,7 @@ import json import markdown import os +import pprint import re import sys @@ -63,6 +64,7 @@ def plugin_docstring_to_description(docstring): def compile_digest(report_path, taxonomy=_config.reporting.taxonomy): evals = [] + payloads = [] setup = defaultdict(str) with open(report_path, "r", encoding="utf-8") as reportfile: for line in reportfile: @@ -75,6 +77,12 @@ def compile_digest(report_path, taxonomy=_config.reporting.taxonomy): run_uuid = record["run"] elif record["entry_type"] == "start_run setup": setup = record + elif record["entry_type"] == "payload_init": + payloads.append( + record["payload_name"] + + " " + + pprint.pformat(record, sort_dicts=True, width=60) + ) calibration = garak.analyze.calibration.Calibration() calibration_used = False @@ -85,10 +93,11 @@ def compile_digest(report_path, taxonomy=_config.reporting.taxonomy): "garak_version": garak_version, "start_time": start_time, "run_uuid": run_uuid, - "setup": repr(setup), + "setup": pprint.pformat(setup, sort_dicts=True, width=60), "probespec": setup["plugins.probe_spec"], "model_type": setup["plugins.model_type"], "model_name": setup["plugins.model_name"], + "payloads": payloads, } ) diff --git a/garak/analyze/templates/digest_header.jinja b/garak/analyze/templates/digest_header.jinja index 00d73492..7ab1e0a2 100644 --- a/garak/analyze/templates/digest_header.jinja +++ b/garak/analyze/templates/digest_header.jinja @@ -71,13 +71,25 @@ h2,h3,h4 {padding-top: 10px; padding-bottom: 10px}

garak run: {{reportfile}}

-

config

-

filename: {{reportfile}}

-

garak version: {{garak_version}}

-

generator: {{model_type}}.{{model_name}}

-

started at: {{start_time}}

-

run config: {{setup}}

-

probe spec: {{probespec}}

+

config details

+
+filename: {{reportfile}}
+
+garak version: {{garak_version}}
+
+generator: {{model_type}}.{{model_name}}
+
+probe spec: {{probespec}}
+
+run started at: {{start_time}}
+
+run config: {{setup}}
+
+{% for payload in payloads %}
+payload: {{payload}}
+
+{% endfor %}
+
{%if model_name %}