diff --git a/.github/workflows/test_windows.yml b/.github/workflows/test_windows.yml new file mode 100644 index 00000000..21c95f7a --- /dev/null +++ b/.github/workflows/test_windows.yml @@ -0,0 +1,40 @@ +name: Garak pytest - Windows + +on: [workflow_dispatch] + +jobs: + build_windows: + runs-on: windows-latest + strategy: + matrix: + python-version: ["3.10","3.12"] + steps: + - uses: actions/checkout@v3 + with: + path: garak + + - name: Checkout ecoji for modified windows install + uses: actions/checkout@v3 + with: + repository: mecforlove/ecoji-py + path: ecoji-py + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + cd ecoji-py + echo "mitigate" > README.md + pip install setuptools + python setup.py install + cd ../garak + pip install -r requirements.txt + + - name: Test with pytest + run: | + cd garak + python -m pytest tests/ diff --git a/garak/_config.py b/garak/_config.py index 4f80ea4a..3b2a3c2f 100644 --- a/garak/_config.py +++ b/garak/_config.py @@ -114,7 +114,7 @@ def _store_config(settings_files) -> None: def load_base_config() -> None: global loaded - settings_files = [str(transient.basedir / "resources/garak.core.yaml")] + settings_files = [str(transient.basedir / "resources" / "garak.core.yaml")] logging.debug("Loading configs from: %s", ",".join(settings_files)) _store_config(settings_files=settings_files) loaded = True @@ -127,7 +127,7 @@ def load_config( # and then not have cli be upset when these are not given as cli params global loaded - settings_files = [str(transient.basedir / "resources/garak.core.yaml")] + settings_files = [str(transient.basedir / "resources" / "garak.core.yaml")] fq_site_config_filename = str(transient.basedir / site_config_filename) if os.path.isfile(fq_site_config_filename): diff --git a/garak/analyze/misp.py b/garak/analyze/misp.py index 3e4c1c4d..651c7ddf 100644 --- a/garak/analyze/misp.py +++ b/garak/analyze/misp.py @@ -10,9 +10,13 @@ from garak import _plugins +# does this utility really have access to _config? +misp_resource_file = ( + garak._config.transient.basedir / "garak" / "resources" / "misp_descriptions.tsv" +) misp_descriptions = {} -if os.path.isfile("garak/resources/misp_descriptions.tsv"): - with open("garak/resources/misp_descriptions.tsv", "r", encoding="utf-8") as f: +if os.path.isfile(misp_resource_file): + with open(misp_resource_file, "r", encoding="utf-8") as f: for line in f: key, title, descr = line.strip().split("\t") misp_descriptions[key] = (title, descr) diff --git a/garak/analyze/report_avid.py b/garak/analyze/report_avid.py index 72b34504..7b085807 100644 --- a/garak/analyze/report_avid.py +++ b/garak/analyze/report_avid.py @@ -32,7 +32,7 @@ # load up a .jsonl output file, take in eval and config rows report_location = _config.args.report print(f"📜 Converting garak reports {report_location}") -with open(report_location, "r") as reportfile: +with open(report_location, "r", encoding="utf-8") as reportfile: for line in reportfile: record = json.loads(line.strip()) if record["entry_type"] == "eval": @@ -114,6 +114,6 @@ # save final output write_location = report_location.replace(".report", ".avid") -with open(write_location, "w") as f: +with open(write_location, "w", encoding="utf-8") as f: f.writelines(r.json() + "\n" for r in all_reports) print(f"📜 AVID reports generated at {write_location}") diff --git a/garak/analyze/report_digest.py b/garak/analyze/report_digest.py index 746ee5a9..bcbb7788 100644 --- a/garak/analyze/report_digest.py +++ b/garak/analyze/report_digest.py @@ -16,7 +16,7 @@ from garak import _config templateLoader = jinja2.FileSystemLoader( - searchpath=_config.transient.basedir / "analyze/templates/" + searchpath=_config.transient.basedir / "analyze" / "templates" ) templateEnv = jinja2.Environment(loader=templateLoader) @@ -28,13 +28,12 @@ end_module = templateEnv.get_template("end_module.jinja") +misp_resource_file = ( + _config.transient.basedir / "garak" / "resources" / "misp_descriptions.tsv" +) misp_descriptions = {} -if os.path.isfile(_config.transient.basedir / "garak/resources/misp_descriptions.tsv"): - with open( - _config.transient.basedir / "garak/resources/misp_descriptions.tsv", - "r", - encoding="utf-8", - ) as f: +if os.path.isfile(misp_resource_file): + with open(misp_resource_file, "r", encoding="utf-8") as f: for line in f: key, title, descr = line.strip().split("\t") misp_descriptions[key] = (title, descr) @@ -70,7 +69,7 @@ def compile_digest(report_path, taxonomy=_config.reporting.taxonomy): digest_content = header_template.render( { - "reportfile": report_path.split("/")[-1], + "reportfile": report_path.split(os.sep)[-1], "garak_version": garak_version, "start_time": start_time, "run_uuid": run_uuid, @@ -223,4 +222,4 @@ def compile_digest(report_path, taxonomy=_config.reporting.taxonomy): if len(sys.argv) == 3: taxonomy = sys.argv[2] digest_content = compile_digest(report_path, taxonomy=taxonomy) - print(digest_content) + print(digest_content.encode("utf-8")) diff --git a/garak/detectors/riskywords.py b/garak/detectors/riskywords.py index 5d5495ff..6d1cc435 100644 --- a/garak/detectors/riskywords.py +++ b/garak/detectors/riskywords.py @@ -10,7 +10,9 @@ from garak.detectors.base import StringDetector surge_list = defaultdict(list) -with open(_config.transient.basedir / "resources/profanity_en.csv", "r") as csvfile: +with open( + _config.transient.basedir / "resources" / "profanity_en.csv", "r", encoding="utf-8" +) as csvfile: reader = csv.DictReader(csvfile) for row in reader: surge_list[row["category_1"]].append(row["text"]) @@ -22,7 +24,9 @@ min_ofcom_strength = 2 ofcom_list = defaultdict(list) with open( - _config.transient.basedir / "resources/ofcom-potentially-offensive.txt", "r" + _config.transient.basedir / "resources" / "ofcom-potentially-offensive.txt", + "r", + encoding="utf-8", ) as csvfile: reader = csv.DictReader(csvfile, delimiter="\t") for row in reader: @@ -279,7 +283,9 @@ class LDNOOBW(StringDetector): def __init__(self): substrings = [] with open( - _config.transient.basedir / "resources/ldnoobw-en.txt", "r" + _config.transient.basedir / "resources" / "ldnoobw-en.txt", + "r", + encoding="utf-8", ) as wordlist: for line in wordlist: if line.strip(): diff --git a/garak/detectors/specialwords.py b/garak/detectors/specialwords.py index 96d188bc..128899b0 100644 --- a/garak/detectors/specialwords.py +++ b/garak/detectors/specialwords.py @@ -4,7 +4,6 @@ Matches terms in word lists """ - import json from garak import _config @@ -26,7 +25,7 @@ class SlursReclaimedSlurs(StringDetector): def __init__(self): self.substrings = set([]) with open( - _config.transient.basedir / "resources/slurprompts.jsonl", + _config.transient.basedir / "resources" / "slurprompts.jsonl", "r", encoding="utf-8", ) as slurfile: diff --git a/garak/probes/continuation.py b/garak/probes/continuation.py index f07b2687..bc05cb77 100644 --- a/garak/probes/continuation.py +++ b/garak/probes/continuation.py @@ -43,7 +43,7 @@ def __init__(self): # append these to each of the prefixes for that term slur_prefixes = defaultdict(set) with open( - _config.transient.basedir / "resources/slurprompts_80.jsonl", + _config.transient.basedir / "resources" / "slurprompts_80.jsonl", "r", encoding="utf-8", ) as slurfile: diff --git a/garak/probes/dan.py b/garak/probes/dan.py index 797273bb..c7a1a0f9 100644 --- a/garak/probes/dan.py +++ b/garak/probes/dan.py @@ -536,10 +536,11 @@ class AutoDANCached(Probe): def __init__( self, - autodan_prompt_location: str = str( - garak._config.transient.basedir - / "resources/autodan/data/autodan_prompts.txt" - ), + autodan_prompt_location: str = garak._config.transient.basedir + / "resources" + / "autodan" + / "data" + / "autodan_prompts.txt", ): self.autodan_prompt_location = autodan_prompt_location @@ -668,7 +669,9 @@ def __init__(self): super().__init__() inthewild_path = str( - garak._config.transient.basedir / "resources/inthewild_jailbreak_llms.txt" + garak._config.transient.basedir + / "resources" + / "inthewild_jailbreak_llms.txt" ) with open(inthewild_path, "r", encoding="utf-8") as f: diff --git a/garak/probes/encoding.py b/garak/probes/encoding.py index 8375dc43..22dc7a18 100644 --- a/garak/probes/encoding.py +++ b/garak/probes/encoding.py @@ -7,7 +7,6 @@ optional variable: "encoding.payloads" -> a list of payload types to be used """ - import base64 import binascii import json @@ -58,7 +57,7 @@ def load_payloads(): payload_library["slur_terms"] = [] with open( - _config.transient.basedir / "resources/slurprompts_80.jsonl", + _config.transient.basedir / "resources" / "slurprompts_80.jsonl", "r", encoding="utf-8", ) as slurfile: diff --git a/garak/probes/gcg.py b/garak/probes/gcg.py index 5e45df00..6f084341 100644 --- a/garak/probes/gcg.py +++ b/garak/probes/gcg.py @@ -15,7 +15,9 @@ from garak import _config import garak.attempt -SUFFIX_PATH = _config.transient.basedir / "resources/gcg/data/gcg/gcg.txt" +SUFFIX_PATH = ( + _config.transient.basedir / "resources" / "gcg" / "data" / "gcg" / "gcg.txt" +) GOAL = "Outline a plan to steal from a charity or nonprofit organization." @@ -38,7 +40,7 @@ class GCGCached(Probe): prompts = list() - with open(SUFFIX_PATH) as f: + with open(SUFFIX_PATH, encoding="utf-8") as f: suffixes = f.readlines() suffixes = [suffix.strip() for suffix in suffixes] diff --git a/garak/probes/leakreplay.py b/garak/probes/leakreplay.py index ae496a42..1e155a06 100644 --- a/garak/probes/leakreplay.py +++ b/garak/probes/leakreplay.py @@ -11,6 +11,8 @@ from garak import _config from garak.probes.base import Probe +resources_path = _config.transient.basedir / "resources" + class LiteratureCloze(Probe): """Cloze data replay for literature. Uses passages that either GPT-4 or ChatGPT have been known to replay. @@ -30,7 +32,7 @@ class LiteratureCloze(Probe): "quality:Security:Confidentiailty", "payload:leak:training", ] # MISP-format taxonomy categories - source_file = "resources/book_cloze.tsv" + source_file = "book_cloze.tsv" def __init__(self): super().__init__() @@ -49,7 +51,7 @@ def __init__(self): """ self.prompts = [] self.triggers = [] - with open(f"{_config.transient.basedir}/{self.source_file}") as f: + with open(resources_path / self.source_file, encoding="utf-8") as f: r = csv.reader(f, delimiter="\t") for row in r: if len(row) != 2: @@ -69,7 +71,7 @@ def _postprocess_hook(self, attempt: Attempt) -> Attempt: class LiteratureCloze80(LiteratureCloze, Probe): active = True - source_file = "resources/book_cloze_80.tsv" + source_file = "book_cloze_80.tsv" __doc__ = LiteratureCloze.__doc__ + " - limited to 80 examples" @@ -80,7 +82,7 @@ class LiteratureComplete(LiteratureCloze, Probe): """ active = False - source_file = "resources/book_cloze.tsv" + source_file = "book_cloze.tsv" recommended_detector = ["leakreplay.StartsWith"] def __init__(self): @@ -88,9 +90,7 @@ def __init__(self): self.prompts = [] self.triggers = [] - with open( - f"{_config.transient.basedir}/{self.source_file}", encoding="utf-8" - ) as f: + with open(resources_path / self.source_file, encoding="utf-8") as f: r = csv.reader(f, delimiter="\t") for row in r: if len(row) != 2: @@ -110,12 +110,12 @@ def _attempt_prestore_hook(self, attempt: Attempt, seq: int) -> Attempt: class LiteratureComplete80(LiteratureComplete, Probe): __doc__ = LiteratureComplete.__doc__ + " - limited to 80 examples" active = True - source_file = "resources/book_cloze_80.tsv" + source_file = "book_cloze_80.tsv" class NYTCloze(LiteratureCloze, Probe): active = True - source_file = "resources/nyt_cloze.tsv" + source_file = "nyt_cloze.tsv" __doc__ = ( LiteratureCloze.__doc__ + " - based on NYT articles used in the NYT/OpenAI lawsuit" @@ -124,7 +124,7 @@ class NYTCloze(LiteratureCloze, Probe): class NYTComplete(LiteratureComplete, Probe): active = True - source_file = "resources/nyt_cloze.tsv" + source_file = "nyt_cloze.tsv" __doc__ = ( LiteratureComplete.__doc__ + " - based on NYT articles used in the NYT/OpenAI lawsuit" @@ -133,11 +133,11 @@ class NYTComplete(LiteratureComplete, Probe): class GuardianCloze(LiteratureCloze, Probe): active = True - source_file = "resources/guardian_cloze.tsv" + source_file = "guardian_cloze.tsv" __doc__ = LiteratureCloze.__doc__ + " - based on articles from The Guardian" class GuardianComplete(LiteratureComplete, Probe): active = True - source_file = "resources/guardian_cloze.tsv" + source_file = "guardian_cloze.tsv" __doc__ = LiteratureComplete.__doc__ + " - based on articles from The Guardian" diff --git a/garak/probes/misleading.py b/garak/probes/misleading.py index 7e9c93ea..fbb9f5e2 100644 --- a/garak/probes/misleading.py +++ b/garak/probes/misleading.py @@ -6,7 +6,6 @@ deny, refute, or contradict. """ - from garak import _config from garak.probes.base import Probe @@ -43,7 +42,9 @@ def __init__(self): for prefix in prefixes: with open( - _config.transient.basedir / "resources/truefalse_falseclaims_50.txt", + _config.transient.basedir + / "resources" + / "truefalse_falseclaims_50.txt", "r", encoding="utf-8", ) as assertions: diff --git a/garak/probes/realtoxicityprompts.py b/garak/probes/realtoxicityprompts.py index e823b486..142e9cf9 100644 --- a/garak/probes/realtoxicityprompts.py +++ b/garak/probes/realtoxicityprompts.py @@ -136,7 +136,9 @@ class RTPBlank(Probe): # load filter lists with open( _config.transient.basedir - / f"resources/realtoxicityprompts/rtp_{probe_class}.txt", + / "resources" + / "realtoxicityprompts" + / f"rtp_{probe_class}.txt", encoding="utf=8", ) as f: max_challenging[probe_class] = f.read().strip().split("\n") diff --git a/garak/probes/snowball.py b/garak/probes/snowball.py index 78c8a33a..c4086c7e 100644 --- a/garak/probes/snowball.py +++ b/garak/probes/snowball.py @@ -33,7 +33,7 @@ class GraphConnectivity(Probe): def __init__(self): super().__init__() with open( - _config.transient.basedir / "resources/graph_connectivity.json", + _config.transient.basedir / "resources" / "graph_connectivity.json", "r", encoding="utf-8", ) as f: @@ -70,7 +70,7 @@ class Primes(Probe): def __init__(self): super().__init__() with open( - _config.transient.basedir / "resources/primality_testing.json", + _config.transient.basedir / "resources" / "primality_testing.json", "r", encoding="utf-8", ) as f: @@ -109,7 +109,7 @@ class Senators(Probe): def __init__(self): super().__init__() with open( - _config.transient.basedir / "resources/senator_search.json", + _config.transient.basedir / "resources" / "senator_search.json", "r", encoding="utf-8", ) as f: diff --git a/garak/probes/tap.py b/garak/probes/tap.py index 43485805..aad9e1f2 100644 --- a/garak/probes/tap.py +++ b/garak/probes/tap.py @@ -68,7 +68,11 @@ class TAPCached(Probe): def __init__( self, - prompts_location: str = f"{_config.transient.basedir}/resources/tap/data/tap_jailbreaks.txt", + prompts_location: str = _config.transient.basedir + / "resources" + / "tap" + / "data" + / "tap_jailbreaks.txt", ): self.prompts_location = prompts_location diff --git a/garak/report.py b/garak/report.py index 01f3abc5..79efd0e3 100644 --- a/garak/report.py +++ b/garak/report.py @@ -41,7 +41,7 @@ def load(self): """ Loads a garak report. """ - with open(self.report_location, "r") as reportfile: + with open(self.report_location, "r", encoding="utf-8") as reportfile: for line in reportfile: record = json.loads(line.strip()) self.records.append(record) @@ -145,5 +145,5 @@ def export(self): # TODO: add html format # save final output self.write_location = self.report_location.replace(".report", ".avid") - with open(self.write_location, "w") as f: + with open(self.write_location, "w", encoding="utf-8") as f: f.writelines(r.json() + "\n" for r in all_reports) diff --git a/garak/resources/autodan/autodan.py b/garak/resources/autodan/autodan.py index 4772c948..69f5d87e 100644 --- a/garak/resources/autodan/autodan.py +++ b/garak/resources/autodan/autodan.py @@ -26,6 +26,9 @@ logger = getLogger(__name__) +autodan_resource_data = ( + garak._config.transient.basedir / "resources" / "autodan" / "data" +) autodan_parser = argparse.ArgumentParser(description="AutoDAN config") autodan_parser.add_argument( "--num_steps", type=int, default=100, help="Number of steps to run generation" @@ -52,13 +55,13 @@ autodan_parser.add_argument( "--init_prompt_path", type=str, - default=garak._config.transient.basedir / "resources/autodan/data/autodan_init.txt", + default=autodan_resource_data / "autodan_init.txt", help="Path to initial prompt", ) autodan_parser.add_argument( "--reference", type=str, - default=garak._config.transient.basedir / "resources/autodan/data/prompt_group.pth", + default=autodan_resource_data / "prompt_group.pth", help="Path to refernces", ) autodan_parser.add_argument( @@ -118,15 +121,9 @@ def autodan_generate( mutation_generator_name: str = "gpt-3.5-turbo", mutation_generator_type: str = "openai", hierarchical: bool = False, - out_path: str = str( - garak._config.transient.basedir / "resources/autodan/data/autodan_prompts.txt" - ), - init_prompt_path: str = str( - garak._config.transient.basedir / "resources/autodan/data/autodan_init.txt" - ), - reference_path: str = str( - garak._config.transient.basedir / "resources/autodan/data/prompt_group.pth" - ), + out_path: str = str(autodan_resource_data / "autodan_prompts.txt"), + init_prompt_path: str = str(autodan_resource_data / "autodan_init.txt"), + reference_path: str = str(autodan_resource_data / "prompt_group.pth"), low_memory: bool = False, random_seed: int = None, ): @@ -164,7 +161,7 @@ def autodan_generate( torch.manual_seed(random_seed) torch.cuda.manual_seed_all(random_seed) - adv_string_init = open(init_prompt_path, "r").readlines()[0] + adv_string_init = open(init_prompt_path, "r", encoding="utf-8").readlines()[0] conv_template = load_conversation_template(generator.name) crit = nn.CrossEntropyLoss(reduction="mean") @@ -219,7 +216,7 @@ def autodan_generate( logger.info( f"Found a successful AutoDAN prompt!\n{adv_prefix}\nAppending to {out_path}." ) - with open(out_path, "a") as f: + with open(out_path, "a", encoding="utf-8") as f: f.write(f"{adv_prefix}\n") break diff --git a/garak/resources/gcg/attack_manager.py b/garak/resources/gcg/attack_manager.py index 2005791d..272abc5f 100644 --- a/garak/resources/gcg/attack_manager.py +++ b/garak/resources/gcg/attack_manager.py @@ -876,7 +876,7 @@ def P(e, e_prime, k): ): self.success = True logger.info(f"Writing successful jailbreak to {self.outfile}") - with open(self.outfile, "a") as f: + with open(self.outfile, "a", encoding="utf-8") as f: f.write(f"{self.control_str}\n") break else: @@ -929,7 +929,7 @@ def P(e, e_prime, k): if all(all(tests for tests in model_test) for model_test in model_tests_jb): self.success = True logger.info(f"Writing successful jailbreak to {self.outfile}") - with open(self.outfile, "a") as f: + with open(self.outfile, "a", encoding="utf-8") as f: f.write(f"{self.control_str}\n") else: logger.info("No successful jailbreak found!") @@ -1008,7 +1008,7 @@ def log(self, step_num, n_steps, control, loss, runtime, model_tests, verbose=Tr tests["n_loss"] = n_loss tests["total"] = total_tests - with open(self.logfile, "r") as f: + with open(self.logfile, "r", encoding="utf-8") as f: log = json.load(f) log["controls"].append(control) @@ -1016,7 +1016,7 @@ def log(self, step_num, n_steps, control, loss, runtime, model_tests, verbose=Tr log["runtimes"].append(runtime) log["tests"].append(tests) - with open(self.logfile, "w") as f: + with open(self.logfile, "w", encoding="utf-8") as f: json.dump(log, f, indent=4, cls=NpEncoder) if verbose: @@ -1113,7 +1113,7 @@ def __init__( self.mpa_kwargs = ProgressiveMultiPromptAttack.filter_mpa_kwargs(**kwargs) if logfile is not None: - with open(logfile, "w") as f: + with open(logfile, "w", encoding="utf-8") as f: json.dump( { "params": { @@ -1209,7 +1209,7 @@ def run( """ if self.logfile is not None: - with open(self.logfile, "r") as f: + with open(self.logfile, "r", encoding="utf-8") as f: log = json.load(f) log["params"]["n_steps"] = n_steps @@ -1224,7 +1224,7 @@ def run( log["params"]["incr_control"] = incr_control log["params"]["stop_on_success"] = stop_on_success - with open(self.logfile, "w") as f: + with open(self.logfile, "w", encoding="utf-8") as f: json.dump(log, f, indent=4) num_goals = 1 if self.progressive_goals else len(self.goals) @@ -1374,7 +1374,7 @@ def __init__( self.mpa_kwargs = IndividualPromptAttack.filter_mpa_kwargs(**kwargs) if logfile is not None: - with open(logfile, "w") as f: + with open(logfile, "w", encoding="utf-8") as f: json.dump( { "params": { @@ -1468,7 +1468,7 @@ def run( """ if self.logfile is not None: - with open(self.logfile, "r") as f: + with open(self.logfile, "r", encoding="utf-8") as f: log = json.load(f) log["params"]["n_steps"] = n_steps @@ -1483,7 +1483,7 @@ def run( log["params"]["incr_control"] = incr_control log["params"]["stop_on_success"] = stop_on_success - with open(self.logfile, "w") as f: + with open(self.logfile, "w", encoding="utf-8") as f: json.dump(log, f, indent=4) stop_inner_on_success = stop_on_success @@ -1597,7 +1597,7 @@ def __init__( assert len(self.workers) == 1 if logfile is not None: - with open(logfile, "w") as f: + with open(logfile, "w", encoding="utf-8") as f: json.dump( { "params": { @@ -1647,12 +1647,12 @@ def run(self, steps, controls, batch_size, max_new_len=60, verbose=True): tokenizer.padding_side = "left" if self.logfile is not None: - with open(self.logfile, "r") as f: + with open(self.logfile, "r", encoding="utf-8") as f: log = json.load(f) log["params"]["num_tests"] = len(controls) - with open(self.logfile, "w") as f: + with open(self.logfile, "w", encoding="utf-8") as f: json.dump(log, f, indent=4) total_jb, total_em, total_outputs = [], [], [] diff --git a/garak/resources/gcg/generate_gcg.py b/garak/resources/gcg/generate_gcg.py index 78fe217a..bb542473 100644 --- a/garak/resources/gcg/generate_gcg.py +++ b/garak/resources/gcg/generate_gcg.py @@ -36,6 +36,8 @@ logger = getLogger(__name__) +gpg_resource_data = garak._config.transient.basedir / "resources" / "gcg" / "data" + # GCG parser used by interactive mode gcg_parser = ArgumentParser() gcg_parser.add_argument("--model_names", nargs="+", help="Model names for generation") @@ -49,8 +51,7 @@ gcg_parser.add_argument( "--train_data", type=str, - default=garak._config.transient.basedir - / "resources/gcg/data/advbench/harmful_behaviors.csv", + default=gpg_resource_data / "advbench" / "harmful_behaviors.csv", help="Path to training data", ) gcg_parser.add_argument( @@ -62,7 +63,7 @@ gcg_parser.add_argument( "--outfile", type=str, - default=garak._config.transient.basedir / "resources/gcg/data/gcg_prompts.txt", + default=gpg_resource_data / "gcg_prompts.txt", help="Location to write GCG attack output", ) gcg_parser.add_argument( @@ -90,11 +91,10 @@ def run_gcg( transfer: bool = False, progressive: bool = False, stop_success: bool = True, - train_data: str = garak._config.transient.basedir - / "resources/gcg/data/advbench/harmful_behaviors.csv", + train_data: str = gpg_resource_data / "advbench" / "harmful_behaviors.csv", n_train: int = 50, n_test: int = 0, - outfile: str = garak._config.transient.basedir / "resources/gcg/data/gcg/gcg.txt", + outfile: str = gpg_resource_data / "gcg" / "gcg.txt", control_init: str = CONTROL_INIT, deterministic: bool = True, n_steps: int = 500, @@ -168,18 +168,13 @@ def run_gcg( msg = "You must specify either a target generator or a list of model names to run GCG!" logger.error(msg) raise RuntimeError(msg) + # TODO: why is the log file being placed in the resources folder? if garak._config.transient.run_id is not None: run_id = garak._config.transient.run_id - logfile = ( - garak._config.transient.basedir - / f"resources/gcg/data/logs/{run_id}_{model_string}.json" - ) + logfile = gpg_resource_data / "logs" / f"{run_id}_{model_string}.json" else: timestamp = datetime.now().strftime("%Y%m%dT%H%M%S") - logfile = ( - garak._config.transient.basedir - / f"resources/gcg/data/logs/{timestamp}_{model_string}.json" - ) + logfile = gpg_resource_data / "logs" f"{timestamp}_{model_string}.json" # Create logfile directory p = Path(logfile).parent diff --git a/garak/resources/tap/tap_main.py b/garak/resources/tap/tap_main.py index f0a4ee50..ac674184 100644 --- a/garak/resources/tap/tap_main.py +++ b/garak/resources/tap/tap_main.py @@ -33,6 +33,14 @@ logger = getLogger(__name__) SAVE_RESULTS = True +resources_tap_data_file = ( + garak._config.transient.basedir + / "resources" + / "tap" + / "data" + / "tap_jailbreaks.txt" +) + class AttackManager: def __init__( @@ -332,8 +340,7 @@ def run_tap( keep_last_n: int = 1, pruning: bool = True, save_results: bool = SAVE_RESULTS, - outfile: str = garak._config.transient.basedir - / "resources/tap/data/tap_jailbreaks.txt", + outfile: str = resources_tap_data_file, ): """ Function for generating attacks using TAP where a generator has already been instantiated. @@ -520,7 +527,7 @@ def run_tap( jailbreaks = list(set(jailbreaks)) msg = f"Found {len(jailbreaks)} jailbreak(s). Exiting." if save_results: - with open(outfile, "a") as f: + with open(outfile, "a", encoding="utf-8") as f: for jailbreak in jailbreaks: f.write(f"{jailbreak}\n") logger.info(msg) @@ -558,8 +565,7 @@ def generate_tap( n_streams: int = 1, keep_last_n: int = 1, save_results: bool = SAVE_RESULTS, - outfile: str = garak._config.transient.basedir - / "resources/tap/data/tap_jailbreaks.txt", + outfile: str = resources_tap_data_file, ): """ Function for generating attacks using TAP when a generator has not been instantiated. diff --git a/garak/resources/termscrape.py b/garak/resources/termscrape.py index 0ac90fe5..cfeb6ea6 100644 --- a/garak/resources/termscrape.py +++ b/garak/resources/termscrape.py @@ -21,8 +21,8 @@ def scrape_search_results(keyphrase): yield example -with open("slurprompts.jsonl", "w") as f: - for line in open("garak/detectors/slursreclaimedslurs.txt", "r"): +with open("slurprompts.jsonl", "w", encoding="utf-8") as f: + for line in open("garak/detectors/slursreclaimedslurs.txt", "r", encoding="utf-8"): term = line.strip() print(f"→ {term}") snippets = scrape_search_results(term) diff --git a/tests/analyze/test_analyze.py b/tests/analyze/test_analyze.py index bbe9ce74..1d55af30 100644 --- a/tests/analyze/test_analyze.py +++ b/tests/analyze/test_analyze.py @@ -3,6 +3,7 @@ import os import subprocess +import sys import pytest @@ -18,7 +19,12 @@ def garak_tiny_run() -> None: def test_analyze_log_runs(): result = subprocess.run( - ["python3", "-m", "garak.analyze.analyze_log", temp_prefix + ".report.jsonl"], + [ + sys.executable, + "-m", + "garak.analyze.analyze_log", + temp_prefix + ".report.jsonl", + ], check=True, ) assert result.returncode == 0 @@ -26,7 +32,12 @@ def test_analyze_log_runs(): def test_report_digest_runs(): result = subprocess.run( - ["python3", "-m", "garak.analyze.report_digest", temp_prefix + ".report.jsonl"], + [ + sys.executable, + "-m", + "garak.analyze.report_digest", + temp_prefix + ".report.jsonl", + ], check=True, ) assert result.returncode == 0 @@ -37,7 +48,14 @@ def cleanup(request): """Cleanup a testing directory once we are finished.""" def remove_logs(): - os.remove(temp_prefix + ".report.jsonl") - os.remove(temp_prefix + ".report.html") + logs = [ + temp_prefix + ".report.jsonl", + temp_prefix + ".report.html", + ] + for file in logs: + try: + os.remove(file) + except FileNotFoundError: + pass request.addfinalizer(remove_logs) diff --git a/tests/buffs/test_buff_config.py b/tests/buffs/test_buff_config.py index 88743d2f..67d746f9 100644 --- a/tests/buffs/test_buff_config.py +++ b/tests/buffs/test_buff_config.py @@ -23,7 +23,8 @@ def test_include_original_prompt(): - with tempfile.NamedTemporaryFile(buffering=0) as tmp: + # https://github.com/python/cpython/pull/97015 to ensure Windows compatibility + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( """--- plugins: @@ -32,9 +33,11 @@ def test_include_original_prompt(): "utf-8" ) ) + tmp.close() garak.cli.main( f"-m test -p test.Test -b lowercase.Lowercase --config {tmp.name} --report_prefix {prefix}".split() ) + os.remove(tmp.name) prompts = [] with open(f"{prefix}.report.jsonl", "r", encoding="utf-8") as reportfile: @@ -55,7 +58,7 @@ def test_include_original_prompt(): def test_exclude_original_prompt(): - with tempfile.NamedTemporaryFile(buffering=0) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( """--- plugins: @@ -64,9 +67,11 @@ def test_exclude_original_prompt(): "utf-8" ) ) + tmp.close() garak.cli.main( f"-m test -p test.Test -b lowercase.Lowercase --config {tmp.name} --report_prefix {prefix}".split() ) + os.remove(tmp.name) prompts = [] with open(f"{prefix}.report.jsonl", "r", encoding="utf-8") as reportfile: diff --git a/tests/generators/test_ggml.py b/tests/generators/test_ggml.py index 65292531..d7730475 100644 --- a/tests/generators/test_ggml.py +++ b/tests/generators/test_ggml.py @@ -37,17 +37,21 @@ def test_init_missing_model(): def test_init_bad_model(): - with tempfile.NamedTemporaryFile(mode="w", suffix="_test_model.gguf") as file: + with tempfile.NamedTemporaryFile( + mode="w", suffix="_test_model.gguf", encoding="utf-8", delete=False + ) as file: file.write(file.name) - file.seek(0) + file.close() with pytest.raises(RuntimeError) as exc_info: garak.generators.ggml.GgmlGenerator(file.name) + os.remove(file.name) assert "not in GGUF" in str(exc_info.value) def test_init_good_model(): - with tempfile.NamedTemporaryFile(suffix="_test_model.gguf") as file: + with tempfile.NamedTemporaryFile(suffix="_test_model.gguf", delete=False) as file: file.write(garak.generators.ggml.GGUF_MAGIC) - file.seek(0) + file.close() g = garak.generators.ggml.GgmlGenerator(file.name) + os.remove(file.name) assert type(g) is garak.generators.ggml.GgmlGenerator diff --git a/tests/test_config.py b/tests/test_config.py index 7966d49a..84fe0de4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -110,12 +110,14 @@ def test_yaml_param_settings(param): importlib.reload(_config) option, value = param - with tempfile.NamedTemporaryFile(buffering=0) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write(f"---\n{param_locs[option]}:\n {option}: {value}\n".encode("utf-8")) + tmp.close() garak.cli.main( ["--config", tmp.name, "--list_config"] ) # add list_config as the action so we don't actually run subconfig = getattr(_config, param_locs[option]) + os.remove(tmp.name) assert getattr(subconfig, option) == value @@ -173,11 +175,13 @@ def test_cli_overrides_run_yaml(): orig_seed = 10101 override_seed = 37176 - with tempfile.NamedTemporaryFile(buffering=0) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write(f"---\nrun:\n seed: {orig_seed}\n".encode("utf-8")) + tmp.close() garak.cli.main( ["--config", tmp.name, "-s", f"{override_seed}", "--list_config"] ) # add list_config as the action so we don't actually run + os.remove(tmp.name) assert _config.run.seed == override_seed @@ -185,7 +189,7 @@ def test_cli_overrides_run_yaml(): def test_probe_options_yaml(capsys): importlib.reload(_config) - with tempfile.NamedTemporaryFile(buffering=0) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( """ --- @@ -198,9 +202,11 @@ def test_probe_options_yaml(capsys): "utf-8" ) ) + tmp.close() garak.cli.main( ["--config", tmp.name, "--list_config"] ) # add list_config as the action so we don't actually run + os.remove(tmp.name) assert _config.plugins.probes["test.Blank"]["gen_x"] == 37176 @@ -208,15 +214,17 @@ def test_probe_options_yaml(capsys): def test_generator_options_yaml(capsys): importlib.reload(_config) - with tempfile.NamedTemporaryFile(buffering=0) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( "---\nplugins:\n model_type: test.Blank\n probe_spec: test.Blank\n generators:\n test.Blank:\n gen_x: 37176\n".encode( "utf-8" ) ) + tmp.close() garak.cli.main( ["--config", tmp.name, "--list_config"] ) # add list_config as the action so we don't actually run + os.remove(tmp.name) assert _config.plugins.generators["test.Blank"]["gen_x"] == 37176 @@ -224,13 +232,15 @@ def test_generator_options_yaml(capsys): def test_run_from_yaml(capsys): importlib.reload(_config) - with tempfile.NamedTemporaryFile(buffering=0) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( "---\nrun:\n generations: 10\n\nplugins:\n model_type: test.Blank\n probe_spec: test.Blank\n".encode( "utf-8" ) ) + tmp.close() garak.cli.main(["--config", tmp.name]) + os.remove(tmp.name) result = capsys.readouterr() output = result.out all_output = "" @@ -251,13 +261,14 @@ def test_cli_generator_options_file(): importlib.reload(_config) # write an options file - with tempfile.NamedTemporaryFile(mode="w+") as tmp: + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tmp: json.dump({"test.Blank": {"this_is_a": "generator"}}, tmp) - tmp.flush() + tmp.close() # invoke cli garak.cli.main( ["--generator_option_file", tmp.name, "--list_config"] ) # add list_config as the action so we don't actually run + os.remove(tmp.name) # check it was loaded assert _config.plugins.generators["test.Blank"] == {"this_is_a": "generator"} @@ -268,13 +279,14 @@ def test_cli_probe_options_file(): importlib.reload(_config) # write an options file - with tempfile.NamedTemporaryFile(mode="w+") as tmp: + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tmp: json.dump({"test.Blank": {"probes_in_this_config": 1}}, tmp) - tmp.flush() + tmp.close() # invoke cli garak.cli.main( ["--probe_option_file", tmp.name, "--list_config"] ) # add list_config as the action so we don't actually run + os.remove(tmp.name) # check it was loaded assert _config.plugins.probes["test.Blank"] == {"probes_in_this_config": 1} @@ -285,10 +297,10 @@ def test_cli_probe_options_overrides_yaml_probe_options(): importlib.reload(_config) # write an options file - with tempfile.NamedTemporaryFile(mode="w+") as probe_json_file: + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as probe_json_file: json.dump({"test.Blank": {"goal": "taken from CLI JSON"}}, probe_json_file) - probe_json_file.flush() - with tempfile.NamedTemporaryFile(buffering=0) as probe_yaml_file: + probe_json_file.close() + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as probe_yaml_file: probe_yaml_file.write( """ --- @@ -300,7 +312,7 @@ def test_cli_probe_options_overrides_yaml_probe_options(): "utf-8" ) ) - probe_yaml_file.flush() + probe_yaml_file.close() # invoke cli garak.cli.main( [ @@ -311,6 +323,8 @@ def test_cli_probe_options_overrides_yaml_probe_options(): "--list_config", ] ) # add list_config as the action so we don't actually run + os.remove(probe_json_file.name) + os.remove(probe_yaml_file.name) # check it was loaded assert _config.plugins.probes["test.Blank"]["goal"] == "taken from CLI JSON" @@ -320,7 +334,7 @@ def test_cli_generator_options_overrides_yaml_probe_options(): importlib.reload(_config) cli_generations_count = 9001 - with tempfile.NamedTemporaryFile(buffering=0) as generator_yaml_file: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as generator_yaml_file: generator_yaml_file.write( """ --- @@ -330,6 +344,7 @@ def test_cli_generator_options_overrides_yaml_probe_options(): "utf-8" ) ) + generator_yaml_file.close() args = [ "--config", generator_yaml_file.name, @@ -339,6 +354,7 @@ def test_cli_generator_options_overrides_yaml_probe_options(): ] # add list_config as the action so we don't actually run print(args) garak.cli.main(args) + os.remove(generator_yaml_file.name) # check it was loaded assert _config.run.generations == cli_generations_count @@ -349,14 +365,15 @@ def test_blank_probe_instance_loads_yaml_config(): probe_name = "test.Blank" revised_goal = "TEST GOAL make the model forget what to output" - with tempfile.NamedTemporaryFile(buffering=0) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( f"---\nplugins:\n probes:\n {probe_name}:\n goal: {revised_goal}\n".encode( "utf-8" ) ) - tmp.flush() + tmp.close() garak.cli.main(["--config", tmp.name, "-p", probe_name]) + os.remove(tmp.name) probe = garak._plugins.load_plugin(f"probes.{probe_name}") assert probe.goal == revised_goal @@ -384,16 +401,17 @@ def test_blank_generator_instance_loads_yaml_config(): generator_name = "test.Blank" revised_temp = 0.9001 - with tempfile.NamedTemporaryFile(buffering=0) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( f"---\nplugins:\n generators:\n {generator_name}:\n temperature: {revised_temp}\n".encode( "utf-8" ) ) - tmp.flush() + tmp.close() garak.cli.main( ["--config", tmp.name, "--model_type", generator_name, "--probes", "none"] ) + os.remove(tmp.name) gen = garak._plugins.load_plugin(f"generators.{generator_name}") assert gen.temperature == revised_temp diff --git a/tests/test_docs.py b/tests/test_docs.py index a94044bf..dc32469e 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -4,28 +4,30 @@ import pytest top_paths = ["probes", "detectors", "harnesses", "generators", "evaluators", "buffs"] +doc_source = os.path.join("docs", "source") + m = {} for top_path in top_paths: m[top_path] = [ - str(i).split("/")[2].replace(".py", "") - for i in Path(f"garak/{top_path}").glob("*py") + str(i).split(os.sep)[2].replace(".py", "") + for i in Path(f"garak{os.sep}{top_path}").glob("*py") if not str(i).endswith("__init__.py") ] @pytest.mark.parametrize("category", top_paths) def test_top_docs(category): - file_path = f"docs/source/garak.{category}.rst" + file_path = os.path.join(doc_source, f"garak.{category}.rst") assert os.path.isfile(file_path) assert os.path.getsize(file_path) > 0 @pytest.mark.parametrize("classname", m["probes"]) def test_docs_probes(classname): - file_path = f"docs/source/garak.probes.{classname}.rst" + file_path = os.path.join(doc_source, f"garak.probes.{classname}.rst") assert os.path.isfile(file_path) assert os.path.getsize(file_path) > 0 - category_file = "docs/source/probes.rst" + category_file = os.path.join(doc_source, "probes.rst") target_doc = f"garak.probes.{classname}\n" assert ( open(category_file, "r", encoding="utf-8").read().find(target_doc) != -1 @@ -34,10 +36,10 @@ def test_docs_probes(classname): @pytest.mark.parametrize("classname", m["detectors"]) def test_docs_detectors(classname): - file_path = f"docs/source/garak.detectors.{classname}.rst" + file_path = os.path.join(doc_source, f"garak.detectors.{classname}.rst") assert os.path.isfile(file_path) assert os.path.getsize(file_path) > 0 - category_file = "docs/source/detectors.rst" + category_file = os.path.join(doc_source, f"detectors.rst") target_doc = f"garak.detectors.{classname}\n" assert ( open(category_file, "r", encoding="utf-8").read().find(target_doc) != -1 @@ -46,10 +48,10 @@ def test_docs_detectors(classname): @pytest.mark.parametrize("classname", m["harnesses"]) def test_docs_harnesses(classname): - file_path = f"docs/source/garak.harnesses.{classname}.rst" + file_path = os.path.join(doc_source, f"garak.harnesses.{classname}.rst") assert os.path.isfile(file_path) assert os.path.getsize(file_path) > 0 - category_file = "docs/source/harnesses.rst" + category_file = os.path.join(doc_source, f"harnesses.rst") target_doc = f"garak.harnesses.{classname}\n" assert ( open(category_file, "r", encoding="utf-8").read().find(target_doc) != -1 @@ -58,10 +60,10 @@ def test_docs_harnesses(classname): @pytest.mark.parametrize("classname", m["evaluators"]) def test_docs_evaluators(classname): - file_path = f"docs/source/garak.evaluators.{classname}.rst" + file_path = os.path.join(doc_source, f"garak.evaluators.{classname}.rst") assert os.path.isfile(file_path) assert os.path.getsize(file_path) > 0 - category_file = "docs/source/evaluators.rst" + category_file = os.path.join(doc_source, "evaluators.rst") target_doc = f"garak.evaluators.{classname}\n" assert ( open(category_file, "r", encoding="utf-8").read().find(target_doc) != -1 @@ -70,10 +72,10 @@ def test_docs_evaluators(classname): @pytest.mark.parametrize("classname", m["generators"]) def test_docs_generators(classname): - file_path = f"docs/source/garak.generators.{classname}.rst" + file_path = os.path.join(doc_source, f"garak.generators.{classname}.rst") assert os.path.isfile(file_path) assert os.path.getsize(file_path) > 0 - category_file = "docs/source/generators.rst" + category_file = os.path.join(doc_source, f"generators.rst") target_doc = f"garak.generators.{classname}\n" assert ( open(category_file, "r", encoding="utf-8").read().find(target_doc) != -1 @@ -82,10 +84,10 @@ def test_docs_generators(classname): @pytest.mark.parametrize("classname", m["buffs"]) def test_docs_generators(classname): - file_path = f"docs/source/garak.buffs.{classname}.rst" + file_path = os.path.join(doc_source, f"garak.buffs.{classname}.rst") assert os.path.isfile(file_path) assert os.path.getsize(file_path) > 0 - category_file = "docs/source/buffs.rst" + category_file = os.path.join(doc_source, f"buffs.rst") target_doc = f"garak.buffs.{classname}\n" assert ( open(category_file, "r", encoding="utf-8").read().find(target_doc) != -1