From 7165cf078232dee620a323e3b971dbdb766cb639 Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Wed, 12 Jun 2024 11:50:47 +0200 Subject: [PATCH 1/2] don't use list/dict as default value, W0102 --- garak/cli.py | 5 ++++- garak/harnesses/probewise.py | 5 ++++- garak/harnesses/pxd.py | 4 +++- garak/report.py | 10 +++++++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/garak/cli.py b/garak/cli.py index c6ae38af..5321e711 100644 --- a/garak/cli.py +++ b/garak/cli.py @@ -6,7 +6,7 @@ command_options = "list_detectors list_probes list_generators list_buffs list_config plugin_info interactive report version".split() -def main(arguments=[]) -> None: +def main(arguments=None) -> None: """Main entry point for garak runs invoked from the CLI""" import datetime @@ -18,6 +18,9 @@ def main(arguments=[]) -> None: _config.transient.starttime_iso = _config.transient.starttime.isoformat() _config.version = __version__ + if arguments is None: + arguments = [] + import garak.command as command import logging import re diff --git a/garak/harnesses/probewise.py b/garak/harnesses/probewise.py index 1f1462ca..56d73bbf 100644 --- a/garak/harnesses/probewise.py +++ b/garak/harnesses/probewise.py @@ -27,7 +27,7 @@ def _load_detector(self, detector_name: str) -> Detector: logging.error(f" detector load failed: {detector_name}, skipping >>") return False - def run(self, model, probenames, evaluator, buff_names=[]): + def run(self, model, probenames, evaluator, buff_names=None): """Execute a probe-by-probe scan Probes are executed in name order. For each probe, the detectors @@ -54,6 +54,9 @@ def run(self, model, probenames, evaluator, buff_names=[]): :type buff_names: List[str] """ + if buff_names is None: + buff_names = [] + if not probenames: msg = "No probes, nothing to do" logging.warning(msg) diff --git a/garak/harnesses/pxd.py b/garak/harnesses/pxd.py index 878ae6ac..9202505f 100644 --- a/garak/harnesses/pxd.py +++ b/garak/harnesses/pxd.py @@ -20,7 +20,9 @@ class PxD(Harness): - def run(self, model, probe_names, detector_names, evaluator, buff_names=[]): + def run(self, model, probe_names, detector_names, evaluator, buff_names=None): + if buff_names is None: + buff_names = [] probe_names = sorted(probe_names) detector_names = sorted(detector_names) print( diff --git a/garak/report.py b/garak/report.py index 79efd0e3..c74d8c65 100644 --- a/garak/report.py +++ b/garak/report.py @@ -29,7 +29,12 @@ class Report: """ def __init__( - self, report_location, records=[], metadata=None, evaluations=None, scores=None + self, + report_location, + records=None, + metadata=None, + evaluations=None, + scores=None, ): self.report_location = report_location self.records = records @@ -37,6 +42,9 @@ def __init__( self.evaluations = evaluations self.scores = scores + if self.records is None: + self.records = [] + def load(self): """ Loads a garak report. From fbe530508b37bc803f20a32ae6245e441b56b3dd Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Wed, 12 Jun 2024 11:55:01 +0200 Subject: [PATCH 2/2] prune unused imports out --- garak/buffs/low_resource_languages.py | 2 -- garak/detectors/perspective.py | 2 -- garak/detectors/visual_jailbreak.py | 3 --- garak/evaluators/base.py | 1 - garak/generators/cohere.py | 2 -- garak/generators/huggingface.py | 1 - garak/generators/nvcf.py | 3 +-- garak/generators/octo.py | 1 - garak/generators/openai.py | 2 -- garak/generators/rest.py | 1 - garak/interactive.py | 1 - garak/probes/dan.py | 1 + garak/probes/goodside.py | 4 ++-- garak/probes/leakreplay.py | 1 - garak/probes/malwaregen.py | 2 +- 15 files changed, 5 insertions(+), 22 deletions(-) diff --git a/garak/buffs/low_resource_languages.py b/garak/buffs/low_resource_languages.py index ba7a9e1b..6be89d18 100644 --- a/garak/buffs/low_resource_languages.py +++ b/garak/buffs/low_resource_languages.py @@ -3,10 +3,8 @@ """ Buff that converts prompts with different encodings. """ -import logging from collections.abc import Iterable from deepl import Translator -from os import getenv import garak.attempt from garak import _config diff --git a/garak/detectors/perspective.py b/garak/detectors/perspective.py index f2c305de..a5d6ac3d 100644 --- a/garak/detectors/perspective.py +++ b/garak/detectors/perspective.py @@ -8,14 +8,12 @@ """ import logging -import os import sys import backoff from garak import _config from garak.detectors import Detector -from garak.exception import APIKeyMissingError perspective_classes = [ { diff --git a/garak/detectors/visual_jailbreak.py b/garak/detectors/visual_jailbreak.py index b084ff2f..07c1ba46 100644 --- a/garak/detectors/visual_jailbreak.py +++ b/garak/detectors/visual_jailbreak.py @@ -2,11 +2,8 @@ Detectors for guessing whether a jailbreak with both image and text prompt has been successful """ -import garak.attempt from garak import _config from garak.detectors.base import StringDetector -from typing import List -import re class FigStep(StringDetector): diff --git a/garak/evaluators/base.py b/garak/evaluators/base.py index 1492d6a9..879b2a19 100644 --- a/garak/evaluators/base.py +++ b/garak/evaluators/base.py @@ -5,7 +5,6 @@ import json import logging -import os from typing import List from colorama import Fore, Style diff --git a/garak/generators/cohere.py b/garak/generators/cohere.py index b7d2af9f..d42a1aa7 100644 --- a/garak/generators/cohere.py +++ b/garak/generators/cohere.py @@ -7,7 +7,6 @@ """ import logging -import os from typing import List, Union import backoff @@ -15,7 +14,6 @@ import tqdm from garak import _config -from garak.exception import APIKeyMissingError from garak.generators.base import Generator diff --git a/garak/generators/huggingface.py b/garak/generators/huggingface.py index 4f92f072..a70157be 100644 --- a/garak/generators/huggingface.py +++ b/garak/generators/huggingface.py @@ -16,7 +16,6 @@ import logging import re -import os from typing import List, Union import warnings diff --git a/garak/generators/nvcf.py b/garak/generators/nvcf.py index 7e7ad543..3dc2c59a 100644 --- a/garak/generators/nvcf.py +++ b/garak/generators/nvcf.py @@ -5,7 +5,6 @@ import json import logging -import os import time from typing import List, Union @@ -13,7 +12,7 @@ import requests from garak import _config -from garak.exception import APIKeyMissingError, ModelNameMissingError +from garak.exception import ModelNameMissingError from garak.generators.base import Generator diff --git a/garak/generators/octo.py b/garak/generators/octo.py index 749a9879..f6482be8 100644 --- a/garak/generators/octo.py +++ b/garak/generators/octo.py @@ -4,7 +4,6 @@ """OctoML LLM interface""" -import os from typing import List, Union import backoff diff --git a/garak/generators/openai.py b/garak/generators/openai.py index 2fca2259..939ccaf2 100644 --- a/garak/generators/openai.py +++ b/garak/generators/openai.py @@ -10,7 +10,6 @@ * https://platform.openai.com/docs/model-index-for-researchers """ -import os import re import logging from typing import List, Union @@ -19,7 +18,6 @@ import backoff from garak import _config -from garak.exception import APIKeyMissingError from garak.generators.base import Generator # lists derived from https://platform.openai.com/docs/models diff --git a/garak/generators/rest.py b/garak/generators/rest.py index 1ef6baf8..645c8994 100644 --- a/garak/generators/rest.py +++ b/garak/generators/rest.py @@ -8,7 +8,6 @@ import json import logging -import os from typing import List, Union import requests diff --git a/garak/interactive.py b/garak/interactive.py index bdd70fe1..309c3d56 100644 --- a/garak/interactive.py +++ b/garak/interactive.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: Apache-2.0 import argparse -import importlib from logging import getLogger import random import sys diff --git a/garak/probes/dan.py b/garak/probes/dan.py index 1fd7025a..da85f9f2 100644 --- a/garak/probes/dan.py +++ b/garak/probes/dan.py @@ -658,6 +658,7 @@ class DanInTheWildMini(DanInTheWild): Tries a set of 100 known jailbreaks, in an attempt to get a model to disregard alignment/system prompt. Success is measured by not getting a safety mitigaton message.""" + active = True def __init__(self, config_root=_config): diff --git a/garak/probes/goodside.py b/garak/probes/goodside.py index 0eab9833..8cb4852a 100644 --- a/garak/probes/goodside.py +++ b/garak/probes/goodside.py @@ -119,10 +119,10 @@ class Tag(Probe): primary_detector = "base.TriggerListDetector" tags = [ "owasp:llm01", - "payload:generic" + "payload:generic", "quality:Security:Adversarial", "quality:Security:PromptStability", - ] + ] goal = "inject invisible instructions" prompts = [] diff --git a/garak/probes/leakreplay.py b/garak/probes/leakreplay.py index efec519c..04eff705 100644 --- a/garak/probes/leakreplay.py +++ b/garak/probes/leakreplay.py @@ -5,7 +5,6 @@ import csv import re -from typing import List from garak.attempt import Attempt from garak import _config diff --git a/garak/probes/malwaregen.py b/garak/probes/malwaregen.py index 56d1cff3..0650c7eb 100644 --- a/garak/probes/malwaregen.py +++ b/garak/probes/malwaregen.py @@ -148,7 +148,7 @@ class Payload(Probe): tags = [ "avid-effect:security:S0403", "quality:Behavioral:ContentSafety:LegalGoodsServices", - ] + ] goal = "write a malware payload" def __init__(self, config_root=_config):