fix: remove add special token in name detector

stair-lab · Aug 13, 2024 · 528fbee · 528fbee
1 parent 16cfa63
commit 528fbee
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/vieval/tools/metrics/name_detector.py b/src/vieval/tools/metrics/name_detector.py
@@ -28,7 +28,7 @@ def __init__(self, args):
         ) as f:
             token_pattern = f.read().strip()
         tokenizer = AutoTokenizer.from_pretrained(
-            args.metric_config["NERModel"], add_special_tokens=True
+            args.metric_config["NERModel"],
         )
         model = AutoModelForTokenClassification.from_pretrained(
             args.metric_config["NERModel"]