From 9f55189f9bfd8ba5805d23ee1d97c841dd6c8d6b Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Wed, 8 Nov 2023 13:38:19 -0800 Subject: [PATCH] [tokenizer] Fixes tokenizer bug (#2843) Fixes #2840 --- .../ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java | 2 +- .../huggingface/tokenizers/HuggingFaceTokenizerTest.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java index f75342b7cb8..fa601e15525 100644 --- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java +++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java @@ -787,7 +787,7 @@ public HuggingFaceTokenizer build() throws IOException { return managed(HuggingFaceTokenizer.newInstance(vocab, merges, options)); } throw new IOException("tokenizer.json file not found."); - } else if (Files.exists(tokenizerPath)) { + } else if (!Files.exists(tokenizerPath)) { throw new IOException("Tokenizer file not exits: " + tokenizerPath); } return managed(HuggingFaceTokenizer.newInstance(tokenizerPath, options)); diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java index dcbef24748d..2bc30d4bddf 100644 --- a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java +++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java @@ -115,6 +115,12 @@ public void testTokenizer() throws IOException { Assert.assertEquals(encodings.length, 2); Assert.assertEquals(encodings[0].getIds(), ids); } + + Assert.assertThrows( + () -> { + Path file = Paths.get("build/tokenizer/non-exists.json"); + HuggingFaceTokenizer.builder().optTokenizerPath(file).build(); + }); } @Test