revert and simplify

huggingface · Jun 28, 2024 · f53e514 · f53e514
1 parent ed3428a
commit f53e514
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 19 deletions.
diff --git a/tokenizers/src/tokenizer/added_vocabulary.rs b/tokenizers/src/tokenizer/added_vocabulary.rs
@@ -216,17 +216,14 @@ impl AddedVocabulary {
     }
 
     /// Get the token matching the given id if it exists
-    #[deprecated(
-        since = "0.19.0",
-        note = "please use `added_vocabulary.simple_id_to_token(id).or_else(|| model.id_to_token(id)` instead"
-    )]
     pub fn id_to_token(&self, id: u32, model: &impl Model) -> Option<String> {
         self.added_tokens_map_r
             .get(&id)
             .map(|t| t.content.clone())
             .or_else(|| model.id_to_token(id))
     }
 
+    //
     pub fn simple_id_to_token(&self, id: u32) -> Option<String> {
         self.added_tokens_map_r.get(&id).map(|t| t.content.clone())
     }

diff --git a/tokenizers/src/tokenizer/mod.rs b/tokenizers/src/tokenizer/mod.rs
@@ -851,24 +851,13 @@ where
             .iter()
             .filter_map(|id| {
                 self.added_vocabulary
-                    .simple_id_to_token(*id)
-                    .and_then(|token| {
-                        if skip_special_tokens && self.added_vocabulary.is_special_token(&token) {
-                            None
-                        } else if let Some(pre_tok) = &self.pre_tokenizer {
-                            let mut string = PreTokenizedString::from(token);
-                            pre_tok.pre_tokenize(&mut string);
-                            println!("Pre-tok String: {}", string.original);
-                            Some(string.original)
-                        } else {
-                            println!("String: {}", token);
-                            Some(token)
-                        }
+                    .id_to_token(*id, &self.model)
+                    .filter(|token| {
+                        !skip_special_tokens || !self.added_vocabulary.is_special_token(token)
                     })
-                    .or_else(|| self.model.id_to_token(*id))
             })
             .collect::<Vec<_>>();
-        println!("This should print: {:?}", tokens);
+
         if let Some(decoder) = &self.decoder {
             decoder.decode(tokens)
         } else {