nit

huggingface · Jun 19, 2024 · dd6e99b · dd6e99b
1 parent 9d389bc
commit dd6e99b
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/tokenizers/src/tokenizer/pre_tokenizer.rs b/tokenizers/src/tokenizer/pre_tokenizer.rs
@@ -20,7 +20,7 @@ pub struct Split {
     /// The underlying `NormalizedString`. Each SubString is represented by a `NormalizedString`
     /// and in the end we might be carrying a lot of SubString representing various parts of the
     /// original input string.
-    normalized: NormalizedString,
+    pub normalized: NormalizedString,
     /// Optional Tokens associated to this Split
     tokens: Option<Vec<Token>>,
 }
@@ -52,7 +52,7 @@ impl From<(NormalizedString, Option<Vec<Token>>)> for Split {
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct PreTokenizedString {
     pub original: String,
-    splits: Vec<Split>,
+    pub splits: Vec<Split>,
 }
 
 impl PreTokenizedString {