-
Notifications
You must be signed in to change notification settings - Fork 10
/
Copy pathmod.rs
299 lines (265 loc) · 9.69 KB
/
mod.rs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
// Copyright 2019 vtext developers
//
// Licensed under the Apache License, Version 2.0,
// <http://apache.org/licenses/LICENSE-2.0>. This file may not be copied,
// modified, or distributed except according to those terms.
/*!
# Tokenization module
This module includes several tokenizers
For instance let's tokenize the following sentence,
```rust
use vtext::tokenize::*;
let s = "The “brown” fox can't jump 32.3 feet, right?";
```
Using a regular expression tokenizer we would get,
```rust
# let s = "The “brown” fox can't jump 32.3 feet, right?";
# use vtext::tokenize::*;
let tokenizer = RegexpTokenizer::new(r"\b\w\w+\b".to_string());
let tokens: Vec<&str> = tokenizer.tokenize(s).collect();
assert_eq!(tokens, &["The", "brown", "fox", "can", "jump", "32", "feet", "right"]);
```
which would remove all punctuation. A more general approach is to apply unicode segmentation,
```rust
# let s = "The “brown” fox can't jump 32.3 feet, right?";
# use vtext::tokenize::*;
let tokenizer = UnicodeSegmentTokenizer::new(true);
let tokens: Vec<&str> = tokenizer.tokenize(s).collect();
assert_eq!(tokens, &["The", "“", "brown", "”", "fox", "can't", "jump", "32.3", "feet", ",", "right", "?"]);
```
Here `UnicodeSegmentTokenizer` object is a thin wrapper around the
[unicode-segmentation](https://github.com/unicode-rs/unicode-segmentation) crate.
This approach produces better results, however for instance the word "can't" should be tokenized
as "ca", "n't" in English. To address such issues, we apply several additional rules on the previous results,
```rust
# let s = "The “brown” fox can't jump 32.3 feet, right?";
# use vtext::tokenize::*;
let tokenizer = VTextTokenizer::new("en");
let tokens: Vec<&str> = tokenizer.tokenize(s).collect();
assert_eq!(tokens, &["The", "“", "brown", "”", "fox", "ca", "n't", "jump", "32.3", "feet", ",", "right", "?"]);
*/
extern crate regex;
extern crate unicode_segmentation;
use regex::Regex;
use std::fmt;
use unicode_segmentation::UnicodeSegmentation;
#[cfg(test)]
mod tests;
pub trait Tokenizer: fmt::Debug {
fn tokenize<'a>(&'a self, text: &'a str) -> Box<dyn Iterator<Item = &'a str> + 'a>;
}
/// Regular expression tokenizer
///
#[derive(Clone)]
pub struct RegexpTokenizer {
pub pattern: String,
regexp: Regex,
}
impl RegexpTokenizer {
/// Create a new instance
pub fn new(pattern: String) -> RegexpTokenizer {
let regexp = Regex::new(&pattern).unwrap();
RegexpTokenizer { pattern, regexp }
}
}
impl Tokenizer for RegexpTokenizer {
/// Tokenize a string
fn tokenize<'a>(&'a self, text: &'a str) -> Box<dyn Iterator<Item = &'a str> + 'a> {
Box::new(self.regexp.find_iter(text).map(|m| m.as_str()))
}
}
impl fmt::Debug for RegexpTokenizer {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "RegexpTokenizer {{ pattern: {} }}", self.pattern)
}
}
/// Unicode Segmentation tokenizer
///
/// This implementation is a thin wrapper around the
/// `unicode-segmentation` crate
///
/// ## References
///
/// * [Unicode® Standard Annex #29](http://www.unicode.org/reports/tr29/)
#[derive(Debug, Clone)]
pub struct UnicodeSegmentTokenizer {
pub word_bounds: bool,
}
impl UnicodeSegmentTokenizer {
/// Create a new instance
pub fn new(word_bounds: bool) -> UnicodeSegmentTokenizer {
UnicodeSegmentTokenizer { word_bounds }
}
}
impl Tokenizer for UnicodeSegmentTokenizer {
/// Tokenize a string
fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = &'a str> + 'a> {
if self.word_bounds {
let res = text.split_word_bounds().filter(|x| x != &" ");
Box::new(res)
} else {
Box::new(text.unicode_words())
}
}
}
/// vtext tokenizer
///
/// This tokenizer a few additional rules on top of word boundaries computed
/// by unicode segmentation.
///
/// Additional language specific rules are implemented for English (en),
/// and French (en). Providing `lang` parameter with any other value, will siletly
/// fallback to `lang="any"`.
///
///
/// ## References
///
/// * [Unicode® Standard Annex #29](http://www.unicode.org/reports/tr29/)
#[derive(Debug, Clone)]
pub struct VTextTokenizer {
pub lang: String,
}
impl VTextTokenizer {
/// Create a new instance
pub fn new(lang: &str) -> VTextTokenizer {
let lang_valid = match lang {
"en" | "fr" => lang,
_ => {
// TODO: add some warning message here
//println!(
// "Warning: Lokenizer for {} \
// is not implemented! Falling back to the \
// language independent tokenizer!",
// lang
//);
"any"
}
};
VTextTokenizer {
lang: lang_valid.to_string(),
}
}
}
impl Tokenizer for VTextTokenizer {
/// Tokenize a string
fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = &'a str> + 'a> {
let tokens = text.split_word_bounds();
let mut res: Vec<&'a str> = Vec::new();
let mut punct_start_seq: i64 = -1;
let mut punct_last = 'X';
let mut str_idx: usize = 0;
for tok in tokens {
let tok_len = tok.len();
str_idx += tok_len;
if (tok_len == 1) & (tok != " ") {
// Handle punctuation
let ch = tok.chars().next().unwrap();
if ch.is_ascii_punctuation() {
if ch != punct_last {
if punct_start_seq >= 0 {
res.push(&text[punct_start_seq as usize..str_idx - tok_len]);
}
punct_start_seq = (str_idx as i64) - (tok_len as i64);
}
punct_last = ch;
continue;
}
}
if punct_start_seq >= 0 {
res.push(&text[punct_start_seq as usize..str_idx - tok_len]);
punct_start_seq = -1;
punct_last = 'X';
}
match self.lang.as_ref() {
"en" => {
// Handle contractions
if let Some(apostroph_idx) = tok.find(&"'") {
let mut apostroph_idx = apostroph_idx;
if tok.ends_with(&"n't") {
// also include the "n" from "n't"
apostroph_idx -= 1;
}
res.push(&tok[..apostroph_idx]);
res.push(&tok[apostroph_idx..]);
continue;
} else if let Some(apostroph_idx) = tok.find(&"’") {
// TODO: refactor to avoid repetitions
let mut apostroph_idx = apostroph_idx;
if tok.ends_with(&"n’t") {
// also include the "n" from "n't"
apostroph_idx -= 1;
}
res.push(&tok[..apostroph_idx]);
res.push(&tok[apostroph_idx..]);
continue;
}
}
"fr" => {
// Handle English contractions
if let Some(apostroph_idx) = tok.find(&"'") {
let apostroph_idx = apostroph_idx;
if apostroph_idx == 1 {
let apostroph_idx = apostroph_idx + "'".len();
res.push(&tok[..apostroph_idx]);
res.push(&tok[apostroph_idx..]);
continue;
}
}
}
_ => {}
};
res.push(tok);
if res.len() >= 3 {
// Merge some sequences
let tok0 = res[res.len() - 3];
let tok1 = res[res.len() - 2];
let tok2 = res[res.len() - 1];
if (tok0 != " ") & (tok2 != " ") & !tok0.is_empty() & !tok2.is_empty() {
let char0_last = tok0.chars().last().unwrap();
let char2_first = tok0.chars().next().unwrap();
let f1 = ((tok1 == "-") | (tok1 == "@") | (tok1 == "&"))
& char0_last.is_alphanumeric()
& char2_first.is_alphanumeric();
let f2 = ((tok1 == "/") | (tok1 == ":"))
& char0_last.is_numeric()
& char2_first.is_numeric();
if f1 | f2 {
res.truncate(res.len() - 3);
res.push(&text[str_idx - tok0.len() - tok1.len() - tok2.len()..str_idx]);
}
}
}
}
if punct_start_seq >= 0 {
res.push(&text[punct_start_seq as usize..]);
}
// remove whitespace tokens
let res = res.into_iter().filter(|x| x != &" ");
Box::new(res)
}
}
/// Character tokenizer
#[derive(Debug, Clone)]
pub struct CharacterTokenizer {
pub window_size: usize,
}
impl CharacterTokenizer {
/// Create a new instance
pub fn new(window_size: usize) -> CharacterTokenizer {
CharacterTokenizer { window_size }
}
}
impl Tokenizer for CharacterTokenizer {
/// Tokenize a string
fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = &'a str> + 'a> {
let res = text
.char_indices()
.zip(
text.char_indices()
.skip(self.window_size)
.chain(Some((text.len(), ' '))),
)
.map(move |((i, _), (j, _))| &text[i..j]);
Box::new(res)
}
}