clean spam samples

umputun · Dec 6, 2023 · e62c5a6 · e62c5a6
1 parent 8bfadc1
commit e62c5a6
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 1,781 deletions.
diff --git a/app/bot/spam.go b/app/bot/spam.go
@@ -315,7 +315,7 @@ func (s *SpamFilter) tokenize(inp string) map[string]int {
 			continue
 		}
 		token = s.cleanEmoji(token)
-		token = strings.Trim(token, ".,!?-:;()")
+		token = strings.Trim(token, ".,!?-:;()#")
 		token = strings.ToLower(token)
 		tokenFrequency[strings.ToLower(token)]++
 	}