Create Vectorizer

Vectorizer/You-Now-Who/Vectorizer

@@ -0,0 +1,54 @@
+
+class CustomCountVectorizer {
+    constructor(lowercase = true, tokenPattern = /\b\w+\b/g) {
+        this.lowercase = lowercase;
+        this.tokenPattern = tokenPattern;
+        this.vocabulary = {};
+    }
+
+    fit(documents) {
+        this.vocabulary = {};
+        for (const doc of documents) {
+            let docText = doc;
+            if (this.lowercase) {
+                docText = doc.toLowerCase();
+            }
+            const tokens = docText.match(this.tokenPattern);
+            // console.log(tokens[0])
+            for (const token of tokens) {
+                if (!(token in this.vocabulary)) {
+                    this.vocabulary[token] = Object.keys(this.vocabulary).length;
+                }
+            }
+        }
+    }
+
+    transform(documents) {
+        if (Object.keys(this.vocabulary).length === 0) {
+            throw new Error("You must fit the vectorizer before transforming data.");
+        }
+
+        const transformedData = [];
+
+        for (const doc of documents) {
+            let docText = doc;
+            if (this.lowercase) {
+                docText = doc.toLowerCase();
+            }
+            const tokens = docText.match(this.tokenPattern);
+
+            const docVector = Array.from({ length: Object.keys(this.vocabulary).length }, () => 0);
+
+            for (const token of tokens) {
+                if (token in this.vocabulary) {
+                    const tokenIdx = this.vocabulary[token];
+                    docVector[tokenIdx]++;
+                }
+            }
+
+            transformedData.push(docVector);
+        }
+
+        return transformedData;
+    }
+}