From 691afca4d88e789fc4ffd0878a24731a45e09b3a Mon Sep 17 00:00:00 2001
From: Andrew DalPino <me@andrewdalpino.com>
Date: Mon, 26 Apr 2021 13:02:43 -0500
Subject: [PATCH] Move Token Hashing Vectorizer to main package

---
 README.md                                     |   2 +-
 .../TokenHashingVectorizerBench.php           |  64 ---------
 composer.json                                 |   2 +-
 src/Transformers/TokenHashingVectorizer.php   | 124 ------------------
 .../TokenHashingVectorizerTest.php            |  63 ---------
 5 files changed, 2 insertions(+), 253 deletions(-)
 delete mode 100644 benchmarks/Transformers/TokenHashingVectorizerBench.php
 delete mode 100644 src/Transformers/TokenHashingVectorizer.php
 delete mode 100644 tests/Transformers/TokenHashingVectorizerTest.php

diff --git a/README.md b/README.md
index 26aa23d..a89521e 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ $ composer require rubix/extras
 ```
 
 ### Requirements
-- [PHP](https://php.net/manual/en/install.php) 7.2 or above
+- [PHP](https://php.net/manual/en/install.php) 7.4 or above
 
 ## License
 The code is licensed [MIT](LICENSE) and the documentation is licensed [CC BY-NC 4.0](https://creativecommons.org/licenses/by-nc/4.0/).
diff --git a/benchmarks/Transformers/TokenHashingVectorizerBench.php b/benchmarks/Transformers/TokenHashingVectorizerBench.php
deleted file mode 100644
index 8540b4a..0000000
--- a/benchmarks/Transformers/TokenHashingVectorizerBench.php
+++ /dev/null
@@ -1,64 +0,0 @@
-<?php
-
-namespace Rubix\ML\Benchmarks\Transformers;
-
-use Rubix\ML\Datasets\Unlabeled;
-use Rubix\ML\Transformers\TokenHashingVectorizer;
-
-/**
- * @Groups({"Transformers"})
- * @BeforeMethods({"setUp"})
- */
-class TokenHashingVectorizerBench
-{
-    protected const DATASET_SIZE = 10000;
-
-    protected const SAMPLE_TEXT = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec at nisl posuere, luctus sapien vel, maximus ex. Curabitur tincidunt, libero at commodo tempor, magna neque malesuada diam, vel blandit metus velit quis magna. Vestibulum auctor libero quam, eu ullamcorper nulla dapibus a. Mauris id ultricies sapien. Integer consequat mi eget vehicula vulputate. Mauris cursus nisi non semper dictum. Quisque luctus ex in tortor laoreet tincidunt. Vestibulum imperdiet purus sit amet sapien dignissim elementum. Mauris tincidunt eget ex eu laoreet. Etiam efficitur quam at purus sagittis hendrerit. Mauris tempus, sem in pulvinar imperdiet, lectus ipsum molestie ante, id semper nunc est sit amet sem. Nulla at justo eleifend, gravida neque eu, consequat arcu. Vivamus bibendum eleifend metus, id elementum orci aliquet ac. Praesent pellentesque nisi vitae tincidunt eleifend. Pellentesque quis ex et lorem laoreet hendrerit ut ac lorem. Aliquam non sagittis est.';
-
-    /**
-     * @var \Rubix\ML\Datasets\Dataset
-     */
-    protected $dataset;
-
-    /**
-     * @var \Rubix\ML\Transformers\TokenHashingVectorizer
-     */
-    protected $transformer;
-
-    /**
-     * @var array[]
-     */
-    protected $aSamples;
-
-    /**
-     * @var array[]
-     */
-    protected $bSamples;
-
-    public function setUp() : void
-    {
-        $samples = [];
-
-        for ($i = 0; $i < self::DATASET_SIZE; ++$i) {
-            $text = self::SAMPLE_TEXT;
-
-            str_shuffle($text);
-
-            $samples[] = [$text];
-        }
-
-        $this->dataset = Unlabeled::quick($samples);
-
-        $this->transformer = new TokenHashingVectorizer(1000);
-    }
-
-    /**
-     * @Subject
-     * @Iterations(3)
-     * @OutputTimeUnit("milliseconds", precision=3)
-     */
-    public function apply() : void
-    {
-        $this->dataset->apply($this->transformer);
-    }
-}
diff --git a/composer.json b/composer.json
index cf7518b..bc0fcdc 100644
--- a/composer.json
+++ b/composer.json
@@ -31,7 +31,7 @@
     "require-dev": {
         "friendsofphp/php-cs-fixer": "2.18.*",
         "league/flysystem-memory": "^2.0",
-        "phpbench/phpbench": "1.0.0-alpha6",
+        "phpbench/phpbench": "1.0.0-alpha8",
         "phpstan/extension-installer": "^1.0",
         "phpstan/phpstan": "0.12.*",
         "phpstan/phpstan-phpunit": "0.12.*",
diff --git a/src/Transformers/TokenHashingVectorizer.php b/src/Transformers/TokenHashingVectorizer.php
deleted file mode 100644
index 40c7a64..0000000
--- a/src/Transformers/TokenHashingVectorizer.php
+++ /dev/null
@@ -1,124 +0,0 @@
-<?php
-
-namespace Rubix\ML\Transformers;
-
-use Rubix\ML\DataType;
-use Rubix\ML\Datasets\Dataset;
-use Rubix\ML\Tokenizers\Word;
-use Rubix\ML\Tokenizers\Tokenizer;
-use Rubix\ML\Exceptions\InvalidArgumentException;
-
-use function count;
-use function is_string;
-
-/**
- * Token Hashing Vectorizer
- *
- * Token Hashing Vectorizer builds token count vectors on the fly by employing a *hashing
- * trick*. It is a stateless transformer that uses the CRC32 (Cyclic Redundancy Check)
- * hashing algorithm to assign token occurrences to a bucket in a vector of user-defined
- * dimensionality. The advantage of hashing over a fixed vocabulary is that there is no
- * memory footprint however there is a chance that certain tokens will collide with other
- * tokens especially in lower-dimensional vector spaces.
- *
- * @category    Machine Learning
- * @package     Rubix/ML
- * @author      Andrew DalPino
- */
-class TokenHashingVectorizer implements Transformer
-{
-    /**
-     * The maximum number of dimensions supported.
-     *
-     * @var int
-     */
-    protected const MAX_DIMENSIONS = 4294967295;
-
-    /**
-     * The dimensionality of the vector space.
-     *
-     * @var int
-     */
-    protected $dimensions;
-
-    /**
-     * The tokenizer used to extract tokens from blobs of text.
-     *
-     * @var \Rubix\ML\Tokenizers\Tokenizer
-     */
-    protected $tokenizer;
-
-    /**
-     * @param int $dimensions
-     * @param \Rubix\ML\Tokenizers\Tokenizer|null $tokenizer
-     * @throws \Rubix\ML\Exceptions\InvalidArgumentException
-     */
-    public function __construct(int $dimensions, ?Tokenizer $tokenizer = null)
-    {
-        if ($dimensions < 1 or $dimensions > self::MAX_DIMENSIONS) {
-            throw new InvalidArgumentException('Dimensions must be'
-                . ' between 0 and ' . self::MAX_DIMENSIONS
-                . ", $dimensions given.");
-        }
-
-        $this->dimensions = $dimensions;
-        $this->tokenizer = $tokenizer ?? new Word();
-    }
-
-    /**
-     * Return the data types that this transformer is compatible with.
-     *
-     * @return \Rubix\ML\DataType[]
-     */
-    public function compatibility() : array
-    {
-        return DataType::all();
-    }
-
-    /**
-     * Transform the dataset in place.
-     *
-     * @param array[] $samples
-     */
-    public function transform(array &$samples) : void
-    {
-        $scale = $this->dimensions / self::MAX_DIMENSIONS;
-
-        foreach ($samples as &$sample) {
-            $vectors = [];
-
-            foreach ($sample as $column => $value) {
-                if (is_string($value)) {
-                    $template = array_fill(0, $this->dimensions, 0);
-
-                    $tokens = $this->tokenizer->tokenize($value);
-
-                    $counts = array_count_values($tokens);
-
-                    foreach ($counts as $token => $count) {
-                        $offset = (int) floor(crc32($token) * $scale);
-
-                        $template[$offset] += $count;
-                    }
-
-                    $vectors[] = $template;
-
-                    unset($sample[$column]);
-                }
-            }
-
-            $sample = array_merge($sample, ...$vectors);
-        }
-    }
-
-    /**
-     * Return the string representation of the object.
-     *
-     * @return string
-     */
-    public function __toString() : string
-    {
-        return "Token Hashing Vectorizer (dimensions: {$this->dimensions},"
-            . " tokenizer: {$this->tokenizer})";
-    }
-}
diff --git a/tests/Transformers/TokenHashingVectorizerTest.php b/tests/Transformers/TokenHashingVectorizerTest.php
deleted file mode 100644
index e5c7010..0000000
--- a/tests/Transformers/TokenHashingVectorizerTest.php
+++ /dev/null
@@ -1,63 +0,0 @@
-<?php
-
-namespace Rubix\ML\Tests\Transformers;
-
-use Rubix\ML\Tokenizers\Word;
-use Rubix\ML\Datasets\Unlabeled;
-use Rubix\ML\Transformers\Transformer;
-use Rubix\ML\Transformers\TokenHashingVectorizer;
-use PHPUnit\Framework\TestCase;
-
-/**
- * @group Transformers
- * @covers \Rubix\ML\Transformers\TokenHashingVectorizer
- */
-class TokenHashingVectorizerTest extends TestCase
-{
-    /**
-     * @var \Rubix\ML\Datasets\Unlabeled
-     */
-    protected $dataset;
-
-    /**
-     * @var \Rubix\ML\Transformers\TokenHashingVectorizer
-     */
-    protected $transformer;
-
-    /**
-     * @before
-     */
-    protected function setUp() : void
-    {
-        $this->dataset = Unlabeled::quick([
-            ['the quick brown fox jumped over the lazy man sitting at a bus stop drinking a can of coke'],
-            ['with a dandy umbrella'],
-        ]);
-
-        $this->transformer = new TokenHashingVectorizer(20, new Word());
-    }
-
-    /**
-     * @test
-     */
-    public function build() : void
-    {
-        $this->assertInstanceOf(TokenHashingVectorizer::class, $this->transformer);
-        $this->assertInstanceOf(Transformer::class, $this->transformer);
-    }
-
-    /**
-     * @test
-     */
-    public function transform() : void
-    {
-        $this->dataset->apply($this->transformer);
-
-        $outcome = [
-            [1, 1, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 0, 0, 2, 1, 0, 1, 5, 0],
-            [0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0],
-        ];
-
-        $this->assertEquals($outcome, $this->dataset->samples());
-    }
-}