WaveNet_STT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "WaveNet_STT.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mohammedterry/NLP_for_ML/blob/master/WaveNet_STT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "tX_X8SoXKZnl",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# WaveNet"
      ]
    },
    {
      "metadata": {
        "id": "34uw9FyTGagI",
        "colab_type": "code",
        "outputId": "8975379e-c770-4174-a6a8-b5e23a7608e7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 292
        }
      },
      "cell_type": "code",
      "source": [
        "!pip3 install tensorflow==1.0.0"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorflow==1.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/48/58/b71480f9ec9d08d581d672a81b15ab5fec36a5fcda2093558a23614d8468/tensorflow-1.0.0-cp36-cp36m-manylinux1_x86_64.whl (44.5MB)\n",
            "\u001b[K    100% |████████████████████████████████| 44.5MB 1.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.0.0) (3.6.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.0.0) (0.33.1)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.0.0) (1.11.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.0.0) (1.14.6)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.1.0->tensorflow==1.0.0) (40.8.0)\n",
            "\u001b[31mstable-baselines 2.2.1 has requirement tensorflow>=1.5.0, but you'll have tensorflow 1.0.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mmagenta 0.3.19 has requirement tensorflow>=1.12.0, but you'll have tensorflow 1.0.0 which is incompatible.\u001b[0m\n",
            "Installing collected packages: tensorflow\n",
            "  Found existing installation: tensorflow 1.13.1\n",
            "    Uninstalling tensorflow-1.13.1:\n",
            "      Successfully uninstalled tensorflow-1.13.1\n",
            "Successfully installed tensorflow-1.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "85wQog3AGOs7",
        "colab_type": "code",
        "outputId": "b7e44694-ea40-449d-9e26-309c0df01c9c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 190
        }
      },
      "cell_type": "code",
      "source": [
        "!pip3 install sugartensor"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sugartensor\n",
            "  Downloading https://files.pythonhosted.org/packages/29/3d/89dc6e0c12dd6dad7392131c3dceee5427a0bf7564a2bff4544e55e00967/sugartensor-1.0.0.2.tar.gz\n",
            "Requirement already satisfied: tqdm>=4.8.4 in /usr/local/lib/python3.6/dist-packages (from sugartensor) (4.28.1)\n",
            "Building wheels for collected packages: sugartensor\n",
            "  Building wheel for sugartensor (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/03/38/76/be0ec682a477b29d34c3d4d115305d2823075738a7a51dd6d9\n",
            "Successfully built sugartensor\n",
            "Installing collected packages: sugartensor\n",
            "Successfully installed sugartensor-1.0.0.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "QEUG17Wu-JUo",
        "colab_type": "code",
        "outputId": "36e10dab-e6dc-4f00-c5e5-7ef5ee6b617e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/buriburisuri/speech-to-text-wavenet"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'speech-to-text-wavenet'...\n",
            "remote: Enumerating objects: 106, done.\u001b[K\n",
            "remote: Total 106 (delta 0), reused 0 (delta 0), pack-reused 106\u001b[K\n",
            "Receiving objects: 100% (106/106), 248.84 KiB | 830.00 KiB/s, done.\n",
            "Resolving deltas: 100% (44/44), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "LzdnvSc0-kvW",
        "colab_type": "code",
        "outputId": "400c9b79-d90e-43b1-95f6-019ee7672828",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.chdir('speech-to-text-wavenet')\n",
        "!ls"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CHANGELOG.md  docker   model.py  preprocess.py\trecognize.py\t  test.py\n",
            "data.py       LICENSE  png\t README.md\trequirements.txt  train.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "e0ekbY6qmCNG",
        "colab_type": "code",
        "outputId": "0449aae8-b51a-47e4-a7d2-fbccf871a753",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "cell_type": "code",
      "source": [
        "!mkdir asset\n",
        "!ls"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "asset\t      docker\tpng\t       recognize.py\t train.py\n",
            "CHANGELOG.md  LICENSE\tpreprocess.py  requirements.txt\n",
            "data.py       model.py\tREADME.md      test.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "bTnWuxgaKtB1",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Load in Pretrained Weights saved in Google Drive"
      ]
    },
    {
      "metadata": {
        "id": "HkRaJbV2_lTm",
        "colab_type": "code",
        "outputId": "e82ae606-a414-4cce-f907-e557a64ca505",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Tr9VrhOZlzrg",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!cp -r ../gdrive/My\\ Drive/Codes_Projects/speech-to-text-wavenet/asset/* asset/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Hzu1fgrTLgRW",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Tests"
      ]
    },
    {
      "metadata": {
        "id": "QDz1PZSgLjUn",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from IPython.display import Audio\n",
        "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "J_nfQnwhL8n-",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Test 1"
      ]
    },
    {
      "metadata": {
        "id": "cYBk7OLTFSju",
        "colab_type": "code",
        "outputId": "14762e55-dbf7-4400-b9b0-26c04348855f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "cell_type": "code",
      "source": [
        "!curl -O https://www.ee.columbia.edu/~dpwe/sounds/sents/sm3_cln.wav\n",
        "Audio(url = \"https://www.ee.columbia.edu/~dpwe/sounds/sents/sm3_cln.wav\")"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100  104k  100  104k    0     0   130k      0 --:--:-- --:--:-- --:--:--  130k\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "                <audio controls=\"controls\" >\n",
              "                    <source src=\"https://www.ee.columbia.edu/~dpwe/sounds/sents/sm3_cln.wav\" type=\"audio/x-wav\" />\n",
              "                    Your browser does not support the audio element.\n",
              "                </audio>\n",
              "              "
            ],
            "text/plain": [
              "<IPython.lib.display.Audio object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "metadata": {
        "id": "mTmqtQeS-YxU",
        "colab_type": "code",
        "outputId": "05071ccd-865a-4377-bcdc-3aa4ca152b04",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "!python3 recognize.py --file sm3_cln.wav"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cotage teese with chives s tolicious\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "T-QViPO2MBT4",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Test 2"
      ]
    },
    {
      "metadata": {
        "id": "bKm1w4Sv9sTN",
        "colab_type": "code",
        "outputId": "d286ccef-51a0-4855-d4ea-811b9e5f2405",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "cell_type": "code",
      "source": [
        "!curl -O https://www.ee.columbia.edu/~dpwe/sounds/sents/sm1_cln.wav\n",
        "Audio(url = \"https://www.ee.columbia.edu/~dpwe/sounds/sents/sm1_cln.wav\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r 44  108k   44 49152    0     0   104k      0  0:00:01 --:--:--  0:00:01  104k\r100  108k  100  108k    0     0   204k      0 --:--:-- --:--:-- --:--:--  203k\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "                <audio controls=\"controls\" >\n",
              "                    <source src=\"https://www.ee.columbia.edu/~dpwe/sounds/sents/sm1_cln.wav\" type=\"audio/x-wav\" />\n",
              "                    Your browser does not support the audio element.\n",
              "                </audio>\n",
              "              "
            ],
            "text/plain": [
              "<IPython.lib.display.Audio object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "metadata": {
        "id": "e72oDVpOMOJL",
        "colab_type": "code",
        "outputId": "d8061bde-3a06-4aec-e6bc-3e3108775b99",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "!python3 recognize.py --file sm1_cln.wav"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cotage chees with chaes is telicious\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "vD9un8wLMszP",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Test 3"
      ]
    },
    {
      "metadata": {
        "id": "URkrfOmGMuaz",
        "colab_type": "code",
        "outputId": "5378342c-9bc5-4e9b-f0b8-66287ba05961",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "cell_type": "code",
      "source": [
        "! curl -O https://www.ee.columbia.edu/~dpwe/sounds/sents/sf3_cln.wav\n",
        "Audio(url = \"https://www.ee.columbia.edu/~dpwe/sounds/sents/sf3_cln.wav\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100 79302  100 79302    0     0   169k      0 --:--:-- --:--:-- --:--:--  169k\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "                <audio controls=\"controls\" >\n",
              "                    <source src=\"https://www.ee.columbia.edu/~dpwe/sounds/sents/sf3_cln.wav\" type=\"audio/x-wav\" />\n",
              "                    Your browser does not support the audio element.\n",
              "                </audio>\n",
              "              "
            ],
            "text/plain": [
              "<IPython.lib.display.Audio object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "metadata": {
        "id": "pQ7OVCLgMx4S",
        "colab_type": "code",
        "outputId": "f203c16d-1b8f-4308-f5d8-f17a9f391313",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "!python3 recognize.py --file sf3_cln.wav"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cottige cheese with chives is delicious\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "kmUiIrk7M47n",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Test 4"
      ]
    },
    {
      "metadata": {
        "id": "pdbQJqU1M6GQ",
        "colab_type": "code",
        "outputId": "0144ffb0-c08f-401f-c1f4-736349b2d630",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "cell_type": "code",
      "source": [
        "!curl -O https://www.ee.columbia.edu/~dpwe/sounds/sents/sf1_cln.wav\n",
        "Audio(url = \"https://www.ee.columbia.edu/~dpwe/sounds/sents/sf1_cln.wav\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100 83398  100 83398    0     0   141k      0 --:--:-- --:--:-- --:--:--  141k\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "                <audio controls=\"controls\" >\n",
              "                    <source src=\"https://www.ee.columbia.edu/~dpwe/sounds/sents/sf1_cln.wav\" type=\"audio/x-wav\" />\n",
              "                    Your browser does not support the audio element.\n",
              "                </audio>\n",
              "              "
            ],
            "text/plain": [
              "<IPython.lib.display.Audio object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "metadata": {
        "id": "dGD-QNHXM-qA",
        "colab_type": "code",
        "outputId": "d9a900d6-a8e6-4527-b5bb-9ad916a2a23d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "!python3 recognize.py --file sf1_cln.wav"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "codies chees with chives is delicious\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ww_xIVu8NGWe",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Test 5"
      ]
    },
    {
      "metadata": {
        "id": "HMJr--k5NH7P",
        "colab_type": "code",
        "outputId": "a49b488b-72e9-4c23-a9f5-5244bc09dca5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "cell_type": "code",
      "source": [
        "!curl -O https://www.ee.columbia.edu/~dpwe/sounds/sents/sm2_cln.wav\n",
        "Audio(url = \"https://www.ee.columbia.edu/~dpwe/sounds/sents/sm2_cln.wav\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r100 90566  100 90566    0     0   189k      0 --:--:-- --:--:-- --:--:--  189k\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "                <audio controls=\"controls\" >\n",
              "                    <source src=\"https://www.ee.columbia.edu/~dpwe/sounds/sents/sm2_cln.wav\" type=\"audio/x-wav\" />\n",
              "                    Your browser does not support the audio element.\n",
              "                </audio>\n",
              "              "
            ],
            "text/plain": [
              "<IPython.lib.display.Audio object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "metadata": {
        "id": "HDSWbi2gNLPG",
        "colab_type": "code",
        "outputId": "e9088e77-e378-4c39-eb36-d0ca85d779b3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "!python3 recognize.py --file sm2_cln.wav"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cottis chees with chis is delicious\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "0JfRwehkj4tj",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Try some more (longer, background noise, etc)"
      ]
    },
    {
      "metadata": {
        "id": "DAFwKVRsj7OB",
        "colab_type": "code",
        "outputId": "218adc8e-391c-4b49-c4dc-cdb6340e68ab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "cell_type": "code",
      "source": [
        "!curl -O https://www.ee.columbia.edu/~dpwe/sounds/musp/mssp2.wav\n",
        "Audio(url = \"https://www.ee.columbia.edu/~dpwe/sounds/musp/mssp2.wav\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r100  215k  100  215k    0     0   868k      0 --:--:-- --:--:-- --:--:--  864k\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "                <audio controls=\"controls\" >\n",
              "                    <source src=\"https://www.ee.columbia.edu/~dpwe/sounds/musp/mssp2.wav\" type=\"audio/x-wav\" />\n",
              "                    Your browser does not support the audio element.\n",
              "                </audio>\n",
              "              "
            ],
            "text/plain": [
              "<IPython.lib.display.Audio object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "metadata": {
        "id": "lkcBVAdxkF9p",
        "colab_type": "code",
        "outputId": "a43f36f9-823c-49e9-9f0d-5c13ea0d90d5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "!python3 recognize.py --file mssp2.wav"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ay its time forsae intr vention here s apout of you cal dont look back\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "x8lzkP1gBLY-",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Phonetic Spellchecking"
      ]
    },
    {
      "metadata": {
        "id": "SfI1rc1uBNr9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "f5891609-4944-4c0e-fa2d-58dd8c2a0994"
      },
      "cell_type": "code",
      "source": [
        "!pip3 install jellyfish\n",
        "from jellyfish import metaphone, soundex, nysiis, match_rating_codex, jaro_distance, levenshtein_distance, damerau_levenshtein_distance, jaro_winkler, hamming_distance"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: jellyfish in /usr/local/lib/python3.6/dist-packages (0.7.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "8iVkyMKxBThI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "19633523-d517-4537-8172-fc7d44cf2c64"
      },
      "cell_type": "code",
      "source": [
        "!wget https://raw.githubusercontent.com/first20hours/google-10000-english/master/google-10000-english.txt\n",
        "with open('google-10000-english.txt') as f:\n",
        "  vocab = [word.strip() for word in f.readlines()]\n",
        "vocab += [\"chive\",\"chives\"]"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-03-07 12:08:42--  https://raw.githubusercontent.com/first20hours/google-10000-english/master/google-10000-english.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 75888 (74K) [text/plain]\n",
            "Saving to: ‘google-10000-english.txt’\n",
            "\n",
            "\rgoogle-10000-englis   0%[                    ]       0  --.-KB/s               \rgoogle-10000-englis 100%[===================>]  74.11K  --.-KB/s    in 0.01s   \n",
            "\n",
            "2019-03-07 12:08:42 (7.37 MB/s) - ‘google-10000-english.txt’ saved [75888/75888]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "eTENO0TIBWvD",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def suggest_corrections(typo, top_n = 3, reverse = False):\n",
        "  typo_phonetic = phonetic_encoder(typo)\n",
        "  scores = [distance_metric(typo_phonetic, phonetic) for phonetic in vocab_phonetic]\n",
        "  ranked_candidates_idx = sorted([(score,idx) for idx,score in enumerate(scores)], reverse= reverse)\n",
        "  ranked_candidates = [vocab[idx] for _,idx in ranked_candidates_idx[:top_n]]\n",
        "  return ranked_candidates\n",
        "\n",
        "def spellcheck(sentence, reverse = False):\n",
        "  return ' '.join([typo if typo in vocab else suggest_corrections(typo,1,reverse)[0] for typo in sentence.split()])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jo-w0IJhHy-D",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "phonetic_encoders = [metaphone, soundex, nysiis, match_rating_codex]\n",
        "inverse_distance_metrics = [jaro_distance, jaro_winkler]\n",
        "distance_metrics = [levenshtein_distance, damerau_levenshtein_distance, hamming_distance]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "wbR_ussbJAax",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 833
        },
        "outputId": "327a49ad-766b-43b7-cddc-0f7b11e09132"
      },
      "cell_type": "code",
      "source": [
        "typo = 'chivez'\n",
        "test_sentence = 'cottige cheeze with chivez is delicius'\n",
        "\n",
        "for phonetic_encoder in phonetic_encoders:\n",
        "  print('\\n'+'-'*10 + phonetic_encoder.__name__.upper() + '-'*10)\n",
        "  vocab_phonetic = [phonetic_encoder(word) for word in vocab]\n",
        "  for distance_metric in distance_metrics:\n",
        "    print(f\"{distance_metric.__name__.upper()}\\n\\t\\t\\t{suggest_corrections(typo)}, {spellcheck(test_sentence)}\")\n",
        "  for distance_metric in inverse_distance_metrics:\n",
        "    print(f\"{distance_metric.__name__.upper()}\\n\\t\\t\\t{suggest_corrections(typo,reverse=True)}, {spellcheck(test_sentence,reverse=True)}\")"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "----------METAPHONE----------\n",
            "LEVENSHTEIN_DISTANCE\n",
            "\t\t\t['chives', 'reviews', 'office'], cottage choose with chives is delicious\n",
            "DAMERAU_LEVENSHTEIN_DISTANCE\n",
            "\t\t\t['chives', 'reviews', 'office'], cottage choose with chives is delicious\n",
            "HAMMING_DISTANCE\n",
            "\t\t\t['chives', 'reviews', 'office'], cottage choose with chives is delicious\n",
            "JARO_DISTANCE\n",
            "\t\t\t['chives', 'chive', 'chevy'], cottage chess with chives is delicious\n",
            "JARO_WINKLER\n",
            "\t\t\t['chives', 'chive', 'chevy'], cottage chess with chives is delicious\n",
            "\n",
            "----------SOUNDEX----------\n",
            "LEVENSHTEIN_DISTANCE\n",
            "\t\t\t['cvs', 'copies', 'chips'], cities check with cvs is delicious\n",
            "DAMERAU_LEVENSHTEIN_DISTANCE\n",
            "\t\t\t['cvs', 'copies', 'chips'], cities check with cvs is delicious\n",
            "HAMMING_DISTANCE\n",
            "\t\t\t['cvs', 'copies', 'chips'], cities check with cvs is delicious\n",
            "JARO_DISTANCE\n",
            "\t\t\t['chives', 'cubic', 'cups'], cottage casey with chives is delicious\n",
            "JARO_WINKLER\n",
            "\t\t\t['chives', 'cubic', 'cups'], cottage casey with chives is delicious\n",
            "\n",
            "----------NYSIIS----------\n",
            "LEVENSHTEIN_DISTANCE\n",
            "\t\t\t['cave', 'cove', 'chive'], cottage case with cave is delicious\n",
            "DAMERAU_LEVENSHTEIN_DISTANCE\n",
            "\t\t\t['cave', 'cove', 'chive'], cottage case with cave is delicious\n",
            "HAMMING_DISTANCE\n",
            "\t\t\t['cave', 'cove', 'chive'], cottage case with cave is delicious\n",
            "JARO_DISTANCE\n",
            "\t\t\t['chives', 'chive', 'cove'], cottages casa with chives is delicious\n",
            "JARO_WINKLER\n",
            "\t\t\t['chives', 'chive', 'cove'], cottages casa with chives is delicious\n",
            "\n",
            "----------MATCH_RATING_CODEX----------\n",
            "LEVENSHTEIN_DISTANCE\n",
            "\t\t\t['chevy', 'chive', 'chives'], cottage china with chevy is delicious\n",
            "DAMERAU_LEVENSHTEIN_DISTANCE\n",
            "\t\t\t['chevy', 'chive', 'chives'], cottage china with chevy is delicious\n",
            "HAMMING_DISTANCE\n",
            "\t\t\t['chevy', 'chive', 'chives'], cottage china with chevy is delicious\n",
            "JARO_DISTANCE\n",
            "\t\t\t['chive', 'chives', 'cho'], cottage cho with chive is delicious\n",
            "JARO_WINKLER\n",
            "\t\t\t['chive', 'chives', 'chevy'], cottage cho with chive is delicious\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "g0k2eHB7B9MH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "outputId": "bf55f708-f3a7-45b4-fa3c-1ad0d904eca1"
      },
      "cell_type": "code",
      "source": [
        "phonetic_encoder = metaphone\n",
        "vocab_phonetic = [phonetic_encoder(word) for word in vocab]\n",
        "distance_metric = damerau_levenshtein_distance\n",
        "\n",
        "tests = [\n",
        "    \"cotage teese with chives s tolicious\",\n",
        "    \"cotage chees with chaes is telicious\",\n",
        "    \"cottige cheese with chives is delicious\",\n",
        "    \"codies chees with chives is delicious\",\n",
        "    \"cottis chees with chis is delicious\",\n",
        "]\n",
        "\n",
        "for test in tests:\n",
        "  print(f\"{test}\\n{spellcheck(test)}\\n\")"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cotage teese with chives s tolicious\n",
            "cottage does with chives s delicious\n",
            "\n",
            "cotage chees with chaes is telicious\n",
            "cottage choose with choose is delicious\n",
            "\n",
            "cottige cheese with chives is delicious\n",
            "cottage cheese with chives is delicious\n",
            "\n",
            "codies chees with chives is delicious\n",
            "kids choose with chives is delicious\n",
            "\n",
            "cottis chees with chis is delicious\n",
            "kids choose with choose is delicious\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}