Text_Mining.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Text Mining.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mohammedterry/NLP_for_ML/blob/master/Text_Mining.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "E5jzBMCalDsT",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "%matplotlib inline\n",
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "\n",
        "def display(ranked_wordlist, n_samples = 15):\n",
        "  sns.barplot(x=\"frequency\", y=\"words\", data=pd.DataFrame({\"frequency\":[x for x,_ in ranked_wordlist[:n_samples]], \"words\":[x for _,x in ranked_wordlist[:n_samples]]}))\n",
        "  \n",
        "def compare(ranked_wordlist_a, ranked_wordlist_b, method_a = \"a\", method_b = \"b\", n_samples = 15):\n",
        "  d_i = pd.DataFrame({\n",
        "    \"method\":[method_a]*n_samples + [method_b]*n_samples,\n",
        "    \"frequency\":[x for x,_ in ranked_wordlist_a[:n_samples]] + [x for x,_ in ranked_wordlist_b[:n_samples]],\n",
        "    \"words\":[x for _,x in ranked_wordlist_a[:n_samples]] + [x for _,x in ranked_wordlist_b[:n_samples]], \n",
        "  })\n",
        "  sns.factorplot(\"frequency\", \"words\", col=\"method\", data=d_i, kind=\"bar\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "VSijAIUcmFeV",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Get some raw text"
      ]
    },
    {
      "metadata": {
        "id": "zNTuY819mE1m",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup as bs\n",
        "\n",
        "def scrape(url):\n",
        "  soup = bs(requests.get(url, headers = {'user-agent': \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\",'referrer': 'https://google.com'}).text, \"html.parser\")\n",
        "  return '\\n'.join([dom.text for dom in soup.find_all(['title','h1','h2','h3','h4','h5','h6','p'])])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "BGTH4tdpmxpd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "outputId": "f306b045-4f25-4733-b084-56cbbe70adc3"
      },
      "cell_type": "code",
      "source": [
        "example_document = scrape(\"http://www.literatureproject.com/alice/alice_7.htm\")\n",
        "\n",
        "print(example_document[:300])"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "7 : CHAPTER VII\n",
            "\n",
            "\n",
            " \n",
            "Prev \n",
            "| Next \n",
            "| Contents \n",
            "CHAPTER VII\n",
            "A Mad Tea-Party\n",
            "              \n",
            "\n",
            "  There was a table set out under a tree in front of the house,\n",
            "and the March Hare and the Hatter were having tea at it:  a\n",
            "Dormouse was sitting between them, fast asleep, and the other two\n",
            "were using it as a c\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "hYcsM4UtkTxL",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Frequency of Terms"
      ]
    },
    {
      "metadata": {
        "id": "-EW8yEm_kX6j",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from collections import defaultdict\n",
        "\n",
        "def rank_words(document):\n",
        "  counter = defaultdict(int)\n",
        "  for word in document.split():\n",
        "      counter[word] += 1\n",
        "  return sorted(zip(counter.values(), counter.keys()), reverse = True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3FsSk0Dwkiaw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "ranked_raw = rank_words(example_document)\n",
        "display(ranked_raw)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "SQu7EDsspUIa",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Strip Punctuation "
      ]
    },
    {
      "metadata": {
        "id": "H2gtUIGipYA_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def clean(text):\n",
        "  return ' '.join(''.join(letter if ord('a') <= ord(letter.lower()) <= ord('z') or letter.isdigit() else ' ' for letter in text).split())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "NQv9ZD_8pqow",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "clean_document = clean(example_document)\n",
        "clean_document[:100]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jSa_IE5UqYoo",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Lower Case"
      ]
    },
    {
      "metadata": {
        "id": "VGt2KbRcqZ5P",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "clean_document = clean_document.lower()\n",
        "clean_document[:100]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ECTKitnotcec",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "ranked_clean = rank_words(clean_document)\n",
        "display(ranked_clean)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "w9ekHDBBpuHl",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "compare(ranked_raw,ranked_clean, \"raw\", \"lower case\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "VogWiYNRtzh4",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Split Strings"
      ]
    },
    {
      "metadata": {
        "id": "ptjMWEbq2vrs",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip3 install wordninja"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "oufIgFRq2tM7",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from wordninja import split\n",
        "\n",
        "x = 'itwasadarkandstormynighttherainfellintorrentsexceptatoccasionalintervalswhenitwascheckedbyaviolentgustofwindwhichsweptupthestreetsforitisinlondonthatoursceneliesrattlingalongthehousetopsandfiercelyagitatingthescantyflameofthelampsthatstruggledagainstthedarkness'\n",
        "' '.join(split(x))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "pDnURB6h28D2",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "split_document = ' '.join(split(clean_document))\n",
        "ranked_split = rank_words(split_document)\n",
        "display(ranked_split)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1jiHNpXL3ND8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "compare(ranked_clean, ranked_split, \"clean\", \"split\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "SU6K0s6q3kAa",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Stop Words"
      ]
    },
    {
      "metadata": {
        "id": "nIFrMoIA31Ez",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "fe3cbdf9-25f3-4791-eb5a-22614a6f861b"
      },
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 81
        }
      ]
    },
    {
      "metadata": {
        "id": "hk8l-vaS36tv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "42e953c3-9088-47ca-80eb-f4a8b4756447"
      },
      "cell_type": "code",
      "source": [
        "from nltk.corpus import stopwords\n",
        "stopWords = stopwords.words('english')\n",
        "stopWords[:10]"
      ],
      "execution_count": 82,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\"]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 82
        }
      ]
    },
    {
      "metadata": {
        "id": "-e3Unw_W4ETJ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "nostopwords_document = ' '.join([word for word in split_document.split() if word not in stopWords])\n",
        "ranked_nostopwords = rank_words(nostopwords_document)\n",
        "display(ranked_nostopwords)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "fuZ-4FqU409-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "compare(ranked_split,ranked_nostopwords, \"split\", \"no stop words\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "TQIP7TAJ31mo",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# N Grams\n",
        "phrases"
      ]
    },
    {
      "metadata": {
        "id": "cF16v7Cg830o",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "list(nltk.bigrams(\"this is a test\".split()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "XEHLxP_o86eb",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "list(nltk.trigrams(\"this is a test\".split()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "w6UBoD8O7b_h",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from collections import defaultdict\n",
        "import nltk\n",
        "\n",
        "def rank_ngrams(document, unigrams = True, bigrams = True, trigrams = True):\n",
        "  counter = defaultdict(int)\n",
        "  if unigrams:\n",
        "    for word in document.split():\n",
        "        counter[word] += 1\n",
        "  if bigrams:\n",
        "    for bigram in nltk.bigrams(document.split()):\n",
        "      counter[' '.join(bigram)] += 1\n",
        "  if trigrams:\n",
        "    for trigram in nltk.trigrams(document.split()):\n",
        "      counter[' '.join(trigram)] += 1\n",
        "  return sorted(zip(counter.values(), counter.keys()), reverse = True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Rbeu-PjQ7LfO",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "ranked_ngrams = rank_ngrams(nostopwords_document)\n",
        "display(ranked_ngrams)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "rrZzqLyl9M-u",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "compare(ranked_nostopwords, ranked_ngrams, method_b = \"n grams\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ozixvPV16BMg",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Stemming\n",
        "chopping off suffixes"
      ]
    },
    {
      "metadata": {
        "id": "n97KKi2y9g7p",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('rslp')\n",
        "\n",
        "from nltk.stem.api import StemmerI\n",
        "from nltk.stem.regexp import RegexpStemmer\n",
        "from nltk.stem.lancaster import LancasterStemmer\n",
        "from nltk.stem.isri import ISRIStemmer\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "from nltk.stem.snowball import SnowballStemmer\n",
        "from nltk.stem.rslp import RSLPStemmer\n",
        "\n",
        "stemmer_sb = SnowballStemmer(\"english\")\n",
        "stemmer_p = PorterStemmer()\n",
        "stemmer_l = LancasterStemmer()\n",
        "stemmer_regex = RegexpStemmer('ing$|s$|e$|able$|ed$|en$',min=4)\n",
        "stemmer_isri = ISRIStemmer()\n",
        "stemmer_rslp = RSLPStemmer()\n",
        "\n",
        "def get_name(model):\n",
        "  name = clean(str(model).split()[0])\n",
        "  name = ''.join([c + ' ' if d.isupper() and e.islower() else c for c,d,e in zip(name,name[1:] + '..',name[2:] + '..')])\n",
        "  return ' '.join(name.split())\n",
        "  \n",
        "def test_stemmers(Xs, Ys, stemmers = [stemmer_sb, stemmer_p, stemmer_l, stemmer_isri, stemmer_rslp, stemmer_regex]):\n",
        "  results = {\"expected\":Ys}\n",
        "  for stemmer in stemmers:\n",
        "    results[get_name(stemmer)] = [stemmer.stem(x.lower()) for x in Xs]\n",
        "  for model,words in results.items():\n",
        "    results[model].append(sum([w1.lower() == w2.lower() for w1,w2 in zip(results['expected'],words)]))  \n",
        "\n",
        "  import pandas as pd\n",
        "  df = pd.DataFrame(results)\n",
        "  df.set_index(\"expected\", inplace = True) \n",
        "  return df"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "K1blZP2vdZLU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "example_sentence = \"Sitting while eating leaves one wondering and then googling\"\n",
        "expected_sentence = \"Sit while eat leave one wonder and then google\"\n",
        "test_stemmers(example_sentence.split(), expected_sentence.split() )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "uA7mDpfYmr7I",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Lemmatising\n",
        "getting to the root of the word"
      ]
    },
    {
      "metadata": {
        "id": "ThRmSgsFd4lN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "1ed973c3-3f5c-461c-d88b-88e1136742ed"
      },
      "cell_type": "code",
      "source": [
        "nltk.download('wordnet')\n",
        "\n",
        "from nltk.stem.wordnet import WordNetLemmatizer \n",
        "from spacy.lemmatizer import Lemmatizer \n",
        "\n",
        "wn_lemmatiser = WordNetLemmatizer()\n",
        "sp_lemmatiser = Lemmatizer()\n",
        "\n",
        "def test_lemmatisers(Xs, Ys):\n",
        "  lemmatiser_results = {\n",
        "      \"expected\":Ys,\n",
        "      \"NLTK\":[wn_lemmatiser.lemmatize(x.lower(),'v') for x in Xs],\n",
        "      \"Spacy\":[sp_lemmatiser(x.lower(), 'VERB')[0] for x in Xs],    \n",
        "  }\n",
        "\n",
        "  for lemmatiser,words in lemmatiser_results.items():\n",
        "    lemmatiser_results[lemmatiser].append(sum([w1.lower() == w2.lower() for w1,w2 in zip(lemmatiser_results['expected'],words)]))  \n",
        "\n",
        "  import pandas as pd\n",
        "  lemma_df = pd.DataFrame(lemmatiser_results)\n",
        "  lemma_df.set_index(\"expected\", inplace = True) \n",
        "  return lemma_df"
      ],
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "e1pv-dQinhy0",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "example_sentence = \"Sitting while eating leaves one wondering and then googling\"\n",
        "expected_sentence = \"Sit while eat leave one wonder and then google\"\n",
        "test_lemmatisers(example_sentence.split(), expected_sentence.split())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ToSiET8ktXjx",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "lemmatised_document = ' '.join([wn_lemmatiser.lemmatize(word,'v') for word in nostopwords_document.split()])\n",
        "ranked_lemma = rank_ngrams(lemmatised_document)\n",
        "display(ranked_lemma)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "xE4GRdmUuE38",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "compare(ranked_ngrams, ranked_lemma,'pre-lemmatisation', 'post-lemmatisation')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3w2U1AKY6vWl",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Tf-Idf (Sklearn)\n",
        "giving more weight to words uncommon to this context"
      ]
    },
    {
      "metadata": {
        "id": "3Wqip1YtvQL4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "alice_wonderland_12chapters = [scrape(f\"http://www.literatureproject.com/alice/alice_{i}.htm\") for i in range(1,13)]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "BuoBpF9Ov1iq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "a5773186-8e19-479a-8afd-19fa9765b252"
      },
      "cell_type": "code",
      "source": [
        "alice_wonderland_12chapters[6][:100]"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'7 : CHAPTER VII\\n\\n\\n\\xa0\\nPrev \\n| Next \\n| Contents \\nCHAPTER VII\\nA Mad Tea-Party\\n              \\n\\n\\xa0\\xa0There wa'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    },
    {
      "metadata": {
        "id": "tzXhwemp6uxI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "def rank_tfidf(document, tfidf):\n",
        "  feature_names = tfidf.get_feature_names()\n",
        "  response = tfidf.transform([document])\n",
        "  tfs = {feature_names[col] : response[0, col]  for col in response.nonzero()[1]}\n",
        "  return sorted(zip(tfs.values(), tfs.keys()), reverse = True)\n",
        "\n",
        "tfidf = TfidfVectorizer(stop_words = 'english', ngram_range = (1,3))\n",
        "tfidf.fit_transform(alice_wonderland_12chapters)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Q8aHKZF6wUY9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "ranked_tfidf = rank_tfidf(example_document, tfidf)\n",
        "display(ranked_tfidf)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "JJKmmnVfxSKB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "max_freq = max(ranked_ngrams)[0] * 3\n",
        "scaled_tfidf = [(freq*max_freq,word) for freq,word in ranked_tfidf]\n",
        "compare(ranked_ngrams, scaled_tfidf, \"pre-tfidf\", \"post-tfidf\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8_7OVjrztIhW",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Montezuma Ranking (Gensim)"
      ]
    },
    {
      "metadata": {
        "id": "Rwg0JdUFtJZk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from gensim.summarization import keywords\n",
        "ranked_gensim = [(f,w) for w,f in keywords(example_document,scores=True)]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4vaRuEBlt7M2",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "display(ranked_gensim)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8L-v5PiFuV_E",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from gensim.summarization import mz_keywords\n",
        "ranked_montezuma = [(f,w) for w,f in mz_keywords(example_document,scores=True, weighted=False)]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "dvJK3vGfvT5w",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "display(ranked_montezuma)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "uUw8vUlMvXyV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "compare(ranked_gensim, ranked_mz)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1DH80yURwuTe",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Document Topic Modelling (DTM - Gensim)"
      ]
    },
    {
      "metadata": {
        "id": "YMXsJFnXwxhR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "17f68dd2-ef54-4ff7-b3e0-533bfb1908c7"
      },
      "cell_type": "code",
      "source": [
        "from gensim import corpora\n",
        "\n",
        "class DTMcorpus(corpora.textcorpus.TextCorpus):\n",
        "    def get_texts(self):\n",
        "        return self.input\n",
        "    def __len__(self):\n",
        "        return len(self.input)\n",
        "\n",
        "!git clone https://github.com/magsilva/dtm"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'dtm' already exists and is not an empty directory.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "TLjsLpP1xggK",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "corpus = DTMcorpus([[wn_lemmatiser.lemmatize(word,'v') for word in clean(document).lower().split() if word not in stopWords] for document in alice_wonderland_12chapters])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "u7FXicCDxvlZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from gensim.models.wrappers.dtmmodel import DtmModel\n",
        "dtm_path = 'dtm/bin/dtm-linux64'\n",
        "time_seq = [1 for _ in range(12)]\n",
        "dtm = DtmModel(dtm_path, corpus, time_seq, num_topics=1, id2word=corpus.dictionary, initialize_lda=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZSWAwQ5Bx2zo",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "ranked_dtm = dtm.show_topic(topicid=0, time=7, topn=30)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "b3_5bD2MyEOM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 333
        },
        "outputId": "4ed6e6b6-3aad-45e2-dfb5-e38a71c8a2c1"
      },
      "cell_type": "code",
      "source": [
        "display(ranked_dtm)"
      ],
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/seaborn/categorical.py:1428: FutureWarning: remove_na is deprecated and is a private function. Do not use.\n",
            "  stat_data = remove_na(group_data)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEGCAYAAACZ0MnKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHexJREFUeJzt3XmYXVWZ7/FvgAQiQlAIQSQYEPp3\nDQgNhEkCSYDL1GAYEkaJQUBsBeNFb3tFpnZAUWjpEG1FkNCNiBiGBqQZbCASGWTUpoUX4RIFQpgk\nCCqEJNV/rFVyUtSwU6mz965zfp/nyVPn7Om8ZxHqzVpr73cN6ejowMzMrIhVqg7AzMwGDycNMzMr\nzEnDzMwKc9IwM7PCnDTMzKyw1aoOoNmWLFna8fLLf646jFp617vegdumZ26fnrltetcK7TNy5FpD\nutve8j2N1VZbteoQastt0zu3T8/cNr1r5fYZ0urPabzwL5e29hc0M+vOlMkrdXrb9jTMzGzgOGmY\nmVlhThpmZlZYbe6ekrQxcCmwlBTXR4BvA2sC7wBOAtYFjoyIo/M53weui4hrKwnazKzN1KmnMQW4\nJSImATOA9wEX5vdfAD4P3AzsKGkNSasAuwA3VhWwmVm7qVPSuBmYJulcYHXgV8AhkuYBZwPrRsRS\n4HpgP2BH4I6IWFxVwGZm7aY2SSMiHga2Bu4AvgZ8BngmIsYDf99w6L8CU4EPA5eVHaeZWTurTdKQ\ndDiwZURcA5wKrAc8kXcfBAwDiIiHgPcCOwA/ryBUM7O2VZukATwGzJJ0K3AGcAVwsqSbgXuADSQd\nk4+9GbgvIvzgnplZiWpz91REPEDqPTT6QMPrawEkDQEmAp8oJzIzM+s0qMqISBoDXAlcERFnFzyt\n44UXXm1eUIPYyJFr4bbpmdunZ26b3rVC+/RURqQ2PY0iImI+sF3VcZiZtas6zWmYmVnNDaqeRn8s\n+PbJVYdQWwuqDqDm3D4962/bDD30Hwc0DiufexpmZlaYk4aZmRVWm+EpSfOBLYETgbkRcVelAZmZ\n2dvUJml0ioivVx2DmZl1r5KkIWltUt2oxrLnnftmA3OAm4BLSNVuXwemAQuBC4BNgaHA6RFxa5mx\nm5m1s6rmNDbg7WXPu/oosDAidgG+TypQeCTwbD7vQOC8kuI1MzOqG556DjhN0udIZdD/1M0x2wL/\nCRARlwNI+hdgV0nj8zHDJQ1zeXQzs3JUlTQ6y54fLWkccE43xyzl7T2hxcBXI+JHzQ7QzMzerqrh\nqW7LnndxL7A7gKT9JZ1CqnY7OW9bX9JZJcRqZmZZVUnjX+lS9hzoWhzrcmBNSXNJPZNLSOXSX5N0\nJ3AdacEmMzMrSSXDUxFxL92UPc+mN7ye1s3pxzUjJjMz69ugKo3eTy6N3oNWKN/cTG6fnrltetcK\n7dNTaXSXETEzs8KcNMzMrLDalREZaHddsH/VIdTW4yV/3mYH+U5ps8HOPQ0zMyvMScPMzApz0jAz\ns8JqO6chaQSp2u1w4AbgeOAY4CzgTeBp4GMR8UZlQZqZtZk69zSmAb+JiPHAItIT498FDouICcDL\npKq3ZmZWkjonjQ8Av8ivrwXeDXRExFN5223ANlUEZmbWruqcNIYAy/Lrjvyn8QnFYQ37zcysBHVO\nGk8A4/LrfUnDUR2SNs7bJgD3VRGYmVm7qnPSmE1acOl2YBRpfY3jgcvytqGkSrhmZlaS2t49RVo/\n/EsRcZOknYEJETEPGN/HeWZm1iR1ThqvkNbcOJ00l/Hp/lxk549fP+irTTZLK1TiNLNy1TZpRMQi\nYO+q4zAzs7fUeU7DzMxqprY9jYEy5+J9qg6hVibs/5OqQzCzQcw9DTMzK8xJw8zMCqskaUg6RNJ0\nSed0s+9yScN7OG+MJD/QZ2ZWkdLnNCSNAY4Aru9uf0QcXmpAZmZWWBUT4d8GdgB+DWwo6UpgLPDN\niPiBpPnAlsAsYAGwHbAxcBTwh86LSNoXOAk4ICKWlvkFzMzaVRXDU98E5gK/BzYFDgUOpPuH91aP\niL2BfyaVSgdA0mbAacARThhmZuWpeiL87vxL/xlgRDf778g/n27YvyZwDXBiRLzS/BDNzKxT1Ulj\nScPrIQX3b0RKJp9sVlBmZta9KpLGMlZuLiVICeP9kvYamJDMzKyIKpLGI8C2wLf6e4GI6ACOA86T\ntNZABWZmZr0b0tHRUXUMTTXn4n1a+wuuoMYyIq5y2zu3T8/cNr1rhfYZOXKt7qYMWr/21JRjbhz0\n//HMzOqi6olwMzMbRJw0zMyssJYfnpr5w/Zax+mIveZUHYKZtTD3NMzMrLBKexqSpgPjgaURcYKk\n3YBHI+J5SR8GbgQ2BOZExLgKQzUzM+rR01gUESfk1x8D1s+vTwaGVROSmZl1pw5zGp1rZHyBVLhw\nC0mzgJ2A/wCO7TxQ0q7AWcCbwFPA8RGxuPyQzczaUx16GgBExC3AQ8AxEXEJsBDYF2hMCjOByRGx\nO/AcMLX0QM3M2lgdehqFSBoFbA5cJQlStdsXKw3KzKzNDJqkQepxPBMRE6sOxMysXdVmeCprrIC7\nXDXciHgZQNLY/PMkSVuVHqGZWRurW9KYC8yRtAVwOzAPWK9h/7HAxZLuIN2qG6VHaGbWxlq+yu3M\nH+7d2l+wixV5IrwVKnE2k9unZ26b3rVC+7RtldtPH3XToP+PZ2ZWF3UbnjIzsxpz0jAzs8Jafnjq\nmKv3qTqEUnxj/E/6PsjMbCW5p2FmZoU5aZiZWWGVJg1J0yWdswLHdxY3NDOzCrinYWZmhdViIlzS\nDODw/PaaiDhb0kbAD0hraiwjPQ3e0XDOvsBJwAERsbTkkM3M2lIdehqbANOBXfOfwyS9H/gScFEu\nUPgd4MzOEyRtBpwGHOGEYWZWnjokjW2AuyNiSUQsAX4BbA2MI9WfArgtHwepJPo1wIkR8UrJsZqZ\ntbU6JI0OoLHGSedwVOP2zm0AGwF3AJ8sK0AzM0vqkDQeBHaWtJqk1YAd87Z7gUn5mAlA511TQUoY\n75e0V9nBmpm1szokjfnABaSy6HcAF0bE74DTgWmSbiXNeZzReUJEdADHAedJWqvsgM3M2lWld09F\nxOyGt9/usm8BaY3wrsbl/U8AY5sWnJmZvU0tbrltposPutGl0c3MBkgdhqfMzGyQcNIwM7PCWn54\nar+rv1J1CE11yfgZVYdgZm3EPQ0zMyustKSxohVtzcysftzTMDOzwiqZ05D0NeBPwBhgJCDgmxFx\nkaSJwFnAm8DTwMeAXwFbkMqKvAxMioj7JN0EfDw/DGhmZk1Wek9D0lRgNCkhfBA4GDiQVOYc4LvA\nYRExgZQgjgTuB7YkFS28j1R2ZBVglBOGmVl5yk4aWwBnk0qAANyVS5s/DYyQ9G6gIyKeyvs7q9vO\nBXYCdgHOJ9Wn+iDwQImxm5m1vRVOGvlf+P01BvhvYEp+v6Rh3xB6rnh7Oylp7ATcAowgJZDbViIW\nMzNbQX0mgHzX0ydzFdp5wJOS/r6fn/dT0hzFacCorjsj4mWgQ9LGedME4L6IeIw0pDUiIl4FFpKG\ntJw0zMxKVKTXcAJwIemX9MOklfYO6+8HRsQLpIq1n+/hkOOByyTdDgwFLs/bnwc65y/uAcZExNP9\njcPMzFZckbun/hIRiyXtB1waEcskdfR5VheNFW0j4nLeSgZExGukoSsiYh4wvpvzj2x4fSEpkZmZ\nWYkK3XIr6dukOYTjJe0MrNHUqAbQDQed6iq3ZmYDpMjw1FHAb4EP5zudxgCfaGZQZmZWTz32NCTt\n1vD2AWCUpFHAM6S7l8zMrM30Njz11fxzddIzEY8Cq5Ke3r4H2K2H82rl765s7amP2bv1+54EM7MV\n1uPwVETsGhG7Ao8Am0TENhGxFbAZ8P/LCtDMzOqjyJzGZhGxsPNNflp7k+aFZGZmdVXk7qkXJf0I\nmEd6Ontn4M9NjaoASVMiYk7VcZiZtZMiPY3DgVtJcxljgbuAqc0Mqi+ShgEnVxmDmVk7KtLTmBER\nX296JICkEcAcYDhwA+np8Gm8VSr9qbztW8AHJX0nIj5ZRmxmZlasp7GlpM2aHkkyDfhNRIwHFpGK\nF84EJkfE7sBzpF7ON4FwwjAzK1eRnsZWwCOSXgIWk6vRRsTGvZ/WLx8gVbQFuBY4F1gPuEoSwJrA\ni034XDMzK6BI0jig6VG8ZQhpsh1SmfTFwMKImNh4kKQxJcZkZmZZkeGpp0kP8n2WNPn8oSaulvcE\nMC6/3pe0ch+SxuafJ0naipRYKlmq1sysnRVJGjOBDwNBqkF1qKR/blI8s4Fdc1n0UcBS4FjgYkl3\nkKrfBvAsMEzST5oUh5mZdaPIv9a3zOt1d5qVf4E3w5rAlyLiplxNd0Iulb5jN8eObVIMZmbWgyJJ\nY5ikVSJiGYCkVQue1x+vACdLOp00v/Hplb3gTw85zqXRzcwGSJFf/tcD90qaS/pFPpGGBZQGUkQs\nAvZuxrXNzGzlFUkah5Oej3gWeB24LCLubWpUZmZWS0WSxp7ApPxnJ2A/STdFxHlNjWyAHDDnqqpD\nGHA/mPC/qw7BzNpUn3dPRcRzeU3vLwPfIJXzOKXZgZmZWf302dOQdBGwKbAQuAP4YkT8V7MDMzOz\n+inynMY7SRPgrwB/AF5oakRmZlZbffY0IuIwAEkfJN05dbGkMRHxgYEKQtJQ4AJSj2Z14HTgO8D3\nSGVMVifNrfy54bihwOkRcetAxWFmZr3rs6chaW1J+wEfAQ4DRgBXD3AcRwCv54cIDwZmkRLaoxGx\nG/AksAdwJPBsREwCDgQGxWS8mVmrKHL31EPAz/KfsyPiD02IYxy5um1ELJD0BrABaQ4FUv2rEaS7\nt3aVND5vHy5pWEQsbkJMZmbWRZHhqU1LiKODNG/SaRipKOGShm1DSFVvvxoRPyohJjMz66LIRHgZ\n7iU9B4Kk0aSEsaib4+4BJufj1pd0VmkRmplZbZLG5cCqkm7Lr0/o4bgrgNck3Qlcx1vDV2ZmVoJa\nrEkREUuA47psHtOw/3MN27seZ2ZmJalF0mim66Yc7Cq3ZmYDpC7DU2ZmNgg4aZiZWWEtPzx10JXz\nqg5hQF2w29ZVh2Bmbcw9DTMzK8xJw8zMCislaUiaLumclTj/PEmbDGRMZma24gbFnEZEfKbqGMzM\nrNyksYmkG4DRwLeA3wJnkVYCfAo4nlSD6lLgfcCdwKERsZGk24ETgSmkwoUC3g98JiL+o8TvYGbW\n1sqc0/gbUt2oicCXgJnA5IjYHXgOmArsA6wRETsBtwIbdnOd0RGxHzCDnsuNmJlZE5TZ05gXEW8C\nL0l6FdgcuEoSwJrAi8B7gV/k429g+Sq3f71O/tlZLt3MzEpSZtLo6PL+2YiY2LhB0ueBpQ3Hdz0H\n3l4u3czMSlJm0thZ0qrAu4F3AK9LGhsRv5F0EjAXeII0bwGwV8nxmZlZH8r8pfwo8BNgM+CLwHzS\neuOLgQWktb8fAz4maR5pJb+XSozPzMz6UErSiIjZwOxudu3Y+EbSu4GLIuJKSe8l9zoahrEebrjm\nw6RJdTMzK0ndhn9eBQ6V9H9Jd3b9n5W94NWHjHdpdDOzAVKrpJHvrjqs6jjMzKx7rj1lZmaF1aqn\n0QyfvvqpqkMYMKeNX6fqEMyszbmnYWZmhTlpmJlZYU4aZmZWWClzGpKGApeQqte+DnwMOBPYFFgd\nOD0ibpb0BPB90vMZjwP3kwoZ/jYijpK0IXARMIxUbuS4iPh9Gd/BzMzK62l8FFgYEbuQksJ04PWI\nmAAcDMzKx60KPABsD+wCzI+IHYBdJa0DfBk4NyL2AM4DTispfjMzo7yksS25em1EXA6sSyoTQkQs\nAN7IT4MD/DIiOkjl0h/M254nVbT9EHBmXl/jC/k6ZmZWkrJuuV3K8gmqg+Ur1A4DluXXjVVsu1a0\nXQxMjYhnmxGkmZn1rqyexr3A7gCS9icVIpyU348GlkXEogLXuQc4MJ+3u6QjmxOumZl1p6yexuXA\nnpLmkpZ3PRY4TdJtpF5G0RX4ziRVxj2C1FuZPvChmplZT8qqcrsYmNZl83HdHDem4fW47l4Dew90\nfGZmVkzLlxGZedBoV7k1MxsgfrjPzMwKc9IwM7PCWn546qo5L1YdwkrbdcLqVYdgZga4p2FmZivA\nScPMzApz0jAzs8KcNMzMrLCmTYRLWhu4ClgDuBU4mlQ/asuIeE3SOcDDwL8BF5DKpA8llUm/VdJY\nUvXbDuBV0tPf65BKrD8BbA08GBFve0jQzMyao5k9jaOBhyJiPPAbli9Q2OhI4NmImESqK3Ve3n4+\ncEIug34z8Km8fTvgFFL59P1yyXQzMytBM2+5/QC5/HnDz+58iLRexvj8frikYcAOwPclQVqo6d68\n//GIWAggaQGpZHqRYodmZraSmpk0hpCGluCtEucdDfuH5p+Lga9GxI8aT5b0Z2BSXlujc9sYli+X\n3vk5ZmZWgmYOTz0K7Jhf75l//hF4j6RVgZ3ytnuAyQCS1pd0Vt7+K2CfvP1wSXs0MVYzMyugmUnj\n34DtJf0c2CpvmwVcR5og/++87QrgNUl35n135O0zgFNyOfXpvLWKn5mZVWRIR0dH30etJEnvBB5u\nLH1elqvmvNj8L9hkzSojMnLkWq4A3Au3T8/cNr1rhfYZOXKtbof+W7721MFT1hv0//HMzOqirEWY\nXgPGlPFZZmbWPH4i3MzMCmv54akHL3y+6hD6baPJw6sOwcxsOe5pmJlZYbXpaUiaDowHlkbECZLm\nk+tUVRmXmZm9pW49jUURcULVQZiZWfdq09PIxki6LyLGdW6QNBq4GjgA2Aw4C3gTeAo4PiIWVxKp\nmVkbqltPo6s1SE+WHx8RzwIzgckRsTvwHDC1yuDMzNpN3XoaXX0XuDYiHpQ0CtgcuCpXvl0TeLHK\n4MzM2k3dk8bTwNGSZpGq4T4TEROrDcnMrH3VfXjqVOBa4IyIeBkgr+iHpJMkbdXbyWZmNrDq3tMA\n+Cpwt6SrgGOBiyUtBhaQlok1M7OS1CZpRMRsYHbD+zENu7dteL0jZmZWidokjWbZ5rj1XeXWzGyA\n1H1Ow8zMasRJw8zMCmv54amF5zxedQgrZNWPjqo6BDOzHrmnYWZmhTlpmJlZYYMiaUgaI+m+braf\nk0uqm5lZCQZF0jAzs3po+kS4pEeBLYAhwMvApIi4T9JNwJ3APvnQayLibEmzgTkRcb2k/YEpwJkN\n1/sI8HlSXaq/AA83+zuYmVlSxt1T9wNbAsOA+4CdJT0A7ASsD2yfj/ulpDm9XUjSENJ6GuNICej+\nZgVtZmZvV0bSmEtKEMOB84GDgZ8DLwF3R8QSAEm/ALbu41rrAq9GxPMN55iZWUnKmNO4nZQ0dgJu\nAUYAuwBnkIasOg0DlgEdDduGdrnWkHxMJ8/JmJmVqOm/dCPiMWA0MCIiXgUWAgcCT5KGqlaTtBqp\nEOGDwB+B9+TTx3e53EvACEnrSBpKSj5mZlaSsv6l/jzwu/z6HmBMRMwjlTafC9wBXBgRvyMt7/o5\nSTeS1gL/q4hYRpoUnwvMwZPgZmalGtLR0dH3UYPYwnMeH1RfsMwyIiNHruUKwL1w+/TMbdO7Vmif\nkSPXGtLd9pavPbXB5zYb9P/xzMzqwhPJZmZWmJOGmZkV1vLDU8/NvL3qEApZ5Yjtqg7BzKxP7mmY\nmVlhThpmZlaYk4aZmRVWizkNSRsDlwJLSTF9BDgd2JRUSuT0iLhV0lhgFqnUyKvA9IhYVE3UZmbt\npy49jSnALRExCZgBTAOeze8PBM7Lx50PnBARewA3A5+qIlgzs3ZVi54GKQFcLWkdUnmQDYFdJXXW\nnhouaRiwA/B9SQCrA/dWEayZWbuqRdKIiIclbQ3sBXwNeB9wSkT8qPE4SX8mLeI0qEqDmJm1iloM\nT0k6HNgyIq4BTiUVKpyc960v6ax86K/IK/1JOlzSHlXEa2bWrmrR0wAeA74r6TXSZPghwAxJdwKr\n8tZyrzOACyT9P9JSr0dWEKuZWduqRdKIiAdI8xWNjuvmuEeAXUsJyszM3qYWSaOZRn16oqvcmpkN\nkFrMaZiZ2eDQ8oswmZnZwHFPw8zMCnPSMDOzwpw0zMysMCcNMzMrzEnDzMwKc9IwM7PCnDTMzKyw\nQf9EuKRvATuRFmaaERH3NuzbEziLVM/qhoj4cl/ntJJ+ts2WwL8D34qIWeVHXY5+ts03SGVsVgO+\nFhFXlR54SVa0fSS9A5gNjALWAL4cEdeXHngJ+vN3J+8bDjxMapvZpQY9gAZ1T0PSBGDziNgZOBaY\n2eWQmaTih7sAe0kaW+CcltDPtlmTtNDVf5YabMn62TaTSJWYdyZVWj6PFtWf9gEOAO6LiAnAocA/\nlRhyafrZNp1OBf5QSqBNNKiTBrAHcA38tZjhuyStDSBpU+APEfFURCwDbsjH93hOi+lP27wB7Acs\nqCbk0vSnbX4OTM3nLwLWlLRq6ZGXY4XbJyJ+HBHfyOePBp6uIO4y9OfvDpL+FzAW+GklUQ+gwZ40\nNgBeaHj/Qt7W3b7ngff0cU4rWeG2iYglEfGXkuKrUn/aZmlE/ClvO5Y09LC06ZFWoz//XwGQlzO4\nDPhMk2OsSn/b5lzg5KZHV4LBnjS6GtKPfb2d00r60zbtonDbSJpMShonNjWieincPhHxIeDDwKWS\n2uHvVZ9tI2kacFdEPFlOSM012JPGApbvJWwIPNvDvvfmbb2d00r60zbtol9tI2lv4IvAvhHxSglx\nVmWF20fSdpJGA0TEQ6SbBUaWEGvZ+vN35++AyZLuJq0TdFqeMB+UBnvSuBmYAiBpW2BBRLwKEBHz\ngbUljZG0GrB/Pr7Hc1pMf9qmXaxw20gaAXwT2D8iBv1kZh/683dnN+Cz+ZxRwDuBF8sPvelWuG0i\n4rCI2D4idgIuJN099bNqwl95g740uqSvk/7CLgM+BWwDvBIRV0vaDTg7H3plRJzT3TkR8avyI2++\nFW0bSduRxl7HkNZpfwY4uBV/SfajbT5OWnb4sYbLTIuI35cYdmn60T7DgYtIk+DDgX+MiOsqCL3p\n+vM7p+HcM4H5g/mW20GfNMzMrDyDfXjKzMxK5KRhZmaFOWmYmVlhThpmZlaYk4aZmRU26KvcmtWB\npC2AWaRihp8g3Yp58mCs9CppDnBBRLTTsztWkHsaZitJ0irApcAnI+INUhmNGYMxYWQnAN+R9M6q\nA7H6cU/DWpKkiaSSH08D2wN3A78GDgLWA/YFNgfOINUIehM4PiKelHQQ8A/A66T/R46OiPmSbgd+\nBnwI+BvgjIj4ITAZeDoiHpF0IrAd8HVJQ0mlsn8MbBoRUyUdCpyUP/MF4LiIeCk/PDgjx/trYOeI\nGJ8/8ysR8TNJY4B5EbGRpHcB3yWV6hgBnBsRl+WHx9YFNsrf77aIOCkntpnAuNxE5wLrALtExPTc\nZocBh0TEoZKuJ5W8aNkS8NY/7mlYK9uBVNpiHHAUsCgiJgH3A9NIv3QPzmtAnA90Pr27DnBYPvYG\nli9O+M6I2I9UtPAf8rZ9gBsB8sJVDwGfjYhr8/7f5oQxmpTI9oyI8cDtwCm5tPbXgEkRsTeweoHv\n9hXgxojYnfR08pckddZ62oZU6mJ74JicYI4CRuVSFvsA04ErSGs+dPYoDiWVuQC4JR9nthz3NKyV\nPdJZAkXSS8CdefvTwFBS2eqrJAGsSlqJDeA54JL8r/MNgLsarnl7/vk74N359Wh6Xyeh83N3zp95\nU/7M1YEnST2C30XE8/m4m4Ft+/huk4DtJX00v38T2CS/npfLtv9F0os5zh07Y4+IRaQiekj6d2BK\nnscYS+pJdX6/MX3EYG3IScNa2ZJe3v8t8PuImNh4QB5S+jGwbUT8Ng83jWs4pPEaRUt/L84/3wB+\nGRH7d/nMcV2OX9bwurHOz7CG12+Q5lDu63Kt/Xj79x6Sr9PdyML3SENVbwCX58WDzHrk4SlrV48B\n6+U10ZG0W55XWIv0S3u+pDVI8xV9DRc9Rept9OVeYAdJG+TPnJrX53gc2ETSevm4AxrO+WPDtXdv\n2D6PNJyEpOGSvpMrq/bkTvJwk6S1Jd0jaVguYz6cNAR3ccPx7wPmF/hO1macNKxd/QX4CHCRpLnA\nl4G5eTjrMtIv+B+TyqHvLmlqj1dK8xl79/WBEbGANNl9vaSfk+ZF7s7DRacDcyXdQpqA7zQLODVv\nX7Nh+5nA5pLmkZaifTAiuvYwGl0BPJlX1rsF+KeI6OwBXQq82qVi7575e5ktx1VuzVZSnvu4Hzgy\nrxu9stebSLpjavzKXqvAZw0BrgXO73wuQ9K6wD3ANi261oytBPc0zFZSngc4mvRsQ5E7n2ohLyJ0\nP/Bwlwf5vkeaL3HCsLdxT8PMzApzT8PMzApz0jAzs8KcNMzMrDAnDTMzK8xJw8zMCvsfJ/2R0XaB\nYXYAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "lGuiQkv7yi-9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 351
        },
        "outputId": "74089811-956b-49fe-b421-06bee9d3bf41"
      },
      "cell_type": "code",
      "source": [
        "max_freq = max(ranked_lemma)[0] * 15\n",
        "scaled_dtm = [(freq*max_freq,word) for freq,word in ranked_dtm]\n",
        "compare(scaled_dtm, ranked_lemma,'DTM', 'Lemma')"
      ],
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/seaborn/categorical.py:1428: FutureWarning: remove_na is deprecated and is a private function. Do not use.\n",
            "  stat_data = remove_na(group_data)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAEYCAYAAABRMYxdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xm8XfO9//HXMcQYqSHGRqOl79Kg\nVRVDZn6mIkjMRMytCi7uVUqoXlMNVaKUhIjxEkNRc5GKOcar+Ki4MUZMMdUQSc7vj+867BznJDvn\n7H32WWu/n4/HeZy111577c/3nHx3Pue7vuvzbWhsbMTMzMysSBaodQBmZmZmleYEx8zMzArHCY6Z\nmZkVjhMcMzMzKxwnOGZmZlY4TnDMzMyscBaqdQBmbSVpY+DtiHhF0v3Af0fEPW081xtAn4iY0obX\nTgGmAZ+T+tSrwLER8aqkI4DtskP7A/8AGoG/AY8D9wFbRcQdJefbE7gcWK0t8ZjlSSfrx5tFxMtt\neW/rfJzgWJ7tA/wP8EqtAwH2aPpglDQMmCBp7Yg4Gzg7298IbBoRM7PHA4CXgH2BO0rOtXe236we\ndKZ+bAXiBMdqJvsP/rfAG8DPgUeAZ4EdgOVIIxtvSBoInAA0AF8BBwA/AXYCNpD0H9kpN822fwj8\nLiKukLQCMAZYElgE+ENE3JjtvxZYEHgiO3fz+EpHX5o8HRGHz61dETFO0hbAXsCf5/FjeBToI2mZ\niPhA0qpAV2DqPF5n1ikUtR+XvH4d4Cxg4ezrkIh4Khtt+gfQG1gDOJz0x0kvYFxEnCzpRGDF7Gtd\n4PSszeuT+vh2WcwXAj/K2vZoRBxaTmw2d56DY7W2AXAkqcPvAXwYEQNJH1ZDJS1O6vw7RkR/4Dzg\nzIi4EXgaODIi7s3O1RARvyD9RXh0tu8kYEJEDAAGAxdI6gocBjwSEX2Ay4CVmwcWEWdHxIBmX2V9\nKAIPA2uXcdxs4Pqs7ZA+IK8p8z3MOoui9mOAK4FfZu99MDC65LmGiNgCGEdKXnYDtgD+s+SYNUnJ\n3j5Zu88gJYK9SEnP0sCzEdEvInoDm0vqNR/xWSuc4FitvRARH0TEF8D7wEPZ/jeAbqQPgZWAG7K/\nmI4CurdyrvtLXvudbLs3cDdARLyTPSdS8jEx2/8k8FHFWpR0A2aVeezlpA8/SP85XF3hWMyqrZD9\nWNLy2fuMyeL+E7CUpKb/Ox8sifWJiJjBN21u8nBENGb7p0XE5Ozxm9lxHwI9JD2cvcdKpJEvaydf\norJamzmXxw3Al8Br2V9P83OupqHq5outNWT7GkijJ00WbH6ydg5tbwJcV8ZxRMSzkhaUdADwakRM\nk1TOS806i6L24y+BL1uKO+ujpbE2/xm0tL/5MQ3ArqQRnb4RMVPSpDLisjI4wbHO7iVgOUm9IuI5\nSf2AH0XERaQPtoXn8fpHSEPGT0tamfTXUQDPAxsBf5PUm3Rtfw6lE4Tnh6R9SfMH5udS0+XAaaTr\n+GZFk7t+nL32I0lTJG0dEbdJ+iGwa0Sc1JbztWKF9FYxU9LPgNVJc3GsnZzgWKcWEZ9nt02PkfRF\ntvvA7PvdwF8kzS0pOCF77VbAosCBEfGppD8B10q6F/gn7b+D40pJnwPLAC8DgyLi8/l4/VXASODG\ndsZh1unksB8DzIiIzYFhwLmSfkNKxI5o53s0dx1wi6QJpEteZ2bvt2FETK/we9WVhsbG5iN/ZmZm\nZvnmScZmZmZWOE5wzMzMrHCc4JiZmVnhOMExMzOzwvFdVO0wc+asxunTP6t1GDWx9NKLU69tB7c/\nT+3v3r3rt8r3l6Oe+zfk63dcDfXc/ry1vbU+7gSnHaZfXL8FZ+v93sWqtX/o4GqduaIWWuhb9dQK\np577N7iP13P7a972Cn0O+hKVmZmZFY4THDMzMyucwl+ikrQqcAVp4cOFgD2B84ElgMWBEcCywO4R\nsVf2mouBWyLi5poEbWZmZu1SDyM4Q4G7I2IgcBjwPWB09vgY4GjgLqC3pEWzVWI3Ae6oVcBmZmbW\nPvWQ4NwFDJN0FmkBs2eAIZImAqcDy0bELOBWYGugN/BAtuy9mZmZ5VDhE5yIeA5YF3gAOJW0WvOb\nEdEH+FXJoeOAnYDtSAsfmpmZWU7VwxycXYFXIuImSe8BuwDPZk/vAHQBiIinJa0CLA8cW865u/9q\nT95995MqRN35de/etW7bDm5/Pajn/g3+N17P7S9K2ws/ggO8BIySdC9wAnAtcISku4BHgRUl7ZMd\nexcwKSK8xLqZmVmOFX4EB7gB6AUcAkyIiIeBNUuevxlAUgMwAPhluSeedsFZlYsyZ6bVOoD5tMDQ\nA2sdguVMPfdvyF8fr7TO1H5/frVNPSQ4AETEaa09J6kncD1wbUS83GFBmZmZWVUUKsGRtBRpgnBp\njZum58YC44E7gctIt4t/AQwDXgeeBraUtC0wMiLu7dDgzczMrGKKNgdnRb5d46a5vYG3I2IT4GLS\nXVO7A1Oz120PnNNB8ZqZmVkVFGoEh3TZ9HhJR5Fq3vy7hWPWA/4OEBHXAEi6AOgrqU92zGKSurgW\njpmZWT4VLcFpqnGzl6T1gTNbOGYW3x65mgGcHBH1vXywmZlZQRTtEtVywORs++saN808DgwCkLSN\npGNJt4sPzvYtL+mUDojVzMzMqqRoIzjjgHGSdgJGAbsBDc2OuQbYTNIE4CvSnJxpwCBJDwELAieW\n82Yr/OrIQhRDaouiFIIya009929wH6/39hdBoRKciHicFmrcZIaXbA9r4eX7VyMmMzMz63iFSnA6\n2uvn7VHrENpk0V0vrHUIZp1eXvt3xYy4stYRmLVL0ebgmJmZmdXnCI6kbqSif4sBtwEHAPsAp5Dm\n5bwB7BsRX9YsSDMzM2uzeh3BGQY8HxF9gA9JE5EvBHaJiP7AdFLxPzMzM8uhek1w1gQezLZvBpYB\nGiPi9WzffcBPaxGYmZmZtV+9JjgNwOxsuzH7Kr2dvEvJ82ZmZpYz9ZrgTAbWz7a3Il2SapS0arav\nPzCpFoGZmZlZ+9XlJGNgLPBXSfcDd5OWbzgAuErSTFICdM28TtJjxJUuBGVWUO7fZvlWrwnOEsBJ\nEXGnpI2A/hExEegzj9eZmZlZDhQqwZE0BOgK9IqIo5o9dw2wT0R8DnwEHCFpJGnuzamSJkXE+t86\n6Vw8eeG2FYq8bXoMuaqm729WZLXu361xvzcrT2ESHEk9SWtP3drS8xGxa8n2h8AWzV5rZmZmBVGY\nBAc4H9gAeBZYWdL1wFrAGRFxiaQpQC/SIpxvAT8DVgX2AD5oOomkrYARwLYRMasjG2BmZmaVUaS7\nqM4AJgCvAd8Hdga2Bw5t4dhFImIL4E+ULLwpaXXgeGA3JzdmZmb5VaQEp9QjWYLyJtCthecfyL6/\nUfL8EsBNwCER8VH1QzQzM7NqKWqCM7Nku6HM579LSnwOrlZQZmZm1jGKlODMpn1zioKU3PxA0uaV\nCcnMzMxqoUiTjF8A1iNVIR7TlhNERKOk/YFbJPWOiLlW+Vrvl7e4EJhZQbl/m+VbQ2NjY61jyLPG\nev0A7N69a11/+Lv9+Wl/9+5dW7pMXY667d+Qr99xNdRz+/PW9tb6eJFGcOYgaTipMvGsiDhIUj/g\nxYh4R9J2wB3AysD4+S3w1+Se0VtXLN5yrDv4fzr0/czMzPKqSHNwWvJhRByUbe8LLJ9tH0FaMdzM\nzMwKqLAjOJmekiYBx5Bq4vxY0ihgQ+B2YL+mAyX1BU4BvgJeBw6IiBkdH7KZmZm1V9FHcACIiLuB\np0lrUV0GvA1sBZQmMOcCgyNiEDAN2KnDAzUzM7OKKPoITlkkrQCsAdwgCVLRv/dqGpSZmZm1mROc\nZAbwZkQMqHUgZmZm1n51cYkqU1oIcI6igBExHUDSWtn3EZLW6fAIzczMrCLqaQRnAjBe0mDgfmAi\nMLzk+f2ASyXNIK02ftG8TrjZ/rflqlaAmZlZvXChv/ap20JgeSsEVWluf37a70J/bZOn33E11HP7\n89b2ei301ysijirz+J7MZ9G/8Zdu2bbg2qD/Ntd12HuZmZnlXT3NwTEzM7M6UdgRnCaSDgN2zR7e\nFBGnS/oucAmpmvFs0vybxpLXbAWMALaNiFkdHLKZmZm1U9FHcFYjTSTum33tIukHwEnAmOy28D8D\nJza9QNLqwPHAbk5uzMzM8qnoCc5PgUciYmZEzAQeBNYF1ifdSQVwX3YcpAJ/NwGHRMRHHRyrmZmZ\nVUjRE5xGoHR2ddMlqdL9TfsAvgs8ABzcUQGamZlZ5RU9wXkK2EjSQpIWAnpn+x4HBmbH9AcmZdtB\nSm5+IGnzjg7WzMzMKqPok4ynkC5BTSAlc6Mj4lVJI4Exkg4gLdOwH7AwQEQ0StofuEVS74hotRjA\n0H3uyFWtADMzs3rhQn/tMGbcFh3yw9tuq/Ed8TbzJW+FoCrN7c9P+9ta6K+j+ne1tPdzI0+/42qo\n5/bnre2t9fFCXKKSNFzSmbWOw8zMzDqHQiQ4ZmZmZqUKNwdH0qnAv4GeQHdAwBkRMUbSAOAU4Cvg\nDWBf4Bngx6S7qqYDAyNikqQ7gQMj4tUOb4SZmZm1S6FGcCTtBPQgJS9rAzsC25OqEgNcCOwSEf1J\nyczuwBNAL1ItnEmku64WAFZwcmNmZpZPRUpwfgycDuyfPX44q0T8BtBN0jJAY0S8nj3fVOBvArAh\nsAlwHulW8rWBJzswdjMzM6ugIiU4PYF/AkOzxzNLnmug9aJ/95MSnA2Bu4FupGTnvqpGa2ZmZlVT\npATnb6Q5NccDKzR/MiKmA42SVs129QcmRcRLpMta3bKaN2+TLms5wTEzM8upQk0yjoh3JZ1AWkDz\nkhYOOQC4StJMYDJwTbb/HeDjbPtRoH9EvDGv99tv2J25qhVgZuVz/zbLNxf6a4czr+6YQmB7b+ZC\nf52N25+f9re10B/QmJc2VkOefsfVUM/tz1vbC13or9IkDZ33UWZmZtZZOcFpRlIX4Ihax2FmZmZt\nV6g5OHMjqRswHlgMuI00H2cY3xT+ez3b90dgbUl/joiDaxSumZmZtUM9jeAMA56PiD7Ah6Rbxs8F\nBkfEIGAasBNwBhBObszMzPKrbkZwgDVJNW8AbgbOApYDbpAEsATwXk0iMzMzs4qqpwSngVTYD1LR\nvxnA2xExoPQgST07NiwzMzOrtHpKcCYD65Pm4WxFWosKSWtFxPOSRpCWbfiQMn8uR+3mOhlmZmad\nUT3NwRkL9JV0P6nS8SxgP+BSSQ8AfYAApgJdJF1XozjNzMysneppBGcJ4KSIuFPSRqRqxRNJi2s2\nt1Y5Jzz8+i0rGd+3/LafcywzM7O2qKcE5yPgCEkjSfNxDq1xPGZmZlYlhU5wJC0MXAR8H1gEGEla\np+ovwNmSFgE2Az4rOW5hYGRE3FuToM3MzKzdij4HZzfgi4joD+wIjCIldS9GRD/g/4BNgd2BqREx\nkLSS+Dk1itfMzMwqoOgJzvpktW8i4i3gS2AZ4IHs+TeAbsDGwPbZBOTxwGLZkg1mZmaWQ4W+REWq\nd1O6ymgXUi2cmSX7Gkg1cU6OiKs7MDYzMzOrkqKP4DwODASQ1IOU3HzYwnGPAoOz45aXdEqHRWhm\nZmYVV/QRnGuAAZLuI43eHASMa+G4a4FBkh4CFgROLOfk5wy5w4X+zMzMOqFCJzgRMRPYv9nuniXP\nH1Wyv/lxZmZmllO5T3AkDQd6NUtW5uf15wB/ioj/m9/XbvXX/drylmUbt7Fv5jIzM2uL3Cc47RUR\nh9c6BjMzM6usoiQ4q0m6DegB/BH4F3AK8BXwOnAA6Y6qK4DvAQ8BO0fEd7Nbww8BhpJuGRfwA+Dw\niLi9g9thZmZmFVCUu6h+SLoLagBwEnAuMDgiBgHTgJ2ALYFFI2JD4F5g5RbO0yMitgYOI01INjMz\nsxwqygjOxIj4Cnhf0ifAGsANkiAtsvkesArwYHb8bcxZC+fr82TfmwoAmpmZWQ4VJcFpbPZ4akQM\nKN0h6WhgVsnxzV8D3y4AaGZmZjlUlARnI0kLkpZhWBz4QtJaEfG8pBHABGAyaZ4NwOYUp+1mZmbW\nTFH+k38RuA5YHfgtMAW4VNIM4C3SSuEvAftKmkhan+r99r7p7YPHuNCfmZlZJ5T7BCcixgJjW3iq\nd+kDScsAYyLiekmrkI3mlFzKeq7knM+RJiybmZlZDuU+wZkPnwA7S/pP0t1j/9HeE25943+3O6jW\nXNbnsKqd28zMrOjqJsHJ7rLapdZxmJmZWfXlPsGRtDBwGamA3xfAvqTFMr8PLAKMjIi7JE0GLiZd\nmnoZeIJUH+dfEbGHpJWBMaRFOWcB+0fEax3cHDMzM6uAIhT62xt4OyI2ISUww4EvIqI/sCMwKjtu\nQeBJ4OfAJsCUiNgA6CvpO8DvgbMiYlPgHOD4Dm2FmZmZVUwREpz1yAr4RcQ1wLKku6SIiLeAL7MJ\nxgCPRUQjqbrxU9m+d0hF/TYGTsyWbjgmO4+ZmZnlUO4vUZEuJ5Umao3MWaSvCzA72y4t5Ne8qN8M\nYKeImFqNIM3MzKzjFGEE53FgEICkbUj1bQZmj3sAsyPiwzLO8yiwffa6QZJ2r064ZmZmVm1FGMG5\nBthM0gTS6uH7AcdLuo80elPuopknkooD7kYaBRo+rxfctsNxLvRnZmbWCc13giNpgYiYPe8jO0ZE\nzACGNdu9fwvH9SzZXr+lbWCLSsdnZmZmHW+eCY6k4aT1nS4iTd7tIem0iLiguqF1fr+4YdS8D2qj\nsX33rtq5zczMiq6cOTgHAaNJ81OeA1bDBfPMzMysEyvnEtXnETFD0tbAFRExW1JjtQNrjaSlgBuA\nRYF7gb1Id0H1iohPJZ1JSsQuJ406fR9YmFTw715Ja5Fq4zSSlm8YDnyHVCxwMrAu8FREfOsyl5mZ\nmeVDWXdRSTqfVBxvgqSNSMlFrewFPB0RfYDnmfOW8FK7A1MjYiBp9OmcbP95wEFZQb+7gF9n+38G\nHEsqBLh1VvzPzMzMcqicEZw9SJekzo2IWZJ6Ar+salRztyZZIb+S7y3ZmFSluE/2eDFJXYANgIsl\nQVrK4fHs+Zcj4m0ASW+Riv+Vc3u5mZmZdTKtJjiS+pU8fBJYQdIKwJuk//xrpYF0eQm+KdZXesls\n4ez7DODkiLi69MWSPgMGZhWNm/b1ZM7Cf03vY2ZmZjk0txGck7PviwBrAy+S1nMSqShev1ZeV20v\nAr2B64HNsn0fAytJegXYkLQMw6PAYOBqScsDh0fEscAzwJbA7ZJ2Bd4lzb0xMzOzgmg1wYmIvgCS\nLgO2K7l804O0MGWtXA7cKOkfwMRs3yjgFiCAf2b7rgUGSXqIlJidmO0/DLhI0m+Az0lzdZZqSyB/\n2/EQF/ozMzPrhMqZg7N6U3IDEBGvS1qtijHNVbbsQtNSDEsCu0fExaSVxJtrqeDfC0DfZrs/AFor\n/mdmZmY5U06C856kq0mjJbOBjYDPqhpVhWXFCvsAsyLiIElTyG4rb895t7l+bLtja8ml/YZU5bxm\nZmb1opwEZ1dgT9I8nAbgYdJloprLEpSeZR7+YUQcVcVwzMzMrJMoJ8E5LCJOq3ok1ddT0qTSy0/Z\nfKIbgW2B1YFTSAt2vg4ckK1zZWZmZjlTTqG/XpJWr3okHW9R0kjUARExFTgXGBwRg4BpwE61DM7M\nzMzarpwRnHWAFyS9T6ot0wA0RsSqVY2s+i4Ebo6Ip7L6PmsAN2QFAJcA3qtlcGZmZtZ25SQ421Y9\nitp4A9hL0ihS4vZmRAyobUhmZmZWCeVconqDVNTvSOAIYOOIeLWqUXWM44CbgRMiYjpAthAnkkZI\nWqeWwZmZmVnblTOCcy6wPGndpwZgZ0kbRsRh1Qysg5wMPCLpBmA/4FJJM4C3SCuRz9WtQ4a70J+Z\nmVknVE6C0ysi+pc8HiXpgWoFVA0RMRYYW/K4Z8nT65Vs956f824z/tr2hPUtl/bfqqLnMzMzq1fl\nXKLqIunr4yQtSHmJUS5I6ilpUgv7z8wKBJqZmVnOlJOo3Ao8LmkC6RLVAOCaagZlZmZm1h7lVjKe\nBkwFvgCuiojHqxpVmSS9CPyYlHhNBwZGxCRJdwIPkVYNB7gpIk6XNBYYHxG3StoGGMo3i3AiaU/g\naNLE6s+B5zqqLWZmZlY55SQ4m5EWtxwIbAhsLenOiDinqpGV5wmgF9AFmARsJOlJUpzLAz/PjntM\n0vi5nUhSA6mS8fqkZOmJagVtZmZm1TXPOTgRMS0irgF+D/yBtJTBsdUOrEwTSMnMJsB5pEnCawPv\nA49ExMyImAk8CKw7j3MtC3wSEe9ExFfZa8zMzCyH5pngSBoj6T7gDKAb8NuIWL7qkZXnflKCsyFw\nNym+TYATSJetmnQhrYTeWLJv4WbnasiOaVLOBGwzMzPrhMq5RLUk6T//j4APgHerGtF8iIiXsgUz\nv4qITyS9DWwPnAQcJampfb1Jl58GAitl+/o0O937QDdJ3wH+TUqUHp7b+986dGfXwTEzM+uEyrlE\ntUu2hMH5QHdSMbwXqh3YfHgHaKqs/CjQMyImkgr1TQAeAEZn1ZcvJyU+d5AutX0tImaTJhxPAMbj\nCcZmZma51dDY2DjXAyQtRRrt6E8a1VgAuD8iOss8nJoZPP62uf/w5sPo/n0rdaoO0b1717oevXL7\n89P+7t27Nsz7qBY15qWN1ZCn33E11HP789b21vp4OZeongbuyb5Oj4gPKhmYmZmZWaXNM8GJiO93\nRCBmZmZmlVKYJRdaI2lV4ApgFqm9ewIjge+T7qQaGRH3ZiuJjyLdafUJMDwiPqxN1GZmZtYe9XAr\n9FDg7ogYCBwGDAOmZo+3B5oKFp4HHBQRmwJ3Ab+uRbBmZmbWfoUfwSElKzdmt3+PB1YG+kpquk18\nMUldgA2AiyUBLAJ0iuUozMzMbP4VPsGJiOckrQtsDpwKfA84NiKuLj1O0mektawqdmeUmZmZ1Ubh\nExxJuwKvRMRNkt4DLgEGA1dLWh44PLvl/RnS4py3Z695NyL+Prdz/3Xo1rm6lc7MzKxeFD7BAV4C\nLpT0KWmi8RDgMEkPAQvyzWrihwEXSfoNaSXx3WsQq5mZmVVArhMcSQOAQyJiaGvHRMSTwAaShkbE\n+Ox1t0fE/s2OewHIV7U9M6uaHa6fWOsQbB4u6jevNZStntXDXVRkk4iPyLZ7ArvVNCAzMzOrqlyP\n4GSWlHQFsC5wHfAQ8HtgBjAd2Bn4I7C2pD+TJhlvIGlktv9SYGnSz2JERDwr6V/AbcA7EXFyRzfI\nzMzM2qcIIzhrAQcCGwEjSMnK7hHRH/gY2AI4A4iIODjbnhARJwGHA3dktW9+BZyVnXNh4HYnN2Zm\nZvlUhBGcJyPiMwBJDcC7wGhJC5GqFd87l9duDHSXtGf2ePGS5x6rRrBmZmZWfUVIcGY2e3wJ8IuI\neEHSqHm8dgbpstTDrTxnZmZmOVSES1TNdQNeyyoXDwS6ALP5Jpkr3X6UtFwDktaSdEQHx2pmZmZV\nUIQRnObOBx4k1b/5A6nOze1AF0nXAQcD60n6I2nRzbGSHiDVxDm0JhGbWadz45A+dV3Is3v3rnXd\nfsu/hsZGr0zQDo31+gFQ7x9+bn9+2t+9e9eGNr60bvs35Ot3XA313P68tb21Pl7TERxJSwLPRUTP\nWsZhZtbcztc/X+sQCun8fj1qHYLViSLOwTEzM7M61+EjOJKWAq4HFgUmZvsGAKcAXwFvAPuSqg1v\nBawM/AY4DZhMurX7AmAdoDdwfkScP5dz9IqIo0pHiyQdDexImnB8S0ScIqlvyetfBw6ICN9JZWZm\nlkO1GMHZk5Ro9AWezvZdCOySFeebzjcLXa4K9APeBH4CHAn8AjgdOA7YFjhgHudoyVHAJqRkaXq2\n71xgcEQMAqYBO7WvmWZmZlYrtUhw1iItpwBwP7AM0BgRr2f77gN+mm0/HhFNs6AnR8T7wFTSEgpv\nkhKRbpLmdo6WjAfuISVHV0paAVgDuEHS/aTby1dpVyvNzMysZmoxybiBdGkIUoLVmO1r0qXk+dJL\nRDNb2W6YyzlKbxFbuGkjIn4l6UekdaruJy3n8GZEDJi/ppiZmVlnVIsRnADWz7YHki4RNUpaNdvX\nH5g0XyeMaO0cHwMrZfv6AEjqJmlkRLyYrUf1ATAre26t7PsISeu0pXFmZmZWe7UYwRkH3Cjp76RJ\nxo2kS0VXSZpJmkh8DWmuzvxo6RyLA7/NLjv9DZgdER9J6i7pMeBT4KGI+EDSfsClkmYAbwEXtbeh\nZpZf1w5ZK1e1QCotb7VQzJpzob/2qdtCYPX+4ef256f9LvTXNnn6HVdDPbc/b23vlIX+OpPsNvND\nImJorWMxs9o78ca3ah1CjbX8H9yv+3Tt4DjM2saF/szMzKxwcjmCI2k4aSLxcsCPgd+SivqtBewB\n7AJsQComeGFEjJY0lnRX1rLArsBlwPeAL4Bh2amXlHQFsC5wXTYJ2czMzHImzyM4awDbAacCxwA7\nZNv7AFMiog/QFyhNUj6IiCHA3sDbEbEJcHF2HkgJ0oHARsCIjmiEmZmZVV6eE5xJWRHAqcCzETGL\nVPhvEWAZSQ8BtwPdS17zWPZ9PeBBgIi4JiIuyPY/GRGfRcSnzFlXx8zMzHIkzwlOa4X/egKDgP5Z\n4b4vS55rKhw4i5bbPrOFfWZmZpYzuZyDMw/rAzdHxFeStgMWlNSl2TGPk5Kg6yRtQ1q48yHMzMys\nEIqY4NwDrCFpAnATcCtp9fFS1wCbZcd8RZqTs0aHRmlmndqJO6ycq1oglZa3WihmzbnQX/vUbSGw\nev/wc/vz034X+mubPP2Oq6Ge25+3thei0F85xfgkTQF6ZROFzcza5Mrr3611CDX2xddbm/dbtIZx\nmLVNnicZm5mZmbWoKiM48yrEFxGPSjqb6hXjO0TS1ln7tiDd8n0VsARpAc4REfGYpH8BtwHvAJcC\nY4AupLus9o+I1yr2QzEzM7MOU80RnNYK8e0maVGqW4zvuYjoB7wKbAqsCIyOiIFZLEdnxy0M3B4R\nJwO/B86KiE2Bc4Dj29l+MzMzq5FqJjitFeLrFhFfUN1ifBOz728C3bL3HSJpInA6aYSo+fttDJwo\n6X5SElR6jJmZmeVINScZt1bVkQ1sAAAPPElEQVSIr0FSf74pxveVpNIJwZUoxjfH+wGHA29GxF6S\n1gfObOH9ZgA7RcTUMs5vZmZmnVitJhkvB7xeZjE+JG0j6dh2vt/kbHsH0jyb5h4Fts/eb5Ck3dvx\nfmZmZlZDtbpN/B7g6A4sxjcOGCdpJ2AUaR7QPs2OORG4VNJuQCMwvI3vZWYFsMeQ7rmqBVJpeauF\nYtacC/21T90WAqv3Dz+3Pz/td6G/tsnT77ga6rn9eWt7IQr9laOcYoBmZvNyz1Uu9Ndk3f/nQn+W\nPy70Z2ZmZoVTuBGczBzFAEkrhf+edKfUdGBn0m3hRwFLAkeSCgoeSboDa1JEHFmDuM3MzKwCijqC\n07wY4NLA7hHRH/iYVN0YYO1sO4DjgEHZMT0kbdLhUZuZmVlFFDXBaV4M8F1gdHZH1kC+KeL3TER8\nSVpOYlXgzqzQ3xqkER0zMzPLoaJeompeDPAS4BcR8YKkUSX7S4v8PRERW2BmZma5V9QRnOa6Aa9J\n+g5pBKd5ob8A1pS0PICk30lapYNjNDMzswop6ghOc+eT1rV6CfgDqajf15WRI+IzSYcDt0n6EngK\neKsGcZpZJ7HZ7i70V8/tt/xzob/2qdtCYPX+4ef256f9bS3099Todwr/4fjdwYu1+lyefsfVUM/t\nz1vb66bQXxNJQ4CuwEcRcWOt4zEzM7OOU8gER1JPYDdXMzYzM6tPhUxwSHNuNpA0GzgUeA44jHR3\n1XrAycCWwE+B/4yImyTtiAv9mZmZFUJR76I6A5gAnFSy7yfAnsAvgdOAfbLt4ZKWxIX+zMzMCqOo\nCU5Lmor6TQVeioh/A9NIt5C70J+ZmVmBFPUSVUtmtrLdgAv9mZmZFUpRE5zZzF/bvi70FxHvSPod\ncFFEvFmd8Myss/vp/svn6lZZM5tTUS9RvUCaTNytnIMj4jOgqdDfg6S1qlzoz8zMLKdc6K99XOiv\nTrn9+Wl/Wwv9vX7W24X5cFx02BLz/Zo8/Y6roZ7bn7e2112hv1KSxgLjSXNvVouIC2obkZmZmVVT\nXSQ4TSLijlrHYGZmZtXXKRIcSasCVwCzSDHtCUwHrgKWABYHRpDmxuweEXtlr7sYuCUibi4519nA\nBsCiwIURMbrkueFAr4g4StJ/AUNJE5KPiYj7JP0a2D3bd1NEnFXVhpuZmVlVdJZJxkOBuyNiIKni\n8ErAisDobN8xwNHAXUBvSYtKWgDYBPh6VEbSosCUiOgD9GXOQn+UHLdG9p4bkpKpPSStlu3rA/QD\nhmSJl5mZmeVMZ0lw7gKGSToLWCQiHiEV4RsiaSJwOrBsRMwCbgW2BnoDD0TEjKaTRMQXwDKSHgJu\nB7q38n4/BR6NiNkR8XJE7E8a9VkDuC/76gr0rHxTzczMrNo6xSWqiHhO0rrA5sCpki4BVgPejIi9\nJK0PnJkdPo40mjOFdAnra5L6A4OA/hHxlaRPW3nLWXw7uZsB/C0iDqpEm8zMzKx2OkWCI2lX4JVs\n0cv3gJ1JFYafzQ7ZAegCEBFPS1oFWB44ttmplgNez5Kb7YAFJXVp4S2fAI6XtBBpXs+FpEtjp0ta\nHPgcOAf4TUR8Xsm2mlk+9DhyxVzdKmtmc+osl6heAkZJuhc4AbiANFJzhKS7gEeBFSXtkx1/F2nF\n7+Z1Ku4B1pA0AfgB6XLWt24Jj4gpwOXAP4CbgHMj4jVSUvMP4BHgbSc3ZmZm+dQpRnAi4knSHJjm\n1izZvhlAUgMwgLQSePPzfNTsPH+cy3ueBTS/S2p5YCzwHHDIvCM3s6J6++znax1Cuyy4V49ah2BW\nU51lBKcsknoCk0h3XL1c43DMzMysk+oUIzjlyi4t/azc4yW9CPyYNJ9nOjAwIiZJuhN4iDSp2TVv\nzMzMCiZXCU4bPAH0Ik1QngRsJOlJUv2bLqSaNwAPSrquNiGamZlZpRU9wZlASmYWA84DdiRNIv6A\nb2regGvemJmZFUrRE5z7SVWQFwPGAPuQqh+PBPo0r3kjaVBHB2hmZmaVl6tJxvMrIl4CegDdIuIT\n4G1ge9LIzkBJi0tqkPQnSYvVMlYzMzOrnKKP4AC8A3ycbT9KqnL8mqSmmjezSJOMP5dUqxjNrJNZ\n8Yi1XOjPLMcaGhub18qz+dBYrx+A3bt3resPf7c/P+3v3r1rQxtfWrf9G/L1O66Gem5/3treWh/P\n9QiOpPciYrkyj90OuCMiZkgaGhHjqxyemeXYtD89WOsQ2mWB3depdQhmNVXoOTjNHEG2nhXwm1oG\nYmZmZtXV4SM4klYFriDNfVkI2JNUhO8qYAlgcWAEaRHM3SNir+x1FwO3RMTNzc53Eqlg3/vAtsDK\npHWmABYG9gY2Jt0ufrukW4F1Jd0QETtKOhnoCywIjIqIqyWNJa0uvmxEDKnKD8LMzMyqphYjOENJ\nSy0MJK3gvRKwIjA623cMcDRpQc3ekhaVtADp9u47mp1rGWB8RGyYba+Tne+k7FyXAAdHxOWkO6i2\niojTgY+y5KYv8L2I6AcMAo4ruZvqAyc3ZmZm+VSLBOcuYJiks4BFIuIRYBowRNJE4HTSyMks0mrg\nWwO9gQciYkazc30cEc9m228C3UiJzKGS/gH8B2kkqDUbAxtKuh+4k/TzWCl77rH2NdPMzMxqpcMT\nnIh4DlgXeAA4VdIw4HDgzYjoA/yq5PBxwE7AdqRLWM3NbPa4ATgJuDMblfndPMKZAYyJiAHZ15oR\n8UrJc2ZmZpZDHZ7gSNoV6BURNwHHAesDywGTs0N2IJsMHBFPA6sAG5Bq1pRjOWCypAZgMN9MLJ7N\nN3OOmtr9KLCtpAWyS2HntblhZmZm1mnU4jbxl4ALJX1Kmmh8KLAkME7STsAoYDdJ+0TEpaRLWl0j\notyCPX8hrTs1Jft+kaTNScs2TJQ0AHhK0mMRsYGk+4CHSaM/f65QG80s51Y4bJNc1QIxszl16kJ/\n2SjM3cAvI+LlWsfTgrotBJa3QlCV5vbnp/0u9Nc2efodV0M9tz9vbc9doT9JPYHrgWsrldxIGhIR\n11fiXGZWbO+Muq3WIbSoYZe+tQ7BLBc6bYITEVOAn1XqfFnCtBspaTIzM7MC67QJThWcD2wg6QRg\nbWBpUvtHRMSzkvYgFRicBfwzIg6sXahmZmbWHvW0VMMZwATS3VR3RMSmpFvSz8qeXwLYMiI2AX4k\nae3ahGlmZmbtVU8jOE02BrpL2jN7vHj2/QPgr5IA1mTuBQLNzMysE6vHBGcG6bLUw007JHUhXcJa\nNyLeztarMjMzs5yqpwSnqdDfo8D2wMOS1gK2BC4DZmbJTQ9S8cEurZ7JzMzMOrV6SnBeANYD/g9Y\nVdIDpBXED42I9yXdLelx4BngD8AfJf0kIr6qXchmVivLH7J1rmqBmNmcOnWhPzMzM7O2qKe7qMzM\nzKxOOMExMzOzwnGCY2ZmZoXjBMfMzMwKxwmOmZmZFY4THDMzMyscJzhmZmZWOPVU6K9iJP0R2BBo\nBA6LiMdrHFLVSfoD0Jf0b+ZU4HHgclKxxKnAXhHxZe0irD5JiwHPAb8H/k4dtV/SHsB/ATOBkcCz\nFLT99di/wX3c/bt4/dsjOPNJUn9gjYjYCNgPOLfGIVWdpIFAr6zNWwLnACcB50dEX+BlYN8ahthR\njiMtygp11H5JywInAH2AbYDBFLT99di/wX084/5dsP7tBGf+bQrcBBARLwBLS1qqtiFV3T+AnbLt\nD4ElgAHAzdm+W4DNOj6sjiPpR8BawN+yXQOon/ZvBtwTEZ9ExNSIOJDitr8e+zfUeR93/y5m//Yl\nqvm3IvBEyeN3s30f1yac6ouIWcC/s4f7AbcBW5QMWb4DrFSL2DrQWcAhwN7Z4yXqqP09gcUl3Qws\nDZxIcdtfd/0b3Mdx/y5k//YITvs11DqAjiJpMOnD75BmTxX6ZyBpGPBwRPxfK4cUuv2k9i0L7AgM\nBy5lzjYXuf1Fbtu31GMfd/8ubv92gjP/3iL9RddkZdIkrEKTtAXwW2CriPgI+DSblAewCunnUlS/\nAAZLegTYHzie+mr/NOChiJgZEZOBT4BPCtr+uuzfUNd93P27oP3bCc78uwsYCiBpPeCtiPiktiFV\nl6RuwBnANhHRNAnvHmBItj0EuKMWsXWEiNglIn4eERsCo0l3WdRN+0n/5gdJWiCbkLgkxW1/3fVv\nqO8+7v5d3P7d0NjYWOsYckfSaUA/YDbw64h4psYhVZWkA0nXZV8q2b036cNgUeBVYJ+I+Krjo+tY\nkk4EpgB3AuOok/ZLOoh06QLgv0m3EBey/fXWv8F9vIn7N1Cg/u0Ex8zMzArHl6jMzMyscJzgmJmZ\nWeE4wTEzM7PCcYJjZmZmheMEx8zMzArHSzVY7ki6Flgd2C4i3qh1PGZWOe7fVilOcCyPhgBLRsTn\ntQ7EzCrO/dsqwnVwLFckjSYVpHqNVIjtYeC5iDhF0inAJsBiwATgv7KX/QVYn1Ru/D3gjYg4TlIj\nsHBEzJQ0HNgsIvaUtA5p8b2Fs69DIuIpSfeTKnxuDPwQOCEirpS0PGn9lm7ALODXwDHA3RExNov7\nAuB/I+LP1fvpmOWb+7dVkufgWK5ExP7Z5qZAD+B32YffTsAqEdE/IjYgDXFvkx33E+DnwA7A2mW8\nzZXALyNiAHAwqZprkyUjYmvSh3DTB+ypwG0R0QcYCexF+tAdDiBpQWCr7Lxm1gr3b6skX6KyPPsg\nIiLbHghslP0VBumvrdVIf6FNjIhZwCxJ983thNlfawLGSGravZSkpj8Gms7/KrBMtt0bOBsgIiaQ\n/rpEUndJqwHfBx7IFjA0s/K4f1u7OMGxPJtRsv0lcFFEnFl6gKSjmr1mdivn6lJyni+zv+7mkH0g\nzizZ1ZB9b6Tl0dCLgT2B7zLnX4lmNm/u39YuvkRlRTER2FHSQgCSRkpaA3gO2DhbKbcLsEXJaz4m\nDYND+guR7K+wKZK2zs7zQ0kj5/HeDwFbZsf3kXRZtn8csD2wbvaXn5m1jfu3zTeP4FhR3ABsCDwk\naRbwJPAKMBnYDXgCmAr8b8lrTgPukvQv4Bm++TAcBpwr6TekIfAj5vHexwOXSto2e3wIQER8IGly\nFouZtZ37t80330VldUXSicBCEXFcB7zXd0h//fWNiPer/X5m9c7920r5EpVZFUjaF3gAOM4ffmbF\n4v6dDx7BMTMzs8LxCI6ZmZkVjhMcMzMzKxwnOGZmZlY4TnDMzMyscJzgmJmZWeH8f/fVCaInMpWr\nAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 576x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "cwsmn5Q0tTNm",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Fuzzy String Matching\n",
        "useful for typos"
      ]
    },
    {
      "metadata": {
        "id": "-bua7f1x25ma",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "qFi3u5prvSjX",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Word Vectors\n",
        "useful for capturing meaning"
      ]
    },
    {
      "metadata": {
        "id": "b9OK3GYN2-UC",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}