CS_688_Assignment_6.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CS 688 Assignment 6.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNd2LdENXD2NRqfG5VCBCM5",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/GuptaNavdeep1983/DeepLearningRepo/blob/main/CS_688_Assignment_6.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G7Jw5OkD-WJl"
      },
      "source": [
        "References:\n",
        "\n",
        "https://datascience.stackexchange.com/questions/54904/how-to-avoid-tokenizing-w-sklearn-feature-extraction\n",
        "\n",
        "\n",
        "https://www.datacamp.com/community/tutorials/discovering-hidden-topics-python\n",
        "\n",
        "https://www.freecodecamp.org/news/an-introduction-to-bag-of-words-and-how-to-code-it-in-python-for-nlp-282e87a9da04/\n",
        "\n",
        "https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html#sklearn.feature_extraction.text.CountVectorizer.transform"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "939EWQDE89Pc"
      },
      "source": [
        "pip install gensim"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S-_jecXwgU3_"
      },
      "source": [
        "import statsmodels.api as sm \n",
        "import pylab as py \n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import urllib.request\n",
        "import datetime\n",
        "from zipfile import ZipFile\n",
        "from gzip import decompress\n",
        "from json import loads\n",
        "from requests import get\n",
        "import requests, zipfile, io\n",
        "from bs4 import BeautifulSoup\n",
        "import math\n",
        "import numpy as np\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "import re\n",
        "\n",
        "from gensim import corpora\n",
        "from gensim.models import LsiModel\n",
        "from gensim.models.coherencemodel import CoherenceModel"
      ],
      "execution_count": 87,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pjTPViKKgafF"
      },
      "source": [
        "def get_crowd_funding_data_links(filter_by_years):\n",
        "    # get the content of the website\n",
        "    content = urllib.request.urlopen('https://webrobots.io/indiegogo-dataset/').read()\n",
        "    soup = BeautifulSoup(content,features='html.parser')\n",
        "    \n",
        "    all_stories = pd.DataFrame([], columns=['Year', 'Month', 'json', 'csv'])\n",
        "    parent_div = soup.find(name='div', attrs={'class':'fusion-text'})\n",
        "\n",
        "    # Iterate through the divs to find the list of Month-Year wise anchor tags\n",
        "    index = 0\n",
        "    for year_data in parent_div.find_all(name='ul'):\n",
        "        all_months = year_data.find_all('li')\n",
        "        for month in all_months:\n",
        "            complete_text = month.text\n",
        "            date_time_obj = datetime.datetime.strptime((complete_text.split('[')[0]).strip(), '%Y-%m-%d')\n",
        "            if date_time_obj.year in filter_by_years:\n",
        "                all_stories.loc[index, \"Year\"] = date_time_obj.year\n",
        "                all_stories.loc[index, \"Month\"] = str(date_time_obj.month) if date_time_obj.month >=10 else f'0{date_time_obj.month}'\n",
        "                all_stories.loc[index, \"json\"] = month.find_all('a')[0][\"href\"]\n",
        "                all_stories.loc[index, \"csv\"] = month.find_all('a')[1][\"href\"]\n",
        "                index = index + 1\n",
        "    return all_stories\n",
        "\n",
        "def get_current_index_data(df, indx, columns):\n",
        "    r = requests.get(df[\"csv\"][indx])\n",
        "    z = zipfile.ZipFile(io.BytesIO(r.content))\n",
        "    csv_data = z.open(\"Indiegogo.csv\")\n",
        "    data_df = pd.read_csv(csv_data, usecols=columns, dtype={'title':str, 'tagline':str})\n",
        "    return data_df"
      ],
      "execution_count": 82,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qhs2TRecglUJ"
      },
      "source": [
        "df = get_crowd_funding_data_links([2020])\n",
        "\n",
        "documents = []\n",
        "for indx in df.head(5).index:\n",
        "    current_month_data = get_current_index_data(df, indx, [\"title\", \"tagline\"])\n",
        "    current_month_data.dropna(axis=0, inplace=True)\n",
        "    only_taglines = current_month_data[\"tagline\"]\n",
        "    all_content = ''.join(str(e) for e in only_taglines)\n",
        "    documents.append(all_content)"
      ],
      "execution_count": 84,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RQSatdh57RcW"
      },
      "source": [
        "vectorizer = CountVectorizer(input='content',stop_words='english', analyzer='word', token_pattern=r'\\b[a-zA-Z]{2,}\\b',max_features=100)\n",
        "df = pd.DataFrame(documents, columns=['content'])\n",
        "vectorizer.fit_transform(df['content'])\n",
        "doc_term_matrix = vectorizer.transform(df['content'])\n",
        "feature_names = vectorizer.get_feature_names()\n",
        "df_dictionary = pd.DataFrame(feature_names)\n",
        "dictionary = df_dictionary.to_dict()[0]\n",
        "df_doc_term_matrix = pd.DataFrame(doc_term_matrix.toarray())"
      ],
      "execution_count": 127,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aBwPMD7G8HOK"
      },
      "source": [
        "def create_gensim_lsa_model(number_of_topics,words, dictionary, doc_term_matrix):\n",
        "    # generate LSA model\n",
        "    lsamodel = LsiModel(doc_term_matrix, num_topics=number_of_topics, id2word = dictionary)  # train model\n",
        "    print(lsamodel.print_topics(num_topics=number_of_topics, num_words=words))\n",
        "    return lsamodel"
      ],
      "execution_count": 111,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QIMsN6F59m9c"
      },
      "source": [
        "from sklearn.utils.extmath import randomized_svd\n",
        "\n",
        "U, Sigma, VT = randomized_svd(doc_term_matrix, \n",
        "                              n_components=15,\n",
        "                              n_iter=5,\n",
        "                              random_state=42)"
      ],
      "execution_count": 130,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GlogTBQUJm14",
        "outputId": "25b275c3-151e-498c-f11b-ec23e4b86edd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(U)\n",
        "print(Sigma)\n",
        "print(VT)"
      ],
      "execution_count": 132,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 0.44119157  0.33396529 -0.25739524  0.70795352 -0.35548093]\n",
            " [ 0.45334147 -0.21263494 -0.0274168   0.22701965  0.83485226]\n",
            " [ 0.44708    -0.32083231  0.77908288  0.00519094 -0.30031488]\n",
            " [ 0.44109815  0.709614    0.07723072 -0.53654338  0.08964934]\n",
            " [ 0.45319246 -0.48658829 -0.56573959 -0.39919688 -0.28005176]]\n",
            "[14432.06974845   592.00897557   419.54608135   277.75737672\n",
            "   184.56997074]\n",
            "[[ 5.33105524e-02  4.16495387e-02  7.31670609e-02  8.16533354e-02\n",
            "   1.08808563e-01  4.72136286e-02  7.66663040e-02  7.73980539e-02\n",
            "   5.55769560e-02  5.72689155e-02  4.19836497e-02  1.40814176e-01\n",
            "   8.21949133e-02  6.04172305e-02  4.15027052e-02  4.54405123e-02\n",
            "   8.02186558e-02  5.47050780e-02  4.87292898e-02  7.68992059e-02\n",
            "   9.30249633e-02  8.42743254e-02  4.31051945e-02  6.67835082e-02\n",
            "   6.84453641e-02  4.20573434e-02  4.84213229e-02  4.94049825e-02\n",
            "   4.79311410e-02  5.37512983e-02  4.51573105e-02  4.43856201e-02\n",
            "   8.33111048e-02  6.85668848e-02  9.89530521e-02  5.40271790e-02\n",
            "   7.46789106e-02  6.90996712e-02  5.23792564e-02  1.59549744e-01\n",
            "   4.46109712e-02  4.26600642e-02  5.74326600e-01  7.02164049e-02\n",
            "   8.46718168e-02  4.71823414e-02  5.32054059e-02  4.45772321e-02\n",
            "   6.29685995e-02  1.27302730e-01  5.41340169e-02  5.95002492e-02\n",
            "   5.30726563e-02  7.76109885e-02  1.42537431e-01  4.48425709e-02\n",
            "   5.17526059e-02  6.17184463e-02  1.15302314e-01  1.33854712e-01\n",
            "   5.99148073e-02  3.20762509e-01  1.22870205e-01  4.46902549e-02\n",
            "   6.74547643e-02  6.56288703e-02  5.57913819e-02  8.05865847e-02\n",
            "   7.09618312e-02  5.59355108e-02  6.55534882e-02  5.04348049e-02\n",
            "   4.95671484e-02  5.78044432e-02  4.24296458e-02  9.35167496e-02\n",
            "   4.48723263e-02  4.24584898e-02  7.86199033e-02  4.63382445e-02\n",
            "   5.35536623e-02  5.74850796e-02  6.48214471e-02  1.49401121e-01\n",
            "   4.24763293e-02  1.03822594e-01  4.55208036e-02  5.93131454e-02\n",
            "   5.41792542e-02  4.85893637e-02  5.93724434e-02  6.32642033e-02\n",
            "   8.61096616e-02  4.73416326e-02  6.19629771e-02  6.95715804e-02\n",
            "   5.41111014e-02  2.62792381e-01  4.36559037e-02  4.91702568e-02]\n",
            " [-1.11569070e-02  6.45668963e-02 -3.64779848e-01  6.79420335e-02\n",
            "  -2.45792838e-01  1.47472806e-01 -2.20852429e-02  4.92940050e-02\n",
            "   5.56198325e-02  1.34429809e-01  1.25903152e-01 -3.50427357e-01\n",
            "  -3.13763642e-02  8.04753237e-02  6.32112084e-02 -1.87988993e-02\n",
            "   2.08674532e-01 -1.80600755e-02  3.77495389e-02  7.20644221e-02\n",
            "   1.31255893e-01 -1.41723931e-02  1.30426857e-02  4.90054825e-02\n",
            "   8.63190319e-02  5.06981987e-02  2.92893245e-02  1.03284301e-01\n",
            "   8.20643402e-02  3.22139070e-02  8.49651705e-03 -2.07576738e-02\n",
            "  -3.18875749e-01  8.61116448e-02  1.43481101e-01  3.16751666e-02\n",
            "  -6.42204305e-02  1.08714969e-02  6.44535250e-02  5.82532444e-02\n",
            "   1.35461711e-02  1.75003274e-02  2.65691389e-02  4.48768947e-02\n",
            "   2.87799443e-02 -1.50942302e-03  6.87581429e-02  4.31343558e-02\n",
            "   2.45252539e-02 -7.99850688e-02  7.65093015e-02  6.88984961e-02\n",
            "  -4.25434889e-02 -3.81716842e-02  2.63477128e-02 -1.84038464e-02\n",
            "  -3.60595200e-03 -3.34689925e-02 -4.27271398e-02  4.72620998e-02\n",
            "   3.19576417e-02 -2.47722107e-01  1.06658624e-01  5.30236372e-02\n",
            "  -1.55060393e-01 -5.68432788e-02  1.07003252e-01  9.71207202e-02\n",
            "  -1.19874421e-01  9.99211297e-02 -1.17038145e-02 -3.14515098e-02\n",
            "   4.29443115e-02  6.57925548e-02  4.09242826e-02 -1.30900406e-01\n",
            "   3.30096323e-03 -1.55313998e-01  8.31272285e-02 -2.88472802e-03\n",
            "   1.50097583e-01  6.84748671e-02 -1.39140715e-01  3.07788053e-02\n",
            "   1.94441184e-02  4.32476833e-02  4.51059740e-02 -2.86186270e-03\n",
            "   5.15653850e-02  4.96964560e-03  2.82183594e-02  9.60754215e-02\n",
            "   6.59731397e-02 -4.94689302e-02  1.89484744e-01  9.63172267e-03\n",
            "  -3.26263322e-02  5.54998299e-02  2.20528861e-02 -5.10727672e-02]\n",
            " [ 3.35430389e-02  5.60784047e-02 -7.83435867e-03 -5.17426936e-02\n",
            "  -4.35375705e-02  2.30295849e-01  1.33518042e-01  2.08186050e-02\n",
            "   1.96861954e-02  3.42368611e-02  1.66008066e-01  2.89632766e-01\n",
            "   3.79242475e-02  2.89796909e-02 -5.84297018e-02  5.42716715e-03\n",
            "   2.65518336e-01  2.31905474e-02 -1.67483257e-02 -1.49525959e-01\n",
            "  -2.52532658e-01 -7.31154706e-02 -4.58727344e-02  4.77519283e-02\n",
            "   1.40337910e-02  3.59973168e-02 -4.00735794e-02  7.75803883e-02\n",
            "   2.61641038e-02  3.37398392e-02 -5.56391605e-02  4.60308125e-02\n",
            "   3.22592956e-02 -1.74961537e-01 -7.84656774e-02  4.76206869e-02\n",
            "  -4.92381652e-02 -7.96334913e-02 -8.30147240e-02  4.06327990e-01\n",
            "  -1.11850154e-02 -2.73199225e-02 -2.73354336e-01  4.69217879e-02\n",
            "   6.53922379e-02 -5.92008169e-02  2.04742143e-02 -1.24766479e-02\n",
            "  -3.97401881e-02  5.20707729e-02  1.87712147e-02  2.33403094e-02\n",
            "  -9.81729419e-03  5.66534086e-02  4.39585251e-03  4.24220855e-02\n",
            "  -1.80836877e-02 -2.60209121e-02  2.30531039e-01 -2.73088648e-02\n",
            "  -2.65022083e-02  2.17953827e-02 -1.41093940e-01  2.17646209e-02\n",
            "   3.81019238e-02  7.93010072e-02  5.79749898e-02  1.51342290e-01\n",
            "  -5.82825349e-02  5.69408056e-02 -7.61564054e-02 -4.26855591e-02\n",
            "  -3.40265513e-02 -5.67298234e-02 -5.32047793e-02  1.56491628e-01\n",
            "  -8.58645312e-03 -2.36983758e-02  1.21623576e-01 -3.78596623e-03\n",
            "   2.34745152e-01  1.45020514e-02  5.61748346e-02 -9.79766369e-02\n",
            "  -6.92044334e-02  6.37758511e-03 -9.73662078e-02 -3.13822684e-02\n",
            "   4.57368805e-03  1.28267154e-02 -9.52365037e-02 -9.29218916e-03\n",
            "   1.28606496e-02  4.21543478e-02  1.73066896e-01 -2.17814388e-02\n",
            "  -4.57316936e-02  1.15793595e-02 -5.79033182e-03  3.52603507e-02]\n",
            " [ 1.90186590e-03  1.56118428e-02  1.04482488e-01 -7.13741324e-02\n",
            "   1.77436519e-01  1.72491674e-01  4.42803101e-03  5.20440181e-02\n",
            "  -7.51026228e-02  1.13079046e-01  1.13773790e-01 -1.92399207e-02\n",
            "   8.12821703e-03 -2.86672806e-03 -2.37347520e-02 -6.45123452e-02\n",
            "   1.63179993e-01 -5.78084878e-02 -9.47902546e-04  4.74115664e-02\n",
            "  -1.96737480e-02 -8.92817035e-02  4.00110837e-03  3.46109678e-02\n",
            "  -3.23680048e-02  1.17969414e-02 -6.66570659e-02  3.81687881e-02\n",
            "   1.09275971e-02 -2.20473869e-02 -4.70883441e-02  6.48388068e-02\n",
            "   1.67106062e-01  2.65580788e-02  1.06570287e-01 -1.19000445e-02\n",
            "  -6.57009699e-03  4.04371731e-02 -6.38320432e-02 -5.50719788e-01\n",
            "  -1.46934435e-02  7.29832069e-02  7.41552976e-02  4.06817428e-02\n",
            "  -2.35158031e-02 -6.05071320e-02  3.79132926e-02 -3.83551319e-02\n",
            "   4.41168576e-02 -9.50250064e-02 -2.19698234e-02 -6.04614857e-02\n",
            "   3.11172615e-03 -5.76630588e-02  1.17119269e-02  1.42914771e-02\n",
            "  -8.22949252e-02 -4.60806208e-03  3.05252309e-01  6.73880842e-02\n",
            "  -2.12047453e-02  5.14003894e-02 -1.54414784e-01  1.14385560e-02\n",
            "  -1.83893441e-02  6.27419993e-02  3.28337233e-02 -1.23072080e-01\n",
            "  -1.09209844e-01  2.81757156e-02 -3.75565843e-02  1.34918345e-02\n",
            "  -3.33395404e-02  1.58722704e-02  4.55525347e-02 -2.07084328e-01\n",
            "  -5.92506363e-02  1.73514256e-01  5.54983547e-02 -1.18851204e-01\n",
            "   1.91871529e-01 -8.21582031e-03 -7.71456505e-02  1.58947571e-02\n",
            "   6.77114427e-02  2.73999965e-04 -1.08564100e-01 -2.77873466e-02\n",
            "  -4.37485917e-02 -2.18710190e-02 -4.29909503e-02 -1.31706604e-04\n",
            "  -3.49727476e-02 -1.32885394e-01  2.57641036e-01  5.46164985e-03\n",
            "   9.02574293e-02 -1.01706916e-01  7.24394255e-02  5.01018812e-02]\n",
            " [ 9.13037031e-02  1.48604381e-02  1.74936034e-02 -1.99700487e-01\n",
            "  -9.42776164e-02  1.65060335e-01 -2.67688285e-02  1.60778590e-01\n",
            "   6.06324751e-02  4.21505692e-02  2.53082259e-02 -1.48397481e-01\n",
            "  -5.18559681e-03 -1.08019982e-01 -2.43771265e-02 -9.97948650e-03\n",
            "   1.44007287e-01 -1.80403183e-03 -4.02357064e-02  6.64359591e-02\n",
            "  -5.00888218e-03 -2.83809448e-02  1.19375264e-02 -6.75310917e-02\n",
            "  -7.62290031e-02 -2.67024719e-02 -1.32228102e-01 -4.38227469e-02\n",
            "  -4.96769034e-02 -5.46938854e-03  4.51041993e-02  8.06557019e-02\n",
            "   8.83740231e-02  1.78749016e-01  1.31314499e-02 -3.37193550e-02\n",
            "  -1.09253118e-01 -1.11576360e-01 -8.12245717e-02  2.78113963e-01\n",
            "  -1.51673407e-01 -7.02416931e-02  2.78431257e-01 -1.99742806e-01\n",
            "  -1.04619474e-01  1.31395602e-01 -3.63714242e-02 -1.26461711e-02\n",
            "   1.02969609e-01 -9.97848683e-02 -3.86001982e-02 -2.94351563e-02\n",
            "  -1.23351610e-02 -2.79022038e-02 -5.15585744e-02 -5.91829204e-02\n",
            "  -9.78150970e-02  1.93108064e-02 -2.32536691e-03 -2.46113143e-03\n",
            "  -1.29450297e-01  1.24520593e-01 -8.66850653e-02 -4.66560349e-02\n",
            "  -1.96346336e-01  1.13007974e-01 -6.21431533e-03 -3.39569801e-03\n",
            "  -4.64204326e-02 -2.23963473e-01 -3.16377441e-02 -2.77663471e-02\n",
            "  -2.71532270e-02  3.52662175e-02 -1.68314241e-01 -1.60156165e-02\n",
            "  -3.58967887e-02  3.53286214e-02 -2.46736484e-01  3.83705251e-02\n",
            "   7.32270563e-04 -2.46642105e-02 -9.15642492e-02  9.25774699e-02\n",
            "  -7.48011528e-02 -7.64190590e-02  7.51305667e-02 -9.10810424e-02\n",
            "  -1.41076423e-02 -4.87539666e-03 -8.34515751e-03 -1.09809093e-01\n",
            "  -2.12396351e-01 -4.60973845e-02 -7.66113427e-02  3.52694616e-02\n",
            "  -1.71036014e-01 -1.67612565e-01  4.87779985e-02  4.61627027e-02]]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}