SKLearn 11,  Mengenal TF- IDF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 11 Mengenal TF-IDF (Term Frequency - Inverse Document Frequency)\n",
    "TF-IDF (Term Frequency - Inverse Document Frequency) merupakan salah satu metode statistik yang digunakan untuk mengukur seberapa penting suatu kata terhadap suatu dokumen tertentu dari sekumpulan dokumen atau corpus.\n",
    "\n",
    "Referensi:\n",
    "\n",
    "https://en.wikipedia.org/wiki/Tf%E2%80%93idf\n",
    "https://scikit-learn.org/stable/modules/feature_extraction.html#text-feature-extraction\n",
    "\n",
    "### Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['the house had a tiny little mouse',\n",
       " 'the cat saw the mouse',\n",
       " 'the mouse ran away from the house',\n",
       " 'the cat finally ate the mouse',\n",
       " 'the end of the mouse story']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus = [\n",
    "    'the house had a tiny little mouse', \n",
    "    'the cat saw the mouse',\n",
    "    'the mouse ran away from the house', \n",
    "    'the cat finally ate the mouse',\n",
    "    'the end of the mouse story'\n",
    "]\n",
    "\n",
    "corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TF-IDF Weights dengan TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 7)\t0.2808823162882302\n",
      "  (0, 6)\t0.5894630806320427\n",
      "  (0, 11)\t0.5894630806320427\n",
      "  (0, 5)\t0.47557510189256375\n",
      "  (1, 9)\t0.7297183669435993\n",
      "  (1, 2)\t0.5887321837696324\n",
      "  (1, 7)\t0.3477147117091919\n",
      "  (2, 1)\t0.5894630806320427\n",
      "  (2, 8)\t0.5894630806320427\n",
      "  (2, 7)\t0.2808823162882302\n",
      "  (2, 5)\t0.47557510189256375\n",
      "  (3, 0)\t0.5894630806320427\n",
      "  (3, 4)\t0.5894630806320427\n",
      "  (3, 2)\t0.47557510189256375\n",
      "  (3, 7)\t0.2808823162882302\n",
      "  (4, 10)\t0.6700917930430479\n",
      "  (4, 3)\t0.6700917930430479\n",
      "  (4, 7)\t0.3193023297639811\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "vectorizer = TfidfVectorizer(stop_words='english')\n",
    "response = vectorizer.fit_transform(corpus)\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ate',\n",
       " 'away',\n",
       " 'cat',\n",
       " 'end',\n",
       " 'finally',\n",
       " 'house',\n",
       " 'little',\n",
       " 'mouse',\n",
       " 'ran',\n",
       " 'saw',\n",
       " 'story',\n",
       " 'tiny']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "         0.4755751 , 0.58946308, 0.28088232, 0.        , 0.        ,\n",
       "         0.        , 0.58946308],\n",
       "        [0.        , 0.        , 0.58873218, 0.        , 0.        ,\n",
       "         0.        , 0.        , 0.34771471, 0.        , 0.72971837,\n",
       "         0.        , 0.        ],\n",
       "        [0.        , 0.58946308, 0.        , 0.        , 0.        ,\n",
       "         0.4755751 , 0.        , 0.28088232, 0.58946308, 0.        ,\n",
       "         0.        , 0.        ],\n",
       "        [0.58946308, 0.        , 0.4755751 , 0.        , 0.58946308,\n",
       "         0.        , 0.        , 0.28088232, 0.        , 0.        ,\n",
       "         0.        , 0.        ],\n",
       "        [0.        , 0.        , 0.        , 0.67009179, 0.        ,\n",
       "         0.        , 0.        , 0.31930233, 0.        , 0.        ,\n",
       "         0.67009179, 0.        ]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "response.todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>D1</th>\n",
       "      <th>D2</th>\n",
       "      <th>D3</th>\n",
       "      <th>D4</th>\n",
       "      <th>D5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ate</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.589463</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>away</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.589463</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.588732</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.475575</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>end</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.670092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>finally</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.589463</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>house</th>\n",
       "      <td>0.475575</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.475575</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>little</th>\n",
       "      <td>0.589463</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mouse</th>\n",
       "      <td>0.280882</td>\n",
       "      <td>0.347715</td>\n",
       "      <td>0.280882</td>\n",
       "      <td>0.280882</td>\n",
       "      <td>0.319302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ran</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.589463</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>saw</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.729718</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>story</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.670092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tiny</th>\n",
       "      <td>0.589463</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               D1        D2        D3        D4        D5\n",
       "ate      0.000000  0.000000  0.000000  0.589463  0.000000\n",
       "away     0.000000  0.000000  0.589463  0.000000  0.000000\n",
       "cat      0.000000  0.588732  0.000000  0.475575  0.000000\n",
       "end      0.000000  0.000000  0.000000  0.000000  0.670092\n",
       "finally  0.000000  0.000000  0.000000  0.589463  0.000000\n",
       "house    0.475575  0.000000  0.475575  0.000000  0.000000\n",
       "little   0.589463  0.000000  0.000000  0.000000  0.000000\n",
       "mouse    0.280882  0.347715  0.280882  0.280882  0.319302\n",
       "ran      0.000000  0.000000  0.589463  0.000000  0.000000\n",
       "saw      0.000000  0.729718  0.000000  0.000000  0.000000\n",
       "story    0.000000  0.000000  0.000000  0.000000  0.670092\n",
       "tiny     0.589463  0.000000  0.000000  0.000000  0.000000"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame(response.todense().T,\n",
    "                 index=vectorizer.get_feature_names(),\n",
    "                 columns=[f'D{i+1}' for i in range(len(corpus))])\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}