Train_Word2Vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4ac5faa6-954b-4e58-abc1-79f3a18f6482",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import gensim\n",
    "import numpy as np\n",
    "from gensim.models import word2vec\n",
    "import preprocessing\n",
    "from gensim.models import Phrases"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eea702f9-2950-48cf-8c20-9c7001e6dcae",
   "metadata": {},
   "source": [
    "<h1>Import preprocessed data</h1>\n",
    "\n",
    "Data from file created in get_stack_data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2480a672-a442-415e-855e-068a5a258cae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>tags</th>\n",
       "      <th>score</th>\n",
       "      <th>answers_processed</th>\n",
       "      <th>questions_processed</th>\n",
       "      <th>processed_title</th>\n",
       "      <th>questions</th>\n",
       "      <th>post_corpus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>742</td>\n",
       "      <td>Class views in Django</td>\n",
       "      <td>django|views|oop</td>\n",
       "      <td>0.006875</td>\n",
       "      <td>needed use class based views wanted able use f...</td>\n",
       "      <td>view points function problem want change bit f...</td>\n",
       "      <td>class views django</td>\n",
       "      <td>view points to a function, which can be a pr...</td>\n",
       "      <td>class views django view points function proble...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2933</td>\n",
       "      <td>Create a directly-executable cross-platform GU...</td>\n",
       "      <td>user-interface|deployment|tkinter|release-mana...</td>\n",
       "      <td>0.039978</td>\n",
       "      <td>pysimplegui wraps tkinter works python also ru...</td>\n",
       "      <td>python works multiple platforms used desktop w...</td>\n",
       "      <td>create directly executable cross platform gui ...</td>\n",
       "      <td>Python works on multiple platforms and can be...</td>\n",
       "      <td>create directly executable cross platform gui ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19339</td>\n",
       "      <td>Transpose/Unzip Function (inverse of zip)?</td>\n",
       "      <td>list|matrix|transpose</td>\n",
       "      <td>0.093733</td>\n",
       "      <td>since returns tuples trick seems clever useful...</td>\n",
       "      <td>list item tuples like convert lists first cont...</td>\n",
       "      <td>transpose unzip function inverse zip</td>\n",
       "      <td>I have a list of 2-item tuples and I'd like t...</td>\n",
       "      <td>transpose unzip function inverse zip list item...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21961</td>\n",
       "      <td>Date/time conversion using time.mktime seems w...</td>\n",
       "      <td>datetime</td>\n",
       "      <td>0.000650</td>\n",
       "      <td>local time fancy time tuple incidentally seem ...</td>\n",
       "      <td>return number seconds since epoch since giving...</td>\n",
       "      <td>date time conversion using time mktime seems w...</td>\n",
       "      <td>should return the number of seconds sinc...</td>\n",
       "      <td>date time conversion using time mktime seems w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>36139</td>\n",
       "      <td>How to sort a list of strings?</td>\n",
       "      <td>string|sorting</td>\n",
       "      <td>0.067745</td>\n",
       "      <td>proper way sort strings previous example work ...</td>\n",
       "      <td>best way creating alphabetically sorted list p...</td>\n",
       "      <td>sort list strings</td>\n",
       "      <td>What is the best way of creating an alphabeti...</td>\n",
       "      <td>sort list strings best way creating alphabetic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134908</th>\n",
       "      <td>69059605</td>\n",
       "      <td>How to check if a column of DataFrame contain ...</td>\n",
       "      <td>pandas|dataframe|types</td>\n",
       "      <td>-0.000338</td>\n",
       "      <td>try even outputs create subset dataframe check...</td>\n",
       "      <td>example got data frame gt gt question filter e...</td>\n",
       "      <td>check column dataframe contain float type</td>\n",
       "      <td>For example, I've got this data frame.       ...</td>\n",
       "      <td>check column dataframe contain float type exam...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134909</th>\n",
       "      <td>69059907</td>\n",
       "      <td>How to increase/change a value in a dataset wi...</td>\n",
       "      <td>pandas|for-loop</td>\n",
       "      <td>-0.000536</td>\n",
       "      <td>create boolean auxiallry column multiply value...</td>\n",
       "      <td>want create loop iterates data probability end...</td>\n",
       "      <td>increase change value dataset condition python</td>\n",
       "      <td>What I want to do is: create a for loop that ...</td>\n",
       "      <td>increase change value dataset condition python...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134910</th>\n",
       "      <td>69059948</td>\n",
       "      <td>Check if elements in numpy array are between c...</td>\n",
       "      <td>numpy</td>\n",
       "      <td>-0.000338</td>\n",
       "      <td>direct way add axis broadcast think broadcasti...</td>\n",
       "      <td>numpy array arbitrary size numpy array exactly...</td>\n",
       "      <td>check elements numpy array columns column nump...</td>\n",
       "      <td>I have a 1-D numpy array  with arbitrary size...</td>\n",
       "      <td>check elements numpy array columns column nump...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134911</th>\n",
       "      <td>69060114</td>\n",
       "      <td>Pytorch geometric: how to explain the input in...</td>\n",
       "      <td>machine-learning|graph|pytorch|data-science</td>\n",
       "      <td>-0.000536</td>\n",
       "      <td>okay think got understanding output dimensions...</td>\n",
       "      <td>reading pytorch geometric documentation page c...</td>\n",
       "      <td>pytorch geometric explain input code snippet</td>\n",
       "      <td>I am reading PyTorch geometric documentation ...</td>\n",
       "      <td>pytorch geometric explain input code snippet r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134912</th>\n",
       "      <td>69060696</td>\n",
       "      <td>How do I extract two dates from a string in py...</td>\n",
       "      <td>regex|web-scraping|re|findall</td>\n",
       "      <td>-0.000437</td>\n",
       "      <td>try instead output</td>\n",
       "      <td>returns gives first date get second date</td>\n",
       "      <td>extract two dates string python</td>\n",
       "      <td>returns:      This gives me the first date...</td>\n",
       "      <td>extract two dates string python returns gives ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>134913 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id                                              title  \\\n",
       "0            742                              Class views in Django   \n",
       "1           2933  Create a directly-executable cross-platform GU...   \n",
       "2          19339         Transpose/Unzip Function (inverse of zip)?   \n",
       "3          21961  Date/time conversion using time.mktime seems w...   \n",
       "4          36139                     How to sort a list of strings?   \n",
       "...          ...                                                ...   \n",
       "134908  69059605  How to check if a column of DataFrame contain ...   \n",
       "134909  69059907  How to increase/change a value in a dataset wi...   \n",
       "134910  69059948  Check if elements in numpy array are between c...   \n",
       "134911  69060114  Pytorch geometric: how to explain the input in...   \n",
       "134912  69060696  How do I extract two dates from a string in py...   \n",
       "\n",
       "                                                     tags     score  \\\n",
       "0                                        django|views|oop  0.006875   \n",
       "1       user-interface|deployment|tkinter|release-mana...  0.039978   \n",
       "2                                   list|matrix|transpose  0.093733   \n",
       "3                                                datetime  0.000650   \n",
       "4                                          string|sorting  0.067745   \n",
       "...                                                   ...       ...   \n",
       "134908                             pandas|dataframe|types -0.000338   \n",
       "134909                                    pandas|for-loop -0.000536   \n",
       "134910                                              numpy -0.000338   \n",
       "134911        machine-learning|graph|pytorch|data-science -0.000536   \n",
       "134912                      regex|web-scraping|re|findall -0.000437   \n",
       "\n",
       "                                        answers_processed  \\\n",
       "0       needed use class based views wanted able use f...   \n",
       "1       pysimplegui wraps tkinter works python also ru...   \n",
       "2       since returns tuples trick seems clever useful...   \n",
       "3       local time fancy time tuple incidentally seem ...   \n",
       "4       proper way sort strings previous example work ...   \n",
       "...                                                   ...   \n",
       "134908  try even outputs create subset dataframe check...   \n",
       "134909  create boolean auxiallry column multiply value...   \n",
       "134910  direct way add axis broadcast think broadcasti...   \n",
       "134911  okay think got understanding output dimensions...   \n",
       "134912                                try instead output    \n",
       "\n",
       "                                      questions_processed  \\\n",
       "0       view points function problem want change bit f...   \n",
       "1       python works multiple platforms used desktop w...   \n",
       "2       list item tuples like convert lists first cont...   \n",
       "3       return number seconds since epoch since giving...   \n",
       "4       best way creating alphabetically sorted list p...   \n",
       "...                                                   ...   \n",
       "134908  example got data frame gt gt question filter e...   \n",
       "134909  want create loop iterates data probability end...   \n",
       "134910  numpy array arbitrary size numpy array exactly...   \n",
       "134911  reading pytorch geometric documentation page c...   \n",
       "134912          returns gives first date get second date    \n",
       "\n",
       "                                          processed_title  \\\n",
       "0                                     class views django    \n",
       "1       create directly executable cross platform gui ...   \n",
       "2                   transpose unzip function inverse zip    \n",
       "3       date time conversion using time mktime seems w...   \n",
       "4                                      sort list strings    \n",
       "...                                                   ...   \n",
       "134908         check column dataframe contain float type    \n",
       "134909    increase change value dataset condition python    \n",
       "134910  check elements numpy array columns column nump...   \n",
       "134911      pytorch geometric explain input code snippet    \n",
       "134912                   extract two dates string python    \n",
       "\n",
       "                                                questions  \\\n",
       "0         view points to a function, which can be a pr...   \n",
       "1        Python works on multiple platforms and can be...   \n",
       "2        I have a list of 2-item tuples and I'd like t...   \n",
       "3             should return the number of seconds sinc...   \n",
       "4        What is the best way of creating an alphabeti...   \n",
       "...                                                   ...   \n",
       "134908   For example, I've got this data frame.       ...   \n",
       "134909   What I want to do is: create a for loop that ...   \n",
       "134910   I have a 1-D numpy array  with arbitrary size...   \n",
       "134911   I am reading PyTorch geometric documentation ...   \n",
       "134912      returns:      This gives me the first date...   \n",
       "\n",
       "                                              post_corpus  \n",
       "0       class views django view points function proble...  \n",
       "1       create directly executable cross platform gui ...  \n",
       "2       transpose unzip function inverse zip list item...  \n",
       "3       date time conversion using time mktime seems w...  \n",
       "4       sort list strings best way creating alphabetic...  \n",
       "...                                                   ...  \n",
       "134908  check column dataframe contain float type exam...  \n",
       "134909  increase change value dataset condition python...  \n",
       "134910  check elements numpy array columns column nump...  \n",
       "134911  pytorch geometric explain input code snippet r...  \n",
       "134912  extract two dates string python returns gives ...  \n",
       "\n",
       "[134913 rows x 9 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('DB/Preprocessed_data.csv')\n",
    "\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d904a59a-3760-4b48-a428-13b1088afdb3",
   "metadata": {},
   "source": [
    "From the dataframe tokenize sentence from the post corpus already preprocessed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c49edc8c-5315-4506-a1fd-7ba11c442dd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = [_text.split() for _text in np.array(data.post_corpus)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c9bb679-7e2d-4731-b34b-5d6a9ecf318b",
   "metadata": {
    "tags": []
   },
   "source": [
    "<h1>Train Word Embeddings</h1>\n",
    "\n",
    "Generate word to vec model.\n",
    "\n",
    "Parameter:\n",
    "- Vector size : dimensionality of the word vectors.\n",
    "- Window :  maximum distance between the current and predicted word within a sentence.\n",
    "- Min Count : ignores all words with total frequency lower than this.\n",
    "- Workers : put here the number of cores of your cpu to speed up model creation\n",
    "\n",
    "Build vocabulary and then train and save to file the model pre created with documents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8eb79c45-7e1f-4b5d-bd96-055826c30832",
   "metadata": {},
   "outputs": [],
   "source": [
    "VECTOR_SIZE = 300\n",
    "WINDOW_SIZE = 7\n",
    "EPOCH = 32\n",
    "MIN_COUNT = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5fa038ed-dbde-4763-b400-45d221f0c161",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size vocab: 16346\n"
     ]
    }
   ],
   "source": [
    "word_2_vec_model = gensim.models.word2vec.Word2Vec(vector_size = VECTOR_SIZE, \n",
    "                                                   window = WINDOW_SIZE, min_count = MIN_COUNT, workers = 4 )\n",
    "\n",
    "word_2_vec_model.build_vocab(documents)\n",
    "words = list(word_2_vec_model.wv.index_to_key)\n",
    "sizeVocab = len(words)\n",
    "\n",
    "words\n",
    "print('Size vocab:', sizeVocab)\n",
    "\n",
    "word_2_vec_model.train(documents, total_examples = len(documents), epochs = EPOCH)\n",
    "\n",
    "word_2_vec_model.save('DB/word2vec_trained.bin')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d51901e5-a579-4cb9-b3c4-a629fb8a1d6b",
   "metadata": {
    "tags": []
   },
   "source": [
    "<h2>Sanity checks on train word embeddings</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "db53d205-4e01-457a-aeb6-ec207dac01d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Terms most similar to \"django\"\n",
      "[('flask', 0.5515910387039185), ('mezzanine', 0.5386962890625), ('djnago', 0.5331975817680359), ('app', 0.5121320486068726), ('djangorestframework', 0.506696879863739), ('drf', 0.5031201243400574), ('wagtail', 0.4983934164047241), ('djano', 0.49775344133377075), ('framework', 0.49766576290130615), ('admin', 0.48812422156333923)]\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Terms most similar to \"pandas\"\n",
      "[('panda', 0.6834497451782227), ('dataframe', 0.5815026164054871), ('pyspark', 0.54014652967453), ('pd', 0.5125634670257568), ('columns', 0.5017369985580444), ('column', 0.47391214966773987), ('groupby', 0.4635959565639496), ('covert', 0.45588040351867676), ('dataframes', 0.4424757659435272), ('df', 0.4414061903953552)]\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Terms most similar to \"csv\"\n",
      "[('excel', 0.708406388759613), ('cvs', 0.6375762820243835), ('tsv', 0.6104674339294434), ('file', 0.5861501097679138), ('txt', 0.5790210366249084), ('xlsx', 0.5718708038330078), ('dat', 0.5048170685768127), ('csvs', 0.5023595690727234), ('vcf', 0.4677131175994873), ('read', 0.46131888031959534)]\n"
     ]
    }
   ],
   "source": [
    "word_2_vec_model = gensim.models.word2vec.Word2Vec.load('DB/word2vec_trained.bin')\n",
    "\n",
    "\n",
    "print('Terms most similar to \"django\"')\n",
    "print(word_2_vec_model.wv.most_similar(\"django\"))\n",
    "print('-'*100)\n",
    "\n",
    "print('Terms most similar to \"pandas\"')\n",
    "print(word_2_vec_model.wv.most_similar(\"pandas\"))\n",
    "print('-'*100)\n",
    "\n",
    "print('Terms most similar to \"csv\"')\n",
    "print(word_2_vec_model.wv.most_similar(\"csv\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}