Create_Embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gensim version: \t3.8.1\n",
      "TensorFlow version: \t2.0.0\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "import tensorflow as tf\n",
    "\n",
    "print('gensim version: \\t%s' % gensim.__version__)\n",
    "print('TensorFlow version: \\t%s' % tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "\n",
    "# For displaying gensim logs\n",
    "logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.INFO)\n",
    "\n",
    "# Directory with raw txt-files\n",
    "TEXT_DIR  = 'data/yelp/train'\n",
    "\n",
    "# Directory for saving checkpoint and metadata\n",
    "MODEL_DIR = 'emb_yelp/'\n",
    "\n",
    "# Word2vec\n",
    "EMBEDDING_SIZE = 300"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents: 200000\n"
     ]
    }
   ],
   "source": [
    "import os, re, string\n",
    "\n",
    "\n",
    "def clean_doc(doc):\n",
    "    \"\"\"\n",
    "    Cleaning a document by several methods\n",
    "    \"\"\"\n",
    "    # Lowercase\n",
    "    doc = doc.lower()\n",
    "    # Remove numbers\n",
    "    doc = re.sub(r\"[0-9]+\", \"\", doc)\n",
    "    # Split in tokens\n",
    "    tokens = doc.split()\n",
    "    # Remove punctuation\n",
    "    tokens = [w.translate(str.maketrans('', '', string.punctuation)) for w in tokens]\n",
    "    # Tokens with less then two characters will be ignored\n",
    "    tokens = [word for word in tokens if len(word) > 1]\n",
    "    return ' '.join(tokens)\n",
    "\n",
    "\n",
    "def read_files(path):\n",
    "    \"\"\"\n",
    "    Read in text files\n",
    "    \"\"\"\n",
    "    documents = list()\n",
    "    tokenize  = lambda x: gensim.utils.simple_preprocess(x)\n",
    "    \n",
    "    # Read in all files in directory\n",
    "    if os.path.isdir(path):\n",
    "        for filename in os.listdir(path):\n",
    "            with open('%s/%s' % (path, filename), encoding='utf-8') as f:\n",
    "                doc = f.read()\n",
    "                doc = clean_doc(doc)\n",
    "                documents.append(tokenize(doc))\n",
    "    return documents\n",
    "\n",
    "docs = read_files(TEXT_DIR)\n",
    "print('Number of documents: %i' % len(docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : collecting all words and their counts\n"
     ]
    }
   ],
   "source": [
    "model = gensim.models.Word2Vec(docs, size=EMBEDDING_SIZE, min_count=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : saving Word2Vec object under emb_yelp/word2vec, separately None\n",
      "INFO : not storing attribute vectors_norm\n",
      "INFO : not storing attribute cum_table\n",
      "INFO : saved emb_yelp/word2vec\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists(MODEL_DIR):\n",
    "    os.makedirs(MODEL_DIR)\n",
    "model.save(os.path.join(MODEL_DIR,'word2vec'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating checkpoint and metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of weights: (42113, 300)\n",
      "Vocabulary size: 42113\n",
      "Embedding size: 300\n",
      "WARNING:tensorflow:Saver is deprecated, please switch to tf.train.Checkpoint or tf.keras.Model.save_weights for training checkpoints. When executing eagerly variables do not necessarily have unique names, and so the variable.name-based lookups Saver performs are error-prone.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING : Saver is deprecated, please switch to tf.train.Checkpoint or tf.keras.Model.save_weights for training checkpoints. When executing eagerly variables do not necessarily have unique names, and so the variable.name-based lookups Saver performs are error-prone.\n"
     ]
    }
   ],
   "source": [
    "from tensorboard.plugins import projector\n",
    "\n",
    "weights     = model.wv.vectors\n",
    "index_words = model.wv.index2word\n",
    "\n",
    "vocab_size    = weights.shape[0]\n",
    "embedding_dim = weights.shape[1]\n",
    "\n",
    "print('Shape of weights:', weights.shape)\n",
    "print('Vocabulary size: %i' % vocab_size)\n",
    "print('Embedding size: %i'  % embedding_dim)\n",
    "\n",
    "with open(os.path.join(MODEL_DIR,'metadata.tsv'), 'w') as f:\n",
    "    f.writelines(\"\\n\".join(index_words))\n",
    "\n",
    "config = projector.ProjectorConfig()\n",
    "embedding = config.embeddings.add()\n",
    "embedding.tensor_name = 'embeddings'\n",
    "embedding.metadata_path = './metadata.tsv'\n",
    "projector.visualize_embeddings(MODEL_DIR, config)\n",
    "\n",
    "tensor_embeddings = tf.Variable(model.wv.vectors, name='embeddings')\n",
    "\n",
    "checkpoint = tf.compat.v1.train.Saver([tensor_embeddings])\n",
    "checkpoint_path = checkpoint.save(sess=None, global_step=None, save_path=os.path.join(MODEL_DIR, \"model.ckpt\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : precomputing L2-norms of word weight vectors\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('espresso', 0.6709840893745422),\n",
       " ('latte', 0.6611574292182922),\n",
       " ('cappuccino', 0.6460868716239929),\n",
       " ('tea', 0.643097996711731),\n",
       " ('lattes', 0.613446056842804),\n",
       " ('coffees', 0.612466037273407),\n",
       " ('teas', 0.5807890295982361),\n",
       " ('chai', 0.567467451095581),\n",
       " ('mocha', 0.565311074256897),\n",
       " ('gelato', 0.5606527328491211)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.most_similar(positive=['coffee'], topn=10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}