Tutorials/word2vec, part1. processing corpus.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "В этом туториале будем готовить данные для последующего обучения word2vec\n",
    "\n",
    "Возьмём тот же корпус, что и в предыдущем туториале"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from string import punctuation\n",
    "import random\n",
    "\n",
    "from datasets import load_dataset\n",
    "import nltk\n",
    "from nltk.tokenize import ToktokTokenizer\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pymorphy2 import MorphAnalyzer\n",
    "from tqdm import tqdm\n",
    "\n",
    "morph = MorphAnalyzer()\n",
    "tokenizer = ToktokTokenizer()\n",
    "punct = punctuation + \"«»—\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = load_dataset('cedr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = corpus['train']['text']\n",
    "print(f'Текстов: {len(corpus)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text):\n",
    "    return tokenizer.tokenize(text.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = [tokenize(text) for text in corpus]\n",
    "print(f'Слов: {sum([len(t) for t in corpus])}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Соберем словарь лемм\n",
    "\n",
    "Лемматизировать весь корпус будет очень долго. Давайте лучше соберем словарь уникальных слов и будет лемматизировать только уникальные слова."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatize(word):\n",
    "    word_data = morph.parse(word)[0]\n",
    "    return word_data.normal_form"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok2lemma = {}\n",
    "\n",
    "for text in tqdm(corpus):\n",
    "    for tok in text:\n",
    "        if tok not in tok2lemma:\n",
    "            tok2lemma[tok] = lemmatize(tok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ключ - уникальное слово в нашем корпусе, значение - его лемма\n",
    "tok2lemma['уехали']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(tok2lemma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords = nltk.corpus.stopwords.words('russian')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Замена слов леммами\n",
    "Так как теперь к каждому слову из нашего корпуса мы знаем лемму, то давайте каждое слово заменим на его лемму и уберем стоп-слова и пунктуацию.\n",
    "Это работает гораздо(!) быстрее, чем если бы мы в корпусе для каждого слова каждый раз рассчитывали лемму (с помощью пайморфи), \n",
    "потому что теперь нам надо вызвать пайморфи 29147 раз вместо 128233."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmas_corpus = [[tok2lemma[tok] for tok in text if tok not in stopwords and tok and tok not in punct]\n",
    "                 for text in tqdm(corpus)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Соберем частотный словарь"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "freq = {}\n",
    "\n",
    "for text in tqdm(lemmas_corpus):\n",
    "    for tok in text:\n",
    "        if tok in freq:\n",
    "            freq[tok] += 1\n",
    "        else:\n",
    "            freq[tok] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "freq_df = pd.DataFrame(data={'word': list(freq.keys()), 'n_entries': list(freq.values())})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "freq_df.sort_values(by=['n_entries'], ascending=False, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "freq_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# уникальных слов в словаре\n",
    "freq_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "freq_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_words = freq_df.n_entries.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Замена редких слов\n",
    "В нашем корпусе осталось много слов, которые встречаются очень редко. Давайте мы редкие слова заменим на специальный токен UNK - unknown. Так мы разительно сократим размер нашего словаря с незначительной потерей информации."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Доля слов, которые мы заменим на UNK:')\n",
    "\n",
    "for threshold in np.arange(5, 36, 5):\n",
    "    \n",
    "    sub_df = freq_df[freq_df.n_entries < threshold]\n",
    "    \n",
    "    unk_freq = sub_df['n_entries'].sum() * 100 / n_words\n",
    "    \n",
    "    print('Порог отсечения - {}, доля UNK - {:.2f} %, останется слов в словаре - {}, удалили - {} слов'.format(\n",
    "        threshold, unk_freq, freq_df.shape[0] - sub_df.shape[0], sub_df.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# корпус у нас не большой, поэтому возьмем минимум\n",
    "threshold = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = freq_df[freq_df.n_entries >= threshold]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = set(vocab.word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'Мы сократили наш словарь в {:.2f} раз с потерей 1.51 % всех слов'.format(freq_df.shape[0] / len(words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_with_unk(word):\n",
    "    return word if word in words else 'UNK'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# заменим слово токеном UNK, если его нет в нашем новом словаре\n",
    "processed_corpus = [[replace_with_unk(tok) for tok in text] for text in tqdm(lemmas_corpus)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_duplicate_unks(tokens):\n",
    "    \n",
    "    output_tokens = []\n",
    "    \n",
    "    for tok in tokens:\n",
    "        \n",
    "        if tok == 'UNK' and output_tokens and output_tokens[-1] == 'UNK':\n",
    "            continue\n",
    "            \n",
    "        output_tokens.append(tok)\n",
    "            \n",
    "    return output_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_text = 'думать далее милый барышня UNK UNK тинькоф звонить неделя'.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_duplicate_unks(sample_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# дедублируем подряд идущие унки (оставим только один)\n",
    "processed_corpus = [drop_duplicate_unks(sample) for sample in tqdm(processed_corpus)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts_with_unk = [text for text in processed_corpus if 'UNK' in text]\n",
    "'Текстов с унками - {:.2f} %'.format(len(texts_with_unk) * 100 / len(processed_corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# посмотрим на тексты с унками\n",
    "for text in random.sample(texts_with_unk, k=5):\n",
    "    print(' '.join(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.shuffle(processed_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/processed_corpus.json', 'w') as f:\n",
    "    json.dump(processed_corpus, f, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}