TextMining_spaCyTokenizer_TopicModeling_TextClassification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "57c04311-2ff5-4c55-b898-8c912f300a30",
   "metadata": {},
   "source": [
    "# Basic Natural Language Processing: spaCy Tokenizer, Topic Modeling, Text Classification Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0ec30333-ed89-4047-aa8f-1fe63bf22ccd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(r\"C:\\Users\\user\\Documents\\RenewableEnergy_patents_2023_abstract_cpc.csv\", index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c6e08122-60a8-4322-a870-7b10d4dddb6a",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>assignee</th>\n",
       "      <th>inventor</th>\n",
       "      <th>priority_date</th>\n",
       "      <th>filing_date</th>\n",
       "      <th>publication_date</th>\n",
       "      <th>grant_date</th>\n",
       "      <th>result_link</th>\n",
       "      <th>representative_figure_link</th>\n",
       "      <th>...</th>\n",
       "      <th>kind_code</th>\n",
       "      <th>abstract</th>\n",
       "      <th>cpc_1</th>\n",
       "      <th>cpc_2</th>\n",
       "      <th>cpc_3</th>\n",
       "      <th>cpc_1_num</th>\n",
       "      <th>cpc_2_num</th>\n",
       "      <th>cpc_3_num</th>\n",
       "      <th>file_grant_days</th>\n",
       "      <th>cpc_1_GHY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>US11967653B2</td>\n",
       "      <td>Phased solar power supply system</td>\n",
       "      <td>Ampt, Llc</td>\n",
       "      <td>['Anatoli Ledenev']</td>\n",
       "      <td>2013-03-15</td>\n",
       "      <td>2023-09-05</td>\n",
       "      <td>2024-04-23</td>\n",
       "      <td>2024-04-23</td>\n",
       "      <td>https://patents.google.com/patent/US11967653B2/en</td>\n",
       "      <td>https://patentimages.storage.googleapis.com/c0...</td>\n",
       "      <td>...</td>\n",
       "      <td>B2</td>\n",
       "      <td>A high efficiency solar power system combining...</td>\n",
       "      <td>['F', 'G', 'H', 'Y']</td>\n",
       "      <td>['H02', 'F03', 'G05', 'F05', 'Y02']</td>\n",
       "      <td>['H02J', 'F03B', 'F03D', 'G05B', 'H02S', 'F05B...</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>231</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id                              title   assignee  \\\n",
       "0  US11967653B2  Phased solar power supply system   Ampt, Llc   \n",
       "\n",
       "              inventor priority_date filing_date publication_date  grant_date  \\\n",
       "0  ['Anatoli Ledenev']    2013-03-15  2023-09-05       2024-04-23  2024-04-23   \n",
       "\n",
       "                                         result_link  \\\n",
       "0  https://patents.google.com/patent/US11967653B2/en   \n",
       "\n",
       "                          representative_figure_link  ... kind_code  \\\n",
       "0  https://patentimages.storage.googleapis.com/c0...  ...        B2   \n",
       "\n",
       "                                            abstract                 cpc_1  \\\n",
       "0  A high efficiency solar power system combining...  ['F', 'G', 'H', 'Y']   \n",
       "\n",
       "                                 cpc_2  \\\n",
       "0  ['H02', 'F03', 'G05', 'F05', 'Y02']   \n",
       "\n",
       "                                               cpc_3 cpc_1_num  cpc_2_num  \\\n",
       "0  ['H02J', 'F03B', 'F03D', 'G05B', 'H02S', 'F05B...         4          5   \n",
       "\n",
       "   cpc_3_num  file_grant_days  cpc_1_GHY  \n",
       "0          8              231          1  \n",
       "\n",
       "[1 rows x 21 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf126dc1-401e-42c7-9ba5-f0d80725c22a",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Customized Tokenizer: spaCy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "458c1164-0844-4516-b671-c97f5c9b3e92",
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy.matcher import Matcher\n",
    "from spacy.tokens import Span\n",
    "from spacy.language import Language\n",
    "from spacy.util import filter_spans\n",
    "\n",
    "# Load spaCy model\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "# Define rule-based matcher\n",
    "matcher = Matcher(nlp.vocab)\n",
    "patterns = [\n",
    "    [{'POS': 'NOUN'}, {'POS': 'NOUN'}, {'POS': 'NOUN'}],\n",
    "    [{'POS': 'NOUN'}, {'POS': 'NOUN'}],\n",
    "    [{'POS': 'ADJ'}, {'POS': 'NOUN'}]\n",
    "]\n",
    "matcher.add(\"Patterns\", patterns)\n",
    "\n",
    "# Custom pipeline component for rule-based matching\n",
    "@Language.component('merge_rule_based_matches')\n",
    "def merge_rule_based_matches(doc):\n",
    "    matches = matcher(doc)\n",
    "    spans = [Span(doc, start, end, label=match_id) for match_id, start, end in matches]\n",
    "    # Filter spans to remove overlaps\n",
    "    filtered_spans = filter_spans(spans)\n",
    "    with doc.retokenize() as retokenizer:\n",
    "        for span in filtered_spans:\n",
    "            retokenizer.merge(span)\n",
    "    return doc\n",
    "\n",
    "# Add the custom component to the pipeline before the 'ner' component\n",
    "nlp.add_pipe('merge_rule_based_matches', before=\"ner\")\n",
    "\n",
    "# Custom tokenizer to handle hyphenated words and integrated with rule-based matching\n",
    "def custom_tokenizer(nlp):\n",
    "    prefix_re = spacy.util.compile_prefix_regex(nlp.Defaults.prefixes) # use default\n",
    "    suffix_re = spacy.util.compile_suffix_regex(nlp.Defaults.suffixes) # use default\n",
    "    infix_re = spacy.util.compile_infix_regex(nlp.Defaults.infixes + [r'\\w+-\\w+'])\n",
    "    return spacy.tokenizer.Tokenizer(nlp.vocab, \n",
    "                                     prefix_search = prefix_re.search,\n",
    "                                     suffix_search = suffix_re.search,\n",
    "                                     infix_finditer=infix_re.finditer, \n",
    "                                     token_match=None)\n",
    "\n",
    "nlp.tokenizer = custom_tokenizer(nlp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bd3a4998-0260-472b-b8d4-10e369b45af8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocessing\n",
    "\n",
    "# Words to exclude\n",
    "exclude_words = {\"include\", \"have\", \"provide\", \"receive\", \"method\", \"comprise\", \"configure\", \n",
    "                 \"define\", \"disclose\", \"determine\", \"identify\", \"second\", \"relate\", \"enable\", \n",
    "                 \"present\", \"plurality\", \"e.g\", \"thereof\", \"involve\", \"describe\", \"substantially\",\n",
    "                 \"particularly\", \"1\", \"2\", \"relative\", \"particular\", \"aspect\", \"correspond\"}\n",
    "\n",
    "def custom_tokenizer_preprocessed(text):\n",
    "    doc = nlp(text)\n",
    "    tokens = [\n",
    "        token.lemma_.lower().strip()\n",
    "        for token in doc\n",
    "        if token.lemma_.lower().strip() not in exclude_words\n",
    "        and not token.is_stop\n",
    "        and not token.is_punct\n",
    "        and not token.is_space\n",
    "    ]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb3aedba-fcd7-421c-a398-94b038e7c7a2",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Bag of Words & Tf-Idf: Scikit learn + spaCy Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "669115e5-b299-485b-bf08-47f915d08d80",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\miniconda3\\Lib\\site-packages\\sklearn\\feature_extraction\\text.py:523: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "\n",
    "# Use CountVectorizer with the custom spaCy tokenizer\n",
    "vectorizer = CountVectorizer(tokenizer=custom_tokenizer_preprocessed, min_df=5, max_df=100, ngram_range=(1,1))\n",
    "bof = vectorizer.fit_transform(df['abstract'])\n",
    "\n",
    "# Use TfidfTransformer to transform bag of words\n",
    "tfidf = TfidfTransformer()\n",
    "tfidf_bof = tfidf.fit_transform(bof)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e96cbf93-4655-4d1c-84d0-58a3f3a3cd50",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Topic Modeling with scikit learn "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8d0b53a3-27b8-41b8-9e69-4e9db8962637",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "\n",
    "# Perform topic modeling using Latent Dirichlet Allocation (LDA)\n",
    "lda = LatentDirichletAllocation(n_components=20, random_state=42)\n",
    "document_topics = lda.fit(tfidf_bof)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5ce0c685-52a9-403d-aae7-e408be0c3978",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic #0: reduce | content | prevent | connect | hold | report | electrically | substrate | utilize | convert\n",
      "Topic #1: second set | object | first set | channel | array | detect | automatically | area | number | mean\n",
      "Topic #2: electronic device | component | power | battery | represent | face | use | technology | production | material\n",
      "Topic #3: base | select | image | value | neural network | signal | set | device | compute | detect\n",
      "Topic #4: couple | pattern | say | instruction | position | space | source | measurement | contain | apart\n",
      "Topic #5: wireless | print | responsive | share | access | generate | client device | drive | array | network\n",
      "Topic #6: emit | housing | mount | invention | couple | light | core | function | pixel | transistor\n",
      "Topic #7: display | datum | base | improve | information | signal | interest | environment | apparatus | resource\n",
      "Topic #8: extend | frame | opening | portion | base | accord | distribute | surface | air | datum\n",
      "Topic #9: form | layer | display device | compound | controller | camera | insulate | server | parameter | contact\n",
      "Topic #10: content | pair | heat | electrode | contain | structure | range | space | composition | present disclosure\n",
      "Topic #11: dispose | substrate | element | modify | temperature | direction | light-emitting element | set | update | pixel\n",
      "Topic #12: device | treat | connect | datum | adapt | cell | information | allow | pair | arrange\n",
      "Topic #13: patient | set | sensor | control | position | portion | produce | current | face | detect\n",
      "Topic #14: element | pixel | change | circuit | array | characteristic | power | module | response | embodiment\n",
      "Topic #15: user | device | associate | embodiment | operate | generate | electrode | indicate | analyze | multiple\n",
      "Topic #16: shape | surface | mobile | having | monitor | condition | sensor | present invention | contain | record\n",
      "Topic #17: user | capture | apparatus | image | facilitate | code | technique | interface | structure | various embodiment\n",
      "Topic #18: process | execute | allow | present disclosure | target | address | position | disclosure | transmit | model\n",
      "Topic #19: assembly | subject | application | retrieve | dispose | material | embodiment | mount | composition | module\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Inspect the topic modeling results\n",
    "def print_top_words(model, feature_names, n_top_words):\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        message = f\"Topic #{topic_idx}: \"\n",
    "        message += \" | \".join([feature_names[i] for i in topic.argsort()[:-n_top_words - 1:-1]])\n",
    "        print(message)\n",
    "    print()\n",
    "\n",
    "# Print top words in each topic\n",
    "n_top_words = 10\n",
    "tf_feature_names = vectorizer.get_feature_names_out()\n",
    "print_top_words(lda, tf_feature_names, n_top_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e93ebec8-a7b9-438d-8e77-2373956ad4a8",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Text Classification Model: Predicting CPC code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "72b8a6b9-24f6-41b7-946e-b174c2cb929e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split, KFold, GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a270ce2a-c67c-4627-94d8-455b19e924d0",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "#### Logistic Regression: using BoW as features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4f2dae48-83b0-4f47-a7bc-819838370701",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best model hyperparameter:  {'C': 0.001}\n",
      "best validation accuracy:  0.6663641863278887\n",
      "test accuracy:  0.5833333333333334\n"
     ]
    }
   ],
   "source": [
    "# target, features, train test split\n",
    "y = df['cpc_1_GHY']\n",
    "X = bof\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle = True, random_state=42)\n",
    "\n",
    "# cross validation and grid search settings\n",
    "cv = KFold(n_splits=5, shuffle=True, random_state=0)\n",
    "param_grid = {'C': [0.001, 0.01, 0.1, 1, 10]}\n",
    "\n",
    "# grid search\n",
    "grid_search = GridSearchCV(LogisticRegression(), param_grid, cv=cv, scoring='accuracy', n_jobs = -1)\n",
    "grid_search.fit(X_train, y_train)\n",
    "best_model = grid_search.best_estimator_\n",
    "best_score = grid_search.best_score_\n",
    "best_params = grid_search.best_params_\n",
    "print(\"best model hyperparameter: \", best_params)\n",
    "print(\"best validation accuracy: \", best_score)\n",
    "\n",
    "# Evaluate the best model on the test set\n",
    "y_pred = best_model.predict(X_test)\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "print(\"test accuracy: \", accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1e9c019-4dbb-4da8-a4d0-ff4530cc4ce5",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "#### Logistic Regression: using Tf-Idf as features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "67cf646c-04db-49c4-8778-95a947e56697",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best model hyperparameter:  {'C': 0.001}\n",
      "best validation accuracy:  0.6663641863278887\n",
      "test accuracy:  0.5833333333333334\n"
     ]
    }
   ],
   "source": [
    "# target, features, train test split\n",
    "y = df['cpc_1_GHY']\n",
    "X = tfidf_bof\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle = True, random_state=42)\n",
    "\n",
    "# cross validation and grid search settings\n",
    "cv = KFold(n_splits=5, shuffle=True, random_state=0)\n",
    "param_grid = {'C': [0.001, 0.01, 0.1, 1, 10]}\n",
    "\n",
    "# grid search\n",
    "grid_search = GridSearchCV(LogisticRegression(), param_grid, cv=cv, scoring='accuracy', n_jobs = -1)\n",
    "grid_search.fit(X_train, y_train)\n",
    "best_model = grid_search.best_estimator_\n",
    "best_score = grid_search.best_score_\n",
    "best_params = grid_search.best_params_\n",
    "print(\"best model hyperparameter: \", best_params)\n",
    "print(\"best validation accuracy: \", best_score)\n",
    "\n",
    "# Evaluate the best model on the test set\n",
    "y_pred = best_model.predict(X_test)\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "print(\"test accuracy: \", accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cedf0e6a-9808-4501-9673-22b36717a51e",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "#### Support Vector Classifier: using BoW as features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "61165e57-5a77-4ba7-8073-0f3e36b7e4bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best model hyperparameter:  {'C': 1, 'gamma': 'scale', 'kernel': 'rbf'}\n",
      "best validation accuracy:  0.6732607380520267\n",
      "test accuracy:  0.5833333333333334\n"
     ]
    }
   ],
   "source": [
    "# target, features, train test split\n",
    "y = df['cpc_1_GHY']\n",
    "X = bof\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle = True, random_state=42)\n",
    "\n",
    "# cross validation and grid search settings\n",
    "cv = KFold(n_splits=5, shuffle=True, random_state=0)\n",
    "param_grid = {'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100], \n",
    "              'gamma':['scale', 'auto'], \n",
    "              'kernel': ['linear', 'poly', 'rbf']}\n",
    "\n",
    "# grid search\n",
    "grid_search = GridSearchCV(SVC(), param_grid, cv=cv, scoring='accuracy', n_jobs = -1)\n",
    "grid_search.fit(X_train, y_train)\n",
    "best_model = grid_search.best_estimator_\n",
    "best_score = grid_search.best_score_\n",
    "best_params = grid_search.best_params_\n",
    "print(\"best model hyperparameter: \", best_params)\n",
    "print(\"best validation accuracy: \", best_score)\n",
    "\n",
    "# Evaluate the best model on the test set\n",
    "y_pred = best_model.predict(X_test)\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "print(\"test accuracy: \", accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d42d0f7-7113-4456-8109-3f926d12d7f1",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "#### Support Vector Classifier: using Tf-Idf as features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "44c373ae-c44e-46d9-bfeb-a05a0d441558",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best model hyperparameter:  {'C': 1, 'gamma': 'scale', 'kernel': 'rbf'}\n",
      "best validation accuracy:  0.6664246823956443\n",
      "test accuracy:  0.5833333333333334\n"
     ]
    }
   ],
   "source": [
    "# target, features, train test split\n",
    "y = df['cpc_1_GHY']\n",
    "X = tfidf_bof\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle = True, random_state=42)\n",
    "\n",
    "# cross validation and grid search settings\n",
    "cv = KFold(n_splits=5, shuffle=True, random_state=0)\n",
    "param_grid = {'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100], \n",
    "              'gamma':['scale', 'auto'], \n",
    "              'kernel': ['linear', 'poly', 'rbf']}\n",
    "\n",
    "# grid search\n",
    "grid_search = GridSearchCV(SVC(), param_grid, cv=cv, scoring='accuracy', n_jobs = -1)\n",
    "grid_search.fit(X_train, y_train)\n",
    "best_model = grid_search.best_estimator_\n",
    "best_score = grid_search.best_score_\n",
    "best_params = grid_search.best_params_\n",
    "print(\"best model hyperparameter: \", best_params)\n",
    "print(\"best validation accuracy: \", best_score)\n",
    "\n",
    "# Evaluate the best model on the test set\n",
    "y_pred = best_model.predict(X_test)\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "print(\"test accuracy: \", accuracy)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}