Term_Extraction_Sequence_Classifier.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Term Extraction Sequence Classifier.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "9OsjSGOr0bSA"
      ],
      "authorship_tag": "ABX9TyMoG9pFKSz7mLYf/Ef0v5Bc",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c4da41ffca2d4809a64ca7c3b4375bab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f8aa0656efa64e5385ec59a765939770",
              "IPY_MODEL_59dfeb1cd7f042eba3faa1ce8263eb0f"
            ],
            "layout": "IPY_MODEL_c0aa55048c3b41f097d1583b02dc3c45"
          }
        },
        "f8aa0656efa64e5385ec59a765939770": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "Downloading: 100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dc06957d389e4995acbd22e23bdc8cef",
            "max": 5069051,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_53cf87e674334b27ad3a48422ec20030",
            "value": 5069051
          }
        },
        "59dfeb1cd7f042eba3faa1ce8263eb0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9257c9f9130d4f47a05e3066eec6fffd",
            "placeholder": "​",
            "style": "IPY_MODEL_134178a4421b41de93598e8e0f08dcfb",
            "value": " 5.07M/5.07M [00:01&lt;00:00, 2.72MB/s]"
          }
        },
        "c0aa55048c3b41f097d1583b02dc3c45": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dc06957d389e4995acbd22e23bdc8cef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "53cf87e674334b27ad3a48422ec20030": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "9257c9f9130d4f47a05e3066eec6fffd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "134178a4421b41de93598e8e0f08dcfb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/lwachowiak/Term-Extraction-With-Language-Models/blob/main/Term_Extraction_Sequence_Classifier.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0yMTmZptEkHC"
      },
      "source": [
        "# Imports\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aaxWLY9GFE2W"
      },
      "source": [
        "!pip install transformers\n",
        "!pip install sacremoses\n",
        "!pip install sentencepiece"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m9fYtB3_FHuK"
      },
      "source": [
        "#torch and tranformers for model and training\n",
        "import torch  \n",
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "from torch.utils.data import TensorDataset\n",
        "from transformers import XLMRobertaTokenizer              \n",
        "from transformers import XLMRobertaForSequenceClassification\n",
        "from transformers import AdamW                            \n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "import sentencepiece\n",
        "\n",
        "#sklearn for evaluation\n",
        "from sklearn import preprocessing                       \n",
        "from sklearn.metrics import classification_report        \n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.model_selection import ParameterGrid         \n",
        "from sklearn.model_selection import ParameterSampler      \n",
        "from sklearn.utils.fixes import loguniform\n",
        "\n",
        "#nlp preprocessing\n",
        "from nltk import ngrams                                 \n",
        "from spacy.pipeline import SentenceSegmenter\n",
        "from spacy.lang.en import English\n",
        "from spacy.pipeline import Sentencizer\n",
        "from sacremoses import MosesTokenizer, MosesDetokenizer\n",
        "\n",
        "\n",
        "#utilities\n",
        "import pandas as pd\n",
        "import glob, os\n",
        "import time\n",
        "import datetime\n",
        "import random\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "% matplotlib inline\n",
        "import seaborn as sns\n",
        "import pickle"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kpY66eTVxQNH",
        "outputId": "ba5a7610-f5ce-44cc-b19c-c2d9db65909f"
      },
      "source": [
        "# connect to GPU \n",
        "device = torch.device('cuda')\n",
        "\n",
        "print('Connected to GPU:', torch.cuda.get_device_name(0))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Connected to GPU: Tesla P100-PCIE-16GB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3RPZ14sYHHUm"
      },
      "source": [
        "# Prepare Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TKqV3YfXHSNz"
      },
      "source": [
        "Training Data: corp, wind\n",
        "\n",
        "Valid: equi\n",
        "\n",
        "Test Data: htfl"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ERUBsPPOFfe1"
      },
      "source": [
        "#load terms\n",
        "\n",
        "#en\n",
        "df_corp_terms_en=pd.read_csv('ACTER-master/ACTER-master/en/corp/annotations/corp_en_terms_nes.ann', delimiter=\"\\t\", names=[\"Term\", \"Label\"])  \n",
        "df_equi_terms_en=pd.read_csv('ACTER-master/ACTER-master/en/equi/annotations/equi_en_terms_nes.ann', delimiter=\"\\t\", names=[\"Term\", \"Label\"])  \n",
        "df_htfl_terms_en=pd.read_csv('ACTER-master/ACTER-master/en/htfl/annotations/htfl_en_terms_nes.ann', delimiter=\"\\t\", names=[\"Term\", \"Label\"])  \n",
        "df_wind_terms_en=pd.read_csv('ACTER-master/ACTER-master/en/wind/annotations/wind_en_terms_nes.ann', delimiter=\"\\t\", names=[\"Term\", \"Label\"])  \n",
        "\n",
        "#fr\n",
        "df_corp_terms_fr=pd.read_csv('ACTER-master/ACTER-master/fr/corp/annotations/corp_fr_terms_nes.ann', delimiter=\"\\t\", names=[\"Term\", \"Label\"])  \n",
        "df_equi_terms_fr=pd.read_csv('ACTER-master/ACTER-master/fr/equi/annotations/equi_fr_terms_nes.ann', delimiter=\"\\t\", names=[\"Term\", \"Label\"])  \n",
        "df_htfl_terms_fr=pd.read_csv('ACTER-master/ACTER-master/fr/htfl/annotations/htfl_fr_terms_nes.ann', delimiter=\"\\t\", names=[\"Term\", \"Label\"])  \n",
        "df_wind_terms_fr=pd.read_csv('ACTER-master/ACTER-master/fr/wind/annotations/wind_fr_terms_nes.ann', delimiter=\"\\t\", names=[\"Term\", \"Label\"])  \n",
        "\n",
        "#nl\n",
        "df_corp_terms_nl=pd.read_csv('ACTER-master/ACTER-master/nl/corp/annotations/corp_nl_terms_nes.ann', delimiter=\"\\t\", names=[\"Term\", \"Label\"])  \n",
        "df_equi_terms_nl=pd.read_csv('ACTER-master/ACTER-master/nl/equi/annotations/equi_nl_terms_nes.ann', delimiter=\"\\t\", names=[\"Term\", \"Label\"])  \n",
        "df_htfl_terms_nl=pd.read_csv('ACTER-master/ACTER-master/nl/htfl/annotations/htfl_nl_terms_nes.ann', delimiter=\"\\t\", names=[\"Term\", \"Label\"])  \n",
        "df_wind_terms_nl=pd.read_csv('ACTER-master/ACTER-master/nl/wind/annotations/wind_nl_terms_nes.ann', delimiter=\"\\t\", names=[\"Term\", \"Label\"])  \n",
        "\n",
        "labels=[\"Random\", \"Term\"]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "tw11QcsHF8Gc",
        "outputId": "a16e39d2-4ca0-4127-b7b0-414a028ef98f"
      },
      "source": [
        "# example terms\n",
        "df_wind_terms_en"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Term</th>\n",
              "      <th>Label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>48/600</td>\n",
              "      <td>Named_Entity</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4energia</td>\n",
              "      <td>Named_Entity</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4energy</td>\n",
              "      <td>Named_Entity</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ab \"lietuvos energija\"</td>\n",
              "      <td>Named_Entity</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ab lietuvos elektrine</td>\n",
              "      <td>Named_Entity</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1529</th>\n",
              "      <td>zhiquan</td>\n",
              "      <td>Named_Entity</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1530</th>\n",
              "      <td>çetinkaya</td>\n",
              "      <td>Named_Entity</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1531</th>\n",
              "      <td>çeti̇nkaya</td>\n",
              "      <td>Named_Entity</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1532</th>\n",
              "      <td>çeşme</td>\n",
              "      <td>Named_Entity</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1533</th>\n",
              "      <td>özgen</td>\n",
              "      <td>Named_Entity</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1534 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                        Term         Label\n",
              "0                     48/600  Named_Entity\n",
              "1                   4energia  Named_Entity\n",
              "2                    4energy  Named_Entity\n",
              "3     ab \"lietuvos energija\"  Named_Entity\n",
              "4      ab lietuvos elektrine  Named_Entity\n",
              "...                      ...           ...\n",
              "1529                 zhiquan  Named_Entity\n",
              "1530               çetinkaya  Named_Entity\n",
              "1531              çeti̇nkaya  Named_Entity\n",
              "1532                   çeşme  Named_Entity\n",
              "1533                   özgen  Named_Entity\n",
              "\n",
              "[1534 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sU7NMPaDvbWt"
      },
      "source": [
        "**Functions for preprocessing and creating of Training Data**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3_stqlIDvZxA"
      },
      "source": [
        "#load all text files from folder into a string\n",
        "def load_text_corpus(path):\n",
        "  text_data=\"\"\n",
        "  print(glob.glob(path))\n",
        "  for file in glob.glob(path+\"*.txt\"):\n",
        "      print(file)\n",
        "      with open(file) as f:\n",
        "        temp_data = f.read()\n",
        "        print(len(temp_data))\n",
        "        text_data=text_data+\" \"+temp_data\n",
        "  print(len(text_data))\n",
        "  return text_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4nXtHwAyPoK0"
      },
      "source": [
        "#split in sentences and tokenize\n",
        "def preprocess(text):\n",
        "  #sentenize (from spacy)\n",
        "  sentencizer = Sentencizer()\n",
        "  nlp = English()\n",
        "  nlp.add_pipe(sentencizer)\n",
        "  doc = nlp(text)\n",
        "\n",
        "  #tokenize\n",
        "  sentence_list=[]\n",
        "  mt = MosesTokenizer(lang='en')\n",
        "  for s in doc.sents:\n",
        "    tokenized_text = mt.tokenize(s, return_str=True)\n",
        "    sentence_list.append((tokenized_text.split(), s))     #append tuple of tokens and original senteence\n",
        "  return sentence_list\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1qBA_KhoQkhB"
      },
      "source": [
        "#input is list of sentences and dataframe containing terms\n",
        "def create_training_data(sentence_list, df_terms, n):\n",
        "\n",
        "  #create empty dataframe\n",
        "  training_data = pd.DataFrame(columns=['n_gram', 'Context', 'Label', \"Termtype\"])\n",
        "\n",
        "  md = MosesDetokenizer(lang='en')\n",
        "\n",
        "\n",
        "  print(len(sentence_list))\n",
        "  count=0\n",
        "\n",
        "  for sen in sentence_list:\n",
        "    count+=1\n",
        "    if count%100==0:print(count)\n",
        "\n",
        "    s=sen[0]  #take first part of tuple, i.e. the tokens\n",
        "\n",
        "    # 1-gram up to n-gram\n",
        "    for i in range(1,n+1):\n",
        "      #create n-grams of this sentence\n",
        "      n_grams = ngrams(s, i)\n",
        "\n",
        "      #look if n-grams are in the annotation dataset\n",
        "      for n_gram in n_grams: \n",
        "        n_gram=md.detokenize(n_gram) \n",
        "        context=str(sen[1]).strip()\n",
        "        #if yes add an entry to the training data\n",
        "        if n_gram.lower() in df_terms.values:\n",
        "          #append positive sample\n",
        "          #get termtype like common term\n",
        "          termtype=\"/\"#df_terms.loc[df_terms['Term'] == n_gram.lower()].iloc[0][\"Label\"]\n",
        "          training_data = training_data.append({'n_gram': n_gram, 'Context': context, 'Label': 1, \"Termtype\":termtype}, ignore_index=True)\n",
        "        else:\n",
        "          #append negative sample\n",
        "          training_data = training_data.append({'n_gram': n_gram, 'Context': context, 'Label': 0, \"Termtype\":\"None\"}, ignore_index=True)\n",
        "\n",
        "  return training_data\n",
        "\n",
        "  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4HhBTwYl1-dy"
      },
      "source": [
        "**Create Training Data**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UemCf-2xPrn1"
      },
      "source": [
        "# en \n",
        "#create trainings data for all corp texts\n",
        "corp_text_en=load_text_corpus(\"ACTER-master/ACTER-master/en/corp/texts/annotated/\") # load test\n",
        "corp_s_list=preprocess(corp_text_en)                                                # preprocess\n",
        "train_data_corp_en=create_training_data(corp_s_list, df_corp_terms_en, 6)           # create training data\n",
        "\n",
        "#create trainings data for all wind texts\n",
        "wind_text_en=load_text_corpus(\"ACTER-master/ACTER-master/en/wind/texts/annotated/\") # load test\n",
        "wind_s_list=preprocess(wind_text_en)                                                # preprocess\n",
        "train_data_wind_en=create_training_data(wind_s_list, df_wind_terms_en, 6)           # create training data\n",
        "\n",
        "#create trainings data for all equi texts\n",
        "equi_text_en=load_text_corpus(\"ACTER-master/ACTER-master/en/equi/texts/annotated/\") # load test\n",
        "equi_s_list=preprocess(equi_text_en)                                                # preprocess\n",
        "train_data_equi_en=create_training_data(equi_s_list, df_equi_terms_en, 6)           # create training data\n",
        "\n",
        "#create trainings data for all htfl texts\n",
        "htfl_text_en=load_text_corpus(\"ACTER-master/ACTER-master/en/htfl/texts/annotated/\") # load test\n",
        "htfl_s_list=preprocess(htfl_text_en)                                                # preprocess\n",
        "train_data_htfl_en=create_training_data(htfl_s_list, df_htfl_terms_en, 6)           # create training data "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "stFqQ_Sd2gAN"
      },
      "source": [
        "#fr\n",
        "corp_text_fr=load_text_corpus(\"ACTER-master/ACTER-master/fr/corp/texts/annotated/\") # load text\n",
        "corp_s_list=preprocess(corp_text_fr)                                                # preprocess\n",
        "train_data_corp_fr=create_training_data(corp_s_list, df_corp_terms_fr, 6)           # create training data\n",
        "\n",
        "wind_text_fr=load_text_corpus(\"ACTER-master/ACTER-master/fr/wind/texts/annotated/\") # load text\n",
        "wind_s_list=preprocess(wind_text_fr)                                                # preprocess\n",
        "train_data_wind_fr=create_training_data(wind_s_list, df_wind_terms_fr, 6)           # create training data\n",
        "\n",
        "equi_text_fr=load_text_corpus(\"ACTER-master/ACTER-master/fr/equi/texts/annotated/\") # load text\n",
        "equi_s_list=preprocess(equi_text_fr)                                                # preprocess\n",
        "train_data_equi_fr=create_training_data(equi_s_list, df_equi_terms_fr, 6)           # create training data\n",
        "\n",
        "htfl_text_fr=load_text_corpus(\"ACTER-master/ACTER-master/fr/htfl/texts/annotated/\") # load text\n",
        "htfl_s_list=preprocess(htfl_text_fr)                                                # preprocess\n",
        "train_data_htfl_fr=create_training_data(htfl_s_list, df_htfl_terms_fr, 6)           # create training data "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z2PI4ngj2gKZ"
      },
      "source": [
        "#nl\n",
        "corp_text_nl=load_text_corpus(\"ACTER-master/ACTER-master/nl/corp/texts/annotated/\") # load text\n",
        "corp_s_list=preprocess(corp_text_nl)                                                # preprocess\n",
        "train_data_corp_nl=create_training_data(corp_s_list, df_corp_terms_nl, 6)           # create training data\n",
        "\n",
        "wind_text_nl=load_text_corpus(\"ACTER-master/ACTER-master/nl/wind/texts/annotated/\") # load text\n",
        "wind_s_list=preprocess(wind_text_nl)                                                # preprocess\n",
        "train_data_wind_nl=create_training_data(wind_s_list, df_wind_terms_nl, 6)           # create training data\n",
        "\n",
        "equi_text_nl=load_text_corpus(\"ACTER-master/ACTER-master/nl/equi/texts/annotated/\") # load text\n",
        "equi_s_list=preprocess(equi_text_nl)                                                # preprocess\n",
        "train_data_equi_nl=create_training_data(equi_s_list, df_equi_terms_nl, 6)           # create training data\n",
        "\n",
        "htfl_text_nl=load_text_corpus(\"ACTER-master/ACTER-master/nl/htfl/texts/annotated/\") # load text\n",
        "htfl_s_list=preprocess(htfl_text_nl)                                                # preprocess\n",
        "train_data_htfl_nl=create_training_data(htfl_s_list, df_htfl_terms_nl, 6)           # create training data "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GXrT0L_DNCE_",
        "outputId": "6eed88af-a6eb-43e1-e18c-66c3fd432754"
      },
      "source": [
        "print(train_data_corp_en.groupby('Label').count())\n",
        "print(train_data_wind_en.groupby('Label').count())\n",
        "print(train_data_equi_en.groupby('Label').count())\n",
        "print(train_data_htfl_en.groupby('Label').count())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "       n_gram  Context  Termtype\n",
            "Label                           \n",
            "0      274139   274139    274139\n",
            "1        8708     8708      8708\n",
            "       n_gram  Context  Termtype\n",
            "Label                           \n",
            "0      311535   311535    311535\n",
            "1       10542    10542     10542\n",
            "       n_gram  Context  Termtype\n",
            "Label                           \n",
            "0      298863   298863    298863\n",
            "1       13891    13891     13891\n",
            "       n_gram  Context  Termtype\n",
            "Label                           \n",
            "0      290334   290334    290334\n",
            "1       14376    14376     14376\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "S4_Q9krEESA2",
        "outputId": "5fe80a12-619a-47f5-e7f1-cfd4a9a36d11"
      },
      "source": [
        "train_data_equi_en"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>n_gram</th>\n",
              "      <th>Context</th>\n",
              "      <th>Label</th>\n",
              "      <th>Termtype</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Pirouette</td>\n",
              "      <td>Pirouette (dressage)\\n\\nA Pirouette is a Frenc...</td>\n",
              "      <td>1</td>\n",
              "      <td>Specific_Term</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>(</td>\n",
              "      <td>Pirouette (dressage)\\n\\nA Pirouette is a Frenc...</td>\n",
              "      <td>0</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>dressage</td>\n",
              "      <td>Pirouette (dressage)\\n\\nA Pirouette is a Frenc...</td>\n",
              "      <td>1</td>\n",
              "      <td>Common_Term</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>)</td>\n",
              "      <td>Pirouette (dressage)\\n\\nA Pirouette is a Frenc...</td>\n",
              "      <td>0</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>A</td>\n",
              "      <td>Pirouette (dressage)\\n\\nA Pirouette is a Frenc...</td>\n",
              "      <td>1</td>\n",
              "      <td>Specific_Term</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>312749</th>\n",
              "      <td>about it when he's done</td>\n",
              "      <td>Stop and let your horse think about it when he...</td>\n",
              "      <td>0</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>312750</th>\n",
              "      <td>it when he's done something</td>\n",
              "      <td>Stop and let your horse think about it when he...</td>\n",
              "      <td>0</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>312751</th>\n",
              "      <td>when he's done something right</td>\n",
              "      <td>Stop and let your horse think about it when he...</td>\n",
              "      <td>0</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>312752</th>\n",
              "      <td>he's done something right.</td>\n",
              "      <td>Stop and let your horse think about it when he...</td>\n",
              "      <td>0</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>312753</th>\n",
              "      <td>'s done something right. \"</td>\n",
              "      <td>Stop and let your horse think about it when he...</td>\n",
              "      <td>0</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>312754 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                n_gram  ...       Termtype\n",
              "0                            Pirouette  ...  Specific_Term\n",
              "1                                    (  ...           None\n",
              "2                             dressage  ...    Common_Term\n",
              "3                                    )  ...           None\n",
              "4                                    A  ...  Specific_Term\n",
              "...                                ...  ...            ...\n",
              "312749         about it when he's done  ...           None\n",
              "312750     it when he's done something  ...           None\n",
              "312751  when he's done something right  ...           None\n",
              "312752      he's done something right.  ...           None\n",
              "312753      's done something right. \"  ...           None\n",
              "\n",
              "[312754 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WRYG7Q_sDnNw"
      },
      "source": [
        "**Undersample**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qvHscNxQCmvJ"
      },
      "source": [
        "#undersample class 0 so the amount of trainingsample is the same as label 1 \n",
        "\n",
        "def undersample(train_data):\n",
        "# Class count\n",
        "  print(\"Before\")\n",
        "  print(train_data.Label.value_counts())\n",
        "  count_class_0, count_class_1 = train_data.Label.value_counts()\n",
        "\n",
        "  # Divide by class\n",
        "  df_class_0 = train_data[train_data['Label'] == 0]\n",
        "  df_class_1 = train_data[train_data['Label'] == 1]\n",
        "\n",
        "  df_class_0_under = df_class_0.sample(count_class_1)\n",
        "  df_test_under = pd.concat([df_class_0_under, df_class_1], axis=0)\n",
        "\n",
        "  print(\"After\")\n",
        "  print(df_test_under.Label.value_counts())\n",
        "\n",
        "  return df_test_under"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-wi80YrXFHj4",
        "outputId": "270e8a84-7e17-4952-fad3-a9998718f99b"
      },
      "source": [
        "# undersample the trainingsdata\n",
        "\n",
        "#en\n",
        "train_data_corp_en=undersample(train_data_corp_en)\n",
        "\n",
        "train_data_wind_en=undersample(train_data_wind_en)\n",
        "\n",
        "\n",
        "#fr\n",
        "train_data_corp_fr=undersample(train_data_corp_fr)\n",
        "\n",
        "train_data_wind_fr=undersample(train_data_wind_fr)\n",
        "\n",
        "\n",
        "#nl\n",
        "train_data_corp_nl=undersample(train_data_corp_nl)\n",
        "\n",
        "train_data_wind_nl=undersample(train_data_wind_nl)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Before\n",
            "0    274139\n",
            "1      8708\n",
            "Name: Label, dtype: int64\n",
            "After\n",
            "1    8708\n",
            "0    8708\n",
            "Name: Label, dtype: int64\n",
            "Before\n",
            "0    311535\n",
            "1     10542\n",
            "Name: Label, dtype: int64\n",
            "After\n",
            "1    10542\n",
            "0    10542\n",
            "Name: Label, dtype: int64\n",
            "Before\n",
            "0    325242\n",
            "1      7443\n",
            "Name: Label, dtype: int64\n",
            "After\n",
            "1    7443\n",
            "0    7443\n",
            "Name: Label, dtype: int64\n",
            "Before\n",
            "0    356805\n",
            "1      9293\n",
            "Name: Label, dtype: int64\n",
            "After\n",
            "1    9293\n",
            "0    9293\n",
            "Name: Label, dtype: int64\n",
            "Before\n",
            "0    283267\n",
            "1      7071\n",
            "Name: Label, dtype: int64\n",
            "After\n",
            "1    7071\n",
            "0    7071\n",
            "Name: Label, dtype: int64\n",
            "Before\n",
            "0    287361\n",
            "1      5582\n",
            "Name: Label, dtype: int64\n",
            "After\n",
            "1    5582\n",
            "0    5582\n",
            "Name: Label, dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VSy8hZggPQpf",
        "outputId": "a8893623-b01f-4794-e20d-79e5d6a2136a"
      },
      "source": [
        "#concat trainingsdata\n",
        "trainings_data_df = pd.concat([train_data_corp_en, train_data_wind_en,  train_data_corp_fr, train_data_wind_fr, train_data_corp_nl, train_data_wind_nl])\n",
        "\n",
        "valid_data_df = train_data_equi_en #pd.concat([train_data_equi_en, train_data_equi_fr, train_data_equi_nl ])\n",
        "\n",
        "test_data_df_en = train_data_htfl_en\n",
        "test_data_df_fr = train_data_htfl_fr\n",
        "test_data_df_nl = train_data_htfl_nl\n",
        "\n",
        "print(len(trainings_data_df))\n",
        "print(len(valid_data_df))\n",
        "print(len(test_data_df_en))\n",
        "print(len(test_data_df_fr))\n",
        "print(len(test_data_df_nl))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "97278\n",
            "312754\n",
            "304710\n",
            "303069\n",
            "292615\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jKtVpCjIWPvO"
      },
      "source": [
        "**Tokenizer**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "c4da41ffca2d4809a64ca7c3b4375bab",
            "f8aa0656efa64e5385ec59a765939770",
            "59dfeb1cd7f042eba3faa1ce8263eb0f",
            "c0aa55048c3b41f097d1583b02dc3c45",
            "dc06957d389e4995acbd22e23bdc8cef",
            "53cf87e674334b27ad3a48422ec20030",
            "9257c9f9130d4f47a05e3066eec6fffd",
            "134178a4421b41de93598e8e0f08dcfb"
          ]
        },
        "id": "pJjnroUuWOdg",
        "outputId": "aad8ace8-3731-49da-d753-6649fb6ecd52"
      },
      "source": [
        "xlmr_tokenizer = XLMRobertaTokenizer.from_pretrained(\"xlm-roberta-base\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c4da41ffca2d4809a64ca7c3b4375bab",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=5069051.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9v7WbIW6WV8D"
      },
      "source": [
        "def tokenizer_xlm(data, max_len):\n",
        "  labels_ = []\n",
        "  input_ids_ = []\n",
        "  attn_masks_ = []\n",
        "\n",
        "  # for each datasample:\n",
        "  for index, row in data.iterrows():\n",
        "\n",
        "      sentence = row['n_gram']+\". \"+row[\"Context\"]\n",
        "      #print(sentence)\n",
        "     \n",
        "      # create requiered input, i.e. ids and attention masks\n",
        "      encoded_dict = xlmr_tokenizer.encode_plus(sentence,\n",
        "                                                max_length=max_len, \n",
        "                                                padding='max_length',\n",
        "                                                truncation=True, \n",
        "                                                return_tensors='pt')\n",
        "\n",
        "      # add encoded sample to lists\n",
        "      input_ids_.append(encoded_dict['input_ids'])\n",
        "      attn_masks_.append(encoded_dict['attention_mask'])\n",
        "      labels_.append(row['Label'])\n",
        "      \n",
        "  # Convert each Python list of Tensors into a 2D Tensor matrix.\n",
        "  input_ids_ = torch.cat(input_ids_, dim=0)\n",
        "  attn_masks_ = torch.cat(attn_masks_, dim=0)\n",
        "\n",
        "  # labels to tensor\n",
        "  labels_ = torch.tensor(labels_)\n",
        "\n",
        "  print('Encoder finished. {:,} examples.'.format(len(labels_)))\n",
        "  return input_ids_, attn_masks_, labels_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RcCexBG1ZuP_",
        "outputId": "c2641990-2539-4ee3-b53e-aee04dfb052b"
      },
      "source": [
        "#tokenize input for the different training/test sets\n",
        "max_len=64\n",
        "\n",
        "input_ids_train, attn_masks_train, labels_all_train = tokenizer_xlm(trainings_data_df, max_len)\n",
        "\n",
        "input_ids_valid, attn_masks_valid, labels_all_valid = tokenizer_xlm(valid_data_df, max_len)\n",
        "\n",
        "input_ids_test_en, attn_masks_test_en, labels_test_en = tokenizer_xlm(test_data_df_en, max_len)\n",
        "input_ids_test_fr, attn_masks_test_fr, labels_test_fr = tokenizer_xlm(test_data_df_fr, max_len)\n",
        "input_ids_test_nl, attn_masks_test_nl, labels_test_nl = tokenizer_xlm(test_data_df_nl, max_len)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Encoder finished. 97,278 examples.\n",
            "Encoder finished. 312,754 examples.\n",
            "Encoder finished. 304,710 examples.\n",
            "Encoder finished. 303,069 examples.\n",
            "Encoder finished. 292,615 examples.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nLCLiW9-Nkd-"
      },
      "source": [
        "# create datasets\n",
        "train_dataset = TensorDataset(input_ids_train, attn_masks_train, labels_all_train)\n",
        "\n",
        "valid_dataset = TensorDataset(input_ids_valid, attn_masks_valid, labels_all_valid)\n",
        "\n",
        "test_dataset_en = TensorDataset(input_ids_test_en, attn_masks_test_en, labels_test_en)\n",
        "test_dataset_fr = TensorDataset(input_ids_test_fr, attn_masks_test_fr, labels_test_fr)\n",
        "test_dataset_nl = TensorDataset(input_ids_test_nl, attn_masks_test_nl, labels_test_nl)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Si-ng4T8Ny2O"
      },
      "source": [
        "# create dataloaders\n",
        "batch_size = 32\n",
        "\n",
        "train_dataloader = DataLoader(train_dataset, sampler = RandomSampler(train_dataset), batch_size = batch_size) #random sampling\n",
        "valid_dataloader = DataLoader(valid_dataset, sampler = SequentialSampler(valid_dataset),batch_size = batch_size ) #sequential sampling\n",
        "\n",
        "test_dataloader_en = DataLoader(test_dataset_en, sampler = SequentialSampler(test_dataset_en),batch_size = batch_size ) #sequential sampling\n",
        "test_dataloader_fr = DataLoader(test_dataset_fr, sampler = SequentialSampler(test_dataset_fr),batch_size = batch_size ) #sequential sampling\n",
        "test_dataloader_nl = DataLoader(test_dataset_nl, sampler = SequentialSampler(test_dataset_nl),batch_size = batch_size ) #sequential sampling"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hart2Y_ia5qD"
      },
      "source": [
        "#Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sF72Sc2ur-ds"
      },
      "source": [
        "def create_model(lr, eps, train_dataloader, epochs, device):\n",
        "  xlmr_model = XLMRobertaForSequenceClassification.from_pretrained(\"xlm-roberta-base\", num_labels=2)\n",
        "  desc = xlmr_model.to(device)\n",
        "  print('Connected to GPU:', torch.cuda.get_device_name(0))\n",
        "  optimizer = AdamW(xlmr_model.parameters(),\n",
        "                  lr = lr,   \n",
        "                  eps = eps       \n",
        "                )\n",
        "  total_steps = len(train_dataloader) * epochs\n",
        "  scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0,   \n",
        "                                            num_training_steps = total_steps)\n",
        "  return xlmr_model, optimizer, scheduler"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R7acJSCUtHN6"
      },
      "source": [
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YsxS3wVltI5i"
      },
      "source": [
        "def validate(validation_dataloader, validation_df, xlmr_model, verbose, print_cm): \n",
        "  \n",
        "  # put model in evaluation mode \n",
        "  xlmr_model.eval()\n",
        "\n",
        "  #extract terms and compute scores\n",
        "  extracted_terms_equi=extract_terms(train_data_equi_en, xlmr_model)\n",
        "  extracted_terms_equi_en = set([item.lower() for item in extracted_terms_equi_en])\n",
        "  gold_set_equi_en=set(df_equi_terms_en[\"Term\"])\n",
        "  true_pos=extracted_terms_equi_en.intersection(gold_set_equi_en)\n",
        "  recall=len(true_pos)/len(gold_set_equi_en)\n",
        "  precision=len(true_pos)/len(extracted_terms_equi_en)\n",
        "  f1=2*(precision*recall)/(precision+recall)\n",
        "\n",
        "  return recall, precision, f1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UYBNMpiszm_h"
      },
      "source": [
        "def extract_terms(validation_df, xlmr_model): \n",
        "  print(len(validation_df))\n",
        "  term_list=[]\n",
        "\n",
        "  # put model in evaluation mode \n",
        "  xlmr_model.eval()\n",
        "\n",
        "  for index, row in validation_df.iterrows():\n",
        "    sentence = row['n_gram']+\". \"+row[\"Context\"]\n",
        "    label=validation_df[\"Label\"]\n",
        "\n",
        "    encoded_dict = xlmr_tokenizer.encode_plus(sentence, \n",
        "                                                  max_length=max_len, \n",
        "                                                  padding='max_length',\n",
        "                                                  truncation=True, \n",
        "                                                  return_tensors='pt') \n",
        "    input_id=encoded_dict['input_ids'].to(device)\n",
        "    attn_mask=encoded_dict['attention_mask'].to(device)\n",
        "    label=torch.tensor(0).to(device)    \n",
        "\n",
        "    with torch.no_grad():                \n",
        "      output = xlmr_model(input_id, \n",
        "                                      token_type_ids=None, \n",
        "                                      attention_mask=attn_mask,\n",
        "                                      labels=label)\n",
        "      loss=output.loss\n",
        "      logits=output.logits\n",
        "      \n",
        "    logits = logits.detach().cpu().numpy()\n",
        "    pred=labels[logits[0].argmax(axis=0)]\n",
        "    if pred==\"Term\":\n",
        "      term_list.append(row['n_gram'])\n",
        "\n",
        "  return set(term_list)\n",
        "      "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zs7cOPFJtLUG"
      },
      "source": [
        "def train_model(epochs, xlmr_model, train_dataloader, validation_dataloader, validation_df, random_seed, verbose, optimizer, scheduler):\n",
        "\n",
        "  seed_val = random_seed\n",
        "\n",
        "  random.seed(seed_val)\n",
        "  np.random.seed(seed_val)\n",
        "  torch.manual_seed(seed_val)\n",
        "  torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "  # mostly contains scores about how the training went for each epoch\n",
        "  training_stats = []\n",
        "\n",
        "  # total training time\n",
        "  total_t0 = time.time()\n",
        "\n",
        "  print('\\033[1m'+\"================ Model Training ================\"+'\\033[0m')\n",
        "\n",
        "  # For each epoch...\n",
        "  for epoch_i in range(0, epochs):\n",
        "\n",
        "      print(\"\")\n",
        "      print('\\033[1m'+'======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs)+'\\033[0m')\n",
        "\n",
        "      t0 = time.time()\n",
        "\n",
        "      # summed training loss of the epoch\n",
        "      total_train_loss = 0\n",
        "\n",
        "\n",
        "      # model is being put into training mode as mechanisms like dropout work differently during train and test time\n",
        "      xlmr_model.train()\n",
        "\n",
        "      # iterrate over batches\n",
        "      for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "          # unpack training batch at load it to gpu (device)  \n",
        "          b_input_ids = batch[0].to(device)\n",
        "          b_input_mask = batch[1].to(device)\n",
        "          b_labels = batch[2].to(device)\n",
        "\n",
        "          # clear gradients before calculating new ones\n",
        "          xlmr_model.zero_grad()        \n",
        "\n",
        "          # forward pass with current batch\n",
        "          output = xlmr_model(b_input_ids, \n",
        "                              token_type_ids=None, \n",
        "                              attention_mask=b_input_mask, \n",
        "                              labels=b_labels)\n",
        "          \n",
        "          loss=output.loss\n",
        "          logits=output.logits\n",
        "\n",
        "          # add up the loss\n",
        "          total_train_loss += loss.item()\n",
        "\n",
        "          # calculate new gradients\n",
        "          loss.backward()\n",
        "\n",
        "          # gradient clipping (not bigger than)\n",
        "          torch.nn.utils.clip_grad_norm_(xlmr_model.parameters(), 1.0)\n",
        "\n",
        "          # Update the networks weights based on the gradient as well as the optimiziers parameters\n",
        "          optimizer.step()\n",
        "\n",
        "          # lr update\n",
        "          scheduler.step()\n",
        "\n",
        "      # avg loss over all batches\n",
        "      avg_train_loss = total_train_loss / len(train_dataloader)            \n",
        "      \n",
        "      # training time of this epoch\n",
        "      training_time = format_time(time.time() - t0)\n",
        "\n",
        "      print(\"\")\n",
        "      print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "      print(\"  Training epoch took: {:}\".format(training_time))\n",
        "          \n",
        "  \n",
        "      # VALIDATION\n",
        "      print(\"evaluate\")\n",
        "      if epoch_i==epochs-1:print_cm=True #Print out cm in final iteration\n",
        "      else: print_cm=False\n",
        "      recall, precision, f1 = validate(validation_dataloader, validation_df, xlmr_model, verbose, print_cm)   \n",
        "       \n",
        "\n",
        "      #print('\\033[1m'+ \"  Validation Loss All: {0:.2f}\".format(avg_val_loss) + '\\033[0m')\n",
        "\n",
        "      training_stats.append(\n",
        "          {\n",
        "              'epoch': epoch_i + 1,\n",
        "              'Training Loss': avg_train_loss,\n",
        "              \"precision\": precision,\n",
        "              \"recall\": recall,\n",
        "              \"f1\": f1,\n",
        "              'Training Time': training_time,\n",
        "          }\n",
        "      )\n",
        "\n",
        "      print(\"Precicion\", precision)\n",
        "      print(\"Recall\", recall)\n",
        "      print(\"F1\", f1)\n",
        "\n",
        "  print(\"\\n\\nTraining complete!\")\n",
        "  print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))\n",
        "  \n",
        "  return training_stats\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V1VTPA1anQ2w"
      },
      "source": [
        "lr=2e-5\n",
        "eps=1e-8\n",
        "epochs=3\n",
        "device = torch.device('cuda')\n",
        "xlmr_model, optimizer, scheduler = create_model(lr=lr, eps=eps, train_dataloader=train_dataloader, epochs=epochs, device=device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "56767talsn4M"
      },
      "source": [
        "training_stats=train_model(epochs=epochs,\n",
        "                           xlmr_model=xlmr_model,\n",
        "                           train_dataloader=train_dataloader,\n",
        "                           validation_dataloader=valid_dataloader,\n",
        "                           validation_df=train_data_htfl_en,\n",
        "                           random_seed=42,\n",
        "                           verbose=True,\n",
        "                           optimizer=optimizer,\n",
        "                           scheduler=scheduler)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0-9PbANQp4Uj"
      },
      "source": [
        "# Test Set Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kEniE8WRdjF3"
      },
      "source": [
        "extracted_terms_htfl_en=extract_terms(train_data_htfl_en, xlmr_model)\n",
        "extracted_terms_htfl_fr=extract_terms(train_data_htfl_fr, xlmr_model)\n",
        "extracted_terms_htfl_nl=extract_terms(train_data_htfl_nl, xlmr_model)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kB_M7qk9xbj5"
      },
      "source": [
        "def computeTermEvalMetrics(extracted_terms, gold_df):\n",
        "  #make lower case cause gold standard is lower case\n",
        "  extracted_terms = set([item.lower() for item in extracted_terms])\n",
        "  gold_set=set(gold_df)\n",
        "  true_pos=extracted_terms.intersection(gold_set)\n",
        "  recall=len(true_pos)/len(gold_set)\n",
        "  precision=len(true_pos)/len(extracted_terms)\n",
        "\n",
        "  print(\"Intersection\",len(true_pos))\n",
        "  print(\"Gold\",len(gold_set))\n",
        "  print(\"Extracted\",len(extracted_terms))\n",
        "  print(\"Recall:\", recall)\n",
        "  print(\"Precision:\", precision)\n",
        "  print(\"F1:\", 2*(precision*recall)/(precision+recall))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cABUXjRY1ZHI"
      },
      "source": [
        "computeTermEvalMetrics(extracted_terms_htfl_en, df_htfl_terms_en[\"Term\"])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z1B0JipczW_7"
      },
      "source": [
        "computeTermEvalMetrics(extracted_terms_htfl_fr, df_htfl_terms_fr[\"Term\"])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KKTVrV0AzXEt"
      },
      "source": [
        "computeTermEvalMetrics(extracted_terms_htfl_nl, df_htfl_terms_nl[\"Term\"])"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}