compute_scores.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "fst = load_dataset(\"it5/datasets\", \"fst\", use_auth_token='(<YOUR_TOKEN>)')\n",
    "hg = load_dataset(\"it5/datasets\", \"hg\", use_auth_token='(<YOUR_TOKEN>)')\n",
    "ns = load_dataset(\"it5/datasets\", \"ns\", use_auth_token='(<YOUR_TOKEN>)')\n",
    "qa = load_dataset(\"it5/datasets\", \"qa\", use_auth_token='(<YOUR_TOKEN>)')\n",
    "qg = load_dataset(\"it5/datasets\", \"qg\", use_auth_token='(<YOUR_TOKEN>)')\n",
    "st_g2r = load_dataset(\"it5/datasets\", \"st_g2r\", use_auth_token='(<YOUR_TOKEN>)')\n",
    "st_r2g = load_dataset(\"it5/datasets\", \"st_r2g\", use_auth_token='(<YOUR_TOKEN>)')\n",
    "wits = load_dataset(\"it5/datasets\", \"wits\", use_auth_token='(<YOUR_TOKEN>)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"references/informal.txt\", 'w') as f:\n",
    "    for s in fst[\"test_0\"][\"informal\"]:\n",
    "        f.write(s.lower() + '\\n')\n",
    "\n",
    "for i in range(4):\n",
    "    with open(f\"references/formal{i}.txt\", 'w') as f:\n",
    "        for s in fst[f\"test_{i}\"][\"formal\"]:\n",
    "            f.write(s.lower() + '\\n')\n",
    "\n",
    "with open(f\"references/hg.txt\", 'w') as f:\n",
    "    for s in hg[f\"test\"][\"target\"]:\n",
    "        f.write(s.lower() + '\\n')\n",
    "\n",
    "for name in [\"fanpage\", \"ilpost\"]:\n",
    "    with open(f\"references/{name}.txt\", 'w') as f:\n",
    "        for s in ns[f\"test_{name}\"][\"target\"]:\n",
    "            f.write(s.replace(\"\\n\", \"\").lower() + '\\n')\n",
    "\n",
    "with open(f\"references/qa.txt\", 'w') as f:\n",
    "    for s in qa[f\"test\"][\"target\"]:\n",
    "        f.write(s.lower() + '\\n')\n",
    "\n",
    "with open(f\"references/qg.txt\", 'w') as f:\n",
    "    for s in qg[f\"test\"][\"target\"]:\n",
    "        f.write(s.lower() + '\\n')\n",
    "\n",
    "with open(f\"references/st_g2r.txt\", 'w') as f:\n",
    "    for s in st_g2r[f\"test\"][\"headline\"]:\n",
    "        f.write(s.lower() + '\\n')\n",
    "\n",
    "with open(f\"references/st_r2g.txt\", 'w') as f:\n",
    "    for s in st_r2g[f\"test\"][\"headline\"]:\n",
    "        f.write(s.lower() + '\\n')\n",
    "\n",
    "with open(f\"references/wits.txt\", 'w') as f:\n",
    "    for s in wits[f\"test\"][\"summary\"]:\n",
    "        f.write(s.lower() + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-02-18 14:54:29.237870: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2022-02-18 14:54:29.237892: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_metric\n",
    "\n",
    "rouge = load_metric(\"rouge\")\n",
    "bertscore = load_metric(\"bertscore\")\n",
    "\n",
    "rouge_kwargs = {\n",
    "    'rouge_types': [\"rouge1\", \"rouge2\", \"rougeL\"],\n",
    "    'use_agregator': False,\n",
    "    'use_stemmer': False,\n",
    "}\n",
    "bertscore_kwargs = {\n",
    "    \"model_type\": \"dbmdz/bert-base-italian-xxl-uncased\",\n",
    "    \"lang\": \"it\",\n",
    "    \"num_layers\": 10,\n",
    "    \"batch_size\": 16,\n",
    "    \"rescale_with_baseline\": True,\n",
    "    \"baseline_path\": \"configs/bertscore_baseline_ita.tsv\",\n",
    "    \"use_fast_tokenizer\": True\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_i2f(model, metric, kwargs):\n",
    "    with open(\"references/formal0.txt\", 'r') as f:\n",
    "        ref0 = [s.strip() for s in f.readlines()]\n",
    "    with open(\"references/formal1.txt\", 'r') as f:\n",
    "        ref1 = [s.strip() for s in f.readlines()]\n",
    "    with open(\"references/formal2.txt\", 'r') as f:\n",
    "        ref2 = [s.strip() for s in f.readlines()]\n",
    "    with open(\"references/formal3.txt\", 'r') as f:\n",
    "        ref3 = [s.strip() for s in f.readlines()]\n",
    "    with open(f\"model_predictions/{model}-informal-to-formal{'-ita' if 'mt5' in model else ''}_test_0.txt\", 'r') as f:\n",
    "        hyp = [s.strip() for s in f.readlines()]\n",
    "    scores0 = {k:[s.fmeasure for s in v] for k, v in metric.compute(predictions=hyp, references=ref0, **kwargs).items()}\n",
    "    scores1 = {k:[s.fmeasure for s in v] for k, v in metric.compute(predictions=hyp, references=ref1, **kwargs).items()}\n",
    "    scores2 = {k:[s.fmeasure for s in v] for k, v in metric.compute(predictions=hyp, references=ref2, **kwargs).items()}\n",
    "    scores3 = {k:[s.fmeasure for s in v] for k, v in metric.compute(predictions=hyp, references=ref3, **kwargs).items()}\n",
    "    return {\n",
    "        s:sum([max(x0, x1, x2, x3) for x0, x1, x2, x3 in zip(scores0[s], scores1[s], scores2[s], scores3[s])])/len(scores0[s])\n",
    "        for s in scores0.keys()\n",
    "    }\n",
    "\n",
    "print(\"fst_i2f\")\n",
    "print(\"mt5-small\", compute_i2f(\"mt5-small\", rouge, rouge_kwargs))\n",
    "print(\"mt5-base\", compute_i2f(\"mt5-base\", rouge, rouge_kwargs))\n",
    "print(\"it5-small\", compute_i2f(\"it5-small\", rouge, rouge_kwargs))\n",
    "print(\"it5-base\", compute_i2f(\"it5-base\", rouge, rouge_kwargs))\n",
    "print(\"it5-large\", compute_i2f(\"it5-large\", rouge, rouge_kwargs))\n",
    "print(\"it5-efficient-small-el32\", compute_i2f(\"it5-efficient-small-el32\", rouge, rouge_kwargs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```shell\n",
    "fst_i2f\n",
    "mt5-small {'rouge1': 0.6380633185849459, 'rouge2': 0.4458092701802957, 'rougeL': 0.6202414964282672}\n",
    "mt5-base {'rouge1': 0.6618245718094221, 'rouge2': 0.4710014012177415, 'rougeL': 0.6419942941682181}\n",
    "it5-small {'rouge1': 0.6468545736780796, 'rouge2': 0.4514449451743173, 'rougeL': 0.6280476143942586}\n",
    "it5-base {'rouge1': 0.5826060015283062, 'rouge2': 0.40296946773710746, 'rougeL': 0.561013620053419}\n",
    "it5-large {'rouge1': 0.6627748711489103, 'rouge2': 0.47466091339493427, 'rougeL': 0.6446160596543035}\n",
    "it5-efficient-small-el32 {'rouge1': 0.4303607453244526, 'rouge2': 0.22149827051593776, 'rougeL': 0.4080934363306528}\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_f2i(model, metric, kwargs):\n",
    "    with open(\"references/informal.txt\", 'r') as f:\n",
    "        ref = [s.strip() for s in f.readlines()]\n",
    "    with open(f\"model_predictions/{model}-formal-to-informal{'-ita' if 'mt5' in model else ''}_test_0.txt\", 'r') as f:\n",
    "        hyp0 = [s.strip() for s in f.readlines()]\n",
    "    with open(f\"model_predictions/{model}-formal-to-informal{'-ita' if 'mt5' in model else ''}_test_1.txt\", 'r') as f:\n",
    "        hyp1 = [s.strip() for s in f.readlines()]\n",
    "    with open(f\"model_predictions/{model}-formal-to-informal{'-ita' if 'mt5' in model else ''}_test_2.txt\", 'r') as f:\n",
    "        hyp2 = [s.strip() for s in f.readlines()]\n",
    "    with open(f\"model_predictions/{model}-formal-to-informal{'-ita' if 'mt5' in model else ''}_test_3.txt\", 'r') as f:\n",
    "        hyp3 = [s.strip() for s in f.readlines()]\n",
    "    scores0 = {k:[s.fmeasure for s in v] for k, v in metric.compute(predictions=hyp0, references=ref, **kwargs).items()}\n",
    "    scores1 = {k:[s.fmeasure for s in v] for k, v in metric.compute(predictions=hyp1, references=ref, **kwargs).items()}\n",
    "    scores2 = {k:[s.fmeasure for s in v] for k, v in metric.compute(predictions=hyp2, references=ref, **kwargs).items()}\n",
    "    scores3 = {k:[s.fmeasure for s in v] for k, v in metric.compute(predictions=hyp3, references=ref, **kwargs).items()}\n",
    "    return {\n",
    "        s:sum([max(x0, x1, x2, x3) for x0, x1, x2, x3 in zip(scores0[s], scores1[s], scores2[s], scores3[s])])/len(scores0[s])\n",
    "        for s in scores0.keys()\n",
    "    }\n",
    "\n",
    "print(\"fst_f2i\")\n",
    "print(\"mt5-small\", compute_f2i(\"mt5-small\", rouge, rouge_kwargs))\n",
    "print(\"mt5-base\", compute_f2i(\"mt5-base\", rouge, rouge_kwargs))\n",
    "print(\"it5-small\", compute_f2i(\"it5-small\", rouge, rouge_kwargs))\n",
    "print(\"it5-base\", compute_f2i(\"it5-base\", rouge, rouge_kwargs))\n",
    "print(\"it5-large\", compute_f2i(\"it5-large\", rouge, rouge_kwargs))\n",
    "print(\"it5-efficient-small-el32\", compute_f2i(\"it5-efficient-small-el32\", rouge, rouge_kwargs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```shell\n",
    "fst_f2i\n",
    "mt5-small {'rouge1': 0.6507888177602231, 'rouge2': 0.44986764030237103, 'rougeL': 0.6312564448356363}\n",
    "mt5-base {'rouge1': 0.6526409092033428, 'rouge2': 0.44919264158684963, 'rougeL': 0.6316971160855954}\n",
    "it5-small {'rouge1': 0.6496212216212729, 'rouge2': 0.4495710077185604, 'rougeL': 0.6314902152941851}\n",
    "it5-base {'rouge1': 0.6520272511796381, 'rouge2': 0.44582644850047853, 'rougeL': 0.6315355975325985}\n",
    "it5-large {'rouge1': 0.6107063934564821, 'rouge2': 0.40917491561108527, 'rougeL': 0.5864980585629622}\n",
    "it5-efficient-small-el32 {'rouge1': 0.4594815280791639, 'rouge2': 0.24431311278933063, 'rougeL': 0.4346579925274669}\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rouge_kwargs = {\n",
    "    'rouge_types': [\"rouge1\", \"rouge2\", \"rougeL\"],\n",
    "}\n",
    "\n",
    "def compute_rouge(ref_file, hyp_file, kwargs):\n",
    "    with open(f\"references/{ref_file}.txt\", 'r') as f:\n",
    "        ref = [s.strip() for s in f.readlines()]\n",
    "    with open(f\"model_predictions/{hyp_file}.txt\", 'r') as f:\n",
    "        hyp = [s.strip() for s in f.readlines()]\n",
    "    return {k:v.mid.fmeasure for k,v in rouge.compute(predictions=hyp, references=ref, **kwargs).items()}\n",
    "\n",
    "print(\"hg\")\n",
    "print(\"mt5-small\", compute_rouge(\"hg\", \"mt5-small-headline-generation-ita_test\", rouge_kwargs))\n",
    "print(\"mt5-base\", compute_rouge(\"hg\", \"mt5-base-headline-generation-ita_test\", rouge_kwargs))\n",
    "print(\"it5-base\", compute_rouge(\"hg\", \"it5-base-headline-generation_test\", rouge_kwargs))\n",
    "print(\"it5-small\", compute_rouge(\"hg\", \"it5-small-headline-generation_test\", rouge_kwargs))\n",
    "print(\"it5-large\", compute_rouge(\"hg\", \"it5-large-headline-generation_test\", rouge_kwargs))\n",
    "\n",
    "print(\"qg\")\n",
    "print(\"mt5-small\", compute_rouge(\"qg\", \"mt5-small-question-generation-ita_test\", rouge_kwargs))\n",
    "print(\"mt5-base\", compute_rouge(\"qg\", \"mt5-base-question-generation-ita_test\", rouge_kwargs))\n",
    "print(\"it5-base\", compute_rouge(\"qg\", \"it5-base-question-generation_test\", rouge_kwargs))\n",
    "print(\"it5-small\", compute_rouge(\"qg\", \"it5-small-question-generation_test\", rouge_kwargs))\n",
    "print(\"it5-large\", compute_rouge(\"qg\", \"it5-large-question-generation_test\", rouge_kwargs))\n",
    "\n",
    "print(\"ns_ilpost\")\n",
    "print(\"mt5-small\", compute_rouge(\"ilpost\", \"mt5-small-news-summarization-ita_test_ilpost\", rouge_kwargs))\n",
    "print(\"mt5-base\", compute_rouge(\"ilpost\", \"mt5-base-news-summarization-ita_test_ilpost\", rouge_kwargs))\n",
    "print(\"it5-base\", compute_rouge(\"ilpost\", \"it5-base-news-summarization_test_ilpost\", rouge_kwargs))\n",
    "print(\"it5-small\", compute_rouge(\"ilpost\", \"it5-small-news-summarization_test_ilpost\", rouge_kwargs))\n",
    "print(\"it5-large\", compute_rouge(\"ilpost\", \"it5-large-news-summarization_test_ilpost\", rouge_kwargs))\n",
    "\n",
    "print(\"ns_fanpage\")\n",
    "print(\"mt5-small\", compute_rouge(\"fanpage\", \"mt5-small-news-summarization-ita_test_fanpage\", rouge_kwargs))\n",
    "print(\"mt5-base\", compute_rouge(\"fanpage\", \"mt5-base-news-summarization-ita_test_fanpage\", rouge_kwargs))\n",
    "print(\"it5-base\", compute_rouge(\"fanpage\", \"it5-base-news-summarization_test_fanpage\", rouge_kwargs))\n",
    "print(\"it5-small\", compute_rouge(\"fanpage\", \"it5-small-news-summarization_test_fanpage\", rouge_kwargs))\n",
    "print(\"it5-large\", compute_rouge(\"fanpage\", \"it5-large-news-summarization_test_fanpage\", rouge_kwargs))\n",
    "\n",
    "print(\"wits\")\n",
    "print(\"mt5-small\", compute_rouge(\"wits\", \"mt5-small-wiki-summarization-ita_test\", rouge_kwargs))\n",
    "print(\"mt5-base\", compute_rouge(\"wits\", \"mt5-base-wiki-summarization-ita_test\", rouge_kwargs))\n",
    "print(\"it5-base\", compute_rouge(\"wits\", \"it5-base-wiki-summarization_test\", rouge_kwargs))\n",
    "print(\"it5-small\", compute_rouge(\"wits\", \"it5-small-wiki-summarization_test\", rouge_kwargs))\n",
    "print(\"it5-large\", compute_rouge(\"wits\", \"it5-large-wiki-summarization_test\", rouge_kwargs))\n",
    "\n",
    "print(\"st_g2r\")\n",
    "print(\"mt5-small\", compute_rouge(\"st_g2r\", \"mt5-small-ilgiornale-to-repubblica-ita_test\", rouge_kwargs))\n",
    "print(\"mt5-base\", compute_rouge(\"st_g2r\", \"mt5-base-ilgiornale-to-repubblica-ita_test\", rouge_kwargs))\n",
    "print(\"it5-base\", compute_rouge(\"st_g2r\", \"it5-base-ilgiornale-to-repubblica_test\", rouge_kwargs))\n",
    "print(\"it5-small\", compute_rouge(\"st_g2r\", \"it5-small-ilgiornale-to-repubblica_test\", rouge_kwargs))\n",
    "print(\"it5-large\", compute_rouge(\"st_g2r\", \"it5-large-ilgiornale-to-repubblica_test\", rouge_kwargs))\n",
    "\n",
    "print(\"st_r2g\")\n",
    "print(\"mt5-small\", compute_rouge(\"st_r2g\", \"mt5-small-repubblica-to-ilgiornale-ita_test\", rouge_kwargs))\n",
    "print(\"mt5-base\", compute_rouge(\"st_r2g\", \"mt5-base-repubblica-to-ilgiornale-ita_test\", rouge_kwargs))\n",
    "print(\"it5-base\", compute_rouge(\"st_r2g\", \"it5-base-repubblica-to-ilgiornale_test\", rouge_kwargs))\n",
    "print(\"it5-small\", compute_rouge(\"st_r2g\", \"it5-small-repubblica-to-ilgiornale_test\", rouge_kwargs))\n",
    "print(\"it5-large\", compute_rouge(\"st_r2g\", \"it5-large-repubblica-to-ilgiornale_test\", rouge_kwargs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```shell\n",
    "hg\n",
    "it5-efficient-small-el32 {'rouge1': 0.29889591003219945, 'rouge2': 0.10798865461180047, 'rougeL': 0.26373469404680516}\n",
    "qg\n",
    "it5-efficient-small-el32 {'rouge1': 0.38164340232821636, 'rouge2': 0.20196402932507035, 'rougeL': 0.3567517057878152}\n",
    "ns_ilpost\n",
    "it5-efficient-small-el32 {'rouge1': 0.3603759169791969, 'rouge2': 0.1830199980738273, 'rougeL': 0.2976065389133221}\n",
    "ns_fanpage\n",
    "it5-efficient-small-el32 {'rouge1': 0.34656690725891093, 'rouge2': 0.1623011537990659, 'rougeL': 0.25938109914379837}\n",
    "wits\n",
    "it5-efficient-small-el32 {'rouge1': 0.34603869832030937, 'rouge2': 0.19613477406967467, 'rougeL': 0.3139591766306542}\n",
    "st_g2r\n",
    "it5-efficient-small-el32 {'rouge1': 0.2863701051169089, 'rouge2': 0.09911822484913996, 'rougeL': 0.25276371176133805}\n",
    "st_r2g\n",
    "it5-efficient-small-el32 {'rouge1': 0.26925410997857024, 'rouge2': 0.08722601921218263, 'rougeL': 0.23505332088093583}\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bertscore\n",
    "\n",
    "The following results were obtaining by running the following code on Google Colab:\n",
    "\n",
    "### i2f\n",
    "\n",
    "- mt5-small 0.6840756445527076\n",
    "\n",
    "- mt5-base 0.7121593571454287\n",
    "\n",
    "- it5-small 0.7023371724784374\n",
    "\n",
    "- it5-base 0.6409667591750622\n",
    "\n",
    "- it5-large 0.7144228211343289\n",
    "\n",
    "- it5-efficient-small-el32 0.6303993126545101\n",
    "\n",
    "### f2i\n",
    "\n",
    "- mt5-small 0.6656641669273377\n",
    "\n",
    "- mt5-base 0.667281984001398\n",
    "\n",
    "- it5-small 0.6633332076966763\n",
    "\n",
    "- it5-base 0.6654541707187891\n",
    "\n",
    "- it5-large 0.6129066628664732\n",
    "\n",
    "- it5-efficient-small-el32 0.738897417768836\n",
    "\n",
    "### hg\n",
    "\n",
    "- mt5-small 0.4082620157602709\n",
    "\n",
    "- mt5-base 0.427382238739694\n",
    "\n",
    "- it5-base 0.4326194681830937\n",
    "\n",
    "- it5-small 0.413982511658594\n",
    "\n",
    "- it5-large 0.4299530544915586\n",
    "\n",
    "- it5-efficient-small-el32 0.4269566555540543\n",
    "\n",
    "### qg\n",
    "\n",
    "- mt5-small 0.4625033930285283\n",
    "\n",
    "- mt5-base 0.49497215210106765\n",
    "\n",
    "- it5-base 0.5157709655244594\n",
    "\n",
    "- it5-small 0.5052396924401265\n",
    "\n",
    "- it5-large 0.5221246630159012\n",
    "\n",
    "- it5-efficient-small-el32 0.5169696821015387\n",
    "\n",
    "\n",
    "### ns_ilpost\n",
    "\n",
    "- mt5-small 0.3797239664886993\n",
    "\n",
    "- mt5-base 0.3990554356227883\n",
    "\n",
    "- it5-base 0.06487068192530966\n",
    "\n",
    "- it5-small 0.3949358254239302\n",
    "\n",
    "- it5-large 0.31312756502520644\n",
    "\n",
    "- it5-efficient-small-el32 0.4206761705460535\n",
    "\n",
    "### ns_fanpage\n",
    "\n",
    "- mt5-small 0.36950530688046235\n",
    "\n",
    "- mt5-base 0.3867714759344327\n",
    "\n",
    "- it5-base 0.024452764880952244\n",
    "\n",
    "- it5-small 0.3773716842996095\n",
    "\n",
    "- it5-large 0.3163143912487514\n",
    "\n",
    "- it5-efficient-small-el32 0.3991745289850481\n",
    "\n",
    "### wits\n",
    "\n",
    "- mt5-small 0.5171176899618644\n",
    "\n",
    "- mt5-base 0.5195369849113791\n",
    "\n",
    "- it5-base 0.5302166683937306\n",
    "\n",
    "- it5-small 0.5037536359048216\n",
    "\n",
    "- it5-large 0.5078669870631595\n",
    "\n",
    "- it5-efficient-small-el32 0.513421691011038\n",
    "\n",
    "### st_g2r\n",
    "\n",
    "- mt5-small 0.3818697871814689\n",
    "\n",
    "- mt5-base 0.41139199032320173\n",
    "\n",
    "- it5-base 0.42534179535519795\n",
    "\n",
    "- it5-small 0.4044020188880153\n",
    "\n",
    "- it5-large 0.40020734380937645\n",
    "\n",
    "- it5-efficient-small-el32 0.4219772753238678\n",
    "\n",
    "### st_r2g\n",
    "\n",
    "- mt5-small 0.3629064853049815\n",
    "\n",
    "- mt5-base 0.38393800050299615\n",
    "\n",
    "- it5-base 0.3962114055787213\n",
    "\n",
    "- it5-small 0.3801775599762797\n",
    "\n",
    "- it5-efficient-small-el32 0.3949996523973532"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_i2f(model, metric, kwargs):\n",
    "    with open(\"refs/formal0.txt\", 'r') as f:\n",
    "        ref0 = [s.strip() for s in f.readlines()]\n",
    "    with open(\"references/formal1.txt\", 'r') as f:\n",
    "        ref1 = [s.strip() for s in f.readlines()]\n",
    "    with open(\"references/formal2.txt\", 'r') as f:\n",
    "        ref2 = [s.strip() for s in f.readlines()]\n",
    "    with open(\"references/formal3.txt\", 'r') as f:\n",
    "        ref3 = [s.strip() for s in f.readlines()]\n",
    "    with open(f\"model_predictions/{model}-informal-to-formal{'-ita' if 'mt5' in model else ''}_test_0.txt\", 'r') as f:\n",
    "        hyp = [s.strip() for s in f.readlines()]\n",
    "    \n",
    "    scores0 = metric.compute(predictions=hyp, references=ref0, **kwargs)[\"f1\"]\n",
    "    scores1 = metric.compute(predictions=hyp, references=ref1, **kwargs)[\"f1\"]\n",
    "    scores2 = metric.compute(predictions=hyp, references=ref2, **kwargs)[\"f1\"]\n",
    "    scores3 = metric.compute(predictions=hyp, references=ref3, **kwargs)[\"f1\"]\n",
    "    return sum([max(x0, x1, x2, x3) for x0, x1, x2, x3 in zip(scores0, scores1, scores2, scores3)])/len(scores0)\n",
    "\n",
    "print(\"bertscore i2f\")\n",
    "print(\"mt5-small\", compute_i2f(\"mt5-small\", bertscore, bertscore_kwargs))\n",
    "print(\"mt5-base\", compute_i2f(\"mt5-base\", bertscore, bertscore_kwargs))\n",
    "print(\"it5-small\", compute_i2f(\"it5-small\", bertscore, bertscore_kwargs))\n",
    "print(\"it5-base\", compute_i2f(\"it5-base\", bertscore, bertscore_kwargs))\n",
    "print(\"it5-large\", compute_i2f(\"it5-large\", bertscore, bertscore_kwargs))\n",
    "print(\"it5-efficient-small-el32\", compute_i2f(\"it5-efficient-small-el32\", bertscore, bertscore_kwargs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_f2i(model, metric, kwargs):\n",
    "    with open(\"references/informal.txt\", 'r') as f:\n",
    "        ref = [s.strip() for s in f.readlines()]\n",
    "    with open(f\"model_predictions/{model}-formal-to-informal{'-ita' if 'mt5' in model else ''}_test_0.txt\", 'r') as f:\n",
    "        hyp0 = [s.strip() for s in f.readlines()]\n",
    "    with open(f\"model_predictions/{model}-formal-to-informal{'-ita' if 'mt5' in model else ''}_test_1.txt\", 'r') as f:\n",
    "        hyp1 = [s.strip() for s in f.readlines()]\n",
    "    with open(f\"model_predictions/{model}-formal-to-informal{'-ita' if 'mt5' in model else ''}_test_2.txt\", 'r') as f:\n",
    "        hyp2 = [s.strip() for s in f.readlines()]\n",
    "    with open(f\"model_predictions/{model}-formal-to-informal{'-ita' if 'mt5' in model else ''}_test_3.txt\", 'r') as f:\n",
    "        hyp3 = [s.strip() for s in f.readlines()]\n",
    "    scores0 = metric.compute(predictions=hyp0, references=ref, **kwargs)[\"f1\"]\n",
    "    scores1 = metric.compute(predictions=hyp1, references=ref, **kwargs)[\"f1\"]\n",
    "    scores2 = metric.compute(predictions=hyp2, references=ref, **kwargs)[\"f1\"]\n",
    "    scores3 = metric.compute(predictions=hyp3, references=ref, **kwargs)[\"f1\"]\n",
    "    return sum([max(x0, x1, x2, x3) for x0, x1, x2, x3 in zip(scores0, scores1, scores2, scores3)])/len(scores0)\n",
    "\n",
    "print(\"bertscore f2i\")\n",
    "print(\"mt5-small\", compute_f2i(\"mt5-small\", bertscore, bertscore_kwargs))\n",
    "print(\"mt5-base\", compute_f2i(\"mt5-base\", bertscore, bertscore_kwargs))\n",
    "print(\"it5-small\", compute_f2i(\"it5-small\", bertscore, bertscore_kwargs))\n",
    "print(\"it5-base\", compute_f2i(\"it5-base\", bertscore, bertscore_kwargs))\n",
    "print(\"it5-large\", compute_f2i(\"it5-large\", bertscore, bertscore_kwargs))\n",
    "print(\"it5-efficient-small-el32\", compute_f2i(\"it5-efficient-small-el32\", bertscore, bertscore_kwargs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_bertscore(ref_file, hyp_file, kwargs):\n",
    "    with open(f\"references/{ref_file}.txt\", 'r') as f:\n",
    "        ref = [s.strip() for s in f.readlines()]\n",
    "    with open(f\"model_predictions/{hyp_file}.txt\", 'r') as f:\n",
    "        hyp = [s.strip() for s in f.readlines()]\n",
    "    scores = bertscore.compute(predictions=hyp, references=ref, **kwargs)[\"f1\"]\n",
    "    return sum(scores) / len(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"hg\")\n",
    "print(\"mt5-small\", compute_bertscore(\"hg\", \"mt5-small-headline-generation-ita_test\", bertscore_kwargs))\n",
    "print(\"mt5-base\", compute_bertscore(\"hg\", \"mt5-base-headline-generation-ita_test\", bertscore_kwargs))\n",
    "print(\"it5-base\", compute_bertscore(\"hg\", \"it5-base-headline-generation_test\", bertscore_kwargs))\n",
    "print(\"it5-small\", compute_bertscore(\"hg\", \"it5-small-headline-generation_test\", bertscore_kwargs))\n",
    "print(\"it5-large\", compute_bertscore(\"hg\", \"it5-large-headline-generation_test\", bertscore_kwargs))\n",
    "print(\"it5-efficient-small-el32\", compute_bertscore(\"hg\", \"it5-efficient-small-el32-headline-generation_test\", bertscore_kwargs))\n",
    "\n",
    "print(\"qg\")\n",
    "print(\"mt5-small\", compute_bertscore(\"qg\", \"mt5-small-question-generation-ita_test\", bertscore_kwargs))\n",
    "print(\"mt5-base\", compute_bertscore(\"qg\", \"mt5-base-question-generation-ita_test\", bertscore_kwargs))\n",
    "print(\"it5-base\", compute_bertscore(\"qg\", \"it5-base-question-generation_test\", bertscore_kwargs))\n",
    "print(\"it5-small\", compute_bertscore(\"qg\", \"it5-small-question-generation_test\", bertscore_kwargs))\n",
    "print(\"it5-large\", compute_bertscore(\"qg\", \"it5-large-question-generation_test\", bertscore_kwargs))\n",
    "print(\"it5-efficient-small-el32\", compute_bertscore(\"qg\", \"it5-efficient-small-el32-question-generation_test\", bertscore_kwargs))\n",
    "\n",
    "print(\"ns_ilpost\")\n",
    "print(\"mt5-small\", compute_bertscore(\"ilpost\", \"mt5-small-news-summarization-ita_test_ilpost\", bertscore_kwargs))\n",
    "print(\"mt5-base\", compute_bertscore(\"ilpost\", \"mt5-base-news-summarization-ita_test_ilpost\", bertscore_kwargs))\n",
    "print(\"it5-base\", compute_bertscore(\"ilpost\", \"it5-base-news-summarization_test_ilpost\", bertscore_kwargs))\n",
    "print(\"it5-small\", compute_bertscore(\"ilpost\", \"it5-small-news-summarization_test_ilpost\", bertscore_kwargs))\n",
    "print(\"it5-large\", compute_bertscore(\"ilpost\", \"it5-large-news-summarization_test_ilpost\", bertscore_kwargs))\n",
    "print(\"it5-efficient-small-el32\", compute_bertscore(\"ilpost\", \"it5-efficient-small-el32-news-summarization_test_ilpost\", bertscore_kwargs))\n",
    "\n",
    "print(\"ns_fanpage\")\n",
    "print(\"mt5-small\", compute_bertscore(\"fanpage\", \"mt5-small-news-summarization-ita_test_fanpage\", bertscore_kwargs))\n",
    "print(\"mt5-base\", compute_bertscore(\"fanpage\", \"mt5-base-news-summarization-ita_test_fanpage\", bertscore_kwargs))\n",
    "print(\"it5-base\", compute_bertscore(\"fanpage\", \"it5-base-news-summarization_test_fanpage\", bertscore_kwargs))\n",
    "print(\"it5-small\", compute_bertscore(\"fanpage\", \"it5-small-news-summarization_test_fanpage\", bertscore_kwargs))\n",
    "print(\"it5-large\", compute_bertscore(\"fanpage\", \"it5-large-news-summarization_test_fanpage\", bertscore_kwargs))\n",
    "print(\"it5-efficient-small-el32\", compute_bertscore(\"fanpage\", \"it5-efficient-small-el32-news-summarization_test_fanpage\", bertscore_kwargs))\n",
    "\n",
    "print(\"wits\")\n",
    "print(\"mt5-small\", compute_bertscore(\"wits\", \"mt5-small-wiki-summarization-ita_test\", bertscore_kwargs))\n",
    "print(\"mt5-base\", compute_bertscore(\"wits\", \"mt5-base-wiki-summarization-ita_test\", bertscore_kwargs))\n",
    "print(\"it5-base\", compute_bertscore(\"wits\", \"it5-base-wiki-summarization_test\", bertscore_kwargs))\n",
    "print(\"it5-small\", compute_bertscore(\"wits\", \"it5-small-wiki-summarization_test\", bertscore_kwargs))\n",
    "print(\"it5-large\", compute_bertscore(\"wits\", \"it5-large-wiki-summarization_test\", bertscore_kwargs))\n",
    "print(\"it5-efficient-small-el32\", compute_bertscore(\"wits\", \"it5-efficient-small-el32-wiki-summarization_test\", bertscore_kwargs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CHANGE-it Classifiers Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install transformers==3.0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget -O h-a-classifier.h5 https://unipiit-my.sharepoint.com/:u:/g/personal/l_demattei_studenti_unipi_it/Ee6SwfKkLzFJk-_1ho3LXFUB0fZlADKGdod42qM0dmvGZA?download=1\n",
    "!wget -O h-h-classifier.h5 https://unipiit-my.sharepoint.com/:u:/g/personal/l_demattei_studenti_unipi_it/EZhDwNJSfz1No5WT9t5hOjMB7zrXdJBBu2dGEZCxElIhqA?download=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import transformers\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from transformers import AutoModel, AutoTokenizer\n",
    "\n",
    "# Defining some key variables that will be used later on in the training\n",
    "HH_MAX_LEN = 64\n",
    "HH_VALID_BATCH_SIZE = 128\n",
    "HA_MAX_LEN = 512\n",
    "HA_VALID_BATCH_SIZE = 8\n",
    "HH_MODEL_PATH = \"/content/h-h-classifier.h5\"\n",
    "HA_MODEL_PATH = \"/content/h-a-classifier.h5\"\n",
    "tokenizer = AutoTokenizer.from_pretrained('bert-base-multilingual-cased')\n",
    "\n",
    "class BERTClass(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BERTClass, self).__init__()\n",
    "        self.l1 = AutoModel.from_pretrained('bert-base-multilingual-cased')\n",
    "        self.l2 = torch.nn.Dropout(0.3)\n",
    "        self.l3 = torch.nn.Linear(768, 1)\n",
    "    \n",
    "    def forward(self, ids, mask):\n",
    "        output_1= self.l1(ids, mask)\n",
    "        output_2 = self.l2(output_1[0])\n",
    "        output = self.l3(output_2)\n",
    "        return output\n",
    "\n",
    "class Triage(Dataset):\n",
    "    def __init__(self, dataframe, tokenizer, max_len, field):\n",
    "        self.len = len(dataframe)\n",
    "        self.data = dataframe\n",
    "        self.tokenizer = tokenizer\n",
    "        self.max_len = max_len\n",
    "        self.field = field\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        if self.field == 'fulltext':\n",
    "            orginal = str(self.data.original_fulltext[index])\n",
    "        else:\n",
    "            orginal = str(self.data.original_headlines[index])\n",
    "        orginal = \" \".join(orginal.split())\n",
    "        \n",
    "        \n",
    "        headline_generated = str(self.data.generated_headlines[index])\n",
    "        headline_generated = \" \".join(headline_generated.split())\n",
    "        \n",
    "        inputs = self.tokenizer.encode_plus(\n",
    "            orginal,\n",
    "            headline_generated,\n",
    "            add_special_tokens=True,\n",
    "            max_length=self.max_len,\n",
    "            pad_to_max_length=True,\n",
    "            return_token_type_ids=True,\n",
    "            truncation=True\n",
    "        )\n",
    "        ids = inputs['input_ids']\n",
    "        mask = inputs['attention_mask']\n",
    "\n",
    "        return {\n",
    "            'ids': torch.tensor(ids, dtype=torch.long),\n",
    "            'mask': torch.tensor(mask, dtype=torch.long),\n",
    "        } \n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.len\n",
    "\n",
    "\n",
    "def valid(model, testing_loader):\n",
    "    model.eval()\n",
    "    n_correct = 0; n_wrong = 0; total = 0\n",
    "    y_target = []\n",
    "    y_predicted = []\n",
    "    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n",
    "    with torch.no_grad():\n",
    "        for _, data in enumerate(testing_loader, 0):\n",
    "            ids = data['ids'].to(device, dtype = torch.long)\n",
    "            mask = data['mask'].to(device, dtype = torch.long)\n",
    "            outputs = model(ids, mask).squeeze()\n",
    "            big_val, big_idx = torch.max(outputs.data, dim=1)\n",
    "            total += outputs.size(0)\n",
    "            n_correct += big_idx.sum().item()\n",
    "            \n",
    "    return ((total-n_correct)*100.0)/total\n",
    "\n",
    "\n",
    "\n",
    "def compute_hh_ha(dataset):\n",
    "    # Creating the dataset and dataloader for the neural networkkenizer, MAX_LEN)\n",
    "    hh_testing_set = Triage(dataset.reset_index(drop=True), tokenizer, HH_MAX_LEN, \"headline\")\n",
    "    hh_test_params = {\n",
    "        'batch_size': HH_VALID_BATCH_SIZE,\n",
    "        'shuffle': True,\n",
    "        'num_workers': 0\n",
    "    }\n",
    "    hh_testing_loader = DataLoader(hh_testing_set, **hh_test_params)\n",
    "    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n",
    "    model = BERTClass()\n",
    "    model.to(device)\n",
    "    model.load_state_dict(torch.load(HH_MODEL_PATH, map_location=device))\n",
    "    hh_accuracy = valid(model, hh_testing_loader)\n",
    "    ha_testing_set = Triage(dataset.reset_index(drop=True), tokenizer, HH_MAX_LEN, \"fulltext\")\n",
    "    ha_test_params = {\n",
    "        'batch_size': HH_VALID_BATCH_SIZE,\n",
    "        'shuffle': True,\n",
    "        'num_workers': 0\n",
    "    }\n",
    "    ha_testing_loader = DataLoader(ha_testing_set, **ha_test_params)\n",
    "    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n",
    "    model = BERTClass()\n",
    "    model.to(device)    \n",
    "    model.load_state_dict(torch.load(HH_MODEL_PATH, map_location=device))\n",
    "    ha_accuracy = valid(model, ha_testing_loader)\n",
    "    print(f\"Headlines-Headlines match accuracy {hh_accuracy}\")\n",
    "    print(f\"Headlines-Article match accuracy {ha_accuracy}\")\n",
    "\n",
    "\n",
    "def prepare_g2r(model):\n",
    "    dataset = pd.DataFrame(columns=['original_headlines', 'original_fulltext', 'generated_headlines'])\n",
    "    dataset['original_headlines'] = open('references/st_g2r.txt', 'r').read().splitlines()\n",
    "    dataset['original_fulltext'] = [t.lower() for t in st_g2r[\"test\"].to_pandas()[\"full_text\"]]\n",
    "    dataset['generated_headlines'] = open(f\"model_predictions/{model}-ilgiornale-to-repubblica{'-ita' if 'mt5' in model else ''}_test.txt\", 'r').read().splitlines()\n",
    "    return dataset\n",
    "\n",
    "\n",
    "def prepare_r2g(model):\n",
    "    dataset = pd.DataFrame(columns=['original_headlines', 'original_fulltext', 'generated_headlines'])\n",
    "    dataset['original_headlines'] = open('references/st_r2g.txt', 'r').read().splitlines()\n",
    "    dataset['original_fulltext'] = [t.lower() for t in st_r2g[\"test\"].to_pandas()[\"full_text\"]]\n",
    "    dataset['generated_headlines'] = open(f\"model_predictions/{model}-repubblica-to-ilgiornale{'-ita' if 'mt5' in model else ''}_test.txt\", 'r').read().splitlines()\n",
    "    return dataset\n",
    "\n",
    "def compute_g2r(model):\n",
    "    dataset = prepare_g2r(model)\n",
    "    compute_hh_ha(dataset)\n",
    "\n",
    "def compute_r2g(model):\n",
    "    dataset = prepare_r2g(model)\n",
    "    compute_hh_ha(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compute_g2r(\"it5-small\")\n",
    "compute_g2r(\"it5-base\")\n",
    "compute_g2r(\"it5-large\")\n",
    "compute_g2r(\"mt5-small\")\n",
    "compute_g2r(\"mt5-base\")\n",
    "compute_g2r(\"it5-efficient-small-el32\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### it5-small\n",
    "\n",
    "Headlines-Headlines match accuracy 90.86\n",
    "\n",
    "Headlines-Article match accuracy 86.92\n",
    "\n",
    "### it5-base\n",
    "\n",
    "Headlines-Headlines match accuracy 92.52\n",
    "\n",
    "Headlines-Article match accuracy 85.22\n",
    "\n",
    "### it5-large\n",
    "\n",
    "Headlines-Headlines match accuracy 90.56\n",
    "\n",
    "Headlines-Article match accuracy 84.1\n",
    "\n",
    "### mt5-small\n",
    "\n",
    "Headlines-Headlines match accuracy 78.52\n",
    "\n",
    "Headlines-Article match accuracy 76.7\n",
    "\n",
    "### mt5-base\n",
    "\n",
    "Headlines-Headlines match accuracy 81.48\n",
    "\n",
    "Headlines-Article match accuracy 77.26\n",
    "\n",
    "### it5-efficient-small-el32\n",
    "\n",
    "Headlines-Headlines match accuracy 83.62\n",
    "\n",
    "Headlines-Article match accuracy 76.26"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compute_r2g(\"it5-small\")\n",
    "compute_r2g(\"it5-base\")\n",
    "compute_r2g(\"it5-large\")\n",
    "compute_r2g(\"mt5-small\")\n",
    "compute_r2g(\"mt5-base\")\n",
    "compute_r2g(\"it5-efficient-small-el32\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### it5-small\n",
    "\n",
    "Headlines-Headlines match accuracy 88.68\n",
    "\n",
    "Headlines-Article match accuracy 89.44\n",
    "\n",
    "### it5-base\n",
    "\n",
    "Headlines-Headlines match accuracy 88.34\n",
    "\n",
    "Headlines-Article match accuracy 88.0\n",
    "\n",
    "### it5-large\n",
    "\n",
    "Headlines-Headlines match accuracy 88.34\n",
    "\n",
    "Headlines-Article match accuracy 88.0\n",
    "\n",
    "### mt5-small\n",
    "\n",
    "Headlines-Headlines match accuracy 76.92\n",
    "\n",
    "Headlines-Article match accuracy 84.72\n",
    "\n",
    "### mt5-base\n",
    "\n",
    "Headlines-Headlines match accuracy 77.46\n",
    "\n",
    "Headlines-Article match accuracy 82.54\n",
    "\n",
    "### it5-efficient-small-el32\n",
    "\n",
    "Headlines-Headlines match accuracy 80.78\n",
    "\n",
    "Headlines-Article match accuracy 80.98"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SQUAD eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import string\n",
    "import sys\n",
    "from collections import Counter\n",
    "\n",
    "squad_it = load_dataset(\"squad_it\")\n",
    "squad_it[\"test\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_answer(txt):\n",
    "  def remove_articles(text):\n",
    "      return re.sub(r'\\b(il|lo|la|i|gli|le|l)\\b', ' ', text)\n",
    "  def remove_prepositions(text):\n",
    "      return re.sub(r'\\b(di|a|da|in|con|su|per|tra|fra)\\b', ' ', text)\n",
    "  def white_space_fix(text):\n",
    "      return ' '.join(text.split())\n",
    "  def remove_punc(text):\n",
    "      exclude = set(string.punctuation)\n",
    "      return ''.join(ch for ch in text if ch not in exclude)\n",
    "  def lower(text):\n",
    "      return text.lower()\n",
    "  return white_space_fix(\n",
    "      remove_punc(\n",
    "          remove_prepositions(remove_articles(lower(txt)))\n",
    "      )\n",
    "  )\n",
    "\n",
    "\n",
    "def f1_score(prediction, ground_truth):\n",
    "    prediction_tokens = normalize_answer(prediction).split()\n",
    "    ground_truth_tokens = normalize_answer(ground_truth).split()\n",
    "    common = Counter(prediction_tokens) & Counter(ground_truth_tokens)\n",
    "    num_same = sum(common.values())\n",
    "    if num_same == 0:\n",
    "        return 0\n",
    "    precision = 1.0 * num_same / len(prediction_tokens)\n",
    "    recall = 1.0 * num_same / len(ground_truth_tokens)\n",
    "    f1 = (2 * precision * recall) / (precision + recall)\n",
    "    return f1\n",
    "\n",
    "\n",
    "def exact_match_score(prediction, ground_truth):\n",
    "    return normalize_answer(prediction) == normalize_answer(ground_truth)\n",
    "\n",
    "\n",
    "def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):\n",
    "    return max([metric_fn(prediction, ground_truth) for ground_truth in ground_truths])\n",
    "\n",
    "\n",
    "def evaluate(predictions):\n",
    "    f1 = []\n",
    "    exact_match = []\n",
    "    for ref, prediction in zip(squad_it[\"test\"], predictions):\n",
    "        exact_match.append(metric_max_over_ground_truths(exact_match_score, prediction, ref[\"answers\"][\"text\"]))\n",
    "        f1.append(metric_max_over_ground_truths(f1_score, prediction, ref[\"answers\"][\"text\"]))\n",
    "    exact_match = sum(exact_match) / len(exact_match)\n",
    "    f1 = sum(f1) / len(f1)\n",
    "\n",
    "    return {\"exact_match\": exact_match, \"f1\": f1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"it5-small\", evaluate(open('model_predictions/it5-small-question-answering_test.txt', 'r').read().splitlines()))\n",
    "print(\"it5-base\", evaluate(open('model_predictions/it5-base-question-answering_test.txt', 'r').read().splitlines()))\n",
    "print(\"it5-large\", evaluate(open('model_predictions/it5-large-question-answering_test.txt', 'r').read().splitlines()))\n",
    "print(\"mt5-small\", evaluate(open('model_predictions/mt5-small-question-answering-ita_test.txt', 'r').read().splitlines()))\n",
    "print(\"mt5-base\", evaluate(open('model_predictions/mt5-base-question-answering-ita_test.txt', 'r').read().splitlines()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```shell\n",
    "it5-small {'exact_match': 0.6195295045341044, 'f1': 0.7161013980350992}\n",
    "it5-base {'exact_match': 0.6626363516887895, 'f1': 0.7611178426059487}\n",
    "it5-large {'exact_match': 0.6912866342489158, 'f1': 0.7804222430601728}\n",
    "mt5-small {'exact_match': 0.5606518596399002, 'f1': 0.6602759773423416}\n",
    "mt5-base {'exact_match': 0.6625049283742936, 'f1': 0.7579589870271762}\n",
    "it5-efficient-small-el32 {'exact_match': 0.6446313576028387, 'f1': 0.747087969387605}\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_14493/1562658397.py:20: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
      "  g.set_xticklabels(['0','0', '500K','1M','1.5M','2M'])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe4AAAEMCAYAAAACm1XfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA5p0lEQVR4nO3deYBVZcHH8e/Z7jZ3NoYBBkRFFFyTkiQTzVBfMUXRXHBp0cwtjTI1tQRR03DL3LJseevVNzMTTTE1tXJLXxdcM0VFQRiYYZjt3rnbWd4/7nADERxg5t653N/nD2fuds5z7iPzO8/znPM8RhAEASIiIlIWzFIXQERERPpOwS0iIlJGFNwiIiJlRMEtIiJSRhTcIiIiZUTBLSIiUkYU3CIiImXELnUB+qq9PYnv988t5w0NcdraEv2yLdk0qoPSUx2Unuqg9AZjHZimQX191XpfL5vg9v2g34J79faktFQHpac6KD3VQemVWx2oq1xERKSMKLhFRETKSNl0lYuISPGkUkkSiQ48zy11UQZUS4uJ7/sl2bdl2cTjdUSj6x/P/jgKbhERWUsqlaS7u526ukYcJ4RhGKUu0oCxbRPXLX5wB0FALpelo6MVYKPCW13lIiKylkSig7q6RkKh8BYd2qVkGAahUJi6ukYSiY6N+qyCW0RE1uJ5Lo4TKnUxKoLjhDZ6OKLiusr/+PZ9VH0Y4UtbHVTqooiIDFpqaRfHpnzPFdfi/jCxjPc7lpS6GCIiMoCam5cxefJEXPeTW7MPPng/Z5zxjSKUqn9UXHAbGPhBed1sLyKypTvqqGnst9/n6OjoWOv5k046nsmTJ9LcvKw0BRuEKi+4DZMgKM2l/yIisn5NTSN59NGHCo/fffcd0ul0CUs0OFVccJsYBGpxi4gMOgcd9CUeeujBwuO//OUBpk49pPA4kUhw2WWzOPTQA/jylw/lv//7l4V7sD3P46abrueQQ/bn6KMP55lnnlpr24lEgiuvvJTDDz+I6dMP5he/uAXP84pzYP2s4oLbMAx8FNwiIoPNLrvsRjKZ4P33F+F5Ho899ggHHfSlwus/+clVJJMJ7rrrPm666Rc89NCDPPjgnwG4//55PPPMk/z613fwq1/9jr///bG1tv2jH12CZVnceee9/OY3d/D8889y//33FvPw+k3lBbda3CIig1a+1T2f559/jm222ZahQxsB8H2fxx57hNNOO4tYrIqmppHMmHFCoYX++OOPcswxxzF8+Ahqamr5yldOKmxz1ao2nn32aWbO/B7RaJT6+iEcc8zxPPbYIyU5xs1VtNvBMpkMV1xxBf/85z8Jh8NMmDCByy67rFi7LzAMg6DMVoIREakUU6cewre+9U2am5eu1U3e0dGB67qMGNFUeG7EiCZWrszPPLZyZSvDho1Y47X//L58eTOu63L44VMLz/l+wLBhwwfyUAZM0YL76quvJhwO8/DDD2MYBitXrizWrtdiYOCji9NERAajESOaaGoayT//+QwXXDCr8HxdXR22bbN8eTNjxmwHwIoVywst8oaGobS0LC+8f8WK//w+bNgIHCfEAw88im2X//QlRekqTyaT3HvvvcycObNws/nQoUOLset1GIa6ykVEBrMLL5zFDTf8jGg0WnjONE2mTDmQX/ziFnp6kixf3swf/nBHYQx8ypQDufvuP9DSsoKuri5uv/23hc8OHTqUPfecxE03XU8ymcD3fZYu/ZAFC14s+rH1h6KceixZsoS6ujpuuukmnnvuOaqqqpg5cyYTJ04sxu7XoqvKRUQGt1GjtvrY57/znfO4/vqrOeaYwwmFwkybNp1DDjkMgGnTprNkyQd8/evHU1VVxYwZJ/Lii88XPvvDH17KrbfeyIknHkNPT5KRI0dxwglfK8rx9DcjKEKKvfHGGxx55JFcc801TJs2jVdeeYXTTz+dv/71r8Tj8YHe/VquefrnNHe3cO3Ui4u6XxGRcvHGG/9i5MhtSl2MirFs2QfsssvOfX5/UVrcTU1N2LbNoYceCsDuu+9OfX09ixYtYrfdduvTNtraEvj9cFFZLuMRBAGtrd2bvS3ZdI2N1aqDElMdlN5grQPf90uy1GUplGpZzzX5vr/W/wemadDQsP5GbVHGuIcMGcKkSZN4+umnAVi0aBFtbW1ss00Jzug0xi0iImWsaJfXzZkzh4suuoi5c+di2zZXXXUVNTU1xdp9gamrykVEpIwVLbhHjx7N//zP/xRrd+tlGAaaOE1ERMpVRc6cpilPRUSkXFVecBuGVgcTEZGyVXnBrfu4RUSkjFVecGt1MBERKWPlP2nrRtLMaSIi5eeoo6Zx1VU/Yd68P/Haa68A8P777zFy5ChCoTAAv/rV/3DssdMJhUKF584442wmTdprne0tX97Mtdf+mJaWFQRBgOOE+MEPZrPddtv3S3mbm5dxyilfYf78/PKikydP5JFHniAWi232tisuuDVXuYhI+fre975f+P2oo6Zx+eVz1wnbj3vuo6699sd87nOf59hjj8N1fVpbW8pmAZLyKGU/0lXlIiIb5+nXmnnq1eYB2fbkTzWx925Nn/zGftbS0sLQocMKjxsb//P7j350CY7j8OGHS1i69EO+8IUvsvfe+/KrX/2clpYVHHPM8RxzzHEA3HTT9bz88kvkcjnq6uq48MJZay09OhAqL7gNU1eVi4hswebMuRgI2G23CZx22reorq5e5z0nnPBVLr98Nnff/Xt22mlXvvjF/dlpp10Kry9a9B4//enP8H2fo46aRiKR4KabfkFb20qOP/7LHHro4cRiMU488eucddZ3ALj//nv52c9uYM6cKwf0+CovuDXGLSKyUfberTSt4k1x8823MXz4CLLZLDfccC0/+clVzJp12Trv+6//OphJk/ZiwYIXeOmlF/n2t0/n/PN/wIEHTgVgn332IxQKAbD11tuw1157Y5omjY3DqK6uobW1hW222ZZnn32ae+75I6lUD57nFeUYKy+4dVW5iMgWa/jwEQCEQiGOOOJoLrjgnPW+t7a2jgMO+C/22+8Ahg0bzqOPPlwI7nA4VHifaZqFi91WP/Y8l+XLm7nxxuu47bbfMXLkKF577RXmzPnhAB3Zf1Tc7WC6qlxEZMuUSqVIJBIABEHAo48+zPbbj/vY9z7zzFNkMhkAPM/j3Xffoalp1EbtL5lMYtsODQ0N+L7Pvff+afMOoI8qssWt4BYR2fKsWtXGD394Pr7v43k+2247hu9974KPfe+CBS9y883XY9s2nucxfvxOnHLK6Ru1v7Fjt+eLXzyAE088htraOvbaa29eeWVBfxzKBhlBmaRYf63Hfc/CB3iq+Tmu23fdMQ8pnsG6DnElUR2U3mCtg+XLP2DEiBIsu1wCg2E97o9+34NiPe7BRHOVi4hIOau84NYYt4iIlLHKC25dVS4iImWs4oJbV5WLiEg5q7jg1lXlIiJSziovuDEICBTeIiJSliryPm6AgAADo8SlERGRvujvZT1/9aufM2/e3TQ2NuL7AZFIhHPPvZAddvj4CVsGk8oL7t5OhiAIUG6LiJSX/lrWE2Dq1EP4znfOwXV97rnnj9x6601ce+0N/V7m/lZ5wb1Gi1tERAQgmUwUVhFzXZfzz/8OnZ2dZDIZdt55F8477yIcx+G1117hJz+5Ct8PcF2Xr33tZA48cCrJZIIbb/wJ7767kGw2y6c/PZGzz/4ulmX1e1krLrjN3ma2xrhFRPom9/bT5N56YkC27YzfF2fc3v26zb4s6wnw0EPzefHF/6OzsxPPc7nxxl8AYFkWs2dfTm1tHUEQcPnls5k//z6mTz+KO+74Lccd9xUOPHAqQRAU5ka/8cafMGHCZ7jggovxfZ85c37I/Pl/5rDDjujXY4MKDO7V3eNqcYuIbHn6uqwnrN1V/pe/PMDs2Rfym9/8L77v8/vf386zzz6D73t0d3cTiUQA+MxnJvLb3/6apUs/5LOf/Ry77LIrAE899QRvvvkGd955BwDpdJphw4YPyDFWXHCvviDNV4tbRKRPnHF793ureKBszLKea5oy5QB+9KNLaG9v57nnnuHVV1/mlltuIxar4ne/+zVLliwG4Jhjjmfvvffl+eef4/rrr+Kzn/0cp556JhBwxRXXMGrUVgN1aAWVdzuYxrhFRLZIG7Os50e99NIL1NbWUltbSyLRTW1tHbFYFYlEgr/+9aHC+xYv/oBRo7Zi+vQvc/TRx/Hmm28AsPfe+3L77b/F8zwAOjo6WLZsaT8fYV7FtbjNwqXkCm4RkS3JxizrCf8Z4/b9ANu2ueyyuZimydSph/Lkk09w/PFfpr5+CLvv/unC2t13330nL730Io5j4zghvvvd8wCYOfN73HLLDXz968dhGAaOE+Lb3/4eI0du3BrffVFxy3r+bclT3L3wz1y9zyXEnFg/lEw2xWBdzrCSqA5Kb7DWgZb1LC4t6/kJCmPcanGLiEgZKlpX+ZQpUwiFQoTD+dlszj33XPbZZ59i7b6gMMZdHh0NIiIiaynqGPcNN9zAuHGlnU5udYtbF6eJiEg5qryucrW4RUSkjBW1xX3uuecSBAF77LEH55xzDjU1NcXcPbDGzGlqcYuISBkqWnDfcccdNDU1kc1m+dGPfsSll17KNddc0+fPb+gKu41R0x0FoH5IjKGxj58GT4qjsVHff6mpDkpvMNZBS4uJbVdOh2ypj9U0zY36/6Bowd3U1ATkZ7M5/vjjOeOMMzbq8/11O1iiO38v3sqVCYKos9nbk00zWG+DqSSqg9IbrHXg+37Jb5EqlsFwO5jv+2v9f/BJt4MVJbh7enrwPI/q6mqCIODBBx9kp512Ksau16GZ00REys9ArMedSqX4znf6NiXqYFKU4G5ra+Pss8/G8zx832fs2LHMnj27GLteh6HVwUREylZ/rse9OVzXxbZLM/loUfY6evRo7r333mLs6hP9p8VdGd1AIiKb67nmF/ln8/MDsu29mj7LpKY9BmTbm+KFF/6P2277GdlsBs/z+OpXT+aAAw4C4KyzTmWHHcbzxhuvUVNTw5VXXst1113FggUvUl9fzw47jGPVqjYuv/wqAG6//b/5xz8ex/M8hg4dxve//wMaGoZudhkrbq5y/8N/AWpxi4hsqfq6HvfHGTduR2655ZdYlsWqVW184xtfYc899yrcBbVs2YfccssvsW2bu+++kxUrlnP77XfheR5nn30aw4YNA+Dhhx9k6dKl/Pzn/41pmsybdzc33XQ9s2dfvtnHV3HB3fH+mzBEY9wiIn01qWmPQdUq3pCNWY/743R0tHPllZfy4YeLsSybrq5OFi/+gF133Q2AAw+cWugif+mlF5k69UvYto1t2xxwwEG8+uoCIL8+97///SYnn3wiAJ7nEo/3z91RFRfc2Vz+p9bjFhHZ8mzqetyrXXvtj9l773254oqrMQyDGTOOJJvNFF6PRvu2OFUQBHztaydz6KGHb9T++6JybtTrFRhW/qda3CIiW5TNWY97te7ubpqamjAMg+eff5alS5es972f/vQePPLIX3Bdl0wmw+OP/7Xw2uTJ+zJv3t10dXUBkM1mWbjw7U04qnVVXIvbMPLnKhrjFhHZsmzsetz33XcPjz32SOHx1772Dc444yyuvXYuv/rVL9hpp50ZO3aH9X5++vQv8847b3PiicdQV1fHNttsW3ht6tRD6Ozs4OyzTwXy92ofccTR7LDD5q/XUXHrcf/5rst4eGg3F3x2JqOr+3+Bc+mbwTrxRCVRHZTeYK0Drcfddz09SWKxKrLZLBdccA5f/OIBTJs2faO2sbHrcVdci5vVXeXlcb4iIiKD2MyZZ5LL5chmM0ycuCcHH3zogO+z4oLbMDXGLSIi/eO2235b9H1W3MVpq1vcuqpcRGT91CtZHJvyPVdccJuFFrdmThMR+TiWZZPLZUtdjIqQy2WxrI3r/K644LaM/BeUcXMlLomIyOAUj9fR0dFKNptRy3uABEFANpuho6OVeLxuoz5bcWPcjplfyjOV7SlxSUREBqdotAqAzs6VeJ5b4tIMLNM08f3S9MBalk11dX3h++6rigvukBkCoCeTKnFJREQGr2i0aqMDpRwN1lvyNqTiuspXB7da3CIiUo4qL7jt/OLqqaxa3CIiUn4qLrjDVj64066CW0REyk/FBbdth3D8gLSbLnVRRERENlrFBbdpO4T8gLSb+eQ3i4iIDDIVF9x2KEQ4CMgouEVEpAxVXHCHwmFCvk/G06xAIiJSfiouuMOxOFE/oMfTxWkiIlJ+Ki+449XUuD49gYJbRETKT8UFd6SqmmrXJ0VWc/CKiEjZqbjgtiIxajwP1/BJe7olTEREykvFBbdhh4j1zpnfmSmv+WlFREQqLrgBol5+Te6ubFeJSyIiIrJxKjK4I35+ac+OjIJbRETKS0UGdxX5+cq7suoqFxGR8lKRwR21HWwf2tOdpS6KiIjIRil6cN90002MHz+et99+u9i7LnCc/AVqbT0KbhERKS9FDe433niDl19+mVGjRhVzt+uwHQfHD0hlNV+5iIiUl6IFdzab5dJLL+WSSy4p1i7Xz7IxA/ACv9QlERER2ShFC+6f/vSnHHbYYWy11VbF2uX6mRYmAb6CW0REyoxdjJ0sWLCA119/nXPPPXeTt9HQEO+38rSZ+Ra3ZRs0Nlb323Zl4+i7Lz3VQempDkqv3OqgKMH9/PPP8+6777L//vsDsHz5cr7xjW9w5ZVXMnny5D5to60tge/3z9zihmVj5QIyuRytrbolrBQaG6v13ZeY6qD0VAelNxjrwDSNDTZWixLcp556Kqeeemrh8ZQpU7j11lsZN25cMXa/rt6ucleLjIiISJnp8xj3s88+y5IlSwBoaWnh+9//PhdeeCGtra0DVriBYvRenKYxbhERKTd9Du45c+ZgWfk5vufOnYvruhiGwcUXX7zRO3388cdL19oGDNPCIsBHwS0iIuWlz13lK1asYOTIkbiuy1NPPcXjjz+O4zjss88+A1m+gWHZvVeVq6tcRETKS5+DOx6Ps3LlShYuXMjYsWOpqqoim83iuu5Alm9AGJaNFQQE6ioXEZEy0+fgPvHEEznqqKPI5XJcdNFFALz00ktst912A1a4gWKYtu7jFhGRstTn4D711FM58MADsSyLrbfeGoDhw4dz+eWXD1jhBoph2VhAoDFuEREpMxt1O9iYMWMKvz/77LOYpsmee+7Z74UaaIZl6apyEREpS32+qvzEE0/kxRdfBOAXv/gF55xzDt/73ve49dZbB6xwA8WwHAzQGLeIiJSdPgf3woULmTBhAgB//OMf+d3vfsddd93FnXfeOVBlGzCmZWEGuh1MRETKT5+7yn3fxzAMFi9eTBAEbL/99gB0dpbfmtaG5WCiFreIiJSfPgf3HnvswaWXXkpraysHHnggAIsXL6a+vn7ACjdQVo9xB+g+bhERKS997iq/8sorqampYfz48Zx11lkAvPfee3z1q18dsMINFMO2MdB93CIiUn763OKur6/nnHPOWeu5/fbbr7/LUxSW7eSvKleLW0REykyfW9y5XI4bbriB/fffn912243999+fG264gWw2O5DlGxD5CVh0H7eIiJSfPre4r776al599VXmzJnDyJEjWbZsGbfccguJRKIwk1q5MG1bY9wiIlKW+hzcDz30EPfdd1/hYrTtttuOnXfemcMPP7zsgtswzfyUpwpuEREpM33uKg/Ws5LW+p4fzEzLxlCLW0REylCfg3vq1KmcccYZPPnkk7z77rs88cQTfOtb3+Lggw8eyPINCMMye+/jVnCLiEh56XNX+XnnncfPfvYzLr30UlpaWhg+fDhf+tKXyvLiNNOyMYMAjPx85abR5/MXERGRkupzcIdCIWbOnMnMmTMLz2UyGSZMmMD5558/IIUbKJZlYfT+HvQGuIiISDnYrKamYRhl2d1sWBZWb7E9TcIiIiJlZLP7iA2j/JqrpvmfFreW9hQRkXLyiV3l//znP9f7Wi6X69fCFItl25i9V5RrEhYRESknnxjcP/jBDzb4elNTU78VpmgME1Nd5SIiUoY+Mbgff/zxYpSjuEyrMEZQjmP0IiJSuSryPijDNPO3g6ExbhERKS8VGty6OE1ERMpTRQb3mmPcCm4RESknlRnca4xx+xrjFhGRMlKRwW2Y1hpj3F6JSyMiItJ3FRrcZuHAdTuYiIiUkz7PVb65zjzzTD788ENM0yQWi3HxxRez0047FWv3azNNrN4Wt6cWt4iIlJGiBffcuXOprq4G4NFHH+Wiiy5i3rx5xdr92kwLu3do2/Xd0pRBRERkExStq3x1aAMkEomSznFumFahxa3gFhGRclK0Fjfkp099+umnCYKAX/7yl8Xc9doMc40Wt7rKRUSkfBhBCeb8vPfee5k/fz633XZbsXdd8PdrZnDL6HrOn3w6E0ftXrJyiIiIbIyitrhXmz59OrNmzaK9vZ36+vo+faatLYHv9885RmNjdWE97raOBK2h7n7ZrvRdY2M1ra363ktJdVB6qoPSG4x1YJoGDQ3x9b9ejEIkk0mam5sLjx9//HFqa2upq6srxu4/lhnkx9g1xi0iIuWkKC3uVCrFzJkzSaVSmKZJbW0tt956a0kvUFNwi4hIOSpKcA8dOpS77rqrGLvqM0vBLSIiZagiZ04DWL0+WE7BLSIiZaRygzvIH7pa3CIiUk4qNrhXz1au4BYRkXJSscFt9K7J7WquchERKSMVG9yYJqYPOT9X6pKIiIj0WeUGt2FhBZDz1FUuIiLlo3KD28x3lWddtbhFRKR8VHBwO1gBZDwFt4iIlI+KDe7AiRD2A3py6VIXRUREpM8qNriNUJSoF5DM9pS6KCIiIn1WscHtRGLEfJ+kq+AWEZHyUbHBHY5VUeV5pD11lYuISPmo2OB2olXEfY+snyYI+medbxERkYFWscFNKEKV7xMYPllNwiIiImWiYoPbcKLEfB+AZC5Z4tKIiIj0TQUHd4S4mw/urmx3iUsjIiLSN5Ub3JE4tb3BvbKno7SFERER6aPKDe5YHbVufmWwpR2tJS6NiIhI31RucFfVU+UHGL7B0s62UhdHRESkTyo3uCNxDMMikjNpTbaXujgiIiJ9UrnBbZgYsVpqfIPOXFepiyMiItInFRvcAEZVHUP8gEyQ0CQsIiJSFio6uM1YPUM8j8BO09qhOctFRGTwq+jgNmJ1NObSGGbA/727uNTFERER+USVHdxVdTSmUwC8suSDEpdGRETkk1V0cJtV9QzPuAAs6WomnXVLXCIREZENq+zgrhtJ3A+IGWGCaCevv7eq1EUSERHZoMoO7vpRgMFYqwq7poO/LVha6iKJiIhsUEUHt+GEMWqHMybtQqiHfzcv48331eoWEZHBqyjB3d7ezje/+U0OOuggpk2bxllnncWqVYMjIO2mcWzTuhyA6mEJ7v7He7qnW0REBq2iBLdhGJxyyik8/PDD3H///YwePZprrrmmGLv+RFbTjjQlE0TNEKPG9LCouYsX3tKiIyIiMjgVJbjr6uqYNGlS4fGECRNYtmxZMXb9iaym8ZjALs4QVniLGD08xu8e+jeJVK7URRMREVlH0ce4fd/n97//PVOmTCn2rj+WGW/AqB3Bp7p6SLlp9tsnRE/G5ad/fIWetG4PExGRwcUIijygO2fOHFasWMFNN92EaQ6Oa+NW/e0OVv7zXq7ccTR7jPwUE6IHcu0dL7LD6HrmnLoX0bBd6iKKiIgARQ7uuXPn8tZbb3HrrbcSCoU26rNtbQl8v3+K2thYTWtrd+Gx17aYnj/NYt5un+GVbCtXTr6YVxd2cOt9b7DNiDinHb4rw+qi/bJvyftoHUjxqQ5KT3VQeoOxDkzToKEhvv7Xi1WQ6667jtdff52bb755o0N7oJlDRmPWj+LTK1aQ9jI8v3wBE3ccxumH78LyVSku/+0L/N+bK0pdTBERkeIE98KFC/n5z39OS0sLM2bM4PDDD+db3/pWMXbdJ4ZhENr9YLZpWcrW4QYefP9Rsl6WiTsO4/vHf5rGuii33vcGN9/zGktbE6UuroiIVLCij3FvqoHsKgcIPJfknefzfl0dP4smOHTMf3HwmAMAcD2fB555nz8//T6GAZ/fdQSH7rUtjXVRTNPolzJVmsHYPVVpVAelpzoovcFYB5/UVa6rrnoZlp1vdT9zB5/afQ/mL/orw6uG8Zlhn8K2TKbvsx177TqCR/5vCX9/eSlPv7achpoIUz4zii9MGEUsoq9SREQGnlrcawg8l567f0gPPrdtO4IeN82cvb6PZVprva+1I8WTrzbzr/dX8d6yLkzDYExTNbtu18Ae4xsZNbQKw1BLfEMG41lupVEdlJ7qoPQGYx2oxb0RDMsmvPeJ+A9ew4GZrflvdzkvt77OHsN3X+t9jXVRjtx3O47cdzsWNXfx0tutPPP6cu57ahH3PbWIsGMxpqmabZtq2G27BkYMiVEXDynMRURksym4P8LealecT01lh1cfommHrbnzrXnUR2rZrnbbj33/mKYaxjTVcOS+27F8VQ/vLO1k8YoE7yzt5NEXlvDQc4sBCIcsqqMOY0fVMmpoFU0NMUYPizOsPlbEoxMRkXKn4P4Y4UnHgu9x4r8f4zfbjuQnL93KMeMOZ59Re633M4Zh0NRQRVNDVeG5dNbl9fdW0ZHI0NqRZunKBC++1cJz//pPl39dPER9dYT66jD18TDVMQfLMhg/up6aeIiIY2FbpsbQRUQEUHB/LMMwCO91PE2Ww5mv/YU/brsVd741j45MF1O33R/H7NvXFgnZTNxx2FrPuZ5PT8ZlyYoES1sTfNiapL07zfJVPbz5wSpSGe9jt1UTc4iEbEY0xGhqiBEN2wQBDK2NEAlZWKbJmKZqqmMhDAN1y4uIbKEU3OthGAahPY+mPhLnxBfu4Z5htTz0/mO8uOJlpm9/CLsP3WWTwtG2TGpiIXYZM4RdxgxZ5/VMzqOjO8OK9h5WdWfwvIB01mXpyiQd3Rneb+7i1Xfb1rv9kG1imAZDayOYhsGwuiixiI1hQHUsRHUsRF08RM71MYz8eH1tPEzEsYiG8617hb6IyOCl4N6A/MQsX8IauTPHPvFrdu9czgPDTW577XfsPGQ8x46fztBoQ7/uM+xYDB8SY/iQ9Y99u14+dIMgf4V7d0+OnrRLa2eK1vYUiVSOtq40Ycfiw5VJsjmPIAjoSubwP+EmgrBjEYvYmAZURR1Mw2DbphoiIYuQbRJyLCIhixFDYlimgWEYOLZJdczBD6AqYhML2wSAqRMAEZF+p9vB+ihwM2Rf+QupVx/i8WqLJ+vjuAbs3fRZPj/qc4yuHjXoW6o518+36BP/acl3JrOksx7ZnEcqm38tm/UIIP97zmf5qh6yrkc25/dpP1bvpDT11WFM0yAasnEcE4L8bQ4jhsQYPrQKxwDLMsnmPKpjDjnXJx51qI6FcD2foXVRamMhLCu/PdsaHIvSbCkG420wlUZ1UHqDsQ4+6XYwBfdG8tPd5F75Cy1v/52/x02eq4kSGAaN4Tq+sPW+7NKwI8NiQ/ulnINNEATkXJ+uZJaVnWmC3uc6E1k6EhlCjkVPxiWdcenqyZLJemRyPgEBnpevu2QqR2tnikzWY2Or0zINQo5JPOpQHw8TAFnXJxa2qYuHGDEkRncqR9ixqIo4VEVs0jkP0zCwLIPaWIhYxMb1A4bVRbFMg6ybPxkZXh8lk/NwvYB41OnfL26QGox/sCqN6qD0BmMdKLg/Rn9UVOD7eB++Ttubj/Jq+0IWVIf5IJpfPKXRibN9/Q5sN2R7xtWPpT5cu84kLpWurj7Gh8s6ybk+pgGJVA7LMulKZklnXUzTYFVXhmQqh+sHeJ5POuvhB0F+KKAzjecHZHMe6ayH6/l0JLLYlonvB584JPBRIdsshHh9dZhY2CbregypjuD5AZ7vUxcPU18dpjoWIuSYZLIejp2/JmBobYSQbZHJeYUhhepY/gQg7FhURR3SWZdIyCYazl9MWGqD8Q9WpVEdlN5grANNwDJADNPE3vpTDN/6U+yfbGfft57g/aUv836qhbejGV5Jd/HPlgUA1FgRPtWwI7VVjezSsCOjq0dhGqX/w11Kjm2t1bKtjYcBGLGBsf1Pks66OLaJaRiksx7JdA6DfGs7k/VY1ZUujL23dKR6y2HS3ZNjZWcKAqiNh1i6Mkk252OQv4YgFrEJOzYt7SneWtxBT8bdnEPHMKC2KkQs4pDoyRKLOFRFbVw3AAOqYw6xsI1jmdi2WfgZdizCjoVlGti2SXXUIRaxC9caBEFATVWIuniYdNYjFrHJuX7vNQuDexhHRPpOLe5+FuTS+Ks+JPvByyxa8Rrvp1p4KxZmSdgm0ztGG8dmTKSB6qpGRg7ZjhGxYYyMj6A2XDMgZRqMBuNZbl95vk8m6xEJ5VvlQQBtXWkSPTmqYw7JtEs665Jz8/+/ZnIuiZSL7wcEBKQzHu3dGbp7stTGw/Skc3T35AiHLIIAOhMZMq6P6/q4nk/O9cn1/twUtmUQBBBy8ic14VDvSZNhEHEsDPInMAHkr3gEAqC2KsxJX9pR1xYMoHL+d7ClGIx1oBZ3kRlOBGv49kSHb8/OHMWOyXYOSrThrXyfruY3eTPxIf82EizOZUgnm3mm7fXCZ+vNMNVmmKHRIWzfMJ541VAc02b7ujHEHM2wNlhYpkkskg+zSCj/T2irxvX/I+svvh+Qznpkch5Z1yOX8+nuyQKQcX1CtsnKzjSdySxVEZu2zjQhJ999bxiQy/kEASQzORI9OUJhm87uNJZhkEy7+fv/ofc/FE5KRGRwUXAPMLOqHqrqsYZvz9BdDmAfYLLv4S1fiNe2mK6OpSxPLOODnhV86KTpskwWptt5qfv9tbYzJsiH+laxYYxrGEd1pI7a6uFEakYM+qvZpX+YpkEsYvfbLHqDsaUhIp9MwV0Chmlhj9wRe+SONAKNwK6+D16WIJfGa/uQla1vk+xazvJsJy1eD2+ZGRYbKV5NdvFg8p3CtmKeT7UPEUxCdoSaUJx6u4qYE6UpNpy66hHUxhuJx4dh2mGFvIhImVNwDxKGaYIZwXAimLE6mkbvCsD2a7wnCALaW9/m3VXvkswmSKQ6aMt00JFLEAQB2VyaN/we0pYJKaDr37A8/1kzCIh7ATWBSbVhU4NFtREiG4ow2q6mPjaEeHwE/8qtZLvabRlbMxqjaggEPoZVGbdHiYiUAwV3GTEMgyHDxjNk2Pj1vsfLpUkmWskm22h3k3R2t9CZbqcrm6DTTdHl9tARZFlClqSRJvC6wWuFzHvQ3ruR5c8Sdz18w6Ax6xE1HeqwGW5G2ctuwLHDGDXDwLAwTAMjPhQjEseww+CEMcJxjHAVGGb+hERERPqNgnsLYzkRaupHQ/1oPmkaGM/3CHyX5p4WupMrSXStIJPppj3dQYeXIsj20OZk6fIyLMalx+jmtWw3B7em2eqtLjbY6W4YgAl2CLOqPh/ktgNWCCNSTVt9PVkjhhGJE+QymFVDwDQwqoZghOP5ln6kGsMJ99+XIyKyBVBwVzDLtMC0GF0zGmpGQ9OG3//MsueZ984D3ByC2Mh6mmLDGB5toIkw9YFJlR0lHhjEs1mcnm6MwIMgIEiuIsimwM0RJNvxW96l670MgZv95EI6ESB/tb4RiYMTwQjHMaM1YIfyrfpIHCMcy4e+kx9uIBTBcKIQ+OD7GOEYhGIa4xeRsqfglj77/MjP8plhu/Fiyyt80PUhy5MreLntTZ5xU+u81zYsqpwqqkNxGkbUY5v5xViiTpRh0aEMqYmT7koR8n0cO0yNF2ACNek0RjaFZdqkMp3EUilyfg7b9TCzSYJcmqB7Jbnlb4Pv4bkZNmpOulAMIxTFqKqHIMAIRcFzMetHErhZjFgdxGoxorW4bgYrHMMKV2OYFmbtMAwnimHpn42IlI7+AslGidgR9h45ib1HTgLAD3y6s0lWpdvpynaRyCXpyaVI5npI5JJ0Z7tZ0dOKH/gYhkFXtpuUm+7z/gwMAgIMDGKxKJ7v4ccDwiNG4FgO7ekOYnYE1/eImE7+3UF+opMg8DExiJkhOr0UBAExLDJ4ZIIunMDAJoFLgJFaSdI0CBIGJNYthxUE2EHAkJxH2AfHMHEwMTEwDYMqJ4YdgAdUW2E80yIeqSXrZYnaUSJOBNO0MJ0ophMlHKsjFK3FzSQJR+vw8RkSGULYyk+b251NMCRST8QOEwSBegpEpEDBLZvFNExqw9XUhqv79H4/8Em7aWrqIyxv7SDlpknmkmS8DK7v0Z7pwDRMsl6OiBUmkUsQtsJkvCxd2S6CACJ2mJzvkvEyRHtD2zZtsl6WgAATozfoDPzAJ5FLsl2oGsuwSOQSRKwIETtM1s/hem5h3vBqJ44RBOC7kMvgG4DngpfD9V3S6W5WuUlyvks2cEkGHm4Q4OOTDJJ4RkBgGORIgg/0tOYPOrPp328MkwwBIUxcAizDpMoMYWEQNh0cyyEAQlaYwIDqSB2YJiHTwTAsLNPCMW3CVoioHSVmR4k5+Z8pp5F02iflpqlyqjANg5gdLdSrThZEBicFtxSVaZjEnBj10Wrc6JZ3xbnne3iBj2WYJHtWEQrFyaQ7SWUTBG4GL53Az2VoSSyjJ91FgxPHTXfjeVkSeGQJ8HNpYtkMq/wMK8nh+B6u7xPzXFzDoNsy8QyDnGngARiw0jKJ+QHLbItwEJAzIMDAM8A1DHKGQdCHHDbIT3dqAnEzhOlECZkOWT9HbbiGnJejyonhmA5VToyIHcHuPUFY/TPnu5iGSbUTJ2yFcEwbDIOwFSJshTAwcCwH1/eocmJ4vkdNuJqQmX/v6hMGo3ACJiJrUnCL9CPLtLB6R91r4o0AREJRaj/yvq03YduB50IuTeBmwHIg04Pf0w6mTZDqBN/D724jSHVihGL43a0YToQg24OX7ibjpkk5IZK5HlJuinQ4TCKTIOR5pI380EKSgJwRkDUMvJCPNWZXsoGLZdh0ZDqodqpIuWkSuQQrelrIeFm8wMP1PbzAyw+J9N5vELDp86UaGMScKI7pYBsWMSeKZViYhoXru4Qsh4AA27CJ2hFqwzWFBWSGROpJu2lsw8axHEK9PRPr/gzhmA4hyyn8tAyrcLKwehkHnTzIYKPgFikThmWDFcegd170aA1m3Yg+f/6js6mvb8rTwHeBTbsH3w98giDADTzSboaMl8YPfMAg42XIeBmCAHrc/EWHru/hmDY9boq0myblpsn5OUJWiM5MF4Zh4PouiWwSgKyfHw7xgvzqbRk/w6p0O2+uehvDMHu3uemrt5mGiWPahMwQfuCT8tK9PQXhQo+BZVhr9DbY+bnr7fxaAmErhG3a2KaN1XvC4Qc+jukQtkLEnSpCvdvJ+S5BMsszSxewU8M46sN1OkmQPlFwi8haDHPT/yyYhgkGWFi9F9r17dqH/pLzXdJumpgdxQt8sn6WnJcj6+c+8jObv1bBy5Lzc2S93Ed+5k8Qqpwqsl6WjJftPfHI4vke7ZlOMj2tvUMjHolcEgMDL/A2ueyO6VAfqSVshnCsECHTIWSFiNoRqpwYlmERshzCVjj/vBXGNv/Tq5D2MtSEqgsnFLZpFU4gHNPWdQtbEAW3iGwxHNPGCeX7FizyQUcRZux1fRfLsPADn4yXIeWmC0MIfuCT83P4QUAilyTnZelx09imRSRm09GVIOVlSLkpOjNdhROIjJelO5egM9NFxsviB/5mnRgA2KZNXbgWyzCJ2lE83y0MG6zuKXBMGz/wiTpRolYE28wP/cRDcUJmiIyXoToUJ+e7xJ0YITNEPFQFQNgKY2AUeh4sw8wPafSeOEj/UHCLiGwmu7eXwjIsYmasz8vw9mWFttVj7Tk/1/sz31OQcvPDEKt7EUJWiK5sNxkvg+d7uIGL63u4vtt7EuGS9XKsTLUR9G7PMizSXpqMlyGZS+IGHolskrAVIuWm6VljjobNuWYh1DtMYJtW7/CCtcZFjfnHBgZpL021E6cmXE3YCuMHPiHToS5SS8gM0eOmCFkhYnaEAIg7VVhGvmehqndYIu7ECVkOWS+b723YAtdaKEpwz507l4cffpilS5dy//33M27cuGLsVkSk7K3u3g713uMfskJUOTHqi7Bvz/cwDZOc75LMJcn5ORzTIeWmcUwn34Pg50jmenB9t3B9QcpN5Xsbem/P7Mx2rdUL4a1xUpH1s7hufsghYoVZllzBW+3vrHXSsKlWt/4hf/2BG3iEzPzwg2M5WIZJJBQC38Q2bEzDIOvnqHbiWIaJbdoYhpnvQejtUYmH4gSBT0D+1lSC/EnNZ4btTsyJbnaZ+6Iowb3//vvz1a9+lRNOOKEYuxMRkX5g9XaThyyHkFVXeH71SUMjDQO279UXOgJ0ZrvIejmidoScn6PHTWGQn9ApCAJc3yWZ68E0TBK5JBkvS9gKkfVzpNwUBJDxMlimTc7PkcqlyPlufh8EJLMJvMDHC3wc06Y5sYKcn7/ewTTMwlDF6gmhPr68AftutdeAfR9rKkpwT5w4sRi7ERGRLcTqCx0BhkQGrn+hr8MVbuBhGSYpN907xwCk3Qxe4BO2QlSHPnrfxsDRGLeIiMgGGIaBY+TjsmqN6xeidnG6xj+qbIK7oaF/z2YaG4t7m4qsS3VQeqqD0lMdlF651UHZBHdbWwLf3/SrGtfUl64RGViqg9JTHZSe6qD0BmMdmKaxwcaqbqwTEREpI0UJ7ssvv5x9992X5cuXc9JJJ3HIIYcUY7ciIiJbHCNYfb39IKeu8i2L6qD0VAelpzoovcFYB+oqFxER2YKUzcVpptm/k+P39/Zk46kOSk91UHqqg9IbbHXwSeUpm65yERERUVe5iIhIWVFwi4iIlBEFt4iISBlRcIuIiJQRBbeIiEgZUXCLiIiUEQW3iIhIGVFwi4iIlBEFt4iISBmpqOBetGgRxx57LAcddBDHHnss77//fqmLtEWYMmUKU6dO5fDDD+fwww/nySefBODll1/msMMO46CDDuLkk0+mra2t8JkNvTZ+/HiSySQAq1at4ogjjuD6668v6jFtKebOncuUKVMYP348b7/9duH5KVOmMHnyZDzPKzx3zz33MH78eG6//fZSFHWLsL7ve0033ngje+21V+Hfy5w5cwqvXXDBBYwfP56FCxcWnluyZAk77rgj3/72twe8/Fui9vZ2vvnNb3LQQQcxbdo0zjrrLFatWgXk/9Z8+ctfXuv9N954I+PHj+dvf/tbKYrbJxUV3LNnz+b444/n4Ycf5vjjj2fWrFmlLtIW44YbbuC+++7jvvvuY5999sH3fc477zxmzZrFww8/zMSJE7nmmmsANvjampqbmznhhBM47LDD+M53vlPkI9oy7L///txxxx2MGjVqndeGDRvGU089VXg8b948dtlll2IWb4uzoe97TdOnTy/8e5k9e/Zar+2yyy7Mmzev8HjevHnsvPPOA1LeSmAYBqeccgoPP/ww999/P6NHj17r700QBLzzzjuF3x944AHGjRtXquL2ScUEd1tbG//617849NBDATj00EP517/+VTjzkv71+uuvEw6HmThxIgAzZszgoYce+sTXVlu8eDFf+cpXOPnkkznppJOKW/gtyMSJE2lqavrY14444gjuueceIN+q6+npGfR/sAa7DX3ffTV16lQee+wxPM8jCALmz59f+LslG6+uro5JkyYVHk+YMIFly5YVHq/57+C5555j3Lhx1NXVFbuYG6Vigru5uZnhw4djWRYAlmUxbNgwmpubS1yyLcO5557LtGnTuOSSS+jq6qK5uZmRI0cWXh8yZAi+79PR0bHB11b7+te/zumnn87RRx9dzMOoKHvuuSdvv/02nZ2dzJs3j+nTp5e6SBVj/vz5TJs2jZNPPpkFCxas9VosFmPChAk89dRTZRMk5cL3fX7/+98zZcqUwnNrnijNmzePI444ooQl7JuKCW4ZOHfccQd//vOf+dOf/kQQBFx66aWbvc399tuPu+++m0Qi0Q8llI9jGAYHH3ww8+fPV6uuiGbMmMFjjz3G/fffzze+8Q3OPPNM2tvb13rPEUccwb333ls2QVIuLrvsMmKxGCeeeGLhudUnSn/961958cUX2XfffUtYwr6pmOBuampixYoVhYtxPM+jpaVls7u1hMJ3GAqFOP7443nppZdoampaqztq1apVmKZJXV3dBl9b7eKLL2bcuHGccsopCu8BdMQRR3DDDTcwbtw46uvrS12citDY2IjjOADsvffeNDU1rXUxGsCkSZP497//XTZBUg7mzp3LBx98wPXXX49prh19RxxxBLNnz+aAAw7Atu0SlbDvKia4Gxoa2GmnnXjggQcAeOCBB9hpp50YMmRIiUtW3np6euju7gbyF3Y8+OCD7LTTTuy6666k02leeOEFAO68806mTp0KsMHXVjMMgzlz5ii8B9jo0aP57ne/y5lnnlnqolSMFStWFH5/8803Wbp0KWPGjFnrPYZhcOGFF3LhhReWRZAMdtdddx2vv/46N998M6FQaJ3XJ02axGmnncYJJ5xQgtJtvIr6P+KSSy7hggsu4JZbbqGmpoa5c+eWukhlr62tjbPPPhvP8/B9n7FjxzJ79mxM0+Sqq65i9uzZZDIZRo0axdVXXw2wwdfWtDq8Z82axSmnnMIvf/lL4vF4sQ+xrF1++eU88sgjrFy5kpNOOom6ujrmz5+/1nuOPfbYEpVuy7O+7/ub3/wm3/72t9ltt9247rrreOONNzBNE8dxuOqqq2hsbFxnW2pp94+FCxfy85//nG233ZYZM2YAsNVWW3HzzTcX3mMYBieffHKpirjRjCAIglIXQkRERPqmYrrKRUREtgQKbhERkTKi4BYRESkjCm4REZEyouAWEREpIwpuERGRMqLgFqkgL7zwAjNmzGCPPfZgzz33ZMaMGbz66qvcc889HHfccaUunoj0QUVNwCJSyRKJBKeffjqXXHIJBx98MLlcjhdeeOFjZ5ISkcFLLW6RCrFo0SIgv6StZVlEIhEmT56M4zjMnj2bl19+mU9/+tOF5Vaz2Sxz585lv/324/Of/zyzZs0inU4D+eUP9913X2699VYmTZrElClT+POf/1yyYxOpJApukQoxZswYLMvi+9//Pv/4xz/o7OwEYOzYscyZM4cJEyawYMGCwhzy11xzDYsWLeLee+/lkUceoaWlZa1pIleuXEl7eztPPvkkP/7xj5k1axbvvfdeSY5NpJIouEUqRDwe53//938xDIOLL76Yvfbai9NPP52VK1eu894gCLjrrru46KKLqKurIx6Pc9ppp60zz/nMmTMJhULsueeefOELX+Avf/lLsQ5HpGJpjFukgowdO5Yf//jHALz77rucd955XHHFFUyePHmt961atYpUKsWRRx5ZeC4IAnzfLzyuqakhFosVHo8cOZKWlpYBPgIRUXCLVKixY8dy5JFH8oc//IF99tlnrdfq6+uJRCLMnz+f4cOHf+znu7q66OnpKYR3c3MzO+yww4CXW6TSqatcpEK8++67/PrXv2b58uVAPmgfeOABdt99dxoaGlixYgXZbBbIL7169NFHc8UVV9DW1gbk15F+8skn19rmjTfeSDab5YUXXuDvf//7Ouuqi0j/U4tbpELE43FeeeUVfvOb39Dd3U11dTVf/OIXOf/88wmFQmy//fZMnjwZwzB47rnnOO+887j55ps55phjaG9vZ/jw4Rx33HGF1vnQoUOpqalhn332IRqNcskllzB27NgSH6XIlk/rcYvIRlsd7E888USpiyJScdRVLiIiUkYU3CIiImVEXeUiIiJlRC1uERGRMqLgFhERKSMKbhERkTKi4BYRESkjCm4REZEyouAWEREpI/8P8BUzK1ly8iEAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# This uses the downloaded data from the Tensorboard visualization on the\n",
    "# Hugging Face Hub page of the three pre-trained models.\n",
    "small = pd.read_csv(\"small.csv\")\n",
    "base = pd.read_csv(\"base.csv\")\n",
    "large = pd.read_csv(\"large.csv\")\n",
    "\n",
    "small[\"Model\"] = \"IT5 Small\"\n",
    "base[\"Model\"] = \"IT5 Base\"\n",
    "large[\"Model\"] = \"IT5 Large\"\n",
    "\n",
    "data = pd.concat([small, base, large], ignore_index=True)\n",
    "data.rename(columns={\"Value\": \"Loss\"}, inplace=True)\n",
    "\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "sns.set_theme(style=\"darkgrid\")\n",
    "sns.set(rc={\"figure.figsize\":(8, 4)})\n",
    "g = sns.lineplot(data=data, x=\"Step\", y=\"Loss\", hue=\"Model\")\n",
    "g.set_xticklabels(['0','0', '500K','1M','1.5M','2M'])\n",
    "plt.savefig(\"loss.pdf\")"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "f9f85d98b63f393548f3009c8d52d8286e609a1467b1184fe464fb700873fbd3"
  },
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit ('venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}