Evaluation-Adaptive-MT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "A100",
      "gpuClass": "premium",
      "mount_file_id": "1JfPdYB6T_MU3BNgFbY-Xa6JW76TLdPsR",
      "authorship_tag": "ABX9TyOPAwJRRd2R51gMFw0c3TLr",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ymoslem/Adaptive-MT-LLM-Fine-tuning/blob/main/Evaluation-Adaptive-MT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Evaluation with BLEU, chrF++, TER, and COMET\n",
        "\n",
        "This notebook is part of the repository [Adaptive-MT-LLM-Fine-tuning](https://github.com/ymoslem/Adaptive-MT-LLM-Fine-tuning)."
      ],
      "metadata": {
        "id": "71AD5l3zazsF"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load files"
      ],
      "metadata": {
        "id": "xTcWVCFXjDGM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "data_path = \"/content/drive/MyDrive/data/\"\n",
        "directory = os.path.join(data_path, \"spanish\")\n",
        "\n",
        "os.chdir(directory)\n",
        "os.getcwd()"
      ],
      "metadata": {
        "id": "vLhlhV_G3xa8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load test datasets\n",
        "\n",
        "source_test_file = \"all-filtered.es.real.test\"\n",
        "target_test_file = \"all-filtered.en.real.test\"\n",
        "\n",
        "with open(source_test_file, encoding=\"utf-8\") as source, open(target_test_file, encoding=\"utf-8\") as target:\n",
        "  source_sentences = [sent.strip() for sent in source.readlines()]\n",
        "  target_sentences = [sent.strip() for sent in target.readlines()]\n",
        "\n",
        "print(source_sentences[0])\n",
        "print(target_sentences[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lEiTRW7CjIAx",
        "outputId": "fbe30964-cbc0-4a58-97c8-48f7ddb05b21"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Período de validez después de abierto el envase: 10 horas.\n",
            "Shelf life after first opening the container: 10 hours.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Read the translations\n",
        "\n",
        "translations_file_name = \"translations.en\"\n",
        "\n",
        "with open(translations_file_name, encoding=\"utf-8\") as translated:\n",
        "  translations = [sent.strip() for sent in translated.readlines()]\n",
        "\n",
        "print(translations_file_name, \"\\n\")\n",
        "print(*translations[0:5], sep=\"\\n\")"
      ],
      "metadata": {
        "id": "dXgfm6e9jW9-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(translations)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aukHerV4By2J",
        "outputId": "aa823814-694d-44e2-d2e3-d5f38889091a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10000"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check if there are missing translations\n",
        "count = 0\n",
        "for idx, line in enumerate(translations):\n",
        "  if len(line.strip()) == 0:\n",
        "    count += 1\n",
        "    print(idx,\n",
        "          source_sentences[idx].strip(),\n",
        "          target_sentences[idx].strip(),\n",
        "          sep=\"\\n\",\n",
        "          end=\"\\n\\n\")\n",
        "print(\"Missing translations:\", count)"
      ],
      "metadata": {
        "id": "HABK8IcXbKn4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b38eae51-1f8f-42c5-f43d-95a15477fb8c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Missing translations: 0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluation"
      ],
      "metadata": {
        "id": "AzALzfsu7vl4"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Calculate BLEU, chrF++ and TER"
      ],
      "metadata": {
        "id": "8eRtpHae9-mt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install sacrebleu sentencepiece -q"
      ],
      "metadata": {
        "id": "l_NWEb9XZ-2b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a64d7105-b8ce-42e4-d19d-7cbac7138a02"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.7/119.7 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m33.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import sacrebleu\n",
        "\n",
        "references = target_sentences\n",
        "translations = translations\n",
        "\n",
        "\n",
        "# Calculate BLEU\n",
        "bleu = sacrebleu.corpus_bleu(translations, [references])  # for spBLEU: tokenize='flores200'\n",
        "bleu = round(bleu.score, 2)\n",
        "print(\"BLEU:\", bleu)\n",
        "\n",
        "# Calculate chrF++\n",
        "chrf = sacrebleu.corpus_chrf(translations, [references], word_order=2)  # for chrF++ word_order=2\n",
        "chrf = round(chrf.score, 2)\n",
        "print(\"chrF++:\", chrf)\n",
        "\n",
        "# Calculate TER\n",
        "metric = sacrebleu.metrics.TER()\n",
        "ter = metric.corpus_score(translations, [references])\n",
        "ter = round(ter.score, 2)\n",
        "print(\"TER:\", ter)"
      ],
      "metadata": {
        "id": "RhJmD1F47xVi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Calculate COMET"
      ],
      "metadata": {
        "id": "0BOmakuL-HjO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ['TRANSFORMERS_CACHE'] = \"/content/drive/MyDrive/models/\"\n",
        "\n",
        "!pip3 install unbabel-comet -q\n",
        "\n",
        "# !pip3 install git+https://github.com/Unbabel/COMET.git -q"
      ],
      "metadata": {
        "id": "5gq_5b5wyc5C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from comet import download_model, load_from_checkpoint\n",
        "import pandas as pd\n",
        "\n",
        "references = target_sentences\n",
        "translations = translations\n",
        "\n",
        "# Calculate COMET\n",
        "df = pd.DataFrame({\"src\":source_sentences, \"mt\":translations, \"ref\":references})\n",
        "data = df.to_dict('records')\n",
        "# model_path = download_model(\"wmt20-comet-da\")  # to download the model if you did not yet\n",
        "model_path = \"/content/drive/MyDrive/models/wmt20-comet-da/checkpoints/model.ckpt\"\n",
        "model = load_from_checkpoint(model_path)\n",
        "\n",
        "seg_scores, sys_score = model.predict(data, batch_size=128, gpus=1).values()\n",
        "comet = round(sys_score*100, 2)\n",
        "print(\"COMET:\", comet)"
      ],
      "metadata": {
        "id": "PhLOXCyMrOxD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert scores to polars dataframe\n",
        "import polars as pl\n",
        "\n",
        "df = pl.DataFrame({\"BLEU\": bleu,\n",
        "                   \"chrF++\": chrf,\n",
        "                   \"TER\": ter,\n",
        "                   \"COMET\": comet},\n",
        "                  )\n",
        "\n",
        "df.head()"
      ],
      "metadata": {
        "id": "Z6afMaXTHTcT"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}