human_annotation/create_pairwise_comparisons.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datasets\n",
    "import csv\n",
    "import json\n",
    "import numpy as np\n",
    "import random\n",
    "import itertools\n",
    "\n",
    "PREDICTIONS_PATH = \"../predictions\"\n",
    "MODEL_PAIRS = [\n",
    "    ('bart_base_asqa_dpr@3', 'bart_large_asqa_dpr@3'),\n",
    "    ('bart_large_asqa_dpr@3', 'bart_large_eli5_asqa_dpr@3'),\n",
    "    ('t5_base_asqa_dpr@3', 't5_base_msmarco_asqa_dpr@3'),\n",
    "    ('t5_base_asqa_dpr@3', 'bart_base_asqa_dpr@3'),\n",
    "    ('bart_large_asqa_bm25@3', 'bart_large_asqa_dpr@3'),\n",
    "    ('bart_large_asqa_bm25@1', 'bart_large_asqa_bm25@3'),\n",
    "    ('bart_large_asqa_bm25@3', 'bart_large_asqa_bm25@5'),\n",
    "    ('bart_large_asqa_dpr@1', 'bart_large_asqa_dpr@3'),\n",
    "    ('bart_large_asqa_dpr@3', 'bart_large_asqa_dpr@5'),\n",
    "    ('bart_large_eli5_asqa_dpr@3', 'reference'),\n",
    "    ('dpr@1', 'bart_base_asqa_dpr@1'),\n",
    "    ('dpr@1', 'bart_large_asqa_dpr@1'),\n",
    "]\n",
    "NUM_QUESTIONS = 4\n",
    "\n",
    "# Load the dataset\n",
    "with open('../dataset/ASQA.json', 'r') as f:\n",
    "    asqa_dataset = json.load(f)\n",
    "    asqa_dataset = asqa_dataset['dev']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_question_predictions(model, key):\n",
    "    if model == 'reference':\n",
    "        # Get long answer from ASQA dataset\n",
    "        return asqa_dataset[key]['annotations'][0]['long_answer']\n",
    "    \n",
    "    with open(f\"{PREDICTIONS_PATH}/{model}.json\") as f:\n",
    "        predictions = json.load(f)\n",
    "        return predictions[key]\n",
    "\n",
    "def get_random_question(model1, model2):\n",
    "    # Find question which predictions have at least one gold answer in them\n",
    "    random_question = np.random.choice(list(asqa_dataset.keys()), 1)[0]\n",
    "    \n",
    "    gold_answers = set(itertools.chain(*[p['short_answers'] for p in asqa_dataset[random_question]['qa_pairs']]))\n",
    "    prediction1 = get_question_predictions(model1, random_question)\n",
    "    prediction2 = get_question_predictions(model2, random_question)\n",
    "    gold_in_prediction1 = itertools.chain(*[g.split(\" \") for g in gold_answers if g.lower() in prediction1.lower()])\n",
    "    gold_in_prediction2 = itertools.chain(*[g.split(\" \") for g in gold_answers if g.lower() in prediction2.lower()])\n",
    "    \n",
    "    if len(list(gold_in_prediction1)) > 0 or len(list(gold_in_prediction2)) > 0:\n",
    "        asqa_dataset.pop(random_question)\n",
    "        return random_question, prediction1, prediction2\n",
    "    else:\n",
    "        return get_random_question(model1, model2)\n",
    "\n",
    "for (model1, model2) in MODEL_PAIRS:\n",
    "    with open(f\"./comparisons/{model1}_vs_{model2}_setup.tsv\", \"wt\", encoding='cp1252') as out_file:\n",
    "        # write to tsv file\n",
    "        tsv_writer = csv.writer(out_file, delimiter='\\t')\n",
    "        tsv_writer.writerow(['key', 'model1', 'model2', 'prediction1', 'prediction2'])\n",
    "        # select random questions from dataset\n",
    "        for i in range(NUM_QUESTIONS):\n",
    "            # shuffle models so they are in random order\n",
    "            model1_random = random.choice((model1, model2))\n",
    "            model2_random = model1 if model1_random == model2 else model2\n",
    "            sample_id, prediction1, prediction2 = get_random_question(model1_random, model2_random)\n",
    "            tsv_writer.writerow([sample_id, model1_random, model2_random, prediction1, prediction2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "#Make files ready for drive\n",
    "for file in os.listdir(\"./comparisons\"):\n",
    "    out_file = file.replace(\"_setup\", \"\")\n",
    "    os.system(f\"python preparation.py --asqa ../dataset/ASQA.json --setup ./comparisons/{file} --dst ./comparisons_rfd/{out_file}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.makedirs(\"./results_of_comparisons\", exist_ok=True)\n",
    "\n",
    "#Make files ready for drive\n",
    "for file in os.listdir(\"./comparisons\"):\n",
    "    out_file = file.replace(\"_setup.tsv\", \"\")\n",
    "    !python analysis.py --setup ./comparisons/{file} --comparisons ./human_evaluation/{out_file}/AnnotationInterface.xlsx --dst ./results_of_comparisons/{out_file}.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# flatten list of tuples\n",
    "def flatten(l):\n",
    "    return set([item for sublist in l for item in sublist])\n",
    "\n",
    "MODELS = flatten(MODEL_PAIRS)\n",
    "he_metrics = [\"Accuracy\", \"Comprehensiveness\", \"Fluency\", \"Overall\", \"count\"]\n",
    "results_dict = {m : {metric : 0 for metric in he_metrics} for m in MODELS}\n",
    "\n",
    "for file in os.listdir(\"./results_of_comparisons\"):\n",
    "    with open(f\"./results_of_comparisons/{file}\", \"rt\", encoding='cp1252') as f:\n",
    "        tsv_reader = csv.reader(f, delimiter='\\t')\n",
    "        next(tsv_reader)\n",
    "        for row in tsv_reader:\n",
    "            _, _, acc1, acc2, comp, fluency, overall, model1, model2, sample_id = row\n",
    "            for model, acc in zip([model1, model2], [acc1, acc2]):\n",
    "                results_dict[model][\"Accuracy\"] += float(acc)\n",
    "                results_dict[model][\"Comprehensiveness\"] += 0.5 if str(comp) == 'tie' else 0\n",
    "                results_dict[model][\"Fluency\"] += 0.5 if str(fluency) == 'tie' else 0\n",
    "                results_dict[model][\"Overall\"] += 0.5 if str(overall) == 'tie' else 0\n",
    "                results_dict[model][\"count\"] += 1\n",
    "            \n",
    "            if comp == 'left':\n",
    "                results_dict[model1][\"Comprehensiveness\"] += 1\n",
    "            elif comp == 'right':\n",
    "                results_dict[model2][\"Comprehensiveness\"] += 1\n",
    "\n",
    "            if fluency == 'left':\n",
    "                results_dict[model1][\"Fluency\"] += 1\n",
    "            elif fluency == 'right':\n",
    "                results_dict[model2][\"Fluency\"] += 1\n",
    "            \n",
    "            if overall == 'left':\n",
    "                results_dict[model1][\"Overall\"] += 1\n",
    "            elif overall == 'right':\n",
    "                results_dict[model2][\"Overall\"] += 1\n",
    "\n",
    "results_dict = {m : {metric : results_dict[m][metric] / results_dict[m][\"count\"] for metric in he_metrics} for m in MODELS}\n",
    "\n",
    "with open(\"results.json\", \"wt\") as f:\n",
    "    json.dump(results_dict, f, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_json(\"results.json\").T * 100\n",
    "\n",
    "# drop column count\n",
    "df = df.drop(columns=[\"count\"])\n",
    "\n",
    "# map all values to two decimal places\n",
    "df = df.applymap(lambda x: float(f\"{x:.2f}\"))\n",
    "\n",
    "print(df.sort_values(by=['Overall'], ascending=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_automated = {\n",
    "    \"dpr@1\":{\n",
    "        \"rougeLsum\": 31.375407192067286,\n",
    "        \"length\": 103.46729957805907,\n",
    "        \"str_em\": 29.347749648382564,\n",
    "        \"QA-EM\": 14.418073136427568,\n",
    "        \"QA-F1\": 17.534081701452774,\n",
    "        \"QA-Hit\": 3.5864978902953584,\n",
    "        \"ovscore\": 23.4550411025446\n",
    "    },\n",
    "    \"bart_large_asqa_dpr@1\":{\n",
    "        \"rougeLsum\": 34.17367623514718,\n",
    "        \"length\": 54.69409282700422,\n",
    "        \"str_em\": 25.97573839662447,\n",
    "        \"QA-EM\": 14.349507735583686,\n",
    "        \"QA-F1\": 18.122323894945254,\n",
    "        \"QA-Hit\": 3.059071729957806,\n",
    "        \"ovscore\": 24.885868066320903\n",
    "    },\n",
    "    \"bart_large_asqa_dpr@3\": {\n",
    "        \"rougeLsum\": 36.42172293222454,\n",
    "        \"length\": 62.45675105485232,\n",
    "        \"str_em\": 29.952531645569618,\n",
    "        \"QA-EM\": 16.547116736990155,\n",
    "        \"QA-F1\": 20.77815216309937,\n",
    "        \"QA-Hit\": 3.9029535864978904,\n",
    "        \"ovscore\": 27.509563812027398\n",
    "    },\n",
    "    \"bart_large_asqa_dpr@5\": {\n",
    "        \"rougeLsum\": 36.61926015496989,\n",
    "        \"length\": 63.333333333333336,\n",
    "        \"str_em\": 29.755625879043603,\n",
    "        \"QA-EM\": 16.264064697609,\n",
    "        \"QA-F1\": 20.53591318397746,\n",
    "        \"QA-Hit\": 3.481012658227848,\n",
    "        \"ovscore\": 27.4227997732534\n",
    "    },\n",
    "    \"bart_large_asqa_bm25@1\":{  \n",
    "        \"rougeLsum\": 31.81947228874431,\n",
    "        \"length\": 53.792194092827,\n",
    "        \"str_em\": 16.654360056258792,\n",
    "        \"QA-EM\": 8.649789029535864,\n",
    "        \"QA-F1\": 11.384725834565225,\n",
    "        \"QA-Hit\": 1.7932489451476792,\n",
    "        \"ovscore\": 19.03302309692025\n",
    "    },\n",
    "    \"bart_large_asqa_bm25@3\":{\n",
    "        \"rougeLsum\": 33.43383550706495,\n",
    "        \"length\": 59.677215189873415,\n",
    "        \"str_em\": 20.625879043600566,\n",
    "        \"QA-EM\": 10.114275668073136,\n",
    "        \"QA-F1\": 13.518431641657038,\n",
    "        \"QA-Hit\": 2.2151898734177213,\n",
    "        \"ovscore\": 21.259657095556914\n",
    "    },\n",
    "    \"bart_large_asqa_bm25@5\": {\n",
    "         \"rougeLsum\": 33.80983814622092,\n",
    "        \"length\": 58.994725738396625,\n",
    "        \"str_em\": 21.81082981715893,\n",
    "        \"QA-EM\": 11.031997187060478,\n",
    "        \"QA-F1\": 14.552247321904103,\n",
    "        \"QA-Hit\": 2.7426160337552745,\n",
    "        \"ovscore\": 22.181278741708166\n",
    "    },\n",
    "    \"bart_large_eli5_asqa_dpr@3\":{\n",
    "        \"rougeLsum\": 36.64911989014963,\n",
    "        \"length\": 59.71413502109704,\n",
    "        \"str_em\": 30.253164556962027,\n",
    "        \"QA-EM\": 16.51898734177215,\n",
    "        \"QA-F1\": 20.98682683459458,\n",
    "        \"QA-Hit\": 3.2700421940928273,\n",
    "        \"ovscore\": 27.733530838587182\n",
    "    },\n",
    "    \"t5_base_asqa_dpr@3\": {\n",
    "        \"rougeLsum\": 33.652080108780105,\n",
    "        \"length\": 57.82805907172996,\n",
    "        \"str_em\": 26.41350210970464,\n",
    "        \"QA-EM\": 13.959212376933896,\n",
    "        \"QA-F1\": 17.777224882873334,\n",
    "        \"QA-Hit\": 2.7426160337552745,\n",
    "        \"ovscore\": 24.45895737496291\n",
    "    },\n",
    "    \"t5_base_msmarco_asqa_dpr@3\": {\n",
    "        \"rougeLsum\": 33.68414993414289,\n",
    "        \"length\": 55.979957805907176,\n",
    "        \"str_em\": 26.042545710267227,\n",
    "        \"QA-EM\": 14.228199718706048,\n",
    "        \"QA-F1\": 18.058298780795056,\n",
    "        \"QA-Hit\": 2.848101265822785,\n",
    "        \"ovscore\": 24.66330155692563\n",
    "    },\n",
    "    \"bart_base_asqa_dpr@1\":{\n",
    "        \"rougeLsum\": 33.06625765989156,\n",
    "        \"length\": 52.391350210970465,\n",
    "        \"str_em\": 24.226441631504926,\n",
    "        \"QA-EM\": 12.594936708860757,\n",
    "        \"QA-F1\": 16.07544034144928,\n",
    "        \"QA-Hit\": 2.320675105485232,\n",
    "        \"ovscore\": 23.055469032890592\n",
    "    },\n",
    "    \"bart_base_asqa_dpr@3\": {\n",
    "        \"rougeLsum\": 33.88441978057487,\n",
    "        \"length\": 57.11708860759494,\n",
    "        \"str_em\": 25.149437412095637,\n",
    "        \"QA-EM\": 12.645921237693392,\n",
    "        \"QA-F1\": 16.532482741404735,\n",
    "        \"QA-Hit\": 2.848101265822785,\n",
    "        \"ovscore\": 23.66840901338464\n",
    "    }\n",
    "}\n",
    "\n",
    "model_human = df\n",
    "automated_df = pd.DataFrame(model_automated).T\n",
    "human_df = pd.DataFrame(model_human)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save correlation between all automated and human metrics in a dict\n",
    "correlations = {}\n",
    "for human_stat in human_df.columns:\n",
    "    correlations[human_stat] = {}\n",
    "    for automated_stat in automated_df.columns:\n",
    "        correlations[human_stat][automated_stat] = human_df[human_stat].corr(automated_df[automated_stat], method='pearson')\n",
    "\n",
    "pd.DataFrame(correlations).T.drop(columns=[\"length\", \"QA-EM\", \"QA-Hit\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create pairwise comparisons table\n",
    "\n",
    "for file in os.listdir(\"./results_of_comparisons\"):\n",
    "    pairwise_results_dict = {}\n",
    "    with open(f\"./results_of_comparisons/{file}\", \"rt\", encoding='cp1252') as f:\n",
    "        file = file.replace(\".tsv\", \"\")\n",
    "        pairwise_results_dict = {k : {metric : 0 for metric in he_metrics} for k in file.split(\"_vs_\")}\n",
    "        tsv_reader = csv.reader(f, delimiter='\\t')\n",
    "        next(tsv_reader)\n",
    "        for row in tsv_reader:\n",
    "            _, _, acc1, acc2, comp, fluency, overall, model1, model2, sample_id = row\n",
    "            for model, acc in zip([model1, model2], [acc1, acc2]):\n",
    "                pairwise_results_dict[model][\"Accuracy\"] += float(acc)\n",
    "                pairwise_results_dict[model][\"Comprehensiveness\"] += 0.5 if str(comp) == 'tie' else 0\n",
    "                pairwise_results_dict[model][\"Fluency\"] += 0.5 if str(fluency) == 'tie' else 0\n",
    "                pairwise_results_dict[model][\"Overall\"] += 0.5 if str(overall) == 'tie' else 0\n",
    "                pairwise_results_dict[model][\"count\"] += 1\n",
    "            \n",
    "            if comp == 'left':\n",
    "                pairwise_results_dict[model1][\"Comprehensiveness\"] += 1\n",
    "            elif comp == 'right':\n",
    "                pairwise_results_dict[model2][\"Comprehensiveness\"] += 1\n",
    "\n",
    "            if fluency == 'left':\n",
    "                pairwise_results_dict[model1][\"Fluency\"] += 1\n",
    "            elif fluency == 'right':\n",
    "                pairwise_results_dict[model2][\"Fluency\"] += 1\n",
    "            \n",
    "            if overall == 'left':\n",
    "                pairwise_results_dict[model1][\"Overall\"] += 1\n",
    "            elif overall == 'right':\n",
    "                pairwise_results_dict[model2][\"Overall\"] += 1\n",
    "            \n",
    "        pairwise_results_dict = {k: {k2: v2 / v[\"count\"] for k2, v2 in v.items()} for k, v in pairwise_results_dict.items()}\n",
    "\n",
    "        with open(f\"pairwise_results/{file}.json\", \"wt\") as f:\n",
    "            json.dump(pairwise_results_dict, f, indent=4)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "for file in os.listdir(\"./pairwise_results\"):\n",
    "    df = pd.read_json(f\"./pairwise_results/{file}\").T\n",
    "    # drop column count\n",
    "    df = df.drop(columns=[\"count\"])\n",
    "    # map all values to two decimal places\n",
    "    df = df.applymap(lambda x: float(f\"{x:.2f}\"))\n",
    "    display(df)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('ir2')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "b0e461ed7634304746d0b9e086c06a331048511f130eebac394163c37fd80203"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}