summ_t5_encoder_ori.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/uj-user/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import pandas as pd\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "from datasets import load_dataset, DatasetDict, Dataset\n",
    "from transformers import T5Tokenizer, T5ForConditionalGeneration,  DataCollatorWithPadding\n",
    "\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = T5Tokenizer.from_pretrained(\"google/flan-t5-small\")\n",
    "model = T5ForConditionalGeneration.from_pretrained(\"google/flan-t5-small\", output_hidden_states=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset parquet (/home/uj-user/.cache/huggingface/datasets/HuggingFaceH4___parquet/HuggingFaceH4--databricks_dolly_15k-6252f3495e7d2b9d/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n",
      "100%|██████████| 1/1 [00:00<00:00, 768.89it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['category', 'instruction', 'input', 'output'],\n",
       "        num_rows: 15015\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = load_dataset('HuggingFaceH4/databricks_dolly_15k')\n",
    "#dataset = load_dataset('HuggingFaceH4/asss')\n",
    "df = pd.DataFrame(dataset['train'])\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset cnn_dailymail (/home/uj-user/.cache/huggingface/datasets/cnn_dailymail/3.0.0/3.0.0/1b3c71476f6d152c31c1730e83ccb08bcf23e348233f4fcc11e182248e6bf7de)\n",
      "100%|██████████| 3/3 [00:00<00:00, 353.21it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['article', 'highlights', 'id'],\n",
       "        num_rows: 287113\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['article', 'highlights', 'id'],\n",
       "        num_rows: 13368\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['article', 'highlights', 'id'],\n",
       "        num_rows: 11490\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_dataset('cnn_dailymail', '3.0.0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>instruction</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>When did Virgin Australia start operating?</td>\n",
       "      <td>Virgin Australia, the trading name of Virgin A...</td>\n",
       "      <td>Virgin Australia commenced services on 31 Augu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>When was Tomoaki Komorida born?</td>\n",
       "      <td>Komorida was born in Kumamoto Prefecture on Ju...</td>\n",
       "      <td>Tomoaki Komorida was born on July 10,1981.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>information_extraction</td>\n",
       "      <td>If I have more pieces at the time of stalemate...</td>\n",
       "      <td>Stalemate is a situation in chess where the pl...</td>\n",
       "      <td>No. \\nStalemate is a drawn position. It doesn'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>Given a reference text about Lollapalooza, whe...</td>\n",
       "      <td>Lollapalooza /ˌlɒləpəˈluːzə/ (Lolla) is an ann...</td>\n",
       "      <td>Lollapalooze is an annual musical festival hel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>information_extraction</td>\n",
       "      <td>Who was John Moses Browning?</td>\n",
       "      <td>John Moses Browning (January 23, 1855 – Novemb...</td>\n",
       "      <td>John Moses Browning is one of the most well-kn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4593</th>\n",
       "      <td>information_extraction</td>\n",
       "      <td>When did Phil Knight announce he would step do...</td>\n",
       "      <td>Throughout the 1980s, Nike expanded its produc...</td>\n",
       "      <td>Phil Knight announced he would step down in 20...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4594</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>What are common florals found in Zigalga Natio...</td>\n",
       "      <td>Zigalga National Park (Russian: Национальный п...</td>\n",
       "      <td>Zigalga National Park has the majority of its ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4595</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>What is linux Bootloader</td>\n",
       "      <td>A bootloader, also spelled as boot loader or c...</td>\n",
       "      <td>A bootloader is a program written in machine c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4596</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>What is one-child policy?</td>\n",
       "      <td>The term one-child policy refers to a populati...</td>\n",
       "      <td>The \"one-child policy\" was a Chinese populatio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4597</th>\n",
       "      <td>summarization</td>\n",
       "      <td>What is a laser and who created it?</td>\n",
       "      <td>A laser is a device that emits light through a...</td>\n",
       "      <td>A laser is a device that emits light from an e...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4598 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    category  \\\n",
       "0                  closed_qa   \n",
       "1                  closed_qa   \n",
       "2     information_extraction   \n",
       "3                  closed_qa   \n",
       "4     information_extraction   \n",
       "...                      ...   \n",
       "4593  information_extraction   \n",
       "4594               closed_qa   \n",
       "4595               closed_qa   \n",
       "4596               closed_qa   \n",
       "4597           summarization   \n",
       "\n",
       "                                            instruction  \\\n",
       "0            When did Virgin Australia start operating?   \n",
       "1                       When was Tomoaki Komorida born?   \n",
       "2     If I have more pieces at the time of stalemate...   \n",
       "3     Given a reference text about Lollapalooza, whe...   \n",
       "4                          Who was John Moses Browning?   \n",
       "...                                                 ...   \n",
       "4593  When did Phil Knight announce he would step do...   \n",
       "4594  What are common florals found in Zigalga Natio...   \n",
       "4595                           What is linux Bootloader   \n",
       "4596                          What is one-child policy?   \n",
       "4597                What is a laser and who created it?   \n",
       "\n",
       "                                                  input  \\\n",
       "0     Virgin Australia, the trading name of Virgin A...   \n",
       "1     Komorida was born in Kumamoto Prefecture on Ju...   \n",
       "2     Stalemate is a situation in chess where the pl...   \n",
       "3     Lollapalooza /ˌlɒləpəˈluːzə/ (Lolla) is an ann...   \n",
       "4     John Moses Browning (January 23, 1855 – Novemb...   \n",
       "...                                                 ...   \n",
       "4593  Throughout the 1980s, Nike expanded its produc...   \n",
       "4594  Zigalga National Park (Russian: Национальный п...   \n",
       "4595  A bootloader, also spelled as boot loader or c...   \n",
       "4596  The term one-child policy refers to a populati...   \n",
       "4597  A laser is a device that emits light through a...   \n",
       "\n",
       "                                                 output  \n",
       "0     Virgin Australia commenced services on 31 Augu...  \n",
       "1            Tomoaki Komorida was born on July 10,1981.  \n",
       "2     No. \\nStalemate is a drawn position. It doesn'...  \n",
       "3     Lollapalooze is an annual musical festival hel...  \n",
       "4     John Moses Browning is one of the most well-kn...  \n",
       "...                                                 ...  \n",
       "4593  Phil Knight announced he would step down in 20...  \n",
       "4594  Zigalga National Park has the majority of its ...  \n",
       "4595  A bootloader is a program written in machine c...  \n",
       "4596  The \"one-child policy\" was a Chinese populatio...  \n",
       "4597  A laser is a device that emits light from an e...  \n",
       "\n",
       "[4598 rows x 4 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[df != ''].dropna()\n",
    "df.reset_index(drop=True, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "closed_qa                 1823\n",
       "information_extraction    1512\n",
       "summarization             1263\n",
       "Name: category, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['category'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>instruction</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>When did Virgin Australia start operating?</td>\n",
       "      <td>Virgin Australia, the trading name of Virgin A...</td>\n",
       "      <td>Virgin Australia commenced services on 31 Augu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>When was Tomoaki Komorida born?</td>\n",
       "      <td>Komorida was born in Kumamoto Prefecture on Ju...</td>\n",
       "      <td>Tomoaki Komorida was born on July 10,1981.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>Given a reference text about Lollapalooza, whe...</td>\n",
       "      <td>Lollapalooza /ˌlɒləpəˈluːzə/ (Lolla) is an ann...</td>\n",
       "      <td>Lollapalooze is an annual musical festival hel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>Who was Kyle Van Zyl playing against when he s...</td>\n",
       "      <td>Van Zyl joined the Eastern Province Kings Acad...</td>\n",
       "      <td>Kyle Van Zyl was playing against Boland U21 wh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>Who was the first to record the Needles and Pi...</td>\n",
       "      <td>\"Needles and Pins\" is a rock song credited to ...</td>\n",
       "      <td>The first artist to record Needles and Pins so...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1818</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>How many types of cat does the world cat feder...</td>\n",
       "      <td>As of 2023, The International Cat Association ...</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1819</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>What is the history of coffee?</td>\n",
       "      <td>The history of coffee dates back to centuries ...</td>\n",
       "      <td>Much of the early history of coffee was tied t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1820</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>What are common florals found in Zigalga Natio...</td>\n",
       "      <td>Zigalga National Park (Russian: Национальный п...</td>\n",
       "      <td>Zigalga National Park has the majority of its ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1821</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>What is linux Bootloader</td>\n",
       "      <td>A bootloader, also spelled as boot loader or c...</td>\n",
       "      <td>A bootloader is a program written in machine c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1822</th>\n",
       "      <td>closed_qa</td>\n",
       "      <td>What is one-child policy?</td>\n",
       "      <td>The term one-child policy refers to a populati...</td>\n",
       "      <td>The \"one-child policy\" was a Chinese populatio...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1823 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       category                                        instruction  \\\n",
       "0     closed_qa         When did Virgin Australia start operating?   \n",
       "1     closed_qa                    When was Tomoaki Komorida born?   \n",
       "2     closed_qa  Given a reference text about Lollapalooza, whe...   \n",
       "3     closed_qa  Who was Kyle Van Zyl playing against when he s...   \n",
       "4     closed_qa  Who was the first to record the Needles and Pi...   \n",
       "...         ...                                                ...   \n",
       "1818  closed_qa  How many types of cat does the world cat feder...   \n",
       "1819  closed_qa                     What is the history of coffee?   \n",
       "1820  closed_qa  What are common florals found in Zigalga Natio...   \n",
       "1821  closed_qa                           What is linux Bootloader   \n",
       "1822  closed_qa                          What is one-child policy?   \n",
       "\n",
       "                                                  input  \\\n",
       "0     Virgin Australia, the trading name of Virgin A...   \n",
       "1     Komorida was born in Kumamoto Prefecture on Ju...   \n",
       "2     Lollapalooza /ˌlɒləpəˈluːzə/ (Lolla) is an ann...   \n",
       "3     Van Zyl joined the Eastern Province Kings Acad...   \n",
       "4     \"Needles and Pins\" is a rock song credited to ...   \n",
       "...                                                 ...   \n",
       "1818  As of 2023, The International Cat Association ...   \n",
       "1819  The history of coffee dates back to centuries ...   \n",
       "1820  Zigalga National Park (Russian: Национальный п...   \n",
       "1821  A bootloader, also spelled as boot loader or c...   \n",
       "1822  The term one-child policy refers to a populati...   \n",
       "\n",
       "                                                 output  \n",
       "0     Virgin Australia commenced services on 31 Augu...  \n",
       "1            Tomoaki Komorida was born on July 10,1981.  \n",
       "2     Lollapalooze is an annual musical festival hel...  \n",
       "3     Kyle Van Zyl was playing against Boland U21 wh...  \n",
       "4     The first artist to record Needles and Pins so...  \n",
       "...                                                 ...  \n",
       "1818                                                 69  \n",
       "1819  Much of the early history of coffee was tied t...  \n",
       "1820  Zigalga National Park has the majority of its ...  \n",
       "1821  A bootloader is a program written in machine c...  \n",
       "1822  The \"one-child policy\" was a Chinese populatio...  \n",
       "\n",
       "[1823 rows x 4 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[df['category'].str.contains('closed_qa')]\n",
    "df.reset_index(drop=True, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<seaborn.axisgrid.FacetGrid at 0x7f845dd86340>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAHpCAYAAABN+X+UAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA02ElEQVR4nO3de3BV5b3G8WcnkE0iubAJudUEAiigXOQiMa1SkBQIFrXSU6W0xUpBadBKrOXEekGm01Bp0SmN0M7h0o4irR3FFpUOd7QEKkikQckIJ5goSUBisrmGZOc9f3iymk1CgJBkv0m+n5k1k7XWu9f+vazEx3ddXcYYIwAAYKWgQBcAAAAujqAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxglqSMUZer1fcUg4AsA1BLenkyZOKjIzUyZMnA10KAAB+CGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWC2hQZ2dn6+abb1Z4eLhiYmJ09913q6CgwK/NuXPnlJGRoZ49e6p79+6aOnWqysrK/NoUFRXpjjvuUFhYmGJiYvT444+rpqamLbsCAECrCGhQb9++XRkZGdq1a5c2btyo6upqTZgwQadPn3bazJs3T3//+9/16quvavv27Tp69KjuueceZ73P59Mdd9yh8+fPa+fOnfrjH/+o1atX6+mnnw5ElwAAaFEuY4wJdBF1jh8/rpiYGG3fvl1jxoxRZWWlevXqpTVr1ujb3/62JOngwYMaNGiQcnNzdcstt+jtt9/WN7/5TR09elSxsbGSpOXLl2v+/Pk6fvy4QkJCGnxPVVWVqqqqnHmv16vExERVVlYqIiKibToLAMBlsOocdWVlpSTJ4/FIkvbu3avq6mqlpaU5bQYOHKikpCTl5uZKknJzczVkyBAnpCVp4sSJ8nq9OnDgQKPfk52drcjISGdKTExsrS4BAHBVrAnq2tpaPfroo/ra176mwYMHS5JKS0sVEhKiqKgov7axsbEqLS112tQP6br1desak5WVpcrKSmcqLi5u4d5cHp/Pp8LCQhUWFsrn8wWkBgCA3boEuoA6GRkZys/P17vvvtvq3+V2u+V2u1v9ey6lqKhIM3PeliStyEhXcnJygCsCANjGihH13LlztX79em3dulXXXnutszwuLk7nz59XRUWFX/uysjLFxcU5bS68Crxuvq6NzcI8sQrzxF66IQCgUwpoUBtjNHfuXL3++uvasmVLgxHlyJEj1bVrV23evNlZVlBQoKKiIqWmpkqSUlNT9e9//1vHjh1z2mzcuFERERG64YYb2qYjAAC0koAe+s7IyNCaNWv0xhtvKDw83DmnHBkZqdDQUEVGRmrmzJnKzMyUx+NRRESEHn74YaWmpuqWW26RJE2YMEE33HCDvv/97+u5555TaWmpnnzySWVkZFhxeBsAgKsR0KBetmyZJGns2LF+y1etWqX7779fkvT8888rKChIU6dOVVVVlSZOnKgXX3zRaRscHKz169drzpw5Sk1N1TXXXKMZM2Zo4cKFbdUNAABajVX3UQeK1+tVZGRkm91H7fP5VFRUpOLiYv3qneNyuaSl00ZwMRkAoAFrrvruTOqu9j5b+bm6J/TnED0A4KII6gDhSm8AwOWw4vYsAADQOIIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAi/EIUQuYWp+Ki4ud+aSkJAUHBwewIgCALQhqC5yt+FxP/PWoouKO60x5mVZkpPMmLQCAJILaGqE9YtQ9OiHQZQAALMM5agAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALBbQoN6xY4emTJmihIQEuVwurVu3zm+9y+VqdFq8eLHTpk+fPg3WL1q0qI17AgBA6whoUJ8+fVrDhg1TTk5Oo+tLSkr8ppUrV8rlcmnq1Kl+7RYuXOjX7uGHH26L8gEAaHVdAvnl6enpSk9Pv+j6uLg4v/k33nhD48aNU9++ff2Wh4eHN2jblKqqKlVVVTnzXq/3sj8LAEBbajfnqMvKyvTmm29q5syZDdYtWrRIPXv21PDhw7V48WLV1NQ0ua3s7GxFRkY6U2JiYmuVDQDAVQnoiPpK/PGPf1R4eLjuuecev+WPPPKIRowYIY/Ho507dyorK0slJSVasmTJRbeVlZWlzMxMZ97r9RLWAAArtZugXrlypaZPn65u3br5La8fuEOHDlVISIgefPBBZWdny+12N7ott9t90XUAANikXRz6fuedd1RQUKAf/ehHl2ybkpKimpoaHTlypPULAwCglbWLoF6xYoVGjhypYcOGXbJtXl6egoKCFBMT0waVAQDQugJ66PvUqVM6dOiQM19YWKi8vDx5PB4lJSVJ+vL88auvvqrf/OY3DT6fm5ur3bt3a9y4cQoPD1dubq7mzZun733ve+rRo0eb9eNy+Hw+FRUVSZKKi4tlTIALAgC0CwEN6j179mjcuHHOfN355hkzZmj16tWSpLVr18oYo2nTpjX4vNvt1tq1a7VgwQJVVVUpOTlZ8+bN8ztvbYuioiLNzHlbYZ5YnSg8oO4J/QNdEgCgHQhoUI8dO1bmEkPL2bNna/bs2Y2uGzFihHbt2tUapbWKME+sukcn6Ex5WaBLAQC0E+3iHDUAAJ0VQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYLaFDv2LFDU6ZMUUJCglwul9atW+e3/v7775fL5fKbJk2a5NemvLxc06dPV0REhKKiojRz5kydOnWqDXsBAEDrCWhQnz59WsOGDVNOTs5F20yaNEklJSXO9Morr/itnz59ug4cOKCNGzdq/fr12rFjh2bPnt3apQMA0Ca6BPLL09PTlZ6e3mQbt9utuLi4Rtd99NFH2rBhg9577z2NGjVKkrR06VJNnjxZv/71r5WQkNDo56qqqlRVVeXMe73eZvYAAIDWZf056m3btikmJkYDBgzQnDlzdOLECWddbm6uoqKinJCWpLS0NAUFBWn37t0X3WZ2drYiIyOdKTExsVX7AABAc1kd1JMmTdKf/vQnbd68Wb/61a+0fft2paeny+fzSZJKS0sVExPj95kuXbrI4/GotLT0otvNyspSZWWlMxUXF7dqPwAAaK6AHvq+lPvuu8/5eciQIRo6dKj69eunbdu2afz48c3ertvtltvtbokSAQBoVVaPqC/Ut29fRUdH69ChQ5KkuLg4HTt2zK9NTU2NysvLL3peGwCA9qRdBfWnn36qEydOKD4+XpKUmpqqiooK7d2712mzZcsW1dbWKiUlJVBlAgDQYgJ66PvUqVPO6FiSCgsLlZeXJ4/HI4/Ho2effVZTp05VXFycDh8+rJ/97Gfq37+/Jk6cKEkaNGiQJk2apFmzZmn58uWqrq7W3Llzdd999130im8AANqTgI6o9+zZo+HDh2v48OGSpMzMTA0fPlxPP/20goODtX//ft155526/vrrNXPmTI0cOVLvvPOO3/nll19+WQMHDtT48eM1efJk3XrrrfrDH/4QqC4BANCiAjqiHjt2rIwxF13/j3/845Lb8Hg8WrNmTUuWBQCANdrVOWoAADobghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIBfSlHZ+Dz+VRUVKTi4mI18f4RAAAaRVC3sqKiIs3MeVtnKz9X94T+gS4HANDOENRtIMwTG+gSAADtFOeoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwWECDeseOHZoyZYoSEhLkcrm0bt06Z111dbXmz5+vIUOG6JprrlFCQoJ+8IMf6OjRo37b6NOnj1wul9+0aNGiNu4JAACtI6BBffr0aQ0bNkw5OTkN1p05c0bvv/++nnrqKb3//vt67bXXVFBQoDvvvLNB24ULF6qkpMSZHn744bYoHwCAVtclkF+enp6u9PT0RtdFRkZq48aNfst+97vfafTo0SoqKlJSUpKzPDw8XHFxcZf9vVVVVaqqqnLmvV7vFVYOAEDbaFfnqCsrK+VyuRQVFeW3fNGiRerZs6eGDx+uxYsXq6ampsntZGdnKzIy0pkSExNbsWoAAJovoCPqK3Hu3DnNnz9f06ZNU0REhLP8kUce0YgRI+TxeLRz505lZWWppKRES5Ysuei2srKylJmZ6cx7vV7CGgBgpXYR1NXV1frOd74jY4yWLVvmt65+4A4dOlQhISF68MEHlZ2dLbfb3ej23G73RdcBAGAT6w9914X0J598oo0bN/qNphuTkpKimpoaHTlypG0KBACgFVk9oq4L6Y8//lhbt25Vz549L/mZvLw8BQUFKSYmpg0qbHmm1qfi4mJJUlJSkoKDgwNcEQAgkAIa1KdOndKhQ4ec+cLCQuXl5cnj8Sg+Pl7f/va39f7772v9+vXy+XwqLS2VJHk8HoWEhCg3N1e7d+/WuHHjFB4ertzcXM2bN0/f+9731KNHj0B166qcrfhcT/z1qEJC8rUiI13JycmBLgkAEEABDeo9e/Zo3Lhxznzd+eYZM2ZowYIF+tvf/iZJuummm/w+t3XrVo0dO1Zut1tr167VggULVFVVpeTkZM2bN8/vvHV7FNojhnPoAABJAQ7qsWPHyhhz0fVNrZOkESNGaNeuXS1dFgAA1rD+YjIAADozghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYLFmBXXfvn114sSJBssrKirUt2/fqy4KAAB8qVlBfeTIEfl8vgbLq6qq9Nlnn111UQAA4EtdrqTx3/72N+fnf/zjH4qMjHTmfT6fNm/erD59+rRYcQAAdHZXFNR33323JMnlcmnGjBl+67p27ao+ffroN7/5TYsVBwBAZ3dFQV1bWytJSk5O1nvvvafo6OhWKQoAAHzpioK6TmFhYUvXAQAAGtGsoJakzZs3a/PmzTp27Jgz0q6zcuXKqy4MAAA0M6ifffZZLVy4UKNGjVJ8fLxcLldL1wUAANTMoF6+fLlWr16t73//+y1dDwAAqKdZQX3+/Hl99atfbelaUI+p9am4uNiZT0pKUnBwcAArAgAEQrMeePKjH/1Ia9asaelaUM/Zis/1xF/f18OvvK+ZOW+rqKgo0CUBAAKgWSPqc+fO6Q9/+IM2bdqkoUOHqmvXrn7rlyxZ0iLFdXahPWLUPToh0GUAAAKoWUG9f/9+3XTTTZKk/Px8v3VcWAYAQMtpVlBv3bq1pesAAACN4DWXAABYrFkj6nHjxjV5iHvLli3NLggAAPxHs4K67vx0nerqauXl5Sk/P7/ByzoAAEDzNSuon3/++UaXL1iwQKdOnbqqggAAwH+06Dnq733vezznGwCAFtSiQZ2bm6tu3bq15CYBAOjUmnXo+5577vGbN8aopKREe/bs0VNPPdUihQEAgGYGdWRkpN98UFCQBgwYoIULF2rChAktUhgAAGhmUK9ataql6wAAAI24qnPUe/fu1UsvvaSXXnpJ+/btu+LP79ixQ1OmTFFCQoJcLpfWrVvnt94Yo6efflrx8fEKDQ1VWlqaPv74Y7825eXlmj59uiIiIhQVFaWZM2dy5TkAoMNoVlAfO3ZMt99+u26++WY98sgjeuSRRzRy5EiNHz9ex48fv+ztnD59WsOGDVNOTk6j65977jn99re/1fLly7V7925dc801mjhxos6dO+e0mT59ug4cOKCNGzdq/fr12rFjh2bPnt2cbgEAYJ1mBfXDDz+skydP6sCBAyovL1d5ebny8/Pl9Xr1yCOPXPZ20tPT9Ytf/ELf+ta3GqwzxuiFF17Qk08+qbvuuktDhw7Vn/70Jx09etQZeX/00UfasGGD/ud//kcpKSm69dZbtXTpUq1du1ZHjx5tTtcAALBKs4J6w4YNevHFFzVo0CBn2Q033KCcnBy9/fbbLVJYYWGhSktLlZaW5iyLjIxUSkqKcnNzJX15O1hUVJRGjRrltElLS1NQUJB279590W1XVVXJ6/X6TQAA2KhZQV1bW9vgHdSS1LVrV9XW1l51UZJUWloqSYqNjfVbHhsb66wrLS1VTEyM3/ouXbrI4/E4bRqTnZ2tyMhIZ0pMTGyRmgEAaGnNCurbb79dP/nJT/wOL3/22WeaN2+exo8f32LFtZasrCxVVlY6U3FxcaBLAgCgUc0K6t/97nfyer3q06eP+vXrp379+ik5OVler1dLly5tkcLi4uIkSWVlZX7Ly8rKnHVxcXE6duyY3/qamhqVl5c7bRrjdrsVERHhNwEAYKNm3UedmJio999/X5s2bdLBgwclSYMGDfI7n3y1kpOTFRcXp82bNztv6/J6vdq9e7fmzJkjSUpNTVVFRYX27t2rkSNHSvryFZu1tbVKSUlpsVoCzdT6nFF/UlKSgoODA1wRAKCtXFFQb9myRXPnztWuXbsUERGhb3zjG/rGN74hSaqsrNSNN96o5cuX67bbbrus7Z06dUqHDh1y5gsLC5WXlyePx6OkpCQ9+uij+sUvfqHrrrtOycnJeuqpp5SQkKC7775b0pf/czBp0iTNmjVLy5cvV3V1tebOnav77rtPCQkJV9I1q52t+FxP/PWoQkLytSIjXcnJyYEuCQDQRq4oqF944QXNmjWr0UPFkZGRevDBB7VkyZLLDuo9e/Zo3LhxznxmZqYkacaMGVq9erV+9rOf6fTp05o9e7YqKip06623asOGDX4v/nj55Zc1d+5cjR8/XkFBQZo6dap++9vfXkm32oXQHjFyu92BLgMA0MauKKg/+OAD/epXv7ro+gkTJujXv/71ZW9v7NixMsZcdL3L5dLChQu1cOHCi7bxeDxas2bNZX8nAADtyRVdTFZWVtbobVl1unTpckVPJgMAAE27oqD+yle+ovz8/Iuu379/v+Lj46+6KAAA8KUrCurJkyfrqaee8nvWdp2zZ8/qmWee0Te/+c0WKw4AgM7uis5RP/nkk3rttdd0/fXXa+7cuRowYIAk6eDBg8rJyZHP59PPf/7zVikUAIDO6IqCOjY2Vjt37tScOXOUlZXlXAjmcrk0ceJE5eTkNHjkJwAAaL4rfuBJ79699dZbb+mLL77QoUOHZIzRddddpx49erRGfQAAdGrNejKZJPXo0UM333xzS9YCAAAu0KxnfQMAgLZBUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIs1+1nfuDifz6eioiJJUnFxsf7/JWMAAFwxgroVFBUVaWbO2wrzxOpE4QF1T+gf6JIAAO0Uh75bSZgnVt2jExQaGR3oUgAA7RhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFugS6AFw+U+tTcXGxJCkpKUnBwcEBrggA0NoYUbcjZys+1xN/fV8zc95WUVFRoMsBALQB64O6T58+crlcDaaMjAxJ0tixYxuse+ihhwJcdesJ7RGjME9soMsAALQR6w99v/fee/L5fM58fn6+vvGNb+i//uu/nGWzZs3SwoULnfmwsLA2rREAgNZifVD36tXLb37RokXq16+fvv71rzvLwsLCFBcX19alAQDQ6qw/9F3f+fPn9dJLL+mBBx6Qy+Vylr/88suKjo7W4MGDlZWVpTNnzjS5naqqKnm9Xr8JAAAbWT+irm/dunWqqKjQ/fff7yz77ne/q969eyshIUH79+/X/PnzVVBQoNdee+2i28nOztazzz7bBhUDAHB12lVQr1ixQunp6UpISHCWzZ492/l5yJAhio+P1/jx43X48GH169ev0e1kZWUpMzPTmfd6vUpMTGy9wgEAaKZ2E9SffPKJNm3a1ORIWZJSUlIkSYcOHbpoULvdbrnd7havEQCAltZuzlGvWrVKMTExuuOOO5psl5eXJ0mKj49vg6oAAGhd7WJEXVtbq1WrVmnGjBnq0uU/JR8+fFhr1qzR5MmT1bNnT+3fv1/z5s3TmDFjNHTo0ABWDABAy2gXQb1p0yYVFRXpgQce8FseEhKiTZs26YUXXtDp06eVmJioqVOn6sknnwxQpQAAtKx2EdQTJkyQMabB8sTERG3fvj0AFQEA0DbazTlqAAA6I4IaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBi7eKlHPBnan0qLi525pOSkhQcHBzAigAArYWgbofOVnyuJ/56VFFxx3WmvEwrMtKVnJwc6LIAAK2AoG6nQnvEqHt0QqDLAAC0Ms5RAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAW6xLoAnB1TK1PxcXFkqSkpCQFBwcHuCIAQEtiRN3Ona34XE/89X3NzHlbRUVFgS4HANDCGFF3AKE9YuR2uwNdBgCgFTCiBgDAYlYH9YIFC+RyufymgQMHOuvPnTunjIwM9ezZU927d9fUqVNVVlYWwIoBAGhZVge1JN14440qKSlxpnfffddZN2/ePP3973/Xq6++qu3bt+vo0aO65557AlgtAAAty/pz1F26dFFcXFyD5ZWVlVqxYoXWrFmj22+/XZK0atUqDRo0SLt27dItt9zS1qUCANDirB9Rf/zxx0pISFDfvn01ffp058rmvXv3qrq6WmlpaU7bgQMHKikpSbm5uU1us6qqSl6v128CAMBGVgd1SkqKVq9erQ0bNmjZsmUqLCzUbbfdppMnT6q0tFQhISGKiory+0xsbKxKS0ub3G52drYiIyOdKTExsRV7AQBA81l96Ds9Pd35eejQoUpJSVHv3r31l7/8RaGhoc3eblZWljIzM515r9dLWAMArGT1iPpCUVFRuv7663Xo0CHFxcXp/Pnzqqio8GtTVlbW6Dnt+txutyIiIvwmAABs1K6C+tSpUzp8+LDi4+M1cuRIde3aVZs3b3bWFxQUqKioSKmpqQGsEgCAlmP1oe+f/vSnmjJlinr37q2jR4/qmWeeUXBwsKZNm6bIyEjNnDlTmZmZ8ng8ioiI0MMPP6zU1FSu+AYAdBhWB/Wnn36qadOm6cSJE+rVq5duvfVW7dq1S7169ZIkPf/88woKCtLUqVNVVVWliRMn6sUXXwxw1QAAtByrg3rt2rVNru/WrZtycnKUk5PTRhUBANC22tU5agAAOhuCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAi3UJdAFoGabWp+LiYmc+KSlJwcHBAawIANASCOoO4mzF53rir0cVFXdcZ8rLtCIjXcnJyYEuCwBwlQjqDiS0R4y6RycEugwAQAviHDUAABZjRN2CfD6fioqKVFxcLGMCXQ0AoCMgqFtQUVGRZua8rbOVn6t7Qv9AlwMA6AAI6hYW5okNdAkAgA6Ec9QAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACzG7VmdTN1DWSRe3AEA7YHVI+rs7GzdfPPNCg8PV0xMjO6++24VFBT4tRk7dqxcLpff9NBDDwWoYvvVPZRlZs7bTmADAOxldVBv375dGRkZ2rVrlzZu3Kjq6mpNmDBBp0+f9ms3a9YslZSUONNzzz0XoIrtUPfKy8LCQvl8vgbrwzyxPJgFANoJqw99b9iwwW9+9erViomJ0d69ezVmzBhneVhYmOLi4tq6PGvVvfIyJCSf110CQDtn9Yj6QpWVlZIkj8fjt/zll19WdHS0Bg8erKysLJ05c6bJ7VRVVcnr9fpNHU1ojxhGzQDQAVg9oq6vtrZWjz76qL72ta9p8ODBzvLvfve76t27txISErR//37Nnz9fBQUFeu211y66rezsbD377LNtUTYAAFel3QR1RkaG8vPz9e677/otnz17tvPzkCFDFB8fr/Hjx+vw4cPq169fo9vKyspSZmamM+/1epWYmNg6hQdQ3blqiSu8AaC9ahdBPXfuXK1fv147duzQtdde22TblJQUSdKhQ4cuGtRut1tut7vF67QN56oBoP2zOqiNMXr44Yf1+uuva9u2bZcVNHl5eZKk+Pj4Vq6ufQjtEdMp/qcEADoqq4M6IyNDa9as0RtvvKHw8HCVlpZKkiIjIxUaGqrDhw9rzZo1mjx5snr27Kn9+/dr3rx5GjNmjIYOHRrg6gEAuHpWB/WyZcskfflQk/pWrVql+++/XyEhIdq0aZNeeOEFnT59WomJiZo6daqefPLJAFQLAEDLszqojTFNrk9MTNT27dvbqBoAANpeu7qPGgCAzoagBgDAYgQ1AAAWI6gBALCY1ReTIfB4fzUABBYjajSJ91cDQGAxou4E6j/zu7i4WMZILtflf563cAFA4BDUnUDdM7+j4o7rROEBdU/oz2NFAaCd4NB3JxHaI0bdoxMUGhkd6FIAAFeAoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAi/HAk06q/tPKJJ7jDQC2Iqg7qfpPKztTXqYVGelKTk4OdFkAgAsQ1J1Y3dPKgObgzWpA2+AcNYBm4c1qQNtgRA3nfLXP55MkZ2SUlJQUyLLQDvBmNaD1EdRwzlf7zm5TcGiEouISnfPWAIDAIqgh6cvz1b6QEAVfE8V5awCwCOeoAQCwGCNqNKr+fdbGSC5XgAsCgE6KoEaj/nPe2qvuCf3ldrsDXRIAdEoENS6q7rw1ACBwOEcNAIDFGFHjijX2RCqeUgUArYMRNa5YY0+k4ilVANA6GFGjWcI8sX5XhhcXFyu0R2yjV4cz2gaA5iOo0Wz138B1ovDARa8OrxttS+ItXQBwhQhqXJW6N3CdKS9rsh3PhAaA5uEcNQAAFiOoAQCwGIe+EVBcaAYATWNEjYDiti4AaBojalyWC2/FMqbpdj6fT5IUHBzstK9/61bdSLqp27oaU38ELjEKR/vH7zQuhaDGZWnsVqym2vnOblNwaISi4hIbvXWrbiR9tvLzK3rpR93nwjyxOlNexu1eaPf4ncalENS4bJd7K1bdyzyCr4lqsn39W7bqj9jrj8YbG12EeWLVPTrB7zMXG4XUH600td26dvXbNLXdpjBCwpWq+53GlesM17l0mKDOycnR4sWLVVpaqmHDhmnp0qUaPXp0oMvCZbpwxB4cGqGQkJAmRxd1nwkJyb9ou/qjlaa2W3+EX3ckoLmjG0ZIQNvpDA9U6hBB/ec//1mZmZlavny5UlJS9MILL2jixIkqKChQTExMoMvD/7vUee76I/bga6Iu63B4aI8YhXTt0mBkfeE58Prbbay99J8Rft2RgCvV2HdeTvvG6r6wtpZyOdu/0iMC9fdrXXtJVhxVaMujG201suOITUNX8kCl9jgC7xBBvWTJEs2aNUs//OEPJUnLly/Xm2++qZUrV+q///u/A1wd6lzuee7mbrf+yPrCc+CXat8SmvrOptpLalB3/WUt6XK2f6VHBOrv17r2kqw4qtCWRzfaamTHEZur0x5H4O0+qM+fP6+9e/cqKyvLWRYUFKS0tDTl5uY2+pmqqipVVVU585WVlZIkr9d7VbWcPHlS3pIjOus9oeBTXgUZn04e/1TBp7zynfVedFlT6wLRvlW/MzRC1efOqOb8OZ0sK2qyfVXXrjp4MEwnT550/o0/++wzeUuOqPrcmQbbDTI+HTx4UCdPntRnn32mmqqzzvc0VseF7S/cd2e/ONbg+5vS2Hc2tY269pIa1F1/WUu6nO3Xtak+d0Y1VWebbOf8m9Xt1/9vL+myttHaLrcvLfld0pXtu/q/05fzO9eWfWoP6v79JF3W32v9/XTy5Mmr/u9+nfDwcLku9/aVK2Xauc8++8xIMjt37vRb/vjjj5vRo0c3+plnnnnGSGJiYmJiYmqRqbKystVyrt2PqJsjKytLmZmZznxtba3Ky8vVs2fPq/o/Iq/Xq8TERBUXFysiIqIlSg24jtgnqWP2qyP2SeqY/eqIfZI6d7/Cw8Nb7fvbfVBHR0crODhYZWX+twCVlZUpLi6u0c+43e4GFypFRUW1WE0REREd6pdU6ph9kjpmvzpin6SO2a+O2CeJfrW0dv8I0ZCQEI0cOVKbN292ltXW1mrz5s1KTU0NYGUAAFy9dj+ilqTMzEzNmDFDo0aN0ujRo/XCCy/o9OnTzlXgAAC0Vx0iqO+9914dP35cTz/9tEpLS3XTTTdpw4YNio29/HvrWoLb7dYzzzxz2Y/DbA86Yp+kjtmvjtgnqWP2qyP2SaJfrcVlzMVerwAAAAKt3Z+jBgCgIyOoAQCwGEENAIDFCGoAACxGULeQnJwc9enTR926dVNKSor+9a9/BbokR3Z2tm6++WaFh4crJiZGd999twoKCvzajB07Vi6Xy2966KGH/NoUFRXpjjvuUFhYmGJiYvT444+rpqbGr822bds0YsQIud1u9e/fX6tXr26VPi1YsKBBvQMHDnTWnzt3ThkZGerZs6e6d++uqVOnNngojk39qdOnT58G/XK5XMrIyJDUPvbTjh07NGXKFCUkJMjlcmndunV+640xevrppxUfH6/Q0FClpaXp448/9mtTXl6u6dOnKyIiQlFRUZo5c6ZOnTrl12b//v267bbb1K1bNyUmJuq5555rUMurr76qgQMHqlu3bhoyZIjeeuutVulXdXW15s+fryFDhuiaa65RQkKCfvCDH+jo0aN+22hs/y5atMjafknS/fff36DmSZMm+bWxbX9dqk+N/Y25XC4tXrzYaWPVvmq1h5N2ImvXrjUhISFm5cqV5sCBA2bWrFkmKirKlJWVBbo0Y4wxEydONKtWrTL5+fkmLy/PTJ482SQlJZlTp045bb7+9a+bWbNmmZKSEmeq/+zampoaM3jwYJOWlmb27dtn3nrrLRMdHW2ysrKcNv/7v/9rwsLCTGZmpvnwww/N0qVLTXBwsNmwYUOL9+mZZ54xN954o1+9x48fd9Y/9NBDJjEx0WzevNns2bPH3HLLLearX/2qtf2pc+zYMb8+bdy40UgyW7duNca0j/301ltvmZ///OfmtddeM5LM66+/7rd+0aJFJjIy0qxbt8588MEH5s477zTJycnm7NmzTptJkyaZYcOGmV27dpl33nnH9O/f30ybNs1ZX1lZaWJjY8306dNNfn6+eeWVV0xoaKj5/e9/77T55z//aYKDg81zzz1nPvzwQ/Pkk0+arl27mn//+98t3q+KigqTlpZm/vznP5uDBw+a3NxcM3r0aDNy5Ei/bfTu3dssXLjQb//V/zu0rV/GGDNjxgwzadIkv5rLy8v92ti2vy7Vp/p9KSkpMStXrjQul8scPnzYaWPTviKoW8Do0aNNRkaGM+/z+UxCQoLJzs4OYFUXd+zYMSPJbN++3Vn29a9/3fzkJz+56GfeeustExQUZEpLS51ly5YtMxEREaaqqsoYY8zPfvYzc+ONN/p97t577zUTJ05s2Q6YL4N62LBhja6rqKgwXbt2Na+++qqz7KOPPjKSTG5urjHGvv5czE9+8hPTr18/U1tba4xpf/vpwv9I1tbWmri4OLN48WJnWUVFhXG73eaVV14xxhjz4YcfGknmvffec9q8/fbbxuVymc8++8wYY8yLL75oevTo4fTJGGPmz59vBgwY4Mx/5zvfMXfccYdfPSkpKebBBx9s8X415l//+peRZD755BNnWe/evc3zzz9/0c/Y2K8ZM2aYu+6666KfsX1/Xc6+uuuuu8ztt9/ut8ymfcWh76tU95rNtLQ0Z9mlXrMZaHWv9fR4PH7LX375ZUVHR2vw4MHKysrSmTNnnHW5ubkaMmSI30NkJk6cKK/XqwMHDjht6v871LVprX+Hjz/+WAkJCerbt6+mT5/uvAx+7969qq6u9qtl4MCBSkpKcmqxsT8XOn/+vF566SU98MADfi+LaW/7qb7CwkKVlpb6fX9kZKRSUlL89k1UVJRGjRrltElLS1NQUJB2797ttBkzZoxCQkL8+lBQUKAvvvjCaRPI/VdZWSmXy9XgPQKLFi1Sz549NXz4cC1evNjvtISt/dq2bZtiYmI0YMAAzZkzRydOnPCruT3vr7KyMr355puaOXNmg3W27KsO8WSyQPr888/l8/kaPAUtNjbWeS+vTWpra/Xoo4/qa1/7mgYPHuws/+53v6vevXsrISFB+/fv1/z581VQUKDXXntNklRaWtpoH+vWNdXG6/Xq7NmzCg0NbbF+pKSkaPXq1RowYIBKSkr07LPP6rbbblN+fr5KS0sVEhLS4D+QsbGxl6w1UP1pzLp161RRUaH777/fWdbe9tOF6mpo7Pvr1xcTE+O3vkuXLvJ4PH5tkpOTG2yjbl2PHj0u2s+6bbSmc+fOaf78+Zo2bZrfSxweeeQRjRgxQh6PRzt37lRWVpZKSkq0ZMkSp3bb+jVp0iTdc889Sk5O1uHDh/XEE08oPT1dubm5Cg4Obvf7649//KPCw8N1zz33+C23aV8R1J1MRkaG8vPz9e677/otnz17tvPzkCFDFB8fr/Hjx+vw4cPq169fW5d5Senp6c7PQ4cOVUpKinr37q2//OUvrR6gbWXFihVKT09XQkKCs6y97afOqLq6Wt/5zndkjNGyZcv81tV/ve7QoUMVEhKiBx98UNnZ2dY+dvO+++5zfh4yZIiGDh2qfv36adu2bRo/fnwAK2sZK1eu1PTp09WtWze/5TbtKw59X6XmvGYzUObOnav169dr69atuvbaa5tsm5KSIkk6dOiQJCkuLq7RPtata6pNREREq4dnVFSUrr/+eh06dEhxcXE6f/68KioqGtRyqVrr1jXVpi3688knn2jTpk360Y9+1GS79raf6mpo6u8lLi5Ox44d81tfU1Oj8vLyFtl/rfl3WRfSn3zyiTZu3HjJVyKmpKSopqZGR44ckWRvv+rr27evoqOj/X7n2uv+euedd1RQUHDJvzMpsPuKoL5K7eE1m8YYzZ07V6+//rq2bNnS4HBNY/Ly8iRJ8fHxkqTU1FT9+9//9vuDrPsP0Q033OC0qf/vUNemLf4dTp06pcOHDys+Pl4jR45U165d/WopKChQUVGRU4vt/Vm1apViYmJ0xx13NNmuve2n5ORkxcXF+X2/1+vV7t27/fZNRUWF9u7d67TZsmWLamtrnf8xSU1N1Y4dO1RdXe3XhwEDBqhHjx5Om7bsZ11If/zxx9q0aZN69ux5yc/k5eUpKCjIOXRsY78u9Omnn+rEiRN+v3PtcX9JXx61GjlypIYNG3bJtgHdV1d06RkatXbtWuN2u83q1avNhx9+aGbPnm2ioqL8rrwNpDlz5pjIyEizbds2v1sNzpw5Y4wx5tChQ2bhwoVmz549prCw0Lzxxhumb9++ZsyYMc426m77mTBhgsnLyzMbNmwwvXr1avS2n8cff9x89NFHJicnp9VuZ3rsscfMtm3bTGFhofnnP/9p0tLSTHR0tDl27Jgx5svbs5KSksyWLVvMnj17TGpqqklNTbW2P/X5fD6TlJRk5s+f77e8veynkydPmn379pl9+/YZSWbJkiVm3759ztXPixYtMlFRUeaNN94w+/fvN3fddVejt2cNHz7c7N6927z77rvmuuuu87vdp6KiwsTGxprvf//7Jj8/36xdu9aEhYU1uDWmS5cu5te//rX56KOPzDPPPHNVtzE11a/z58+bO++801x77bUmLy/P7++s7qrgnTt3mueff97k5eWZw4cPm5deesn06tXL/OAHP7C2XydPnjQ//elPTW5uriksLDSbNm0yI0aMMNddd505d+6csw3b9telfgeN+fL2qrCwMLNs2bIGn7dtXxHULWTp0qUmKSnJhISEmNGjR5tdu3YFuiSHpEanVatWGWOMKSoqMmPGjDEej8e43W7Tv39/8/jjj/vdn2uMMUeOHDHp6ekmNDTUREdHm8cee8xUV1f7tdm6dau56aabTEhIiOnbt6/zHS3t3nvvNfHx8SYkJMR85StfMffee685dOiQs/7s2bPmxz/+senRo4cJCwsz3/rWt0xJSYm1/anvH//4h5FkCgoK/Ja3l/20devWRn/fZsyYYYz58hatp556ysTGxhq3223Gjx/foK8nTpww06ZNM927dzcRERHmhz/8oTl58qRfmw8++MDceuutxu12m6985Stm0aJFDWr5y1/+Yq6//noTEhJibrzxRvPmm2+2Sr8KCwsv+ndWdw/83r17TUpKiomMjDTdunUzgwYNMr/85S/9As+2fp05c8ZMmDDB9OrVy3Tt2tX07t3bzJo1q8EgxLb9danfQWOM+f3vf29CQ0NNRUVFg8/btq94zSUAABbjHDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ10MGNHTtWjz76aKDLANBMPJkM6ODKy8vVtWtXhYeHt9l3LliwQOvWrXNeGgKg+XgfNdDBeTyeQJcA4Cpw6Bvo4Oof+u7Tp49++ctf6oEHHlB4eLiSkpL0hz/8wWl75MgRuVwurV27Vl/96lfVrVs3DR48WNu3b3farF69WlFRUX7fsW7dOrlcLmf9s88+qw8++EAul0sul0urV69u7W4CHRZBDXQyv/nNbzRq1Cjt27dPP/7xjzVnzhwVFBT4tXn88cf12GOPad++fUpNTdWUKVN04sSJy9r+vffeq8cee0w33nijSkpKVFJSonvvvbc1ugJ0CgQ10MlMnjxZP/7xj9W/f3/Nnz9f0dHR2rp1q1+buXPnaurUqRo0aJCWLVumyMhIrVix4rK2Hxoaqu7du6tLly6Ki4tTXFycQkNDW6MrQKdAUAOdzNChQ52fXS6X4uLidOzYMb82qampzs9dunTRqFGj9NFHH7VZjQD+g6AGOpmuXbv6zbtcLtXW1l7254OCgnThzSLV1dUtUhuAhghqAA3s2rXL+bmmpkZ79+7VoEGDJEm9evXSyZMndfr0aafNhbdhhYSEyOfztUmtQEdHUANoICcnR6+//roOHjyojIwMffHFF3rggQckSSkpKQoLC9MTTzyhw4cPa82aNQ2u6u7Tp48KCwuVl5enzz//XFVVVQHoBdAxENQAGli0aJEWLVqkYcOG6d1339Xf/vY3RUdHS/ryvuyXXnpJb731loYMGaJXXnlFCxYs8Pv81KlTNWnSJI0bN069evXSK6+8EoBeAB0DTyYD4Dhy5IiSk5O1b98+3XTTTYEuB4AYUQMAYDWCGgAAi3HoGwAAizGiBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFjs/wDsCrShUPs0ewAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 500x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "length = df['input'].apply(len)\n",
    "sns.displot(length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['category', 'instruction', 'input', 'output'],\n",
       "        num_rows: 1458\n",
       "    })\n",
       "    valid: Dataset({\n",
       "        features: ['category', 'instruction', 'input', 'output'],\n",
       "        num_rows: 365\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 카테고리 전처리\n",
    "prep_df = df[df['category']!='']#[:5000]\n",
    "prep_df['category'] = prep_df['category'].apply(lambda x: 'Commonsense/logic' if x==\"Common Sense / Logic\" else x)\n",
    "prep_df = prep_df.sort_values('category')\n",
    "\n",
    "train_dataset, valid_dataset = train_test_split(prep_df, random_state=2022, test_size=0.2)\n",
    "\n",
    "\n",
    "train = Dataset.from_pandas(train_dataset, preserve_index=False)\n",
    "valid = Dataset.from_pandas(valid_dataset, preserve_index=False)\n",
    "dataset = DatasetDict({'train':train, 'valid':valid})\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                 \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['category', 'instruction', 'input', 'output'],\n",
       "        num_rows: 1458\n",
       "    })\n",
       "    valid: Dataset({\n",
       "        features: ['category', 'instruction', 'input', 'output'],\n",
       "        num_rows: 365\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "import emoji\n",
    "from soynlp.normalizer import repeat_normalize\n",
    "\n",
    "def clean(example): \n",
    "    pattern = re.compile(f'[^ .,?!/@$%~％·∼()\\x00-\\x7Fㄱ-ㅣ가-힣\\[\\]]+')\n",
    "    url_pattern = re.compile(r'https?:\\/\\/(www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b([-a-zA-Z0-9()@:%_\\+.~#?&//=]*)')\n",
    "\n",
    "    x_key = ['instruction', 'input', 'output']\n",
    "    \n",
    "    for key in x_key:\n",
    "        x = example[key]\n",
    "        # x = pattern.sub(' ', x)\n",
    "        x = emoji.replace_emoji(x, replace='') #emoji 삭제\n",
    "        x = url_pattern.sub('', x)\n",
    "        x = x.strip()\n",
    "        x = repeat_normalize(x, num_repeats=2)\n",
    "        example[key] = x\n",
    "\n",
    "    return example\n",
    "\n",
    "dataset = dataset.map(clean)\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "prep_df.to_json('./prep_data.json', index=True, orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                 \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['decoder_input_ids', 'decoder_attention_mask', 'input_ids', 'attention_mask'],\n",
       "        num_rows: 1458\n",
       "    })\n",
       "    valid: Dataset({\n",
       "        features: ['decoder_input_ids', 'decoder_attention_mask', 'input_ids', 'attention_mask'],\n",
       "        num_rows: 365\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.tokenize import sent_tokenize\n",
    "\n",
    "def preprocess_func(example):\n",
    "    return tokenizer(example[\"output\"], truncation=True, padding=False)\n",
    "\n",
    "tokenized_datasets = dataset.map(preprocess_func, batched=True)\n",
    "tokenized_datasets['train'] = tokenized_datasets['train'].rename_column(\"input_ids\", \"decoder_input_ids\")\n",
    "tokenized_datasets['train'] = tokenized_datasets['train'].rename_column('attention_mask', 'decoder_attention_mask')\n",
    "tokenized_datasets['valid'] = tokenized_datasets['valid'].rename_column(\"input_ids\", \"decoder_input_ids\")\n",
    "tokenized_datasets['valid'] = tokenized_datasets['valid'].rename_column('attention_mask', 'decoder_attention_mask')\n",
    "\n",
    "\n",
    "def preprocess_function(example):\n",
    "    return tokenizer(f'{example[\"instruction\"]}', truncation=True, padding=False, max_length=128)#, return_tensors='pt')\n",
    "# return tokenizer(f'{example[\"instruction\"]} {example[\"input\"]}', truncation=True, padding=False, max_length=128)#, return_tensors='pt')\n",
    "\n",
    "tokenized_datasets = tokenized_datasets.map(preprocess_function)\n",
    "# tokenized_datasets['train'] = tokenized_datasets['train'].rename_column(\"input_ids\", \"encoder_input_ids\")\n",
    "# tokenized_datasets['train'] = tokenized_datasets['train'].rename_column('attention_mask', 'encoder_attention_mask')\n",
    "\n",
    "tokenized_datasets.set_format(\"torch\")\n",
    "tokenized_datasets = tokenized_datasets.remove_columns(['category', 'instruction', 'input', 'output',])\n",
    "tokenized_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
    "from torchmetrics.text.rouge import ROUGEScore\n",
    "\n",
    "\n",
    "def validation(model, valid_dataloader):\n",
    "    avg_rouge_score = []\n",
    "    for data in valid_dataloader:\n",
    "        data = data.to(device)\n",
    "        # print(data)\n",
    "        # print(data['input_ids'])\n",
    "        # print(data['input_ids'].shape)\n",
    "        # print(data['decoder_input_ids'].shape)\n",
    "        \n",
    "        output = model.generate(input_ids=data['input_ids'])#,labels=data['decoder_input_ids'])\n",
    "        \n",
    "        # question = tokenizer.decode(data['input_ids'][0], skip_special_tokens=True)\n",
    "        pred = tokenizer.decode(output[0], skip_special_tokens=True)\n",
    "        target = tokenizer.decode(data['decoder_input_ids'][0], skip_special_tokens=True)\n",
    "\n",
    "        # print('Q:', question)\n",
    "        # print('A:', pred)\n",
    "        # print('GT:', target, '\\n')\n",
    "        # torch.cuda.empty_cache()\n",
    "        \n",
    "        rouge = ROUGEScore()\n",
    "        rouge_score = rouge(pred, target)\n",
    "        avg_rouge_score.append(rouge_score)\n",
    "    return pd.DataFrame(avg_rouge_score).mean().to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"google/flan-t5-base\"\n",
    "model = T5ForConditionalGeneration.from_pretrained(model_name, output_hidden_states=True).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.15.4"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/uj-user/Yo/dolly-15k-analysis/wandb/run-20230608_134022-ntitnvjn</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/nudago/t5_sum/runs/ntitnvjn' target=\"_blank\">olive-terrain-8</a></strong> to <a href='https://wandb.ai/nudago/t5_sum' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/nudago/t5_sum' target=\"_blank\">https://wandb.ai/nudago/t5_sum</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/nudago/t5_sum/runs/ntitnvjn' target=\"_blank\">https://wandb.ai/nudago/t5_sum/runs/ntitnvjn</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1458/1458 [02:13<00:00, 10.94it/s]\n",
      "/home/uj-user/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/transformers/generation/utils.py:1353: UserWarning: Using `max_length`'s default (20) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:0, Train_loss:2.4376293532911237\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1458/1458 [02:13<00:00, 10.90it/s]\n",
      "/home/uj-user/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/transformers/generation/utils.py:1353: UserWarning: Using `max_length`'s default (20) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:1, Train_loss:2.3510562855531862\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1458/1458 [02:13<00:00, 10.91it/s]\n",
      "/home/uj-user/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/transformers/generation/utils.py:1353: UserWarning: Using `max_length`'s default (20) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:2, Train_loss:2.2871687775567193\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1458/1458 [02:13<00:00, 10.91it/s]\n",
      "/home/uj-user/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/transformers/generation/utils.py:1353: UserWarning: Using `max_length`'s default (20) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:3, Train_loss:2.1943877681841326\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1458/1458 [02:13<00:00, 10.94it/s]\n",
      "/home/uj-user/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/transformers/generation/utils.py:1353: UserWarning: Using `max_length`'s default (20) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:4, Train_loss:2.1434728915845294\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>rouge1_fmeasure</td><td>▆█▃▂▁</td></tr><tr><td>rouge1_precision</td><td>█▆▄▁▁</td></tr><tr><td>rouge1_recall</td><td>▂█▄▁▂</td></tr><tr><td>rouge2_fmeasure</td><td>█▄▁▁▄</td></tr><tr><td>rouge2_precision</td><td>█▅▃▁▂</td></tr><tr><td>rouge2_recall</td><td>█▃▁▁▃</td></tr><tr><td>rougeL_fmeasure</td><td>█▇▅▃▁</td></tr><tr><td>rougeL_precision</td><td>█▅▅▂▁</td></tr><tr><td>rougeL_recall</td><td>▅█▄▂▁</td></tr><tr><td>rougeLsum_fmeasure</td><td>▇█▄▃▁</td></tr><tr><td>rougeLsum_precision</td><td>█▆▄▂▁</td></tr><tr><td>rougeLsum_recall</td><td>▄█▄▂▁</td></tr><tr><td>train_loss</td><td>█▆▄▂▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>rouge1_fmeasure</td><td>0.27468</td></tr><tr><td>rouge1_precision</td><td>0.40049</td></tr><tr><td>rouge1_recall</td><td>0.2662</td></tr><tr><td>rouge2_fmeasure</td><td>0.13656</td></tr><tr><td>rouge2_precision</td><td>0.18675</td></tr><tr><td>rouge2_recall</td><td>0.13155</td></tr><tr><td>rougeL_fmeasure</td><td>0.23947</td></tr><tr><td>rougeL_precision</td><td>0.34642</td></tr><tr><td>rougeL_recall</td><td>0.23549</td></tr><tr><td>rougeLsum_fmeasure</td><td>0.24796</td></tr><tr><td>rougeLsum_precision</td><td>0.36294</td></tr><tr><td>rougeLsum_recall</td><td>0.24224</td></tr><tr><td>train_loss</td><td>2.14347</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">olive-terrain-8</strong> at: <a href='https://wandb.ai/nudago/t5_sum/runs/ntitnvjn' target=\"_blank\">https://wandb.ai/nudago/t5_sum/runs/ntitnvjn</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20230608_134022-ntitnvjn/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)\n",
    "train_epoch = 5\n",
    "\n",
    "\n",
    "train_dataloader = DataLoader(tokenized_datasets['train'], shuffle=False, batch_size=1, collate_fn=data_collator)\n",
    "valid_dataloader = DataLoader(tokenized_datasets[\"valid\"], shuffle=False, batch_size=1, collate_fn=data_collator)\n",
    "\n",
    "import wandb\n",
    "\n",
    "wandb.init(project='t5_sum',\n",
    "           tags=['ori', model_name],\n",
    "           )\n",
    "\n",
    "wandb.watch(model)\n",
    "for e in range(train_epoch):\n",
    "    train_loss = []\n",
    "    total, correct = 0, 0\n",
    "    model.train()\n",
    "\n",
    "    for data in tqdm(train_dataloader):\n",
    "        data = data.to(device)\n",
    "\n",
    "        text_output = model(input_ids=data['input_ids'], \n",
    "                            attention_mask=data['attention_mask'],\n",
    "                            labels=data['decoder_input_ids'][:,:-1].contiguous()\n",
    "                            )\n",
    "\n",
    "        loss = text_output.loss\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "        # print(loss)\n",
    "        train_loss.append(loss.item())\n",
    "        # torch.cuda.empty_cache()\n",
    "        \n",
    "    avg_loss = np.mean(train_loss)\n",
    "    print(f'Epoch:{e}, Train_loss:{avg_loss}')\n",
    "    \n",
    "    valid_log = validation(model, valid_dataloader)\n",
    "    wandb.log({'train_loss':avg_loss})\n",
    "    wandb.log(valid_log)\n",
    "    # print(loss)\n",
    "    # print(logits)\n",
    "    # print(tokenizer.decode(enc_inputs['input_ids'][0], skip_special_tokens=True))\n",
    "    # print(tokenizer.decode(torch.argmax(text_output.logits, dim=2)[0], skip_special_tokens=True))\n",
    "    \n",
    "    \n",
    "# for data in train_dataloader:\n",
    "#     # print(data)\n",
    "#     # print(data['input_ids'])\n",
    "#     # print(data['input_ids'].shape)\n",
    "#     # print(data['decoder_input_ids'].shape)\n",
    "    \n",
    "#     text_output = model.generate(input_ids=data['input_ids'])#,labels=data['decoder_input_ids'])\n",
    "#     print('Q:', tokenizer.decode(data['input_ids'][0], skip_special_tokens=True))\n",
    "#     print('A:', tokenizer.decode(text_output[0], skip_special_tokens=True), '\\n')\n",
    "wandb.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/uj-user/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/transformers/generation/utils.py:1353: UserWarning: Using `max_length`'s default (20) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q: What was Jiang's previous position before becoming the president of China?\n",
      "A: President of China from January 1st, 1921 to December 31, 2003. He was the\n",
      "GT: He was the mayor of Shanghai and then became Communist Party secretary \n",
      "\n",
      "Q: When did Arsenal first reach the first division? How many league titles did they win between 1989 and 2005 and how many FA cups? How many trophies in total for that period?\n",
      "A: Arsenal won four league titles between 1989 and 2005. Arsenal won eight FA Cups between 1989 and\n",
      "GT: Arsenal reached the first division in 1904. Between 1989 and 2005, Arsenal won 5 league titles and 5 FA cups. That’s a total of 10 trophies. \n",
      "\n",
      "Q: Based on the following paragraph on the current use of obsidian, what's the difference between obsidian scalpels and steel scalpels?\n",
      "A: Obsidian is the most common type of scalpel in the world. The difference\n",
      "GT: Obsidian scalpels have a cutting edge that is many times sharper than a steel scalpel. In addition, obsidian scalpels have a smooth blade surface whereas all metal knifes have a jagged, irregular blade surface. \n",
      "\n",
      "Q: Based on this passage, in what field did Raisa earn her PhD?\n",
      "A: Raisa earned her PhD in the field of physiology and physiology. She\n",
      "GT: Based on this passage, Raisa did not earn a PhD. She abandoned her graduate studies in Moscow in order to join Gorbachev in Stavropol. \n",
      "\n",
      "Q: Who is Satoshi Nakamoto?\n",
      "A: Satoshi Nakamoto is a Japanese chess master. He is the current\n",
      "GT: Satoshi Nakamoto is the name used by the presumed inventors of Bitcoin. It is probably a pseudonym or nom de plume for the creator. \n",
      "\n",
      "Q: Who is Charlie McCarthy\n",
      "A: Charlie McCarthy is an American football player who played for the New York Giants. He is the\n",
      "GT: Charles McCarthy (born 1946) is an Irish former hurler who played as a left corner-forward at senior level for the Cork county team. Born in Tower Street, Cork, McCarthy first played competitive hurling during his schooling at Sullivan's Quay CBS. He arrived on the inter-county scene at the age of sixteen when he first linked up with the Cork minor teams as a dual player, before later joining the under-21 hurling side. He made his senior debut during the 1965 championship. McCarthy immediately became a regular member of the starting fifteen and won five All-Ireland medals, nine Munster medals and four National Hurling League medals. The All-Ireland-winning captain of 1978, he was an All-Ireland runner-up on two occasions. As a member of the Munster inter-provincial team on a number of occasions, McCarthy won one Railway Cup medal. At club level he is a two-time All-Ireland medallist with St Finbarr's. In addition to this he has also won four Munster medals and five championship medals, after beginning his career with Redmonds. McCarthy's career tally of 24 goals and 149 points ranks him as Cork's fifth highest championship scorer of all-time. Throughout his career McCarthy made 45 championship appearances. His retirement came following the conclusion of the 1980 championship. In retirement from playing McCarthy became involved in team management and coaching. After serving as coach of the Cork minor and senior teams he guided club side St Finbarr's to championship success. McCarthy is widely regarded as one of the greatest hurlers of his era. During his playing days he won three All-Star awards. He has been repeatedly voted onto teams made up of the sport's greats, including at right corner-forward on the Club Hurling Silver Jubilee Team and the Supreme All-Stars team. McCarthy was also chosen as one of the 125 greatest hurlers of all-time in a 2009 poll. \n",
      "\n",
      "Q: Who is Solveig Margrete Fiske\n",
      "A: Solveig Margrete Fiske is a Norwegian politician and former politician. She is\n",
      "GT: Solveig Margrete Fiske (born 26 October 1952) is a theologian and bishop emerita in the Church of Norway, serving as the bishop of Hamar between 2006 and 2022, when she retired. \n",
      "\n",
      "Q: Given that the current year is 2023, how old is Chevron?\n",
      "A: The current year is 2023, so it is 2023 years old. Chevron is\n",
      "GT: Because the current year is 2023 and Chevron traces its history back to the 1870s, Chevron is roughly between 143 and 153 years old. \n",
      "\n",
      "Q: What is the world's oldest annual marathon based on the reference text below?\n",
      "A: The Marathon runner-up in the world was the 19-year-old Yuri\n",
      "GT: The Boston Marathon is the world's oldest annual marathon, beginning on April 19th 1897. \n",
      "\n",
      "Q: What is handscroll used for\n",
      "A: Handscroll is used to draw a hand on the head. The handcroll\n",
      "GT: The handscroll is a long, narrow, horizontal scroll format in East Asia used for calligraphy or paintings. \n",
      "\n",
      "Q: Given this paragraph, what is the top speed of a Kia Stinger?\n",
      "A: Kia Stinger has a top speed of 155 mph. The top speed of\n",
      "GT: The top speed of a Kia Stinger is 269km/h (167mph) according to this text. \n",
      "\n",
      "Q: Can you tell me something about Stanley J. Goldberg\n",
      "A: Stanley J. Goldberg was a former physicist and a professor of\n",
      "GT: Stanley J. Goldberg (born Maryland, 1939) is a special trial judge of the United States Tax Court. Goldberg attended public schools in Baltimore, MD. He earned a B.S. from the University of Maryland, School of Business and Public Administration in 1960 and an LL.B. from the University of Maryland School of Law in 1964. He did graduate work in Federal Income Taxation at New York University. Goldberg was admitted to practice in Maryland and New Jersey, 1964 and 1967, and Federal District Court. He began his career as a Tax Attorney in January 1965 with the United States Department of Treasury, Office of Chief Counsel, Internal Revenue Service, in New York City and was initially assigned to the General Litigation function. In 1967, he was reassigned to the Tax Litigation function. In 1976, he was promoted to Special Trial Attorney, and then to Assistant District Counsel in 1984. He was appointed a Special Trial Judge of the United States Tax Court on August 4, 1985. In 1992, Goldberg commented on the trend of tax protester arguments being brought before the Tax Court and dismissed with penalties. \n",
      "\n",
      "Q: How old was Dominik Volek when he made his Czech Extraliga debut?\n",
      "A: 16 years old. Dominik Volek made his Czech Extraliga debut when he\n",
      "GT: Dominik Volek was 20 years old. He debuted in 2014 and was born in 1994, so 2014-1994 = 20. \n",
      "\n",
      "Q: What is significant about Octavia Butler?\n",
      "A: Octavia Butler was born in 1939. She was a British actress. She\n",
      "GT: She was the first science fiction writer to receive a MacArthur Fellowship. \n",
      "\n",
      "Q: How big is the Tibetan plateau?\n",
      "A: The Tibetan plateau is about 2.2 million meter wide. It is located in the Indian border\n",
      "GT: The Tibetan Plateau spans about 2,500,000 km2 \n",
      "\n",
      "Q: Given a reference text about bull sharks, tell me the gestation period for bull sharks.\n",
      "A: Bull sharks are considered to be a savanna, but they are not\n",
      "GT: The gestation period for bull sharks is 12 months. \n",
      "\n",
      "Q: Who is Toto?\n",
      "A: Toto is the name of:\n",
      "GT: Toto may refer to many things but a popular reference is to a small dog in the movie The Wonderful Wizard of Oz (1900). Toto belongs to Dorothy Gale, the heroine, and plays a key role at several points in the movie including running away (which causes Dorothy to chase after him and get pulled into the story) and pulling the curtain back to reveal that the Wizard is just an old man. \n",
      "\n",
      "Q: On what month and day was Antwan Deon Odom born?\n",
      "A: November, 1999. He was born in November, 1999. He was born in November, 1999\n",
      "GT: September 24 \n",
      "\n",
      "Q: What was the record for University of Miami Hurricanes in 1980?\n",
      "A: The University of Miami Hurricanes had the record for the 1980 NCAA Division I championship. The record\n",
      "GT: The Hurricanes won 9 games and lost 3 games in 1980 \n",
      "\n",
      "Q: What are the rivers in Rajkot area?\n",
      "A: The Rajkot area is home to the Rajkot River and the Rajkot River. Rajkot\n",
      "GT: There are two rivers - Aaji and Nyari - in the vicinity of Rajkot. \n",
      "\n",
      "Q: Given this paragraph about kubernetes, what is the storage layer of Kubernetes control plane?\n",
      "A: Kubernetes is a storage layer of Kubernetes control plane.\n",
      "GT: ETCD \n",
      "\n",
      "Q: Given a reference text about Sintok, tell me where it is located.\n",
      "A: Sintok is located in the Central Asian region of Asia. The region is located in the\n",
      "GT: Sintok is located 12km from Changlu Town and 52km from Alor Setar City in Malaysia. \n",
      "\n",
      "Q: Given this paragraph, how big are the non-volley zones in pickleball and where are they located?\n",
      "A: The non-volley zones are located in the center of the pickleball field. They\n",
      "GT: 7-foot areas on either side of the net known \n",
      "\n",
      "Q: Given this table of races and dates, how many races took place after July? Give the answer as a number.\n",
      "A: 2 races took place after July.\n",
      "GT: 3 \n",
      "\n",
      "Q: Where is Lugașu de Jos\n",
      "A: Lugasu de Jos is a village in the municipality of Jos, Romania\n",
      "GT: Lugașu de Jos is a commune in Bihor County, Crișana, Romania with a population of 3,580 people. It is composed of three villages: Lugașu de Jos, Lugașu de Sus (Felsolugos) and Urvind (Oervend). \n",
      "\n",
      "Q: What is the first name of Stephen Curry?\n",
      "A: Stephen Curry is the first name of the following people. Stephen Curry was born in the United Kingdom\n",
      "GT: Wardell is the first name of Stephen Curry, who is an American professional basketball player for the Golden State Warriors. It is knowns as one of the greatest basketball players of all time. \n",
      "\n",
      "Q: What is a clique in graph theory?\n",
      "A: A clique is a group of cliques in graph theory that are based on\n",
      "GT: In graph theory, a clique is a subset of vertices of an undirected graph such that every two distinct vertices in the clique are adjacent. \n",
      "\n",
      "Q: Based on this paragraph, how many times has Joe Biden run for president and what characteristics make him unique among all past presidents?\n",
      "A: Joe Biden has run for president three times. He is the only one to have run for\n",
      "GT: Based on this paragraph, Joe Biden has run for president three times, losing in 1988 and 2008 and eventually winning in 2020. As president, he is unique in three ways: the first to have a female vice president, and the first president from the state of Delaware. \n",
      "\n",
      "Q: How many windows does the building have based on the provided paragraph?\n",
      "A: The building has a total of 76 windows. The building has a total of\n",
      "GT: There are a total of 13 windows. The side of the house with the entrance has two four-light windows, the north and south side have four windows each, and the back of the building has three. Therefore, in total, the house has 2 + 4 + 4 + 3 windows or 13 windows. \n",
      "\n",
      "Q: Given this short summary of Giovan Battista Cavagna, how long did it take to build the second Monte di Pieta?\n",
      "A: Giovan Battista Cavagna built the second Monte di Pieta in\n",
      "GT: It took 58 years. Started in 1539, completed in 1597. \n",
      "\n",
      "Q: Given this paragraph about the Last of Us TV Series, who was not cast in the show?\n",
      "A: The show was canceled by the late-admissions cast. The show was\n",
      "GT: Matthew McConaughey turned down the role of Joel which was also reportedly offered to and turned down by Mahershala Ali afterwards. Maisie Williams and Kaitlyn Dever were considered for the role of Ellie but aged out of consideration by the time of production. \n",
      "\n",
      "Q: Given this paragraph about Riot Games' history, which game was used an indication that games could be supported and monetized long-term?\n",
      "A: Riot Games was used an indication that games could be supported and monetized long-\n",
      "GT: Defense of the Ancients (DotA) \n",
      "\n",
      "Q: What is the net worth of one of Taylor Swift's cats?\n",
      "A: $1 million. Taylor Swift's cats are worth around $1 million.\n",
      "GT: One of Taylor Swift's cats is the third richest pet animal in the world, and it's estimated net worth is $97 million. \n",
      "\n",
      "Q: Who built Sigiriya fortress in Sri Lanka?\n",
      "A: Sri Lanka was founded by the Sigiriya fortress. The fortress was built by the\n",
      "GT: King Kashyapa developed Sigiriya city and fortress. \n",
      "\n",
      "Q: Based on this paragraph, what is the diameter of a 10 F Foley catheter in mm?\n",
      "A: The diameter of a 10 F Foley catheter in mm is.5 mm\n",
      "GT: According to this paragraph, the diameter of a 1 F catheter is 1/77mm so the diameter of a 10 F catheter is 10/77mm. \n",
      "\n",
      "Q: Given this reference text about Jane Goodall, tell me how old she is, and her most notable accomplishments.\n",
      "A: Jane Goodall is an American actress, who was a member of the American Council of Women\n",
      "GT: Jane Goodall, 89, is a world-renown primatologist, famous for her study of chimpanzees. She has dedicated her life to conservation and animal welfare issues, and founded the Jane Goodall Institute and the Roots & Shoots programme. She was named a UN Messenger of Peace in 2002, and is an honorary member of the World Future Council. \n",
      "\n",
      "Q: Given this reference test, what is the daily recommended value of magnesium for men in the U.S.?\n",
      "A: The daily recommended value of magnesium for men in the U.S. is 255. The\n",
      "GT: The daily recommended value of magnesium for men in the U.S. is 400 mg. \n",
      "\n",
      "Q: Given this paragraph about a train stabling complex, what was the original name of the Grove Park Sidings?\n",
      "A: Grove Park Sidings was originally known as the \"Sidings of the Grove\".\n",
      "GT: Hither Green Sidings \n",
      "\n",
      "Q: Given this paragraph about the prehistory of Sri Lanka, what are the evidence for prehistoric human colonization in Sri Lanka?\n",
      "A: The evidence for prehistoric human colonization in Sri Lanka is that the human population was mainly\n",
      "GT: The oldest prehistoric evidence of human colonization in Sri Lanka goes back to about 125,000 years to a man named Balangoda Man. He was a Mesolithic hunter-gatherer who lived in caves. Furthermore, human remains dating back to 6000 BCE have also been found in several cites in the country. \n",
      "\n",
      "Q: Based on the reference text about coffee, tell me where coffee plants are primarily grown and the most common bean types.\n",
      "A: The most common bean types are coffee beans grown in the United States. Coffee plants are primarily\n",
      "GT: Coffee plants are cultivated mainly in the equatorial regions of the Americas, Southeast Asia, the Indian subcontinent, and Africa. The most common types of coffee beans are arabica and robusta. \n",
      "\n",
      "Q: Based on the provided text, how many awards did the book \"The Killing Jar\" win and what year were they awarded?\n",
      "A: The Killing Jar won two awards in the year it was published. The first was the Best\n",
      "GT: The Killing Jar won 3 awards and they were awarded in 2006 \n",
      "\n",
      "Q: Given a reference text about Lotus Mark IX, who designed the car?\n",
      "A: The car was designed by Robert lvardsen. It was powered by\n",
      "GT: The Lotus Mark IX was designed by Frank Costin. \n",
      "\n",
      "Q: Whose life is the 1965 film 'The Sound of Music' based on?\n",
      "A: The Sound of Music was based on the life of the composer, John Lyne.\n",
      "GT: It is based on the 1949 memoir of Maria von Trapp \n",
      "\n",
      "Q: Based on this paragraph about dogs, what is a French Bulldog?\n",
      "A: A French Bulldog is a dog that is a breed of dog. It is\n",
      "GT: The French Bulldog is a French breed of companion dog or toy dog. It is is among the most frequently registered dogs in a number of countries including Australia, the United Kingdom, and the United States. \n",
      "\n",
      "Q: Why do sheep keepers paint their sheep?\n",
      "A: They are trying to keep their sheep clean. They are a breed of sheep that are painted\n",
      "GT: Sheep keepers sometimes artificially paint \"smit marks\" onto their sheep in any pattern or color for identification. \n",
      "\n",
      "Q: Is black hole truly black?\n",
      "A: The black hole is a sphere of ice that is a sphere of\n",
      "GT: According to general relativity, a black hole emits no lights at all. Thus it is 100% black by definition. However, when quantum effects are included, a black hole has finite temperature, emitting lights called Hawking radiation. In this sense, a black hole is not truly black. \n",
      "\n",
      "Q: given this article about Enrico Silvestrin, what year did he get evicted from Celebrity Big Brother?\n",
      "A: In the year of his eviction, Enrico Silvestrin was voted\n",
      "GT: 2018 \n",
      "\n",
      "Q: What is the first computer recognized in human history?\n",
      "A: The first computer was the atomic bomb. The atomic bomb was discovered in 1939 by\n",
      "GT: The Antikythera mechanism, an astronomical clock, was built around 100 BC and is considered the world's first computer. \n",
      "\n",
      "Q: What high school did Tyrone Brazelton go to?\n",
      "A: Tyrone Brazelton went to the University of Oxford and graduated with a degree in mathematics\n",
      "GT: Tyrone Brazelton attended Rich East High School and then transferred to Rich Central High School for his senior year. \n",
      "\n",
      "Q: How many member states does the European Union have?\n",
      "A: 27 member states. The EU has a total of 27 member states. There are 57\n",
      "GT: 27 \n",
      "\n",
      "Q: Given this paragraph about the Women's Education Association, which institutions of higher learning had affiliations with the Association?\n",
      "A: The Women's Education Association was a union of the University of Cambridge and the University of\n",
      "GT: The Massachusetts Institute of Technology and Harvard University \n",
      "\n",
      "Q: Given this reference text about Nintendo, when was it founded?\n",
      "A: Nintendo was founded in 1980. Nintendo was a company that was founded in Japan in the early\n",
      "GT: 1889 \n",
      "\n",
      "Q: How many American's signed the Declaration of Independence?\n",
      "A: 20 American's signed the Declaration of Independence. The United States of America has over a\n",
      "GT: 56 \n",
      "\n",
      "Q: Given a reference text about the incudomalleolar joint, tell me its main function.\n",
      "A: The incudomalleolar joint is the main part of the joint. The main function of\n",
      "GT: The function of the incudomalleolar joint is to transfer vibrations between the ossicles in the middle ear. \n",
      "\n",
      "Q: Given the following paragraph about New York City, how many people live in the city?\n",
      "A: 69,569,789, or 77,529,894 people\n",
      "GT: As of 2020, there are over 20.1 million people living in the metropolitan statistical area of New York City. When considering the combined statistical area of New York City as of 2020, it is 23.5 million people. \n",
      "\n",
      "Q: Given these two paragraphs, where did Jimmy Carter go to college?\n",
      "A: The Carters attended the University of Georgia. Jimmy Carter was a member of the Georgia Tech\n",
      "GT: Jimmy Carter received his Bachelor of Science degree from the United States Naval Academy in 1947. However, he began his undergraduate studies at Georgia Southwestern College. A year later, he transferred to the Georgia Institute of Technology before ultimately earning his admission into the Naval Academy where he would complete his studies. \n",
      "\n",
      "Q: What is the stone age?\n",
      "A: Stone Aged was the period of the period from the Bronze Age to the Iron Age. It\n",
      "GT: The Stone age was a prehistoric period that lasted for millions of years and ended in and ended between 4,000 BC and 2,000 BC, with the advent of metalworking. In this period, stone was widely used to make stone tools and weapons with an edge, a point, or a percussion surface. The Stone Age is the first period in the three-age system frequently used in archaeology to divide the timeline of human technological prehistory into functional periods, with the next two being the Bronze Age and the Iron Age, respectively. Though some simple metalworking of malleable metals, particularly the use of gold and copper for purposes of ornamentation, was known in the Stone Age, it is the melting and smelting of copper that marks the end of the Stone Age. The Stone Age is also commonly divided into three distinct periods: Paleolithic era which is the earliest and most primitive; a transitional period with finer tools known as the Mesolithic era; and the final stage known as the Neolithic era. Neolithic peoples were the first to transition away from hunter-gatherer societies into the settled lifestyle of inhabiting towns and villages as agriculture became widespread. \n",
      "\n",
      "Q: Who did Stafford throw touchdown passes to in Super Bowl LV?\n",
      "A: The team threw a touchdown pass to Aaron Thomas. The team threw\n",
      "GT: The text provided does not mention information about Super Bowl LV. However, it does provide information regarding Super Bowl LVI. In Super Bowl LVI, Stafford threw 3 touchdown passes, two to Cooper Kupp and one to Odell Beckham Jr. In the first quarter, Stafford led a long drive that culminated in a Odell Beckham Jr. touchdown reception. Stafford later tossed a touchdown to wide receiver Cooper Kupp. In the fourth quarter, Stafford orchestrated a drive in the final minutes that ended with another touchdown pass to Kupp. \n",
      "\n",
      "Q: Given this paragraph about Simon Bolivar, tell me when and where they were born, and what they are known for\n",
      "A: Simon Bolivar was born in the year of the Roman Empire. He is known for his\n",
      "GT: Simon Bolivar was born in Caracas, Venezuela on 24 July 1783. He is known for leading the wars of independence from the Spanish Empire from what are currently the countries of Colombia, Venezuela, Ecuador, Peru, Panama and Bolivia \n",
      "\n",
      "Q: Given a reference text about David Allan Farrish, tell me how many games he played in the NHL.\n",
      "A: Farrish played in the NHL for the Ottawa Argonauts. Farrish played\n",
      "GT: David Allan Farrish played 430 games in the NHL. \n",
      "\n",
      "Q: Who was injured or killed and in what ways in the accidents?\n",
      "A: The injured person or dead person was killed or injured in the accidents. The victims were in the\n",
      "GT: F.H. Kirchman and his passenger Agnes Louise Brady was also killed. William Gleason Sauerwein had a broken leg and head injuries. Margaret Vaughn had severe head and leg injuries. \n",
      "\n",
      "Q: Given a reference text about Mushulatubbee, when did he sign the Treaty of Dancing Rabbit Creek\n",
      "A: Mushulatubbee signed the Treaty of Dancing Rabbit Creek on December\n",
      "GT: Mushulatubbee signed the Treaty of Dancing Rabbit Creek on September 26, 1830. \n",
      "\n",
      "Q: How large is the Gull Harbour and where exactly is it located\n",
      "A: The Gull Harbour is located in the South East of England. It is located in the north\n",
      "GT: Gull Harbour, that has been formerly named as States Harbour or Great Harbour is the 1.3 km wide bay indenting for 2.8 km the east coast of Weddell Island in the Falkland Islands. It is entered north of Gull Point and south of Mark Point, and is centred at 51°53′56′′S 60°53′20′′W. The principal settlement of the island, Weddell Settlement, is situated at the head of the bay. Until the mid-nineteenth century Gull Harbour was known as Great Harbour. \n",
      "\n",
      "Q: What therapies are available for Autism?\n",
      "A: Autism is a form of neurodegenerative disease that is characterized by a lack of\n",
      "GT: There are multiple therapies available for Autism depending on the spectrum. Intensive, sustained special education programs and behaviour therapy early in life can help children with ASD acquire self-care, social, and job skills, and often can improve functioning and decrease symptom severity and maladaptive behaviours; Available approaches include applied behaviour analysis (ABA), developmental models, structured teaching, speech and language therapy, social skills therapy, and occupational therapy. Occupational therapists work with autistic children by creating interventions that promote social interaction like sharing and cooperation. They also support the autistic child by helping them work through a dilemma as the OT imitates the child and waits for a response from the child. Educational interventions have some effectiveness in children: intensive ABA treatment has demonstrated effectiveness in enhancing global functioning in preschool children and is well-established for improving the intellectual performance of young children. \n",
      "\n",
      "Q: Given the reference text below about Larry Bird, tell me where he was born, what he did for a living, and who his greatest professional rival was.\n",
      "A: Larry Bird was born in New York City. He worked as a professional football player and was\n",
      "GT: Larry Bird was born in French Lick. He played and coached basketball for a living, and his main rival was Magic Johnson. \n",
      "\n",
      "Q: Given this paragraph about the Mercury Seven. Who did not survive to reach retirement and what was the cause of their death?\n",
      "A: The Mercury Seven died in the early 1970s due to heart attack. He was a\n",
      "GT: Only Gus Grissom did not survive to reach retirement. All others survived past retirement from service. The cause of his death is the Apollo 1 fire. \n",
      "\n",
      "Q: Where does the word aurora come from?\n",
      "A: The word aurora originates from the Latin word ареа, meaning\n",
      "GT: The word \"aurora\" is derived from the name of the Roman goddess of the dawn, \n",
      "\n",
      "Q: How tall is Mt. Shakspere?\n",
      "A: Mt. Shakspere is 2.4 m (11 feet) tall. He\n",
      "GT: Mt. Shakespere is 12,174 feet (3,711 meters) tall. \n",
      "\n",
      "Q: Have you heard of Yandovy or ндов?\n",
      "A: Yandovy was a Russian linguist and a spokesman for the\n",
      "GT: Yes, I have. Yandovy or ндов is a rural locality (a village) in Mstyora Urban Settlement, Vyaznikovsky District, Vladimir Oblast, Russia. The population of this village was one as of 2010. It is located 16 km northwest of Vyazniki (the district's administrative centre) by road. Kostenevo is the nearest rural locality. \n",
      "\n",
      "Q: Based on this passage, how many Boeing 747s are required to ship 10 of ASML's best-selling EUVs?\n",
      "A: 10 of the ASML's best-selling EUVs are required to ship 10 of\n",
      "GT: The best-selling EUV is the Twinscan NXE:3600D, and it takes 3 Boeing 747s per EUV, so 30 Boeing 747s are required to ship ten Twinscan NXE:3600Ds. \n",
      "\n",
      "Q: Where is Tesla, Inc. headquarters?\n",
      "A: Tesla, Inc. is headquartered in New York, New York.\n",
      "GT: Austin, Texas \n",
      "\n",
      "Q: What is Lady Gaga's real name?\n",
      "A: Lady Gaga is the real name of the singer and songwriter. She is known for her\n",
      "GT: Lady Gaga is the professional name for Stefani Joanne Angelina Germanotta. \n",
      "\n",
      "Q: What is the Memorial Recreation Park and where is it located\n",
      "A: Memorial Recreation Park is located in the City of New York. It is a public park located\n",
      "GT: Memorial Recreation Park is an athletic and recreation complex in Port Huron, Michigan. The main facility of the complex is a 5,500-seat stadium, home to the Port Huron Northern High School and Port Huron High School football teams. In addition, there are in the 27-acre (11 ha) complex tennis courts, four baseball fields, four softball fields, several football fields, and a quarter-mile track. The complex was built in 1945 and is dedicated to the men and women of Port Huron who served in World War II. \n",
      "\n",
      "Q: What international trophy did the Club Africain football team won in 1971?\n",
      "A: The Club Africain football team won the 1971 FIFA World Cup.\n",
      "GT: The Maghreb Cup Winners Cup \n",
      "\n",
      "Q: What is graph theory?\n",
      "A: graph theory is a theory of the representation of the physical world through the use of graphs\n",
      "GT: Graph theory is the study of graphs, which are mathematical structures used to model pairwise relations between objects. \n",
      "\n",
      "Q: What caused the 2007-2008 financial crisis?\n",
      "A: The 2007-2008 financial crisis was caused by the U.S. recession and the 2008-\n",
      "GT: Arguably the largest contributor to the conditions necessary for the financial collapse was the rapid development of predatory financial products which targeted low-income, low-information homebuyers who largely belonged to racial minorities. \n",
      "\n",
      "Q: When was Mahatma Gandhi born?\n",
      "A: Mahatma Gandhi was born on December 7, 1857. He was the first Indian to\n",
      "GT: Mahatma Gandhi was born on October 2, 1869 \n",
      "\n",
      "Q: Given a reference text about Patrick James Mara, tell me what he was known for.\n",
      "A: Patrick James Mara was a professional footballer who played as a coach. He was\n",
      "GT: Patrick James Mara was most known for being a political adviser to Taoiseach Charles Haughey and serving as Press Secretary and Government Press Secretary. \n",
      "\n",
      "Q: Who recorded the song Ghost Town\n",
      "A: The Ghost Town song was recorded by the band Ghost Town. The song was written and recorded by\n",
      "GT: Ghost Town is a song recorded by American singer Adam Lambert for his third studio album, The Original High (2015). It was released as the album's lead single on April 21, 2015. The single garnered considerable commercial and critical success, becoming somewhat of a sleeper hit in select countries and receiving a substantial amount of radio play in the US. \n",
      "\n",
      "Q: What is use of bezier curve in Robotics?\n",
      "A: Robotics uses a bezier curve to measure the length of a curved object\n",
      "GT: Bézier curves can be used in robotics to produce trajectories of an end-effector due to the virtue of the control polygon’s ability to give a clear indication of whether the path is colliding with any nearby obstacle or object. \n",
      "\n",
      "Q: Is the blue-ring octopus harmless?\n",
      "A: The blue-ring octopus is a stinger. It is\n",
      "GT: No, it is extremely toxic. \n",
      "\n",
      "Q: History of Linux\n",
      "A: History of Linux is a history of the Linux operating system. It was written by the author\n",
      "GT: Linux, computer operating system created in the early 1990s by Finnish software engineer Linus Torvalds and the Free Software Foundation (FSF). While still a student at the University of Helsinki, Torvalds started developing Linux to create a system similar to MINIX, a UNIX operating system. In 1991 he released version 0.02; Version 1.0 of the Linux kernel, the core of the operating system, was released in 1994. About the same time, American software developer Richard Stallman and the FSF made efforts to create an open-source UNIX-like operating system called GNU. In contrast to Torvalds, Stallman and the FSF started by creating utilities for the operating system first. These utilities were then added to the Linux kernel to create a complete system called GNU/Linux, or, less precisely, just Linux. \n",
      "\n",
      "Q: Tell me about the new premier of New South Wales\n",
      "A: The new premier of New South Wales was John Howard. He was the first person to speak at\n",
      "GT: Chris Minns led the Labor party to victory at the 2023 NSW state election and becomes the 47th premier of New South Wales. Minns is married with three sons and is a supporter of the NRL club, St George Illawarra Dragons. \n",
      "\n",
      "Q: Given this paragraph on LeBron James' early career, when did he win his first NBA MVP?\n",
      "A: LeBron James won his first NBA MVP in 1984. He was drafted by the Cleveland\n",
      "GT: LeBron James won his first NBA MVP in 2009. \n",
      "\n",
      "Q: Given this paragraph about the first tour de france, who won the first race\n",
      "A: The first race was the first race of the first tour de France, which was a race\n",
      "GT: The first race in 1903 was won by Maurice Garin, over his rival Hippolyte Aucouturier \n",
      "\n",
      "Q: what is snow skiing\n",
      "A: Snow skiing is a sport of skipping snow to skiing in the air. Snow skiing is\n",
      "GT: skiing is a form of outdoor winter sport. Done in the mountains. You can ski on piste and off piste. \n",
      "\n",
      "Q: Given this paragraph about different types of eyes, how are eye types categorized and why?\n",
      "A: The eye is classified as a synapsis, a ocular,\n",
      "GT: Eyes are categorized into two types, \"simple eyes\" and \"compound eyes\". In this case, \"simple eyes\" do not mean that they have reduced complexity or acuity, but rather that they have one concave photoreceptive surface. This is in contrast to \"compound eyes\" which are comprised of a number of individual lenses on a convex surface. In other words, eyes fall into two groups based on their photoreceptor's cellular construction either being cilliated (as in the vertebrates) or rhabdomeric. \n",
      "\n",
      "Q: Where did Jon Rahm go to college?\n",
      "A: University of California, Berkeley. He went to the University of California, Berkeley in the fall of\n",
      "GT: Arizona State University \n",
      "\n",
      "Q: What was the first road trip after Elson joined Indiana University?\n",
      "A: Elson joined Indiana University in the fall of 2016 on a road trip to New York City\n",
      "GT: The roadtrip was to play the WKU Hilltoppers in Bowling Green, Kentucky \n",
      "\n",
      "Q: When he lived in the mountains of Alaska, was Richard Proenneke in full autonomy?\n",
      "A: Richard Proenneke lived in the mountains of Alaska in 1885. He was a member\n",
      "GT: No, he was getting some of his food by plane. \n",
      "\n",
      "Q: Since it began, how many years has FIVB not been held?\n",
      "A: 10 years ago. FIVB has not been held for any number of years. Since it\n",
      "GT: 16 years. 2008-1993+1 = 16. \n",
      "\n",
      "Q: When did Google purchase YouTube\n",
      "A: Google purchased YouTube on May 29, 2008, and it was acquired by Google on May 29, 2008. Google\n",
      "GT: Google purchased YouTube in October 2006 for $1.65 billion \n",
      "\n",
      "Q: When did King Charles II return to Navarre?\n",
      "A: King Charles II returned to Navarre on 1 May 1921. He accompanied his wife\n",
      "GT: King Charles II returned to Navarre in November 1361. \n",
      "\n",
      "Q: What was the first name of Thomas Attewell's brother and cousin?\n",
      "A: Thomas Attewell was the first name of Thomas Attewell's brother and\n",
      "GT: William and Walter \n",
      "\n",
      "Q: What is the formal definition of a large language model?\n",
      "A: Large language models are defined as a large language model that is a language that is\n",
      "GT: The term large language model (LLM) does not have a formal definition. However, it often refers to deep learning models having a parameter count on the order of billions or more. \n",
      "\n",
      "Q: Given the following paragraph about the history of tallest buildings, which was the first skyscraper and when was it build?\n",
      "A: The first skyscraper was built in 1857. It was built in 1857 and\n",
      "GT: The first skyscraper was the Home Insurance Building in Chicago. It was build in 1885. \n",
      "\n",
      "Q: Given this paragraph about Password-authenticated key exchange, what year were the first provably-secure protocols created?\n",
      "A: The first provably-seasible protocols were created in the year 2000.\n",
      "GT: The first provably-secure protocols for Password-authenticated key exchanges(PAKE) were presented in 2000 at \"Eurocrypt 2000\" by two groups Mihir Bellare, David Pointcheval and Phillip Rogaway, and Victor Boyko, Philip MacKenzie and Sarvar Patel. \n",
      "\n",
      "Q: What is a PFD?\n",
      "A: A PFD is a device that allows a user to enter a PFD into\n",
      "GT: PFD is an abbreviation for process flow diagram. Process flow diagrams specify material flow paths, storage equipment (such as tanks and silos), transformations (such as distillation columns, receiver/head tanks, mixing, separations, pumping, etc.), and flowrates. They also list all pipes and conveyors and their contents, material properties such as density, viscosity, particle-size distribution, flowrates, pressures, temperatures, and materials of construction for the piping and unit operations. \n",
      "\n",
      "Q: Based on the text provided, what does a minimum spanning tree optimize?\n",
      "A: A minimum spanning tree optimizes the length of the tree, which is a minimum of\n",
      "GT: A minimum spanning tree optimizes the a graph in a way that minimizes the sum of all the weights of the edges \n",
      "\n",
      "Q: What is the Maurice Richard Trophy\n",
      "A: Maurice Richard Trophy is a prestigious tennis tournament based on the Maurice Richard Trophy\n",
      "GT: The Maurice \"Rocket\" Richard Trophy, also known as the Rocket Richard Trophy, is awarded annually to the leading goal scorer in the National Hockey League (NHL). It was donated to the NHL by the Montreal Canadiens in 1998–99 and is named in honour of legendary Montreal Canadiens right winger Maurice \"Rocket\" Richard. First won by Teemu Selanne, it is currently held by Auston Matthews, who scored 60 goals during the 2021–22 NHL season. \n",
      "\n",
      "Q: How old was Stefan Richarz when he received his phD?\n",
      "A: Stefan Richarz was a teenager when he received his phd. He\n",
      "GT: Stefan was 45 years old in 1919 when he received his phD from University of Munich, given he was born in 1874. \n",
      "\n",
      "Q: Who directed the Lost in Translation movie?\n",
      "A: Michael G. Scott and John McDerlan are the original writers behind the Lost in Translation\n",
      "GT: Lost in Translation was written and directed by Sofia Coppola. \n",
      "\n",
      "Q: here is a short history of Plainview – Old Bethpage John F. Kennedy High School, which color was changed when the name changed in 1991?\n",
      "A: Plainview – Old Bethpage John F. Kennedy High School was repainted in the\n",
      "GT: The school colors swapped out green for blue and white stayed the same. \n",
      "\n",
      "Q: What sport does Arshdeep play?\n",
      "A: Arshdeep is a professional football player. He plays as a coach. He\n",
      "GT: football \n",
      "\n",
      "Q: What is an ice pop?\n",
      "A: Ice pops are a type of popstack. Ice pops are a\n",
      "GT: An ice-pop is a frozen dessert. It is served on a stick, and is made from sweetened liquid that frozen as a block of ice. \n",
      "\n",
      "Q: Given this paragraph, what is the recommended daily value of magnesium for women in the UK?\n",
      "A: 1.9 mg per day. The recommended daily value of magnesium for women in the UK\n",
      "GT: The daily recommended value for magnesium is 270 mg for women in the UK. \n",
      "\n",
      "Q: How many paralympics did Paraskevi compete in?\n",
      "A: Paraskevi competed in the 20th and 21st Paralympic Games.\n",
      "GT: Paraskevi competed in three paralympics. \n",
      "\n",
      "Q: What does Pinot Noir mean?\n",
      "A: Pinot Noir is a type of wine that is often associated with the ripeness of\n",
      "GT: The name Pinot Noir is derived from the French words for pine and black. \n",
      "\n",
      "Q: Who wrote the TV show Fleabag?\n",
      "A: The writer of the TV show Fleabag is Michael McFlyn. McF\n",
      "GT: Fleabag is a British comedy-drama television series created and written by Phoebe Waller-Bridge. \n",
      "\n",
      "Q: Tell me about the career of Erik Dekker\n",
      "A: Erik Dekker was a physsologist and author. He was\n",
      "GT: Hendrik \"Erik\" Dekker (born 21 August 1970) is a retired Dutch professional road racing cyclist active from 1992 until 2006. He was a member of the Rabobank cycling team from 1992 till 2006. From 2007 to 2015 he was one of Rabobank's team managers. Cycling career Amateur career Dekker rode his first race at eight, and soon became successful. In 1985 he was invited to join the national selection for juniors. As an amateur, his most important results were second places at the youth world championships in Bergamo in 1987 and at the road race in the 1992 Summer Olympics. In that Olympic road race, Dekker got away at 30 km before the finish, together with Fabio Casartelli and Dainis Ozols. Dekker was outsprinted by Casartelli, but was so happy that he won a medal that he also finished with his arms in the air. Directly after the Olympic Games, he became professional, and rode his first race a few weeks later in the Tour de l'Avenir. Professional career His first win as a professional was a stage of the Tour of the Basque Country of 1994, the year he rode his first Tour de France. In 1997 Dekker won the Ronde van Nederland, but a large part of 1998 was lost because of injuries. The year 2000 was Dekker's best. He won three stages in the 2000 Tour de France, although neither a sprinter nor a favourite for the overall win, and was voted most combative cyclist. In the autumn of that year, Dekker won his first classic, the Clásica de San Sebastián. In 2001 Dekker won the Amstel Gold Race and the UCI Road World Cup. In the 2001 Tour de France he took a stage, after having helped his teammate Marc Wauters to a stage win. At the end of the year he was named Dutch Sportsman of the year. The years 2002 and 2003 were less successful because of injuries. He came back in 2004 in the spring classics and a victory in Paris–Tours. He had announced his retirement for the autumn of 2006, but he crashed heavily in the 2006 Tour de France and decided to stop. Managing career In 2007, Dekker started as team manager of \n",
      "\n",
      "Q: Which two people are depicted by carved heads near the windows?\n",
      "A: The two carved heads of the carved heads are depicted by carved heads near\n",
      "GT: St. John the Baptist and the Archbishop Daniel Murray \n",
      "\n",
      "Q: Given this paragraph about John C. Young, where did he go to college?\n",
      "A: Indiana University.\n",
      "GT: Dickinson College and Princeton Theological Seminary \n",
      "\n",
      "Q: Given a reference text about a red letter day, provide an explanation of what it means.\n",
      "A: The red letter day is a recurring word in the English language. The word \"red\n",
      "GT: A red letter day is any day of special significance or opportunity such as holidays and festivals. \n",
      "\n",
      "Q: Given a reference text about No Longer at Ease, tell me the name of the book that precedes it in the trilogy.\n",
      "A: No Longer at Ease is the title of the third book in the series.\n",
      "GT: The book that precedes \"No Longer at Ease\" in the African trilogy is \"Things Fall Apart\". \n",
      "\n",
      "Q: How to make Indian Lemonade?\n",
      "A: Mix 1 cup of water, 1 teaspoon of lemon juice, and 1 teaspoon of salt in\n",
      "GT: Lemonade is typically made with sugar, lemons and water. The Indian version, commonly known as nimbu paani, is made a bit differently. Ingredients - 1 lime's juice - 2 teaspoons granulated sugar - 1 teaspoons black salt - 1 teaspoon cumin powder (optional) - Ice (optional and as needed) - 1 cup water or sparking water or soda water Mix ingredient together and stir for 10-15 seconds in a glass Some tips - If you don't have granulated sugar, you can use simple syrup - You may add some crushed mint leaves \n",
      "\n",
      "Q: Give an example of places where astronauts are trained?\n",
      "A: Astroneo-Marshall, California, NASA astronauts are trained in spacecraft\n",
      "GT: Meteor Crater a tourist destination is one of the places where astronauts are trained. \n",
      "\n",
      "Q: When did Confinity become eBay?\n",
      "A: Confinity became eBay in the United States in the late 1990s. It was acquired by eBay\n",
      "GT: Confinity was acquired by X.com in 1999 which was later renamed to PayPal. PayPal was acquired by eBay in 2002. \n",
      "\n",
      "Q: According to this passage, what percent of the lithography machine market do Ultratech, Canon and Nikon, MKS Instruments, Lam Research and Cadence Design Systems have in aggregate?\n",
      "A: 0.5%. Ultratech, Canon and Nikon, MKS Instruments, Lam Research and Cad\n",
      "GT: If ASML has 67 percent of the worldwide sales of lithography machines and if Ultratech, Canon and Nikon, MKS Instruments, Lam Research and Cadence Design Systems represent their sole competition, than these companies must share the remaining 33 percent of the market. \n",
      "\n",
      "Q: Given this paragraph about flashcards, is flashcard useful for learning painting?\n",
      "A: Flashcards are not useful for learning painting. They can be used to learn a number\n",
      "GT: No. Flashcards are useful for learning things based on memorization via a question-and-answer format. Painting is a creative activity. It's unlikely to be learned by memorization of existing materials. Therefore, flashcard won't be useful for learning \n",
      "\n",
      "Q: Given the following paragraph, how tall is the Leaning Tower of Pisa?\n",
      "A: The Leaning Tower of Pisa is a 2,000 ft. tall\n",
      "GT: The height of the tower is 55.86 metres (183 feet 3 inches) from the ground on the low side and 56.67 m (185 ft 11 in) on the high side. \n",
      "\n",
      "Q: who is Haile Sellassie?\n",
      "A: Haile Sellassie is a former American football player who played as a quarterback for\n",
      "GT: Haile Selassie (Amharic:   ) was a former Emperor of Ethiopia. He ruled the country from 1930 to 1974. His first name before he became emperor was Ras Teferi Mekonnen. The religion movement, Rastafari, after the name of Ras Teferi Mekonnen, emerged in Jamaica shortly after Haile Selassie became emperor. His root is from an extended Solomon and Queen of Sheba(Makeda). \n",
      "\n",
      "Q: Who is the best surfer in the world\n",
      "A: The best surfer in the world is the best surfer in the world is the best sur\n",
      "GT: Kelly Slater \n",
      "\n",
      "Q: Based on this reference text about bull sharks, why can bull sharks live in both salt and fresh water?\n",
      "A: Because bull sharks live in salt water. They can live in both salt and fresh water.\n",
      "GT: Bull sharks can live in both salt and fresh water because they are euryhaline. \n",
      "\n",
      "Q: What types of ritual use Ayahuasca?\n",
      "A: Ayahuasca is a traditional Chinese ritual used to drink water. It is used to\n",
      "GT: Ayahuasca is used both socially and as a ceremonial or shamanic spiritual medicine \n",
      "\n",
      "Q: Is ABAKO no longer an active political party?\n",
      "A: No, ABAKO is no longer an active political party. It was founded in 2001 by\n",
      "GT: Correct, The Alliance of Bakongo, or ABAKO, is no longer a political party. \n",
      "\n",
      "Q: Who was the first African American to go to space?\n",
      "A: African American astronaut John Glennor was the first African American to go to space. He was\n",
      "GT: Guion Stewart Bluford Jr. was the first African American to go to space. He was an American aerospace engineer, retired United States Air force officer and fighter pilot, and former NASA astronaut. He participated in for Space Shuttle flights between 1983 and 1992. \n",
      "\n",
      "Q: In what year was the evidence of an exoplanet first confirmed?\n",
      "A: 1986. The first evidence of an exoplanet was first confirmed in 1913.\n",
      "GT: The detection of an exoplanet was first confirmed in 1992 \n",
      "\n",
      "Q: Given a reference text about Sean Dunphy, tell me where he is buried.\n",
      "A: Sean Dunphy is buried in the Cathedral of Westminster Abbey.\n",
      "GT: Sean Dunphy is buried at Greenogue cemetery in Ashbourne, County Meath. \n",
      "\n",
      "Q: Given these paragraphs about Natural hydrogen, what are other forms of hydrogen obtained using different processes?\n",
      "A: Other forms of hydrogen are derived from a chemical reaction. The chemical reaction is a\n",
      "GT: Green hydrogen, which is produced from renewable energy sources, and from grey, brown or black hydrogen, which is obtained from fossil sources or from the electrolysis of water. \n",
      "\n",
      "Q: Given a reference text that describes about hotpot, from which country does this dish originate?\n",
      "A: China. Hotpot is a dish from China. It originates from the country of China\n",
      "GT: Hot pot or Hotpot is originated from China. \n",
      "\n",
      "Q: Why was Advani concerned about Modi’s appointment as a chief minister of Gujarat?\n",
      "A: He was concerned that Modi would be unable to govern in Gujarat. He was concerned that\n",
      "GT: For Modi being a candidate for the chief minister of Gujarat, Advani was concerned about Modi’s lack of political experience. \n",
      "\n",
      "Q: Given these paragraphs about battery storage power stations, for how long battery power storage was cheaper than open cycle gas turbine power as of 2019?\n",
      "A: The battery storage was cheaper as of 2020 than open cycle gas turbine power as of 2020. The\n",
      "GT: As of 2019, battery power storage is cheaper than open cycle gas turbine power for use up to two hours. \n",
      "\n",
      "Q: Given this paragraph about ext3, tell me why its better than ext2 and its successor.\n",
      "A: It is better than ext2 and its successor, ext3 because it is more flexible\n",
      "GT: ext3 is better than ext2 because of its journaling capabilities, which improve reliability and eliminates the need to check the file system after an unclean shutdown. The successor of ext3 is ext4. \n",
      "\n",
      "Q: What year was Kerala formed?\n",
      "A: Kerala was formed in the 18th century. The first state to be a state was Co\n",
      "GT: 1956 \n",
      "\n",
      "Q: Given the paragraph above paragraph, what is the PMP and who established the exam.\n",
      "A: The PMP is a national body of the United States of America. It established the PM\n",
      "GT: PMP stands for Project Management Professional, and it was established by the Project Management Institute (PMI). The PMP is a professional certification for project managers that is recognized internationally. The PMP is based on the ECO PMP Examination Content Outline. \n",
      "\n",
      "Q: Where is the Birthplace of 'Silicon Valley\n",
      "A: The birthplace of 'Silicon Valley' is San Diego, California. San\n",
      "GT: The HP Garage at 367 Addison Avenue \n",
      "\n",
      "Q: Given this reference text about Jimmy John's, where was the original Jimmy John's located?\n",
      "A: The original Jimmy John's was located in New York City. The location is located in the\n",
      "GT: The original Jimmy John's was located in Charleston, Illinois. \n",
      "\n",
      "Q: Based on the reference text about Bruce Nodwell, what did he create?\n",
      "A: Bruce Nodwell created the \"Sea of Dreams\" and \"The Book of\n",
      "GT: Bruce Nodwell invented the Nodwell 110, a two-tracked vehicle capable of traversing a wide variety of terrain. \n",
      "\n",
      "Q: Given a reference text about Tujhse He Raabta, tell me who played the main role in the film.\n",
      "A: Tujhse He Raabta played the main role in the film.\n",
      "GT: Sara Khan played the lead role in the film Tujhse He Raabta. \n",
      "\n",
      "Q: Who is Rihanna?\n",
      "A: Rihanna is a singer and songwriter. She is best known for being the lead\n",
      "GT: Rihanna (Robyn Rihanna Fenty) is a singer, actress, and businesswoman from Barbados. After auditioning with American record producer Evan Rogers, she signed to Def Jam in 2005 and gained worldwide recognition with her first two studio albums, \"Music of the Sun\" and \"A Girl Like Me\". \n",
      "\n",
      "Q: What is Matong\n",
      "A: Matong is a slang word meaning \"sad\" or \"se\n",
      "GT: Matong is a town in the central east part of the Riverina region of New South Wales, Australia. The town is 37 kilometres (23 mi) east of Narrandera and 26 kilometres (16 mi) west of Coolamon. At the 2016 census, Matong had a population of 164 people. Sport The most popular sport in Matong is Australian rules football, as it lies in the narrow 'canola belt', a geographical triangle stretching from the Grong Grong and Marrar at either end of the Canola Way, to Ungarie, in which Australian football retains a strong following, despite New South Wales being a largely rugby league supporting state. \n",
      "\n",
      "Q: Using this paragraph about Matt Eskandari, did Matt Eskandri make any films before Victim?\n",
      "A: No, Matt Eskandri made a film before Victim. He also made\n",
      "GT: Matt Eskandari filmed \"The Taking\" before his feature debut with the film Victim. \n",
      "\n",
      "Q: What type of products does this company produce?\n",
      "A: Company produces a variety of products including food, clothing, and textiles.\n",
      "GT: maple products \n",
      "\n",
      "Q: Who is Ron Rivest?\n",
      "A: Ron Rivest is an American author of children's books. He is the author of the\n",
      "GT: Ron Rivest is a cryptographer and computer science professor at MIT's Computer Science and Artificial Intelligence Laboratory (CSAIL). He is most well known as the inventor of the RSA public key cryptosystem (with Adi Shamir and Len Adleman). Rivest is also known as the inventor of the RC4 stream cipher and MD5 hash function. \n",
      "\n",
      "Q: Based on this paragraph, who was Donald Trump's physician during 2018 to 2021?\n",
      "A: Donald Trump's physician was Donald Trump's medical advisor. He was the first physician to\n",
      "GT: Sean Patrick Conley \n",
      "\n",
      "Q: Given this paragraph about Arlington, Virginia, what are the 3 public high schools in Arlington?\n",
      "A: Arlington High School, Arlington High School, and Arlington High School are all public high schools. Arlington\n",
      "GT: The 3 public high schools in Arlington, Va are Wakefield High School, Washington-Liberty High School, and Yorktown High School. \n",
      "\n",
      "Q: Given a reference text about Our Common Future, what was the purpose of the report?\n",
      "A: To promote the development of the Common Future. The report was intended to promote the development of the\n",
      "GT: The purpose of the Our Common Future report was to focus on environmental issues on the political agenda and to discuss sustainable development. \n",
      "\n",
      "Q: Given these paragraphs about Multiomics, what is panomics?\n",
      "A: Multiomics is a type of computer science that focuses on the study of the development of\n",
      "GT: The American Society of Clinical Oncology (ASCO) defines panomics as referring to \"the interaction of all biological functions within a cell and with other body functions, combining data collected by targeted tests... and global assays (such as genome sequencing) with other patient-specific information.\" \n",
      "\n",
      "Q: what are the 5 skandhas?\n",
      "A: The 5 skandhas are: 1. The 5 skandhas are:\n",
      "GT: The five skhandas are form, sensations, perceptions, mental activity and consciousness \n",
      "\n",
      "Q: How long was the Nazi German battleship Bismarck active during WWII?\n",
      "A: The Nazi German battleship Bismarck was active during WWII. It was commissioned in 1945\n",
      "GT: The Bismarck was active for only 8 months of World WarII. The Bismarck and an accompanying heavy crusier, the Prinz Eugen engaged with two British battlecruisers, the Hood and the Prince of Wales off the Denmark Strait in May 1941. During the engagement, the Hood was destroyed but the Bismarck suffered damage from fire from the Price of Wales that forced the Nazi battleship to effectively end its operations. \n",
      "\n",
      "Q: Who was Margaret Thatcher?\n",
      "A: Margaret Thatcher was a British politician and author who was the first woman to be elected to\n",
      "GT: Margaret Thatcher was the first woman to be elected prime minister of Great Britain and Northern Ireland, serving in this role from her first election victory in 1979 until she was forced out of office by members of her own party in 1990. During this time she transformed the economic situation in the United Kingdom by introducing economic policies intended to reverse high inflation and weak growth. By constraining the power of trades unions and overseeing extensive deregulation and privatisation, Thatcher helped to create a period of extended growth and economic success. Together with Ronald Reagan, she was also instrumental in recognising and supporting the opportunity to end communist rule in the Soviet Union, where she was dubbed the 'Iron Lady', through her friendship with the Russian leader, Mikhail Gorbachev. Her legacy was to end the so-called 'post-war consensus' in the UK (characterised by high taxes, regulation and nationalisation) and to replace it with policies that encouraged free-enterprise and personal responsibility while acknowledging the importance of the welfare state, provided it was administered responsibly. Although disliked by many on the left for the way that she challenged excessive union power, she is generally acknowledged as the most important British politician of the late twentieth century. \n",
      "\n",
      "Q: per the context given, how would illiterate politicians able to vote?\n",
      "A: illiterate politicians could not vote.\n",
      "GT: They used lettering guide templates made of cardboard with characters of their last name \n",
      "\n",
      "Q: Based on this text, what average temperature in degree celsius can I expect in April in Germany?\n",
      "A: The average temperature in April can be a temperature of -23.0 degrees Celsius.\n",
      "GT: The average temperature in April in Germany is 13.2 °C. \n",
      "\n",
      "Q: What does the term \"tulip mania\" mean?\n",
      "A: The word tulip mania is a common name for a group of people\n",
      "GT: The term \"tulip mania\" is now often used metaphorically to refer to any large economic bubble when asset prices deviate from intrinsic values. \n",
      "\n",
      "Q: Given this paragraph about Thanjavur paintings and its origins, what are the typical features of Thanjavur paintings?\n",
      "A: The Thanjavur paintings are often depicted as a symphonic\n",
      "GT: Thanjavur paintings are easily discernible from the gold coating on wooden panels which form the base. They are characterized by vivid colors, simple composition of icons along with usage of glass beads and at times precious stones. The subjects of paintings have traditionally been Hindus gods though they have evolved to depict secular subjects from other religions. \n",
      "\n",
      "Q: Who are the little Japanese girls who find Totoro?\n",
      "A: The Little Girls are the Japanese girls who find the Japanese girl, Totoro. They find\n",
      "GT: Mei and Satsuki are the two Japanese girls who discover Totoro in the famous Japanese anime My Neighbour Totoro. \n",
      "\n",
      "Q: Is Belgrandia varica found in Europe or Australia?\n",
      "A: Belgrandia varica is found in Australia. It is a species of bee\n",
      "GT: Belgrandia varica is endemic to France, so therefore it is found in Europe. \n",
      "\n",
      "Q: What is Sogndal?\n",
      "A: Sogndal is a village in the South-East of the country in Sweden.\n",
      "GT: Sogndal is a small city in southwestern Norway with a population of about 12,000 \n",
      "\n",
      "Q: Which nation started the Arab Spring and what caused it?\n",
      "A: The Arab Spring began in Egypt and the Arab Spring caused it. The Arab Spring caused Egypt to\n",
      "GT: The Arab Spring began in Tunisia in response to corruption and economic stagnation. From Tunisia, the protests then spread to five other countries: Libya, Egypt, Yemen, Syria and Bahrain. \n",
      "\n",
      "Q: Based on this passage, what is the median number of people per family in Zangisheh?\n",
      "A: The median number of people per household in Zangisheh is 57.81%\n",
      "GT: This passage does not give enough information to conclude what the median number of people per family in Zangisheh is as it only mentions the total population and number of families. We could calculate the arithmetic mean of the number of people per family from this information by dividing the total number of people by the number of families, but the information in the passage is not sufficient to determine the median number of people per family. \n",
      "\n",
      "Q: What city in Ireland is the band U2 from?\n",
      "A: Dublin is the band U2 from Dublin. The band is from Dublin, Ireland. They are\n",
      "GT: Dublin \n",
      "\n",
      "Q: Given a reference text about the Queen Isabella Memorial Bridge, tell me how long it is and when it opened.\n",
      "A: Queen Isabella Memorial Bridge opened on May 3, 1892. It is a sand\n",
      "GT: The Queen Isabella Memorial Bridge opened in 1974 and is 2.5 miles long. \n",
      "\n",
      "Q: Given a reference text about Hackpad, what company acquired it?\n",
      "A: Microsoft acquired the company in May 2015.\n",
      "GT: Hackpad was acquired by Dropbox in April 2014. \n",
      "\n",
      "Q: Given a reference text about Giovanni Dominici, when was he beatified?\n",
      "A: He was beatified in 1253.\n",
      "GT: Giovanni Dominici was beatified on April 9, 1832. \n",
      "\n",
      "Q: When was Santa Monica founded?\n",
      "A: Santa Monica was founded in 1889 by the Spanish explorer Francisco Pizarro. It was\n",
      "GT: Santa Monica was founded in 1886. \n",
      "\n",
      "Q: Where was Mazzy Star formed?\n",
      "A: Mazzy Star was formed in the United States.\n",
      "GT: Mazzy Star was formed in Santa Monica, California in 1988 \n",
      "\n",
      "Q: Who is known as The Invincibles in English football?\n",
      "A: The Invincibles are known as the Invincibles in English football.\n",
      "GT: Preston North End & Arsenal Football Club \n",
      "\n",
      "Q: According to this paragraph, when was the university of virginia founded?\n",
      "A: 1898.\n",
      "GT: The University of Virginia was founded in 1819. \n",
      "\n",
      "Q: Given a reference text about the Kaziranga Elephant Festival, tell me who organizes it.\n",
      "A: The Kaziranga Elephant Festival is organised by the Kaziranga People's Association\n",
      "GT: The Kaziranga Elephant Festival is organized by the Forest and Tourism Departments of Assam. \n",
      "\n",
      "Q: How long did it take for ironworking to be commonplace in West Africa following its introduction in Northern Africa?\n",
      "A: In the early 1900s, ironworking was commonplace in West Africa. It took about\n",
      "GT: Ironworking was introduced in Northern Africa around the first millennium BCE and became commonplace in West Africa by 500 BCE. This means it took 500 years (1000-500) for ironworking to become common in West Africa following its introduction in Northern Africa. \n",
      "\n",
      "Q: Given this text about James Pressdee, tell me what sports they played, which awards they won, when they were born, and if they are still alive.\n",
      "A: James Pressdee played football, basketball, and was a member of the National Football League\n",
      "GT: James Stuart Pressdee played both football and cricket. Though he was an inconsistent bowler, he won the South African Cricket Annual Cricketer of the Year award in 1966. Prior to his professional sports career, James won a Welsh youth cap in football in Swansea Town. Jame was born on June 19th, 1933, and he died on July 20th, 2016 at the age of 83. \n",
      "\n",
      "Q: When was Chhatrapati Shivaji Maharaj born?\n",
      "A: Chhatrapati Shivaji Maharaj was born on 21 August 1939. He was\n",
      "GT: Chhatrapati Shivaji Maharaj was born on 19, February 1630 \n",
      "\n",
      "Q: Which fictional LAPD detective is Titus Welliver known for playing?\n",
      "A: Titus Welliver is known for playing the role of the LAPD detective who played\n",
      "GT: Titus Welliver played Harry Bosch in the TV series Bosch and also in spin off Bosch:Legacy. \n",
      "\n",
      "Q: Given a reference text about magnetism, tell me the definition of magnetism and how magnetic materials are classified.\n",
      "A: Magnetism is the study of magnetic materials. Magnets are materials that are magnetic. Magnet\n",
      "GT: Magnetism is defined as the physical attributes that are mediated by a magnetic field, which is the capacity to induce attractive and repulsive forces. Magnetic materials are classified by their bulk susceptibility. \n",
      "\n",
      "Q: What is Cricket in sports?\n",
      "A: Cricket is a sport that is played in the summer and winter months. Cricket is a\n",
      "GT: Cricket is highly popular game that's originated in south-eastern counties of England and now prominent sport in a number of countries in particular south asian countries like India, Pakistan, Sri Lanka and Bangladesh. It is in essence a bat-and-ball game played between two teams of eleven players on a field. The batting side scores runs by striking the ball with the bat and then running between the wickets, while the bowling and fielding side tries to prevent this and dismiss each batter. Forms of cricket range from Twenty20, one day to Test match lasting up to 5 days. The game originated as children's game is now a highly competitive professional sport followed by billions of people and in last decade commercialised at club level with the introduction of Indian Premier League (IPL). \n",
      "\n",
      "Q: Who is Bill Gates?\n",
      "A: Bill Gates is an inventor and businessperson. He is the founder of Gates Gates\n",
      "GT: Bill Gates is an American business mogul and co-founder of Microsoft. \n",
      "\n",
      "Q: What is Methadone intermediate\n",
      "A: Methadone is an intermediate in the production of a compound called a methylated\n",
      "GT: Methadone intermediate is a methadone precursor scheduled by UN Single Convention on Narcotic Drugs. It is a Schedule II Narcotic controlled substance in the United States and has an ACSCN of 9254. The 2014 annual manufacturing quota was 32 875 kilos. \n",
      "\n",
      "Q: From this reference text, what political party does Taylor Swift belong to?\n",
      "A: Democratic Party (Saturday Night)\n",
      "GT: Taylor Swift is a democrat. \n",
      "\n",
      "Q: How many years was Hutchinson's streak, and what caused it to end?\n",
      "A: Hutchinson's streak was a one year streak, and the streak ended when\n",
      "GT: Hutchinson's streak was 62 years (2019-1958+1) and it ended because of the lock out caused by the COVID-19 pandemic. \n",
      "\n",
      "Q: Approximately how much cashmere is produced each year?\n",
      "A: Approximately 80% of cashmere is produced each year. The average amount of cashmer\n",
      "GT: It is estimated that about 6,500 tons of cashmere is produced each year. \n",
      "\n",
      "Q: Given a reference text about the 2022 Mazar-i-Sharif mosque bombing, how many people were injured and killed?\n",
      "A: The 2022 Mazar-i-Sharif mosque bombing left at least 200 people\n",
      "GT: At least 31 people were killed and more than 87 others were injured in the 2022 Mazar-i-Sharif mosque bombing. \n",
      "\n",
      "Q: What percentage of the world's population did the British Empire hold sway over?\n",
      "A: 80% of the world's population was held by the British Empire. The British Empire\n",
      "GT: 23 per cent of the world population \n",
      "\n",
      "Q: Given a reference text about the 2008 Pittsburgh Pirates season, what was their record for the season?\n",
      "A: The Pirates had a record of 3-2. They were 0-2 in the 2008\n",
      "GT: The 2008 Pittsburgh Pirates finished the season with a record of 67-95. \n",
      "\n",
      "Q: How long after the release of the album “Abbey Road” did The Beatles break up?\n",
      "A: The Beatles broke up in 1969. They broke up in 1969. The Beatles broke up in 1969\n",
      "GT: One year after releasing Abbey Road (1969), the Beatles broke up (1970) \n",
      "\n",
      "Q: Who was Rosa Parks?\n",
      "A: Rosa Parks was a Spanish painter and poet. She was the first woman to paint\n",
      "GT: Rosa Louise McCauley Parks, born on February 4, 1913 was an American activist in the civil rights movement best known for her pivotal role in the Montgomery bus boycott. The United States Congress has honored her as \"the first lady of civil rights\" and \"the mother of the freedom movement\". Parks became a NAACP activist in 1943, participating in several high profile civil rights campaigns. On December 1, 1955, in Montgomery, Alabama, Parks rejected bus driver James F. Blake's order to vacate a row of four seats in the \"colored\" section in favor of a White passenger, once the \"White\" section was filled. Parks was not the first person to resist bus segregation, but the National Association for the Advancement of Colored People (NAACP) believed that she was the best candidate for seeing through a court challenge after her arrest for civil disobedience in violating Alabama segregation laws, and she helped inspire the Black community to boycott the Montgomery buses for over a year. The case became bogged down in the state courts, but the federal Montgomery bus lawsuit Browder v. Gayle resulted in a November 1956 decision that bus segregation is unconstitutional under the Equal Protection Clause of the 14th Amendment to the U.S. Constitution. Parks's act of defiance and the Montgomery bus boycott became important symbols of the movement. She became an international icon of resistance to racial segregation, and organized and collaborated with civil rights leaders, including Edgar Nixon and Martin Luther King Jr. At the time, Parks was employed as a seamstress at a local department store and was secretary of the Montgomery chapter of the NAACP. She had recently attended the Highlander Folk School, a Tennessee center for training activists for workers' rights and racial equality. Although widely honored in later years, she also suffered for her act; she was fired from her job, and received death threats for years afterwards. Shortly after the boycott, she moved to Detroit, where she briefly found similar work. From 1965 to 1988, she served as secretary and receptionist to John Conyers, an African-American US Representative. She was also active in the Black Power movement and the support of political prisoners in the US. After retirement, Parks wrote her autobiography and continued to insist that there was more work to be \n",
      "\n",
      "Q: Based on this passage, why is Chris Kuroda's nickname CK5?\n",
      "A: The nickname CK5 is a nickname given to the former American football player who has played\n",
      "GT: Chris Kuroda's nickname, CK5, adds the number 5 to his initial. That is because his improvised light shows during Phish concerts give him a role comparable to a fifth member of the band. \n",
      "\n",
      "Q: Does Irina Korzhanenko still have her gold medal?\n",
      "A: Irina Korzhanenko has been a member of the European Women's\n",
      "GT: Yes, although she was stripped of the medal she refused to return it. \n",
      "\n",
      "Q: Given a reference text about Joseph A. Garcia, tell me what high school and college he attended.\n",
      "A: Joseph A. Garcia attended the University of Texas at Austin. He attended the University of Texas at\n",
      "GT: Joseph A. Garcia attended Austin E. Lathrop High School and T.C. Williams High School. After high school, he attended the University of Colorado Boulder and Harvard Law School. \n",
      "\n",
      "Q: Given this reference text, why and when did Coke use Splenda as a sweetener in diet Coke?\n",
      "A: Coke used Splenda as a sweetener in diet coke in the\n",
      "GT: Pressure from Walmart pressured Coca-Cola to release a version of diet Coke sweetened with Splenda from 2005 until most distribution ended in 2009. \n",
      "\n",
      "Q: Given this paragraph about a country, what is the most populous Spanish-speaking country?\n",
      "A: Mexico is the most populous country in Spain. It is bordered by the United States\n",
      "GT: Mexico \n",
      "\n",
      "Q: Given this paragraph about the climate in Fairmont, West Virginia, what was the coldest month in the city's history, and what was the average temperature during that month?\n",
      "A: The average temperature in Fairmont, West Virginia was 0.00 °F on January 31s\n",
      "GT: January of 1977 was the coldest month in the history of Fairmont, West Virginia with an average of 16.0 degrees Farenheit. \n",
      "\n",
      "Q: Given a reference text about the Build Back Better Plan, what are the 3 parts that make up the plan?\n",
      "A: The Build Back Better Plan is a plan to build back stronger and more efficient. It includes\n",
      "GT: The three components of the Build Back Better plan are: - American Rescue Plan - American Jobs Plan - American Families Plan \n",
      "\n",
      "Q: Who wrote the TV series Babylon5?\n",
      "A: The series was created by Josh Schwartz and produced by David Lynch. It was directed by David\n",
      "GT: Babylon 5 is an American television series created by writer and producer J. Michael Straczynski. \n",
      "\n",
      "Q: Given a reference text about Connie Passalacqua Hayman, tell me when and where taught journalism.\n",
      "A: Connie Passalacqua Hayman was taught journalism at the University of Oxford from 1939\n",
      "GT: Connie Passalacqua Hayman taught journalism at New York University from 1995 to 2001 and at Marymount Manhattan College from 2007 to 2014. \n",
      "\n",
      "Q: Based on this questio about Kingdom Hearts, what company published it?\n",
      "A: The company that published Kingdom Hearts was a company that was published by Sony. The company\n",
      "GT: Square Enix \n",
      "\n",
      "Q: Given the following note about training in a specific style of karate, tell me why its practitioners consider its fist alignment to offer advantages for their striking.\n",
      "A: The fist alignment allows the practitioner to achieve the best balance and control of the balance.\n",
      "GT: Practitioners of Shrin-ry believe lining up the second and third knuckle, along with the bones of the arm, enables them to deliver faster and more powerful punches. \n",
      "\n",
      "Q: Given this paragraph about Sam Baker, what is the Outland Trophy?\n",
      "A: Outland Trophy is the highest award given out to a person in the United States.\n",
      "GT: The Outland Trophy is given to the best offensive lineman in college football. \n",
      "\n",
      "Q: What horsepower does a BMW 1250GS produce\n",
      "A: a BMW 1250GS produces a maximum of 310 hp. The BMW\n",
      "GT: The 2018 model of the 1250 GS produced 134 peak HP. \n",
      "\n",
      "Q: Did the dome last as long as it was designed for?\n",
      "A: It was designed to last about a decade. The dome was designed to last about a\n",
      "GT: No, it did not. The dome only lasted 13 years, but it was designed to last 15-20 years. \n",
      "\n",
      "Q: Given the reference paragraph, when was the CUBC formed?\n",
      "A: The CUBC was formed in the year 2000.\n",
      "GT: The Cambridge University Boat Club (CUBC) was founded in 1828 \n",
      "\n",
      "Q: Given the reference text about Mother Teresa's international charity, how many languages did Mother Teresa speak?\n",
      "A: Mother Teresa spoke a total of four languages. She spoke five languages: Spanish, English,\n",
      "GT: Mother Teresa was fluent in five languages, namely Bengali, Albanian, Serbian, English and Hindi. \n",
      "\n",
      "Q: Can someone drive on Black Bear Road during the winter?\n",
      "A: Yes, it is a winter road. It is a mountain road in the winter.\n",
      "GT: No, it is only open from late summer to early fall. \n",
      "\n",
      "Q: Based on this article, how did the existence of trains impact the development of Roche Percee?\n",
      "A: The development of the railways was influenced by the development of the railways. The railway\n",
      "GT: The area that became Roche Percee was impacted by trains when the Soo Line connected the village to a train line. This train line allowed for coal mining in the region to develop by providing access for transporting coal. After the development of these coal mines, the area was eventually able to grow enough to become incorporated as a village. \n",
      "\n",
      "Q: Given this mechanism that the Tesla Model Y car uses to heat the interior cabin, what are some pros and cons of this design?\n",
      "A: The mechanism that the Tesla Model Y car uses to heat the interior cabin is very similar to\n",
      "GT: The heat pump in the Tesla Model Y runs the same heating operations as other cars but much more efficiently, significantly increasing driving range for the same battery size. Most other cars use electric resistance heating which consumes much more energy to perform the same job. \n",
      "\n",
      "Q: Other than Tanki, how many other known Amerindian settlements are on Aruba?\n",
      "A: 2 other known Amerindian settlements are on Aruba. The other two are the\n",
      "GT: Three (3). 4 - 1 = 3 \n",
      "\n",
      "Q: What are the seven wonders of the world?\n",
      "A: The seven wonders of the world are the seven wonders of the world. The seven wonder\n",
      "GT: The seven wonders of the world consist of; the Great Pyramid of Giza, The Colossus of Rhodes, the Lighthouse of Alexandria, the Mausoleum at Halicarnassus, the Temple of Artemis, the Statue of Zeus at Olympia, and the Hanging Gardens of Babylon. \n",
      "\n",
      "Q: Given the reference text about moths, what family does Bityla sericea belong to?\n",
      "A: The Bityla sericea belongs to the family of moths. Its\n",
      "GT: Noctuidae \n",
      "\n",
      "Q: Given this paragraph about London, give me a reason why the London Docklands became a focus for regeneration in the 1980s.\n",
      "A: The Docklands became a focus for regeneration in the 1980s. The regeneration of the London\n",
      "GT: One reason why the London Docklands became a focus for regeneration in the 1980s is that London had an increasing role as an international financial centre. \n",
      "\n",
      "Q: What was the Caribbean and Central America Action\n",
      "A: The Caribbean and Central America Action was the Caribbean and Central America Action. It was a strategy\n",
      "GT: Caribbean and Central American Action was a non-profit organization which has hosted the annual Miami Conference on the Caribbean & Central America for 33 years. The organization has a multinational Board of Trustees, currently headed by Cornelius B. Prior, Jr., and is based in Washington, DC. Among the public figures speaking at the 33rd conference, held in 2009 in Miami, were the Prime Minister of Haiti, Jean-Max Bellerive, Canada's Minister of State of Foreign Affairs, Peter Kent and Puerto Rico's Secretary of State, Kenneth McClintock. The organization's policy-oriented programs have focused on tourism, financial services, policy and regulatory environments for business in the Americas, energy security, maritime port security, disaster mitigation, customs, telecommunications, intellectual property rights, information technology, agriculture and telecommunications. Cabinet-level officials have represented the United States at prior conferences. The organization has closed. \n",
      "\n",
      "Q: Why are people eating eggs at Easter\n",
      "A: Easter is celebrated for the first time in the history of the European religion. Eggs are eaten\n",
      "GT: Easter eggs, also called Paschal eggs, are eggs that are decorated for the Christian feast of Easter, which celebrates the resurrection of Jesus. As such, Easter eggs are common during the season of Eastertide (Easter season). The oldest tradition, which continues to be used in Central and Eastern Europe, is to use dyed and painted chicken eggs. Although eggs, in general, were a traditional symbol of fertility and rebirth, in Christianity, for the celebration of Eastertide, Easter eggs symbolize the empty tomb of Jesus, from which Jesus was resurrected. In addition, one ancient tradition was the staining of Easter eggs with the colour red \"in memory of the blood of Christ, shed as at that time of his crucifixion.\" This custom of the Easter egg, according to many sources, can be traced to early Christians of Mesopotamia, and from there it spread into Eastern Europe and Siberia through the Orthodox Churches, and later into Europe through the Catholic and Protestant Churches. Mediaevalist scholars normally conclude that the custom of Easter eggs has its roots in the prohibition of eggs during Lent after which, on Easter, they have been blessed for the occasion. A modern custom in some places is to substitute chocolate eggs wrapped in coloured foil, hand-carved wooden eggs, or plastic eggs filled with confectionery such as chocolate. \n",
      "\n",
      "Q: In what year was pickleball invented?\n",
      "A: 1865. In 1880, pickleball was invented. It was invented in 1880\n",
      "GT: Pickleball was created in 1965 on Bainbridge Island, Washington. \n",
      "\n",
      "Q: How long does it take for a mustard seed to germinate?\n",
      "A: The mustard seed carries about a year to germinate. The seed takes about a\n",
      "GT: Mustard seeds generally take eight to ten days to germinate if placed under the proper conditions, which include a cold atmosphere and relatively moist soil. \n",
      "\n",
      "Q: Who are Industrial Light and Magic?\n",
      "A: Industrial Light and Magic is a 501(c)(3) nonprofit organization founded by the late\n",
      "GT: Industrial Light and Magic are one of the biggest and most innovative VFX (Visual Effects) and CGI (Computer generated imagery) firms. The company was started by George Lucas.and was acquired by Disney. They have contributed Visual special effects for over 300 films including the Star Wars series, Indiana Jones series and the Jurassic Park series. \n",
      "\n",
      "Q: What makes a Squishmallow resellable?\n",
      "A: Squishmallow is a resellable item. The Squishm\n",
      "GT: Squishmallows are a highly demanded toy and also often are sold as limited edition items. As a result, Squishmallows that are more unique and rare can often be sold or higher value later. One example is Jack the Black Cat which has been listed on eBay for nearly $2,000. \n",
      "\n",
      "Q: Given this text, summarize the major events in Call of Duty: Modern Warfare 2 game development and release.\n",
      "A: The major events in Call of Duty: Modern Warfare 2 are: The Battle of the\n",
      "GT: 2008: Call of Duty: Modern Warfare 2 development began Feb. 2009: officially announced March 2009: game testing bagan Nov. 10, 2009: released worldwide for Microsoft Windows, the PlayStation 3 and Xbox 360. 2014: released for macOS 2018: released for Xbox One March 2020: a remastered version is released on the PlayStation 4 April 2020: a remastered version is released on Microsoft WIndows and Xbox One \n",
      "\n",
      "Q: Why is prioritizing diversity hiring in an organization important?\n",
      "A: Diversity hiring is a key way to ensure that the company has a diverse workforce. The\n",
      "GT: Diversity drives creativity, innovation, and personal growth. \n",
      "\n",
      "Q: How to play chess\n",
      "A: Chess is a game that is played on a wooden board. Chess\n",
      "GT: To win the game of chess, a player must capture the opponent's king. This is known as \"Checkmating\" the king. \n",
      "\n",
      "Q: What is the history of the Masters Golf Tournament\n",
      "A: The Masters Golf Tournament was first held in 1899 and has been a major tournament since 19\n",
      "GT: The Masters was started by amateur champion Bobby Jones and investment banker Clifford Roberts.[4] After his grand slam in 1930, Jones acquired the former plant nursery and co-designed Augusta National with course architect Alister MacKenzie.[1] First played 89 years ago in 1934, the tournament is an official money event on the PGA Tour, the European Tour, and the Japan Golf Tour. The field of players is smaller than those of the other major championships because it is an invitational event, held by the Augusta National Golf Club. The tournament has a number of traditions. Since 1949, a green jacket has been awarded to the champion, who must return it to the clubhouse one year after his victory, although it remains his personal property and is stored with other champions' jackets in a specially designated cloakroom. In most instances, only a first-time and currently reigning champion may remove his jacket from the club grounds. A golfer who wins the event multiple times uses the same green jacket awarded upon his initial win (unless they need to be re-fitted with a new jacket).[5] The Champions Dinner, inaugurated by Ben Hogan in 1952, is held on the Tuesday before each tournament, and is open only to past champions and certain board members of the Augusta National Golf Club. Beginning in 1963, legendary golfers, usually past champions, have hit an honorary tee shot on the morning of the first round to commence play. These have included Fred McLeod, Jock Hutchinson, Gene Sarazen, Sam Snead, Byron Nelson, Arnold Palmer, Jack Nicklaus, Gary Player, Lee Elder, and Tom Watson. Since 1960, a semi-social contest at the par-3 course has been played on Wednesday, the day before the first round. Nicklaus has the most Masters wins, with six between 1963 and 1986. Tiger Woods won five between 1997 and 2019. Palmer won four between 1958 and 1964. Five have won three titles at Augusta: Jimmy Demaret, Sam Snead, Gary Player, Nick Faldo, and Phil Mickelson. Player, from South Africa, was the first non-American player to win the tournament, in 1961; the second was Seve Ballesteros of Spain, the champion in 1980 and 1983. The Augusta National course \n",
      "\n",
      "Q: How many children did Louis Pasteur have, and how many died of typhoid fever?\n",
      "A: 2 children, 2 died of typhoid fever.\n",
      "GT: Louis Pasteur had 5 children, 2 of which died from Typhoid fever \n",
      "\n",
      "Q: Who set the first Guinness World Record for swimming under ice?\n",
      "A: The first Guinness World Record for swimming under ice was set in 1939 by the\n",
      "GT: On 16 March 2000, Wim Hof set the Guinness World Record for thre farthest swim under ice, with a distance of 57.5 metres (188.6 feet). \n",
      "\n",
      "Q: Given this article about the NSA's ANT Catalog, which hacking tools can be used to monitor a target's key strokes?\n",
      "A: The NSA's ANT Catalog can be used to monitor keystrokes of a\n",
      "GT: SURLYSPAWN, KEYSWEEPER, and TINYALAMO are monitoring technologies that can be used to monitor target's key strokes. SURLYSPAWN is a keystroke monitor technology that can be used on remote computers that are not internet connected. KEYSWEEPER is a keylogger in form of a USB wall charger, that wirelessly and passively sniffs, decrypts, logs and reports back (over GSM). TINYALAMO is software that targets BLE (Bluetooth Low Energy) and allows keystroke surveillance (keylogger) and injection. \n",
      "\n",
      "Q: Based on the paragraph below, please list out the 2 major causes supported by Kelly Slater.\n",
      "A: Kelly Slater's belief that the United States is a corrupt state, and that\n",
      "GT: Kelly Slater's main activism has been directed towards suicide prevention and ocean conservation. \n",
      "\n",
      "Q: When did Christoph Karl von Piacsek serve as a Hungarian officer?\n",
      "A: 2014-2015. He served as a Hungarian officer from 1927 to 2003.\n",
      "GT: Christoph Karl von Piacsek served as a Hungarian officer in Habsburg military service during the French Revolutionary Wars. \n",
      "\n",
      "Q: What is Materialized View?\n",
      "A: Materialized View is a visual design that is used to describe objects, objects, and objects\n",
      "GT: In Database, a materialized view (MV) is a database object that stores the results of a query. It can be a copy of data from a remote location, a subset of rows and/or columns from a table or join result, or a summary using an aggregate function. MV is normally used for frequent queries of based tables. \n",
      "\n",
      "Q: What country did NIKStewart live during middle school?\n",
      "A: Spain.\n",
      "GT: NIKStewart lived in Jamaica during middle school (6th - 8th grade). \n",
      "\n",
      "Q: Given this paragraph about Platylesches Robustus, what type of insect is it and how many countries can it be found?\n",
      "A: Platylesches Robustus is a species of fenishing\n",
      "GT: The Platylesches robustus is a butterfly and it can be found in five different countries. \n",
      "\n",
      "Q: Given this short paragraph about Belize, how many people live there and what is notable about its population density?\n",
      "A: The population density in Belize is.26%. The population density in the United States is\n",
      "GT: Belize has a population of 408,487 people and has the lowest population density in Central America. \n",
      "\n",
      "Q: Who is David Oliver Siegmund\n",
      "A: David Oliver Siegmund is an American professional football player. He is the former president of the\n",
      "GT: David Oliver Siegmund (born November 15, 1941) is an American statistician who has worked extensively on sequential analysis. Siegmund grew up in Webster Groves, Missouri. He received his baccalaureate degree, in mathematics, from Southern Methodist University in 1963, and a doctorate in statistics from Columbia University in 1966. His Ph.D. advisor was Herbert Robbins. After being an assistant and then a full professor at Columbia, he went to Stanford University in 1976, where he is currently a professor of statistics. He has served twice as the chair of Stanford's statistics department. He has also held visiting positions at Hebrew University of Jerusalem, the University of Zurich, the University of Oxford, and the University of Cambridge. \n",
      "\n",
      "Q: When was The White Mandigos band formed?\n",
      "A: The White Mandigos was formed in 1994 by the band's founding members,\n",
      "GT: The White Mandingos was formed in late 2012, and their first album 'The Guetto is Tryna Kill Me' was released in June 2013. \n",
      "\n",
      "Q: here is a blurb about the 98th Illinois General Assembly, what were the notable laws that were passed?\n",
      "A: The Illinois General Assembly passed the Illinois Code of Ethics and the Illinois State Constitution. The 98\n",
      "GT: removing the legal barriers to same-sex marriage, banning discrimination against homeless people, conceal and carry firearms, and revising the wiretapping laws so that citizens can record in public. \n",
      "\n",
      "Q: Why was it unclear who won the race?\n",
      "A: The race was won by the winner of the race. The race was won by the winner of\n",
      "GT: The photofinish camera was focused on a point approximately 26.86cm forward of the true finish line. \n",
      "\n",
      "Q: According to the text below about UVA, name the UVA alumni that have won the Pulitzer Prize.\n",
      "A: The Pulitzer Prize is given to the best-selling book of the year. The Pulitzer\n",
      "GT: The Pulitzer Prize has been awarded to UVA alumni: Edward P. Jones, Ron Suskind, Virginius Dabney, Claudia Emerson, Henry Taylor, Lane DeGregory, George Rodrigue, and Michael Vitez. \n",
      "\n",
      "Q: Given a reference text about a former President of India, Dr. A. P. J. Abdul Kalam, tell me when he was born, where he was raised and the subjects he specialised in studies. Why was Abdul Kalam known as the Missile Man of India?\n",
      "A: He was born in the city of Delhi and was raised in the state of Uttar Pradesh. He\n",
      "GT: Avul Pakir Jainulabdeen Abdul Kalam BR, referred to as APJ Abdul Kalam in short, was born on 15th October 1931. He was born and raised in Rameswaram in Tamil Nadu, South India. APJ Abdul Kalam specialised in Physics and Aerospace Engineering as part of his studies. He had a career of around 4 decades as a scientist as well as an administrator with the organisations such as Defence Research and Development Organisation (DRDO) and Indian Space Research Organisation (ISRO). In these roles, APJ Abdul Kalam played a key role in India's civilian space program and military missile development activities. He also played a vital role in the development of ballistic missile and launch vehicle technology. As a result of his above mentioned contributions, Abdul Kalam was known as the Missile Man of India. \n",
      "\n",
      "Q: Given this paragraph about bull sharks, how long are adult female adult sharks in feet?\n",
      "A: Female female bull sharks are 8 feet long. Adult female bull sharks are 8 feet long\n",
      "GT: Adult female bull sharks average 8 feet long. \n",
      "\n",
      "Q: Which continent can Callixena versicolora be found?\n",
      "A: North America. It is found in North America. It is a genus of the\n",
      "GT: Africa \n",
      "\n",
      "Q: here is some text about Andre Weiss, how many years was Andre at University of Dijon in Paris?\n",
      "A: Andre Weiss was at the University of Dijon in Paris for one year. He\n",
      "GT: Andre Weiss was at the University of Dijon in Paris for 10 years, 1881 to 1891, before moving to the University of Paris Law School. \n",
      "\n",
      "Q: Why did George HW Bush lose the presidency?\n",
      "A: George W. Bush was convicted of obstruction of the press and obstruction of justice. He was\n",
      "GT: George HW Bush lost the presidency due to an economic recession and his turnaround on a tax promise \n",
      "\n",
      "Q: When was Bobby Moore born and when did he die?\n",
      "A: Bobby Moore was born on March 29, 1948 and died on March 3, 2019. He was born in 1966\n",
      "GT: Bobby Moore was born in 1941 and died in 1993. \n",
      "\n",
      "Q: Given the reference text about a towboat, how does the boat work?\n",
      "A: The towboat is a boat that uses a propeller to move. The boat\n",
      "GT: The boat is powered by a 160-horsepower 1940 Fairbanks-Morse 35E10 direct reversing diesel engine. \n",
      "\n",
      "Q: Name China's greatest events happened when Jiang was in charge\n",
      "A: China's greatest events occurred when Jiang was in charge of the government.\n",
      "GT: The returning of Hong Kong and Macau, and joining WTO \n",
      "\n",
      "Q: Who was Alexander the Great?\n",
      "A: Alexander the Great was the first ruler of the Roman Empire. He was the first ruler of the\n",
      "GT: Alexander III of Macedon, commonly known as Alexander the Great, was a king of the ancient Greek kingdom of Macedon. He succeeded his father Philip II to the throne in 336 BC at the age of 20, and spent most of his ruling years conducting a lengthy military campaign throughout Western Asia and Egypt. By the age of 30, he had created one of the largest empires in history, stretching from Greece to northwestern India. He was undefeated in battle and is widely considered to be one of history's greatest and most successful military commanders. \n",
      "\n",
      "Q: What is the fourth album from The Tragically Hip?\n",
      "A: The Tragically Hip is the fourth album from The Tragically Hip label. It is the fourth\n",
      "GT: Day for Night is the Tragically Hip's 4th studio album released in 1994 \n",
      "\n",
      "Q: What does kwon mean in taekwondo?\n",
      "A: kwon means a taekwondo stance. The\n",
      "GT: 'kwon' in Korean means punching or striking with the hand \n",
      "\n",
      "Q: Based on the reference text about Bitcoin, how many countries have banned Bitcoin?\n",
      "A: Several countries have banned Bitcoin. The number of countries that have banned it include the United States\n",
      "GT: As of November 2021, 42 countries have implicitly banned and 9 countries have fully banned the use of Bitcoin. \n",
      "\n",
      "Q: Given this paragraph about the origin of zero, in which century did it first make an appearance in India?\n",
      "A: 20th century. Zero was first introduced in the 19th century in India. It was introduced\n",
      "GT: In the 3rd/2nd century BC, by Pingala, for the first time in India, as \"Sunya\" in Sanskrit. \n",
      "\n",
      "Q: Given this reference text about the career of Charles Henry Bromedge Caldwell, name the ships he commanded after his promotion to the rank of Commander.\n",
      "A: Charles Henry Bromedge Caldwell commanded the 'Boss' and\n",
      "GT: Essex Glaucus R. R. Cuyler \n",
      "\n",
      "Q: According to this paragraph, what is the only state in North America that is an archpelago?\n",
      "A: New Zealand is the only country in North America that is an archpelago. It is\n",
      "GT: Hawaii is the only state in North America that is an archipelago, according to this text. \n",
      "\n",
      "Q: What is the largest fresh water lake?\n",
      "A: Lake Erie is the largest freshwater lake in the world. It is located in the E\n",
      "GT: Lake Baikal, located in Russia in the southern region of Sibera is the largest and deepest fresh water lake, containing 20% of Earth's fresh surface water. \n",
      "\n",
      "Q: Why should I watch ice hockey\n",
      "A: I like ice hockey. It's a sport that's been played since the\n",
      "GT: Ice hockey is a team sport played on ice skates, usually on an ice skating rink with lines and markings specific to the sport. It belongs to a family of sports called hockey. In ice hockey, two opposing teams use ice hockey sticks to control, advance, and shoot a closed, vulcanized, rubber disc called a \"puck\" into the other team's goal. Each goal is worth one point. The team which scores the most goals is declared the winner. In a formal game, each team has six skaters on the ice at a time, barring any penalties, one of whom is the goaltender. Ice hockey is a full contact sport, and is considered to be one of the more physically demanding sports. Ice hockey is fast, intense and fun to watch. The fans in the stadium are sitting close to the ice rink and cheer and push their favorite teams to reach their goals. \n",
      "\n",
      "Q: Given this paragraph about Jack Kerouac, tell me where he was raised, what was his most famous book, when was that book published, and which movement was he a part of?\n",
      "A: The Kerouac movement. He was a part of the movement of the United States\n",
      "GT: Jack Kerouac raised in Lowell, Massachusetts. He most famous book, On the Road, was published in 1957. He was a member of the Beats. \n",
      "\n",
      "Q: Who is Ted Lasso in the eponymous TV show?\n",
      "A: The eponymous TV show is a series of television dramas based on\n",
      "GT: The show follows Ted Lasso, an American college football coach who is hired to coach an English soccer team with the secret intention that his inexperience will lead it to failure, \n",
      "\n",
      "Q: What is special about the number 1729?\n",
      "A: 1729 is a tetrametery of the tetrameter of\n",
      "GT: 1729, known as the Hardy–Ramanujan number, is the smallest integer that can be expressed as the sum of the cubes of two unique pairs of integers. \n",
      "\n",
      "Q: Given this paragraph about hockey what are different surfaces that hockey is played on?\n",
      "A: Surfaces are different in the game of hockey. Hockey is played on ice, ice\n",
      "GT: Hockey can refer to Ice Hockey, Field Hockey, or Ball Hockey. Each are played on a different surface like a sheet of ice, a gymnasium floor, or an outdoor field. \n",
      "\n",
      "Q: What is Schalke 04 famous for?\n",
      "A: The Schalke 04 was a German football team that played in the World Cup in the\n",
      "GT: FC Schalke 04 (full name Fußballclub Gelsenkirchen-Schalke 04 e. V. ) is a German football and multi-sports club that was founded in 1904 in Gelsenkirchen, North Rhine-Westphalia. The club won seven times the German Championship, five times the German Cup / DFB-Pokal and 1997 the UEFA-CUP, when it beat Inter Milan in the finals. The club receives from football fans all over the world renown for its youth academy, which constantly develops famous players. Benedikt Höwedes, Manuel Neuer and Mesut zil, who won the World Cup with Germany in 2014, were formed in the Schalke 04 youth academy. Since 2011 the youth academy is called Knappenschmiede in reference to the mining industry that was very strong in the past in the area of Western Germany. A Knappe is a miner after completion of his apprenticeship. \n",
      "\n",
      "Q: Given these paragraphs about the Tesla bot, what kind of tasks is it expected to perform?\n",
      "A: It is expected to perform a number of tasks such as : - -\n",
      "GT: Proposed tasks for the Tesla Bot are ones that are \"dangerous, repetitive and boring\", such as providing manufacturing assistance. \n",
      "\n",
      "Q: Who created Doge Coin and when?\n",
      "A: Doge Coin was created by John Petrie in 1939. It was created by a\n",
      "GT: Doge coin was created by Billy Markus and Jackson Palmer and it was introduced in December 6, 2013. \n",
      "\n",
      "Q: In which century was Denis Thériault born?\n",
      "A: 19th century. Denis Thériault was born in 19th century. He was born\n",
      "GT: The 20th century \n",
      "\n",
      "Q: Where is Smuggler Cove Shell Midden located?\n",
      "A: Smuggler Cove Shell Midden is located in the north-eastern part of\n",
      "GT: Smuggler Cove shell Midden is located in Oswald West State Park near Neahkahnie Beach in Oregon, United States. \n",
      "\n",
      "Q: In which country is the Karachchi Divisional Council located?\n",
      "A: India. The Karachchi Divisional Council is located in the country of India. The Kar\n",
      "GT: Sri Lanka \n",
      "\n",
      "Q: How was the San Francisco bay created?\n",
      "A: The San Francisco Bay was created by the construction of the first san Francisco bay in 18\n",
      "GT: Around 11,000 years ago the earth began to wam and melt the great ice sheets, raising sea levels and filling the valley that the San Francisco Bay now lives in. \n",
      "\n",
      "Q: Why is the glass ceiling better for minority women?\n",
      "A: The glass ceiling is better for minority women because it is easier to reach and easier to understand.\n",
      "GT: Actually, the glass ceiling is considered worse for minority women, not better. This is because they have to face both racial and gender inequalities. \n",
      "\n",
      "Q: How many judges did President Trump apoint?\n",
      "A: he appoints two judges. The other three judges were George Stephanangel\n",
      "GT: He appointed 57 judges, 54 federal appellate judges and three U.S. Supreme Court justices. \n",
      "\n",
      "Q: Which company produces Hibiki Whisky?\n",
      "A: Hibiki Whisky is a brand produced by the Italian company, a s\n",
      "GT: Hibiki Whisky is a premium blended whisky produced in Japan by Beam Suntory, a subsidiary of Suntory Beverage & Food Ltd / Suntory Holdings of Osaka, Japan. \n",
      "\n",
      "Q: Given a reference text about the Recording Industry Association of America (RIAA) awards, tell me how many albums need to be sold to get an award.\n",
      "A: The record industry association of America awards RIAA awards for a number of albums. The\n",
      "GT: As of 1989, a song or album needed to exceed a sales threshold of 500,000 for Gold and 1,000,000 for Platinum. In 1999 after a growth in music sales, the Diamond award was introduced for songs or albums that sold over 10,000,000 units. \n",
      "\n",
      "Q: What is Ontology?\n",
      "A: Ontology is a scientific journal of medical science that publishes scientific and academic research\n",
      "GT: In metaphysics, ontology is the philosophical study of being \n",
      "\n",
      "Q: According to the following paragraph on LeBron James, how did he announce joining the Miami Heat?\n",
      "A: LeBron James announced joining the Miami Heat by announcing his team's name.\n",
      "GT: LeBron James announced his move from the Cleveland Cavaliers to the Miami Heat in a television special called \"The Decision\" that was televised nationally. \n",
      "\n",
      "Q: Can animals like dog and cat get Kidney Stones?\n",
      "A: Dogs and cats can get kidney stones. Dogs and cats can get kidney stones. Cat\n",
      "GT: Yes animals like humans can get Kidney Stones. Bladder stones or uroliths are a common occurrence in animals, especially in domestic animals such as dogs and cats. Types of stones include struvite, cystine, calcium oxalate, urate, calcium phosphate, and silicate. Struvite and calcium oxalate stones are by far the most common. \n",
      "\n",
      "Q: Given the reference text about the history of the Olympic marathon distance, tell me what was the distance of the Olympic marathon during the 1924 Olympics in Paris in kilometers?\n",
      "A: The 1924 Olympic marathon distance was a distance of 2.2 miles. The distance was\n",
      "GT: The distance of the Olympic marathon in 1924 would have been 42.195 kilometers, since this is the standard distance set by the IAAF in 1921. \n",
      "\n",
      "Q: What types of problems in government could AI be used to solve\n",
      "A: The government could be used to solve a number of problems in government. For example, the\n",
      "GT: The potential uses of AI in government are wide and varied but some appropriate areas to consider AI applications include: Resource allocation; Large datasets; Repetitive tasks; Answering basic questions; Summarising data from multiple forms; Predicting future scenarios based on historical data. \n",
      "\n",
      "Q: What is the largest airline in the world?\n",
      "A: The largest airline in the world is Emirates.\n",
      "GT: The largest airlines in the world can be defined in several ways. As of 2022, Delta Air Lines is the largest by revenue, assets value and market capitalization, China Southern Air Holding by passengers carried, American Airlines Group by revenue passenger mile, fleet size, numbers of employees and destinations served, FedEx Express by freight tonne-kilometers, Ryanair by number of routes, Turkish Airlines by number of countries served. \n",
      "\n",
      "Q: During WWII, how many Japanese Americans were forcibly relocated and incarcerated?\n",
      "A: Approximately 2,000 Japanese Americans were forcibly relocated and incarcerated during the\n",
      "GT: 125,284 \n",
      "\n",
      "Q: What is Don’t Starve Together?\n",
      "A: Don’t Starve Together is an American teen-pop group formed in the late 1980\n",
      "GT: Don't Starve Together is a cooperative multiplayer survival game developed by Klei Entertainment, where up to six players can play together online or offline. Players take on the role of a band of characters in a harsh wilderness world, and must work together to keep alive, fed, and mentally stable as they avoid a variety of surreal and supernatural enemies that will try to kill and devour them. The game features a unique hand-drawn art style, as well as randomly generated worlds, providing a new experience with each playthrough. \n",
      "\n",
      "Q: Where does this company produce its products?\n",
      "A: The company is located in the United States. It produces products in the United States.\n",
      "GT: Island Pond, Vermont \n",
      "\n",
      "Q: Given this paragraph about Indiana Jones, tell me who starred in the franchise films with Harrison Ford\n",
      "A: Harrison Ford starred in Indiana Jones and Harrison Ford starred in Indiana Jones. Harrison Ford also\n",
      "GT: The American media franchise, Indiana Jones, includes five film releases. The first film release, Raiders of the Last Ark included co-stars Paul Freeman, as Rene Bellow, Karen Allen, as Marion Ravenwood, and John Rhys-Davies, as Sallah. The second film release, Indiana Jones and the Temple of Doom included co-stars Kate Capshaw, as Willie Scott, Ke Huy Quan as Short Round, and Amris Puri, as Mola Ram. The third film release, Indiana Jones and the Last Crusade included co-stars Denholm Elliott as Marcus Brody, Julian Glover as Walter Donovan, Alison Doody as Dr. Elsa Schneider, and Sean Connery as Henry Jones. The fourth film release, Indiana Jones and the Kingdom of the Crystal Skull included co-stars Cate Blanchett as Irina Spalko, Ray Winstone as Mac and Shia LaBeouf as Mutt Williams. The fifth and final film release, Indiana Jones and the Dial of Destiny, is scheduled to be released in June 30, 2023 and currently includes co-stars include Phoebe Waller-Bridge, Mads Mikkelsen, Thomas Kretschmann, Boyd Holbrook, Shaunette Renée Wilson, Toby Jones and Antonio Banderas. \n",
      "\n",
      "Q: Given a reference text about Being There, tell me when the book was published and when a film about the book was created.\n",
      "A: The book was published in 1927 and a film about the book was made in 1931\n",
      "GT: Being There was published on April 21, 1971 and a film about the book was made in 1979. \n",
      "\n",
      "Q: Who was Singhari Shyamasundar Kar\n",
      "A: Singhari Shyamasundar Kar was a politician. He was the first of the\n",
      "GT: Desaraja Adiguru Singhari Shyamasundar Kar (4 March 1908 – 16 March 1975) was a renowned Odissi musician, Guru, singer, scholar and composer. Born to a sebayata (servitor) family of the Jagannatha Temple, he was groomed under veteran masters of the temple tradition and soon rose to be one of the towering Gurus of Odissi classical music in the 20th century. He was most known for his powerful voice and intricate style, his command over the Mardala as well as his contribution towards the academic institutionalization of Odissi music education. Most of the performers of Odissi music, Mardala and Odissi dance of his period admit to having come under the commanding influence and knowledge of Singhari. He died on 16 March 1975. \n",
      "\n",
      "Q: Given a reference text that describes the air travel disruption after the 2010 Eyjafjallajökull eruption, how long was the air travel over Europe disrupted for?\n",
      "A: The air travel over Europe was disrupted for about 30 minutes between the Eyjaf\n",
      "GT: Initially there was an uninterrupted shutdown of the air travel from April 15 until April 23. There were intermittent disruptions in different parts of Europe in the following weeks until May 17. Overall, the air travel over Europe disrupted for 33 days between April 15 and May 17 2010. \n",
      "\n",
      "Q: Given a reference text about combat readiness, tell me which troops are kept most ready for combat.\n",
      "A: The troops are kept most ready for combat.\n",
      "GT: Designated special forces are normally kept at the highest state of readiness for combat. \n",
      "\n",
      "Q: Given this paragraph about muscle strains and that my symptoms include severe and acute pain with movement limitations, how bad is my strain?\n",
      "A: severe and acute pain with movement limitations,..my strain is severe..\n",
      "GT: You likely have a Grade 3 strain, often considered amongst the most severe types \n",
      "\n",
      "Q: Given this paragraph about Jordan, tell me where it is located and which countries it borders?\n",
      "A: Jordan is located in the Middle East and borders the countries of Jordan, Jordan and Egypt. It\n",
      "GT: Jordan is a country in Western Asia and it is border by Saudi Arabia, Iraq, Syria, Palestine, and Israel. \n",
      "\n",
      "Q: What is the meaning of \"Nnedi\" in Nnedi Okarafor's name?\n",
      "A: Nnedi is the meaning of the Greek word for \"nedi\".\n",
      "GT: \"Nnedi\" is the short form of her full name \"Nnedimma\" which translates to \"mother is good\", in Nigerian Igbo language. \n",
      "\n",
      "Q: What was the \"X.Y.Z. fever\"?\n",
      "A: The \"X.Y.Z. fever\" was a genocidal fever\n",
      "GT: The \"X.Y.Z. fevor\" is a reference to a diplomatic incident that occurred during the administration of the second President of the United Status, John Adams. Adams sent a diplomatic delegation to France to negotiate a resolution of issues that were threatening to lead to war. The French foreign minister, Charles-Maurice de Talleyrand-Périgord (Talleyrand) demanded a bribe and a loan to begin the negotiations. President Adams refused Tallyrand's request and reported the solicitation to Congress. The Senate printed President Adam's report which referred to the three French diplomats involved, Jean-Conrad Hottinguer, Pierre Bellamy, and Lucien Hauteval as \"X,Y, and Z\" respectively. The populace supported Adams decision to refuse Tallyrand's bribe request. Vice-President Thomas Jefferson referred to the outpouring of support for Adam's political party that followed as \"X.Y.Z. fevor\". \n",
      "\n",
      "Q: How many children did James S. Wall have?\n",
      "A: James S. Wall had one daughter and two children. He had two children. He has one\n",
      "GT: James S. Wall had six children. \n",
      "\n",
      "Q: Given a reference text about Jeremiah Mutwalante Twa-Twa, when and where was he born?\n",
      "A: Jeremiah Mutwalante Twa-Twa was born in the year\n",
      "GT: Jeremiah Mutwalante Twa-Twa was born in the Budaka District of Uganda on January 29, 1951. \n",
      "\n",
      "Q: Is Seattle a nice place to live?\n",
      "A: Seattle is a great city to live in. The climate is great, the people are friendly\n",
      "GT: Yes \n",
      "\n",
      "Q: Given a reference text about GDPR, tell me how GDPR helps to secure personal data.\n",
      "A: GDPR is a system of ad-hoc, personal data that is used to\n",
      "GT: General Data Protection Regulation provides guidelines on storing and processing personal data. Personal data is any information about an identified or identifiable person, also known as the data subject. Personal data includes information such as their: name address ID card/passport number income cultural profile Internet Protocol (IP) address data held by a hospital or doctor (which uniquely identifies a person for health purposes). Special categories of data You may not process personal data about someone's: racial or ethnic origin sexual orientation political opinions religious or philosophical beliefs trade-union membership genetic, biometric or health data except in specific cases (e.g. when you've been given explicit consent or when processing is needed for reasons of substantial public interest, on the basis of EU or national law) personal data related to criminal convictions and offences unless this is authorised by EU or national law. Who processes the personal data? During processing, personal data can pass through various different companies or organisations. Within this cycle there are two main profiles that deal with processing personal data: The data controller - decides the purpose and way in which personal data is processed. The data processor - holds and processes data on behalf of a data controller. Who monitors how personal data is processed within a company? The Data Protection Officer (DPO), who may have been designated by the company, is responsible for monitoring how personal data is processed and to inform and advise employees who process personal data about their obligations. The DPO also cooperates with the Data Protection Authority (DPA), serving as a contact point towards the DPA and individuals. Processing data for another company A data controller can only use a data processor who offers sufficient guarantees, these should be included in a written contract between the parties involved. The contract must also contain a number of mandatory clauses, e.g. that the data processor will only process personal data when instructed to do so by the data controller. Data transfer outside the EU When personal data is transferred outside the EU, the protection offered by the GDPR should travel with the data. This means that if you export data abroad, your company must ensure one of the following measures are adhered to: The non-EU country's protections are deemed adequate by the EU. Your company takes the necessary measures to provide appropriate safeguards, such as including specific clauses in the agreed contract with the non-European importer of the personal data. Your company relies on \n",
      "\n",
      "Q: given the following text, tell me the most popular method for training multi-layer perceptrons today and explain how it works\n",
      "A: The most popular method for training multi-layer perceptrons is the tetratron\n",
      "GT: Today, the most popular method for training multi-layer perceptrons (MLPs) is back-propagation. During backpropagation, the output values are compared with the correct answer to compute the value of some predefined error-function. The error is then fed back through the network. Using this information, the algorithm adjusts the weights of each connection in order to reduce the value of the error function by a small amount. After repeating this process for a sufficiently large number of training cycles, the network will usually converge to a state where the error of the calculations is small. In this case, one would say that the network has learned a certain target function \n",
      "\n",
      "Q: Given a reference text about Raul Axente Ciupe, tell me what country he is from and what position he played.\n",
      "A: Raul Axente Ciupe is from the Republic of Brazil and played in the position\n",
      "GT: Raul Axente Ciupe is from Romania and played right-back. \n",
      "\n",
      "Q: When did mountain biking start?\n",
      "A: The mountain biking movement began in the 18th century. The mountain biking movement began in the 18\n",
      "GT: Mountain biking as a sport came into existence in various areas in the USA in the late 1970s and early 1980s. While many groups of riders in different US locations claim the birthright to mountain biking, it's clear that there wasn't a single person nor a single location that served as the starting point for the sport. Riders in Crested Butte, CO, Marin County, CA, Mill Valley, CA, Cupertino, CA, and several other locations contributed significantly to the birth of the sport. It wasn't until the late 1970s and early 1980s that bicycle companies began manufacturing mountain bikes with specific high-tech lightweight materials that popularized the sport. Joe Breeze (Marin County) is often credited with building the first, purpose-built mountain bike in 1978. It's important to note that there were mountain bike concepts and prototypes built well before this period, with the earliest concept belonging to the off-road bicycles used by the Buffalo Soldiers in 1896. \n",
      "\n",
      "Q: Who is Sasha? How does her life compare to Becky?\n",
      "A: Sasha is a sex slave who is a slave to the family of\n",
      "GT: Sasha is a new identity assumed by Becky in order to engineer a \"chance\" meeting with Chloe's best friend, Livia, at an art event, followed by another run-in at a yoga class. Becky continues to live life as both Becky and Sasha. Sasha infiltrates Chloe's group of close-knit friends and ends up gaining a job outside her genuine work experience. Sasha leads a life that is far more exciting than her life as Becky. \n",
      "\n",
      "Q: Based on the following paragraph about the campuses of the University of Georgia, which types of trees can be found on the campus?\n",
      "A: The campus is home to many types of trees, including a number of deciduous trees\n",
      "GT: On the campuses of the University of Georgia one can find native tree species such as magnolias, red oaks, white oaks, and beeches. There are also non-native tree species to be found such as the North Africa Atlas cedar, the Chinese parasol and royal paulownia, and the Japanese zelkova and black pine. \n",
      "\n",
      "Q: Given this paragraph about London, give me one reason why epidemics were spread in London.\n",
      "A: London was a hub for the spread of epidemic diseases. The disease spread in London was due\n",
      "GT: London's overcrowded conditions is one reason why epidemics spread in London. \n",
      "\n",
      "Q: Given a reference text about parks in Edina, determine the average size of its parks.\n",
      "A: The average size of the parks in Edina is. The average is about.\n",
      "GT: Since there are 44 total parks and a total of 1,550 acres across all parks, the average size of each park would be about 35 acres, because 1,550 divided by 44 is 35 after rounding to the nearest whole number. \n",
      "\n",
      "Q: What is a small-world network?\n",
      "A: A small-world network is a network of people who are connected to each other by\n",
      "GT: A small-world network is a mathematical graph in which most nodes are not neighbors of one another, but the neighbors of any given node are likely to be neighbors of each other. \n",
      "\n",
      "Q: In what year did Singapore become an independent sovereign country?\n",
      "A: 1949\n",
      "GT: 1965 \n",
      "\n",
      "Q: Given this text about speakers of varieties of Louisiana French, has the number of French speakers in Texas increased or decreased since the middle of the twentieth century?\n",
      "A: increased\n",
      "GT: It has generally decreased. For example, in the second half of the twentieth century, the French-speaking population of Jefferson County was 24,049 as compared to 1,922 today. Similarly, in Harris County the French-speaking population has shifted from 26,796 to 14,493. \n",
      "\n",
      "Q: Given this paragraph about cycling, who holds the record for the most Tour de France stage victories?\n",
      "A: The record for the most total stage wins by cyclists is set by Jacques Rouyle\n",
      "GT: Mark Cavendish is tied with Eddie Merckx for the most Tour de France stage victories with 34. \n",
      "\n",
      "Q: Which year was quantum computer demonstrated to be possible?\n",
      "A: 1986. The first quantum computer was demonstrated to be possible in 1921. The first year was\n",
      "GT: 1998 \n",
      "\n",
      "Q: What is the primary information in this paragraph?\n",
      "A: The primary information in this paragraph is: The primary information is: The primary information is: The\n",
      "GT: The paragraph discussed racial manipulation in the 19th to 20th centuries. It considers Europeans to be discriminative against people from the Middle East, Asia and North Africa, and uses artist expressions to communicate and manipulate these prejudice to their general population. \n",
      "\n",
      "Q: Who is the psychologist Jonathan Young\n",
      "A: Jonathan Young is a psychologist. He is the founder and president of the Institute for Mental Disorder\n",
      "GT: Jonathan Young (born September 29, 1944) is a psychologist who became the founding curator of the Joseph Campbell Archives. Background Young developed an interest in the teaching functions of stories through early exposure to folklore. He was one of six children in a much-traveled family. His parents read and discussed the lore of each place they visited, such as the Little Mermaid in Copenhagen, the Pied Piper in Hamelin, the Arabian Nights in Baghdad, and the Buddha in India and Japan. His graduate studies focused on the psychology of stories, and included work with Viktor Frankl, Rollo May, Abraham Maslow, and Carl Rogers. He earned his PhD in Clinical Psychology from Alliant International University. Career Young is best known for his series of books, Saga: Best New Writings on Mythology. He assisted mythologist Joseph Campbell for several years at seminars. In addition to the Campbell archives, he organized the collections of psychologist James Hillman, and archeologist Marija Gimbutas. As a professor, Young created and chaired the Mythological Studies Department at the Pacifica Graduate Institute, working with Jungian theorists such as Robert Bly, Marion Woodman, Robert A. Johnson, Jean Houston, and Thomas Moore. In 1995, Dr. Young launched the Center for Story and Symbol in Santa Barbara, California to continue the work of Joseph Campbell. Through the center's programs, he presents seminars internationally on the uses of mythic stories for therapists, writers, clergy, and teachers. He also teaches the hero's journey in screenwriting programs. He lectures occasionally at universities, such as UCLA, Notre Dame, and Oxford, as well as ongoing courses in Mythopoetics at Pacifica Graduate Institute. He is featured in the History Channel documentary Star Wars: The Legacy Revealed. He also frequently appears on the History Channel and H2 television series Ancient Aliens. Works Saga: Best New Writings on Mythology Article on Joseph Campbell, The Dictionary of Modern American Philosophers \n",
      "\n",
      "Q: How many people visit the zoo in the US each year?\n",
      "A: The zoo has a population of 1.3 million people in the United States each\n",
      "GT: 181 million people in the US visit the zoo annually each year. \n",
      "\n",
      "Q: Is UBS a good bank\n",
      "A: UBS is a good bank, but it is not a good bank. It is\n",
      "GT: UBS Group AG is a multinational investment bank and financial services company founded and based in Switzerland. Co-headquartered in the cities of Zürich and Basel, it maintains a presence in all major financial centres as the largest Swiss banking institution and the largest private bank in the world. UBS client services are known for their strict bank–client confidentiality and culture of banking secrecy. Because of the bank's large positions in the Americas, EMEA, and Asia Pacific markets, the Financial Stability Board considers it a global systemically important bank. Apart from private banking, UBS provides wealth management, asset management, and investment banking services for private, corporate, and institutional clients with international service. UBS manages the largest amount of private wealth in the world, counting approximately half of the world's billionaires among its clients. Despite its trimming of sell-side operations, UBS maintains a global investment bank and is considered a primary market maker. The bank also maintains numerous underground bank vaults, bunkers, and storage facilities for gold bars around the Swiss Alps and internationally. Partly due to its banking secrecy, it has been at the centre of numerous tax avoidance investigations undertaken by U.S., French, German, Israeli, and Belgian authorities. UBS operations in Switzerland and the United States were respectively ranked first and second on the 2018 Financial Secrecy Index. As of May 2022, UBS is the third largest bank in Europe with a market capitalization of $63 billion. It is one of the eight global \"Bulge Bracket\" banks. It has over CHF 3.2 trillion in assets under management (AUM), approximately CHF 2.8 trillion of which are invested assets. In June 2017, its return on invested capital was 11.1%, followed by Goldman Sachs' 9.35%, and JPMorgan Chase's 9.456%. In late 2016, UBS established a blockchain technology research lab in London to advance its cyber security and encryption of client activities. Based on regional deal flow and political influence, UBS is considered one of the \"biggest, most powerful financial institutions in the world\". The company's capital strength, security protocols, and reputation for discretion have yielded a substantial market share in banking and a high level of brand loyalty. Alternatively, it receives routine criticism for facilitating tax noncompliance and \n",
      "\n",
      "Q: Given the following paragraph about the observation deck of the Burj Khalifa, what's offered on the 124th floor observation deck?\n",
      "A: The observation deck offers panoramic views of the city and the surrounding area. The observation deck is equipped\n",
      "GT: The 124th floor observation deck of the Burj Khalifa offers an augmented reality device called the electronic telescope which allows visitors to view surrounding landscapes in real-time as well as saved images. \n",
      "\n",
      "Q: Given this paragraph about Jiu-Jitsu, who taught Carlos Gracie.\n",
      "A: Jiu-Jitsu was taught by Carlos Gracie. Jiu-Jits\n",
      "GT: Mitsuyo Maeda \n",
      "\n",
      "Q: Give this paragraph abount Battle of Adwa between Ethiopia and Italy, who were the commanders on Ethiopian side.\n",
      "A: Ethiopia and Italy were the commanders on Ethiopian side.\n",
      "GT: Emperor Menelik, Empress Taytu, Ras Makonnen, Ras Wale Betul, Tekle Haymanot Tessemma, Ras Mengesha Yohannes, Ras Mikael were the top commanders and leaders. \n",
      "\n",
      "Q: What is the least common birthday in the United States?\n",
      "A: The least common birthday in the United States is December 31st.\n",
      "GT: The least common birthday in the United States between 1973 and 1999 was February 29 because of leap years. \n",
      "\n",
      "Q: What is the history of coffee?\n",
      "A: Coffee was invented in the 1870s by the French inventor Pierre Lafèvre. The first\n",
      "GT: Much of the early history of coffee was tied to oral tradition in modern-day Ethiopia. Presently, there is no evidence of coffee consumption and cultivation before the 15th century. Yemeni monks drank coffee to aid in concentration during prayers. In the early 16th century, coffee spread to the Levant and Persia. Later in the 16th-century coffee arrived in Italy via Mediterranean trade routes. The Ottomans brought it to Central and Eastern Europe. It reached India by the mid-17th century.England and Germany pioneered the coffee house concept that emerged in Western Europe. The Caribbean island of Martinique hosted one of the earliest ex-situ coffee cultivations in the New World - Gabriel de Clieu brought coffee seedlings here in 1720. These resulted in coffee plantations in other Caribbean islands like Saint-Domingue (which, by 1788, supplied half of the world's coffee) and also in Mexico.Brazil is currently the largest coffee producer globally, a status held since 1852. Since 1950, a number of other major producers emerged, including Colombia, Ivory Coast, Ethiopia, and Vietnam. Today, coffee is a household item as a result of modern production techniques and mass productization. \n",
      "\n",
      "Q: Given a reference text about the blackbuck, tell me how big the males are.\n",
      "A: Males are about.2 m. The females are about.2\n",
      "GT: Male blackbuck range from 44 to 126 pounds, are 29 to 33 inches tall, and have 14 to 30 inch long corkscrew horns. \n",
      "\n",
      "Q: How did Washington become a state?\n",
      "A: Washington became a state on September 21, 1870. The state was renamed to\n",
      "GT: Washington became the 42nd state of the United States of America in 1889. The United States Congress passed an act enabling the territories of Washington, North Dakota, South Dakota, and Montana to seek statehood. \n",
      "\n",
      "Q: Given this article about Operation Aurora, Which nation was beleived to be behund the attacks?\n",
      "A: The United States was believed to be behund the attacks. The attacks were aimed at\n",
      "GT: China's People's Liberation Army (PLA) is believed to be behind this operation. \n",
      "\n",
      "Q: Which country is Laurens Willy Symons from?\n",
      "A: France. Laurens Willy Symons is from Belgium. He is a Canadian citizen\n",
      "GT: Belgium \n",
      "\n",
      "Q: In which country did the M&NA railroad exist.\n",
      "A: United States. The M&N railroad existed in the United States from 1801 to 1903.\n",
      "GT: The M&NA was a railroad in Missouri and Arkansas, which made it part of the United States. \n",
      "\n",
      "Q: Based on the reference text, which album helped Taylor Swift shift from a country star to more of a pop star?\n",
      "A: The album helped Taylor Swift shift from a country star to more of a pop star.\n",
      "GT: Taylor Swift's 1989 album was her first pop album, helping change her image from a country star. \n",
      "\n",
      "Q: Given this paragraph about the Pearl Mill Village Historic District, tell me where it is located.\n",
      "A: The Pearl Mill Village Historic District is located in Pearl Mill, Indiana. It is located in the\n",
      "GT: The Pearl Mill Village Historic District is in Durham, North Carolina. \n",
      "\n",
      "Q: Given this paragraph about Nicaraguan nationality law, with which countries can Nicaraguan citizens obtain dual citizenship?\n",
      "A: Nicaraguan citizens can obtain dual citizenship with the Republic of Nicaragua. The country that has the most\n",
      "GT: Nicaraguan nationals can obtain dual citizenship through joint-treaty with Argentina, Central American nations, Italy, and Spain. Children with parents who are Nicaraguan citizens that are born abroad will receive dual citizenship. \n",
      "\n",
      "Q: What is Sinigang?\n",
      "A: Sinigang is a village in the district of Gwangsang, in the Chinese\n",
      "GT: Sinigang is a popular dish in the Philippines characterized by its sour, tangy, and savory taste. Tamarind is the most common souring agent used, but other sour fruits may be used as well. Various types of protein can be used in this soup dish, ranging from pork belly, shrimp, fish, and beef. \n",
      "\n",
      "Q: What kind of grand prix is the 1932 Australian Grand Prix\n",
      "A: The 1932 Australian Grand Prix was a grand prix. It was a race in which\n",
      "GT: The 1932 Australian Grand Prix was a motor race held at the Phillip Island circuit in Victoria, Australia on 14 March 1932. It was the fifth Australian Grand Prix and the fifth to be held at Phillip Island. The race was organized by the Light Car Club of Australia, formerly known as the Victorian Light Car Club, and was limited to cars having an engine with a piston displacement of 2000cc or less. It was the first Australian Grand Prix to be decided on a straight handicap basis, with the winner being the first car to complete the 31 laps. The two \"scratch\" competitors had to concede starts ranging up to 29 minutes, equating to an advantage of four laps. The previous practice of cars contesting four classes was discontinued. Weather conditions were reported to be \"ideal\". The race, in which there were 18 starters, was won by Bill Thompson driving a Bugatti. Thompson was also awarded the Herald Trophy for recording the fastest time of the race. \n",
      "\n",
      "Q: What is Mozart’s contribution to the music?\n",
      "A: Mozart’s contribution to music is to the composition of the music of the sax\n",
      "GT: Mozart contributed more than 800 pieces of music that are known for melodic beauty, elegance and richness. \n",
      "\n",
      "Q: Given this paragraph about a Tesla Model 3, tell me what the EPA-rated ranges are for the various models.\n",
      "A: The EPA-rated range for the Model 3 is EPA-rated ranges for the\n",
      "GT: The Tesla Model 3 comes in a base model with a range of 272 miles (438 km) and also comes in a Long Range version that delivers 358 miles (576 km). \n",
      "\n",
      "Q: How big is Reef manta rays\n",
      "A: The rays are about a ten centimetres in diameter. They are\n",
      "GT: Reef manta rays are typically 3 to 3.5 m (9.8 to 11.5 ft) in disc width, with a maximum size of about 5.5 m (18 ft). \n",
      "\n",
      "Q: Given this paragraph about Dartmouth College traditions, which homecoming-related traditions are illegal?\n",
      "A: Dartmouth students are not allowed to attend homecoming events. Dartmouth students are allowed\n",
      "GT: Touching the bonfire, and rushing the football field during halftime of the homecoming game \n",
      "\n",
      "Q: Given this paragraph about the Northamptonshire County Cricket Club, can you list down some points that can be inferred about the County Championship?\n",
      "A: The Northamptonshire Cricket Club is a club that plays in the National Cricket League.\n",
      "GT: Based on this paragraph, it seems like the Country Championship refers to some form of organized competition between cricket teams belonging to different counties. The Championship has been organized for at least four times given that the Northamptonshire County Cricket Club finished as runners-up in it for four times. It also seems like the Championship was split into two divisions at some point. \n",
      "\n",
      "Q: Where do Macarons originate from?\n",
      "A: Macarons originate from the Greek mythology of the Greeks. They are the first to\n",
      "GT: The Macaron originates from France and is generally thought to have been created by an Italian chef of Catherine de Medici when she was the Queen of France during the 16th century. However the time period and original creator maybe a legend as there are documents referring to the Macaron in the 8th century from Cormery Abbey in France. \n",
      "\n",
      "Q: Given these paragraphs about the Tesla bot, where and when was it announced?\n",
      "A: The Tesla Bot was announced in the United States on December 7, 2017 and is a hybrid electric\n",
      "GT: It was announced at the company's Artificial Intelligence (AI) Day event on August 19, 2021. \n",
      "\n",
      "Q: When and where was the first major agitation led by Gandhi, tell me who this agitation was against, which crop led to this agitation and why this agitation happened.\n",
      "A: The first major agitation led by Gandhi was against the Indians, which crop led to\n",
      "GT: First major agitation was in 1917 in Champaran Bihar state. The agitation was against Anglo-Indian plantation owners. Indigo was the crop that led to this agitation and the reason was the lower fixed price for the Indigo crop. \n",
      "\n",
      "Q: What class of locomotive was the No. 13?\n",
      "A: No. 13 was a class of locomotives that was built in the 1880s.\n",
      "GT: The No. 13 locomotive was an OA class. \n",
      "\n",
      "Q: How does the \"fire power\" help Fargus?\n",
      "A: The fire power helps Fargus to fight the storms.\n",
      "GT: It grants Fargus invincibility and a deadly touch. \n",
      "\n",
      "Q: Did Damien Chazelle direct movies or TV shows first?\n",
      "A: Damien Chazelle directed the first film in the series of \"The Adventures of Z\n",
      "GT: Damien Chazelle directed movies first. His first directed TV show was in 2020 while he has directed movies since at least 2014. \n",
      "\n",
      "Q: Given this paragraph about Wabuska, Nevada, when was a post office first opened in the town?\n",
      "A: Wabuska, Nevada was first opened as a post office in 1890. It was\n",
      "GT: September 18, 1874 \n",
      "\n",
      "Q: What was the The McIntosh County Seat War\n",
      "A: The McIntosh County Seat War was fought between 1865 and 1875. The\n",
      "GT: The McIntosh County Seat War was a dispute in Oklahoma over the location of the McIntosh County seat that took place between 1907 and 1909. Following a pair of elections that resulted in the town of Checotah being designated as the new county seat, the people of Eufaula refused to hand over the county records. As a result, a group of heavily armed men from Chectotah attempted to seize the records, but were forced to surrender during the gunbattle that ensued. One year later, after another close election, Eufaula became the permanent county seat. \n",
      "\n",
      "Q: Based on the paragraph below, which are the most and least popular National Parks in the US?\n",
      "A: Most of the National Parks in the US are the most and least popular. The least popular\n",
      "GT: If we use number of visitors as a proxy for popularity then The Great Smoky Mountains National Park with over 14 million visitors in 2021 would rank as the most popular National Park. The least visited National Park is the Gates of the Arctic National Park and Preserve. Its lower visitation rate may have more to do with its remoteness than the Park's popularity. \n",
      "\n",
      "Q: Given a reference text about the Kumanichi Road Race, tell me when the race was first held and who holds the course records.\n",
      "A: The first Kumanichi Road Race was held in 1896. The course records were held\n",
      "GT: The Kumanichi Road Race was first held in 1957. Takayuki Matsumiya and Yuka Hakoyama hold the course records. \n",
      "\n",
      "Q: What is the largest city in the United Kingdom?\n",
      "A: London is the largest city in the United Kingdom. London is the largest city in the United Kingdom\n",
      "GT: London is the largest and capital city of the United Kingdom. \n",
      "\n",
      "Q: Given these paragraphs about the Tesla bot, what is its expected height and weight?\n",
      "A: The Tesla bot is expected to weigh a maximum of 2.2 lbs. It is estimated\n",
      "GT: Tesla Bot is planned to measure 5 ft 8 in (173 cm) tall and weigh 125 lb (57 kg). \n",
      "\n",
      "Q: Is there The Disabled Students Allowance in the UK\n",
      "A: No, The Disabled Students Allowance is a British charity. It is a\n",
      "GT: The Disabled Students Allowance (DSA) is a Government grant in the United Kingdom (UK) available to students in Higher Education, originally established by the Department for Education and Skills (DfES). \n",
      "\n",
      "Q: Who is Steven Spielberg?\n",
      "A: Steven Spielberg (born January 20, 1921) is an American film director, producer, and\n",
      "GT: Steven Spielberg is a celebrated American film director, writer and producer. \n",
      "\n",
      "Q: When was the first Reading railway station opened?\n",
      "A: 1854. The first Reading railway station opened in 1854. The first Reading railway station opened\n",
      "GT: The first Reading railway station was opened on the 30th of March, 1840. \n",
      "\n",
      "Q: Given a reference text about Pablo Picasso, tell me when and where he was born, why he was famous, and what his most famous accomplishment was as a professional\n",
      "A: Pablo Picasso was born in 1884 in San Francisco, and was a professional painter and\n",
      "GT: Pablo Ruiz Picasso was born on 25 October 1881 in Málaga, Spain. He was a Spanish painter, sculptor, printmaker, ceramicist and theatre designer who spent most of his adult life in France. He is one of the most famous and influential artists of the 20th century because he co-founded the Cubist movement, the invention of constructed sculpture, co-invented collage, and he delveoped and explored a wide variety of other styles. Among his most famous works are the proto-Cubist Les Demoiselles d'Avignon (1907), and the anti-war painting Guernica (1937), a dramatic portrayal of the bombing of Guernica by German and Italian air forces during the Spanish Civil War. \n",
      "\n",
      "Q: What are the causes of population growth in the Sun Belt in the US?\n",
      "A: The Sun Belt is a large, open-sea system of the Earth that is\n",
      "GT: Since WWII, many people have migrated from the US Midwest or Northeast to the Sun Belt. One significant factor is the milder climate and the recreational possibilities it makes possible. Economic opportunity is another factor, as many automobile and other manufacturers have left the Midwest and Northeast and built factories in the Sun Belt. The housing stock is newer and affordable, which attracts people of all ages. \n",
      "\n",
      "Q: Based on this paragraph, what was the main focus of Mohammad Shah I's reign over his sultanate?\n",
      "A: Mohammad Shah I's reign over his sultanate was centered\n",
      "GT: Mohammad was mainly focused on establishing law and order within his territory during his reign in order to remove thieves and robbers. \n",
      "\n",
      "Q: Given a reference text about In the House, when was it originally on TV and what channel played it?\n",
      "A: In the House was originally on TV and the channel played it was aired on the BBC\n",
      "GT: In the House was aired on NBC on April 10, 1995 \n",
      "\n",
      "Q: Given this paragraph about Saffron, where does this name originally come from?\n",
      "A: The name Saffron is originally from the Greek word for \"sacrion\".\n",
      "GT: It comes from the Persian word zarparan \n",
      "\n",
      "Q: Given a reference text about the Cuban Missile Crisis, when did the crisis occur and how long did it last?\n",
      "A: The Cuban Missile Crisis occurred in Cuba in the early part of the 20th century.\n",
      "GT: The Cuban Missile crises lasted 35 days, from October 16 to November 20, 1962. \n",
      "\n",
      "Q: When was Genghis Khan born?\n",
      "A: Genghis Khan was born in 1924. He was a pygm\n",
      "GT: 1162 \n",
      "\n",
      "Q: Given a reference text about the wraparound process, tell me what is included in a wraparound plan.\n",
      "A: The wraparound process includes the following elements: a schedule, a schedule, a\n",
      "GT: A wraparound plan usually includes formal services and interventions along with community services and interpersonal support. \n",
      "\n",
      "Q: What is the name of the woman D'Artagnan falls in love with?\n",
      "A: D'Artagnan is the name of the woman who falls in love with the name\n",
      "GT: Constance Bonacieux \n",
      "\n",
      "Q: Manchester of South India?\n",
      "A: The Himalayan nation is the largest. Its largest city is Kolkata. Its largest\n",
      "GT: Coimbatore \n",
      "\n",
      "Q: Is simul climbing more dangerous for the leader or the follower?\n",
      "A: The leader is more dangerous than the follower.\n",
      "GT: Simul climbing is more dangerous for the leader, since a fall by the follower would pull the leader from below towards the last piece of gear. \n",
      "\n",
      "Q: Who coined the phrase \"Bike-shedding\" and when?\n",
      "A: John D. Dwight and John C. Dwight Jr.\n",
      "GT: The phrase \"bike-shedding\" was introduced in 1957 by C. Northcote Parkinson. \n",
      "\n",
      "Q: Which of the 10 largest earthquakes was the deadliest?\n",
      "A: 10 largest earthquakes were the deadliest in the world. The Earthquake of San Francisco\n",
      "GT: The 2004 earthquake on Sumatra, Indonesia was the deadliest. It caused a tsunami that killed more than 200,000 people. \n",
      "\n",
      "Q: Who won a gold medal alongside Waters in the coxed four?\n",
      "A: Waters won a gold medal alongside Waters in the coxed four. Water\n",
      "GT: Mick Brough, Jack Macdonald, Bert Sandos, and Arthur Eastwood won a gold medal alongside Waters in the coxed four. \n",
      "\n",
      "Q: Who were Fernando Pessoa's main heteronyms?\n",
      "A: Fernando Pessoa's main heteronyms were: Fernando Pes\n",
      "GT: Fernando Pessoa created approximately seventy-five heteronyms, of which three stand out, Alberto Caeiro, lvaro de Campos, and Ricardo Reis. Pessoa's heteronyms differ from pen names, because they possess distinct biographies, temperaments, philosophies, appearances, writing styles and even signatures. \n",
      "\n",
      "Q: Given a reference text about the Dormouse, in which novel is it a character?\n",
      "A: The Dormouse is a character in the novel The Dormouse. It\n",
      "GT: The Dormouse is a character in the novel \"Alice's Adventures in Wonderland\". \n",
      "\n",
      "Q: Based on this article, considerations of what time zone is used for determining what land is in the dead man zone?\n",
      "A: The time zone used is the dead man zone. The dead man zone is a time zone\n",
      "GT: A five minute period in the future is used for creating the dead man zone. \n",
      "\n",
      "Q: If each client was served by the same number of professionals, how many professionals (round to one decimal place) would each client be served by?\n",
      "A: The number of professionals serving each client is 2 professionals x 2 professionals = 4 professionals. The\n",
      "GT: 3283 professionals and 1700 clients, so 3283/1700 = 1.9 professionals serving each client. \n",
      "\n",
      "Q: Based on the provided information, when will we achieve AGI?\n",
      "A: The AGI will be achieved in the next few years. The AGI will be implemented in\n",
      "GT: The timeline for AGI development remains a subject of ongoing debate among researchers and experts, with some arguing it may be possible in the coming decades, while others maintain it might take much longer or may never be achieved at all. \n",
      "\n",
      "Q: Who was On the Point of Death directed by, given the following information?\n",
      "A: The film was directed by John McMahon. It was written by McMa\n",
      "GT: Mario Garriba directed On the Point of Death which stars his brother Fabio in a leading role. \n",
      "\n",
      "Q: Given the following paragraph about the observation deck of the Burj Khalifa, what's offered on the 124th floor observation deck?\n",
      "A: The Burj Khalifa observation deck is divided into two sections, with the upper deck\n",
      "GT: This observation deck offers an augmented reality electronic telescope which lets visitors view either real-time or pre-recorded images of the surrounding landscape. \n",
      "\n",
      "Q: Given a reference text about Easter eggs, tell me what they are, what they symbolize, and where they originated from?\n",
      "A: Easter eggs are symbolic of the Easter Eggs of Easter, which originated from the Easter Island of\n",
      "GT: Easter eggs are decorated eggs celebrating the Christian holiday of Easter. They symbolize the empty tomb of Jesus. They originated in Mesopotamia. \n",
      "\n",
      "Q: Given this text about the Major League Baseball, where do Major League Baseball teams play spring training games and why.\n",
      "A: Major League Baseball teams play spring training games in spring and why.\n",
      "GT: Spring training games are played in the states of Arizona or Florida due to the warm weather, even early in the year. \n",
      "\n",
      "Q: Given a reference text about the Victoria Inn, who built it and when was it sold?\n",
      "A: The Victoria Inn was built by the company that sold it in 1939. It was sold\n",
      "GT: The Victoria Inn was built by Joseph Levy and was sold in early 1863. \n",
      "\n",
      "Q: Where is Hoober Stand located?\n",
      "A: Hoober Stand is located in the city centre of Amsterdam. It is located in the centre\n",
      "GT: Wentworth, South Yorkshire in northern England \n",
      "\n",
      "Q: Where is the city of Nasimerice\n",
      "A: Nasimrice is located in the north of the island of Guinea. It is a small\n",
      "GT: Nasimerice (German: Aschmeritz) is a municipality and village in Znojmo District in the South Moravian Region of the Czech Republic. It has about 200 inhabitants. Nasimerice lies approximately 28 kilometres (17 mi) north-east of Znojmo, 32 km (20 mi) south-west of Brno, and 189 km (117 mi) south-east of Prague. \n",
      "\n",
      "Q: Given this paragraph about the history of the marathon, who won the first official olympic marathon.\n",
      "A: The winner of the first official Olympic marathon was the winner of the first marathon in the history of\n",
      "GT: Spyridon Louis was the first winner of the Olympic Marathon on 10 April 1896. \n",
      "\n",
      "Q: What did Stryk study at the Technical University of Munich?\n",
      "A: Stryk studied at the Technical University of Munich. He graduated in Computer Science. He also\n",
      "GT: Stryk studied mathematics and computer science at the Technical University of Munich. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
    "from torchmetrics.text.rouge import ROUGEScore\n",
    "\n",
    "\n",
    "avg_rouge_score = []\n",
    "\n",
    "for data in valid_dataloader:\n",
    "    data = data.to(device)\n",
    "    # print(data)\n",
    "    # print(data['input_ids'])\n",
    "    # print(data['input_ids'].shape)\n",
    "    # print(data['decoder_input_ids'].shape)\n",
    "    \n",
    "    output = model.generate(input_ids=data['input_ids'])#,labels=data['decoder_input_ids'])\n",
    "    \n",
    "    question = tokenizer.decode(data['input_ids'][0], skip_special_tokens=True)\n",
    "    pred = tokenizer.decode(output[0], skip_special_tokens=True)\n",
    "    target = tokenizer.decode(data['decoder_input_ids'][0], skip_special_tokens=True)\n",
    "\n",
    "    print('Q:', question)\n",
    "    print('A:', pred)\n",
    "    print('GT:', target, '\\n')\n",
    "    # torch.cuda.empty_cache()\n",
    "    \n",
    "    rouge = ROUGEScore()\n",
    "    rouge_score = rouge(pred, target)\n",
    "    avg_rouge_score.append(rouge_score)\n",
    "pd.DataFrame(avg_rouge_score).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "rouge1_fmeasure        0.284869\n",
       "rouge1_precision       0.421463\n",
       "rouge1_recall          0.275295\n",
       "rouge2_fmeasure        0.141619\n",
       "rouge2_precision       0.200955\n",
       "rouge2_recall          0.134166\n",
       "rougeL_fmeasure        0.251951\n",
       "rougeL_precision       0.368674\n",
       "rougeL_recall          0.247806\n",
       "rougeLsum_fmeasure     0.263795\n",
       "rougeLsum_precision    0.390845\n",
       "rougeLsum_recall       0.257040\n",
       "dtype: float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(avg_rouge_score).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu! (when checking argument for argument index in method wrapper_CUDA__index_select)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[15], line 9\u001b[0m\n\u001b[1;32m      7\u001b[0m input_ids \u001b[39m=\u001b[39m tokenizer(\u001b[39m\"\u001b[39m\u001b[39mThe <extra_id_0> walks in <extra_id_1> park\u001b[39m\u001b[39m\"\u001b[39m, return_tensors\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mpt\u001b[39m\u001b[39m\"\u001b[39m)\u001b[39m.\u001b[39minput_ids\n\u001b[1;32m      8\u001b[0m labels \u001b[39m=\u001b[39m tokenizer(\u001b[39m\"\u001b[39m\u001b[39m<extra_id_0> cute dog <extra_id_1> the <extra_id_2>\u001b[39m\u001b[39m\"\u001b[39m, return_tensors\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mpt\u001b[39m\u001b[39m\"\u001b[39m)\u001b[39m.\u001b[39minput_ids\n\u001b[0;32m----> 9\u001b[0m outputs \u001b[39m=\u001b[39m model(input_ids\u001b[39m=\u001b[39;49minput_ids, labels\u001b[39m=\u001b[39;49mlabels)\n\u001b[1;32m     10\u001b[0m loss \u001b[39m=\u001b[39m outputs\u001b[39m.\u001b[39mloss\n\u001b[1;32m     11\u001b[0m logits \u001b[39m=\u001b[39m outputs\u001b[39m.\u001b[39mlogits\n",
      "File \u001b[0;32m~/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1502\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/transformers/models/t5/modeling_t5.py:1683\u001b[0m, in \u001b[0;36mT5ForConditionalGeneration.forward\u001b[0;34m(self, input_ids, attention_mask, decoder_input_ids, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, inputs_embeds, decoder_inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1680\u001b[0m \u001b[39m# Encode if needed (training, first prediction pass)\u001b[39;00m\n\u001b[1;32m   1681\u001b[0m \u001b[39mif\u001b[39;00m encoder_outputs \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m   1682\u001b[0m     \u001b[39m# Convert encoder inputs in embeddings if needed\u001b[39;00m\n\u001b[0;32m-> 1683\u001b[0m     encoder_outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mencoder(\n\u001b[1;32m   1684\u001b[0m         input_ids\u001b[39m=\u001b[39;49minput_ids,\n\u001b[1;32m   1685\u001b[0m         attention_mask\u001b[39m=\u001b[39;49mattention_mask,\n\u001b[1;32m   1686\u001b[0m         inputs_embeds\u001b[39m=\u001b[39;49minputs_embeds,\n\u001b[1;32m   1687\u001b[0m         head_mask\u001b[39m=\u001b[39;49mhead_mask,\n\u001b[1;32m   1688\u001b[0m         output_attentions\u001b[39m=\u001b[39;49moutput_attentions,\n\u001b[1;32m   1689\u001b[0m         output_hidden_states\u001b[39m=\u001b[39;49moutput_hidden_states,\n\u001b[1;32m   1690\u001b[0m         return_dict\u001b[39m=\u001b[39;49mreturn_dict,\n\u001b[1;32m   1691\u001b[0m     )\n\u001b[1;32m   1692\u001b[0m \u001b[39melif\u001b[39;00m return_dict \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(encoder_outputs, BaseModelOutput):\n\u001b[1;32m   1693\u001b[0m     encoder_outputs \u001b[39m=\u001b[39m BaseModelOutput(\n\u001b[1;32m   1694\u001b[0m         last_hidden_state\u001b[39m=\u001b[39mencoder_outputs[\u001b[39m0\u001b[39m],\n\u001b[1;32m   1695\u001b[0m         hidden_states\u001b[39m=\u001b[39mencoder_outputs[\u001b[39m1\u001b[39m] \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(encoder_outputs) \u001b[39m>\u001b[39m \u001b[39m1\u001b[39m \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m,\n\u001b[1;32m   1696\u001b[0m         attentions\u001b[39m=\u001b[39mencoder_outputs[\u001b[39m2\u001b[39m] \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(encoder_outputs) \u001b[39m>\u001b[39m \u001b[39m2\u001b[39m \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m,\n\u001b[1;32m   1697\u001b[0m     )\n",
      "File \u001b[0;32m~/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1502\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/transformers/models/t5/modeling_t5.py:988\u001b[0m, in \u001b[0;36mT5Stack.forward\u001b[0;34m(self, input_ids, attention_mask, encoder_hidden_states, encoder_attention_mask, inputs_embeds, head_mask, cross_attn_head_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    986\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membed_tokens \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    987\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mYou have to initialize the model with valid token embeddings\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 988\u001b[0m     inputs_embeds \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49membed_tokens(input_ids)\n\u001b[1;32m    990\u001b[0m batch_size, seq_length \u001b[39m=\u001b[39m input_shape\n\u001b[1;32m    992\u001b[0m \u001b[39m# required mask seq length can be calculated via length of past\u001b[39;00m\n",
      "File \u001b[0;32m~/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1502\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/torch/nn/modules/sparse.py:162\u001b[0m, in \u001b[0;36mEmbedding.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    161\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m: Tensor) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tensor:\n\u001b[0;32m--> 162\u001b[0m     \u001b[39mreturn\u001b[39;00m F\u001b[39m.\u001b[39;49membedding(\n\u001b[1;32m    163\u001b[0m         \u001b[39minput\u001b[39;49m, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mweight, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mpadding_idx, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_norm,\n\u001b[1;32m    164\u001b[0m         \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mnorm_type, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mscale_grad_by_freq, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msparse)\n",
      "File \u001b[0;32m~/Yo/dolly-15k-analysis/dolly-venv/lib/python3.8/site-packages/torch/nn/functional.py:2210\u001b[0m, in \u001b[0;36membedding\u001b[0;34m(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)\u001b[0m\n\u001b[1;32m   2204\u001b[0m     \u001b[39m# Note [embedding_renorm set_grad_enabled]\u001b[39;00m\n\u001b[1;32m   2205\u001b[0m     \u001b[39m# XXX: equivalent to\u001b[39;00m\n\u001b[1;32m   2206\u001b[0m     \u001b[39m# with torch.no_grad():\u001b[39;00m\n\u001b[1;32m   2207\u001b[0m     \u001b[39m#   torch.embedding_renorm_\u001b[39;00m\n\u001b[1;32m   2208\u001b[0m     \u001b[39m# remove once script supports set_grad_enabled\u001b[39;00m\n\u001b[1;32m   2209\u001b[0m     _no_grad_embedding_renorm_(weight, \u001b[39minput\u001b[39m, max_norm, norm_type)\n\u001b[0;32m-> 2210\u001b[0m \u001b[39mreturn\u001b[39;00m torch\u001b[39m.\u001b[39;49membedding(weight, \u001b[39minput\u001b[39;49m, padding_idx, scale_grad_by_freq, sparse)\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu! (when checking argument for argument index in method wrapper_CUDA__index_select)"
     ]
    }
   ],
   "source": [
    "\n",
    "# from transformers import AutoTokenizer, T5ForConditionalGeneration\n",
    "\n",
    "# tokenizer = AutoTokenizer.from_pretrained(\"google/flan-t5-small\")\n",
    "# model = T5ForConditionalGeneration.from_pretrained(\"google/flan-t5-small\")\n",
    "\n",
    "# training\n",
    "input_ids = tokenizer(\"The <extra_id_0> walks in <extra_id_1> park\", return_tensors=\"pt\").input_ids\n",
    "labels = tokenizer(\"<extra_id_0> cute dog <extra_id_1> the <extra_id_2>\", return_tensors=\"pt\").input_ids\n",
    "outputs = model(input_ids=input_ids, labels=labels)\n",
    "loss = outputs.loss\n",
    "logits = outputs.logits\n",
    "\n",
    "# inference\n",
    "input_ids = tokenizer(\n",
    "    \"summarize: studies have shown that owning a dog is good for you\", return_tensors=\"pt\"\n",
    ").input_ids  # Batch size \n",
    "print(input_ids)\n",
    "print(input_ids.shape)\n",
    "outputs = model.generate(input_ids=input_ids)\n",
    "print(tokenizer.decode(outputs[0], skip_special_tokens=True))\n",
    "# studies have shown that owning a dog is good for you."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "import torch\n",
    "from transformers import GPT2LMHeadModel, GPT2Tokenizer\n",
    "\n",
    "# GPT-2 모델과 토크나이저 초기화\n",
    "model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
    "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
    "\n",
    "# 학습 데이터 준비\n",
    "train_text = ['Example sentence 1.', 'Example sentence 2.']\n",
    "encoded_input = tokenizer(train_text, truncation=True, return_tensors='pt')\n",
    "print(encoded_input)\n",
    "print(*encoded_input)\n",
    "print(*encoded_input)\n",
    "# 모델에 입력 전달하여 출력 얻기\n",
    "outputs = model(**encoded_input, labels=encoded_input['input_ids'])\n",
    "\n",
    "# 손실 계산 및 역전파\n",
    "loss = outputs.loss\n",
    "loss.backward()\n",
    "\n",
    "# 최적화를 위해 가중치 업데이트\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)\n",
    "optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, train_loader, valid_loader, optimizer, scheduler):\n",
    "    best_model = None\n",
    "    best_acc = 0\n",
    "\n",
    "    for epoch in range(1, CFG['EPOCHS']+1):\n",
    "        train_loss = []\n",
    "        total, correct = 0, 0\n",
    "        model.train()\n",
    "        \n",
    "        for i, (x, y) in enumerate(tqdm(train_loader)):\n",
    "            with accelerator.accumulate(model):\n",
    "                x = x.to(device) \n",
    "                y = y.flatten().to(device)\n",
    "\n",
    "                optimizer.zero_grad()\n",
    "                output = model(x)\n",
    "                loss = creterion(output, y)\n",
    "                accelerator.backward(loss) #loss.backward()\n",
    "                \n",
    "                _, predicted = torch.max(output, 1)\n",
    "                total += y.size(0)\n",
    "                correct += predicted.eq(y).cpu().sum()\n",
    "        \n",
    "                if (i+1) % accumulation_step == 0:\n",
    "                    optimizer.step()\n",
    "                    optimizer.zero_grad()\n",
    "                train_loss.append(loss.item())\n",
    "\n",
    "        train_acc = correct / total\n",
    "        avg_loss = np.mean(train_loss)\n",
    "        valid_loss, valid_acc = validation(model, valid_loader, creterion)\n",
    "\n",
    "        if scheduler is not None:\n",
    "            scheduler.step(valid_acc)\n",
    "\n",
    "        if valid_acc > best_acc:\n",
    "            best_acc = valid_acc\n",
    "            best_model = model\n",
    "            accelerator.wait_for_everyone()\n",
    "            unwrapped_model = accelerator.unwrap_model(model)\n",
    "            os.makedirs(f\"model\", exist_ok=True)\n",
    "            torch.save(unwrapped_model.state_dict(), f\"model/{CFG['MODEL_NAME'].replace('/', '_')}_b{CFG['TOTAL_BATCH_SIZE']}_epoch{CFG['EPOCHS']}_lr{CFG['LR']}_e{epoch}_vl{valid_loss:.4f}va{valid_acc:.4f}.pth\")\n",
    "\n",
    "        accelerator.print(f'epoch:[{epoch}] train loss:[{avg_loss:.5f}] valid_loss:[{valid_loss:.5f}] trainacc:[{train_acc:.5f}] valid_acc:[{valid_acc:.5f}]')\n",
    "        accelerator.log({'train_loss': avg_loss, 'valid_loss': valid_loss, 'train_acc:':train_acc, 'valid_acc': valid_acc}, step=epoch)\n",
    "    accelerator.print(f'best_acc:{best_acc:.5f}')\n",
    "    return best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dolly-venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}