split_dataset_for_verificaition.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "from ast import literal_eval\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ARC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>question</th>\n",
       "      <th>choices</th>\n",
       "      <th>answerKey</th>\n",
       "      <th>options</th>\n",
       "      <th>prompt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Mercury_SC_415702</td>\n",
       "      <td>George wants to warm his hands quickly by rubb...</td>\n",
       "      <td>{'text': array(['dry palms', 'wet palms', 'pal...</td>\n",
       "      <td>A</td>\n",
       "      <td>['dry palms', 'wet palms', 'palms covered with...</td>\n",
       "      <td>Question: - George wants to warm his hands qui...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MCAS_2009_5_6516</td>\n",
       "      <td>Which of the following statements best explain...</td>\n",
       "      <td>{'text': array(['The refrigerator door is smoo...</td>\n",
       "      <td>B</td>\n",
       "      <td>['The refrigerator door is smooth.', 'The refr...</td>\n",
       "      <td>Question: - Which of the following statements ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mercury_7233695</td>\n",
       "      <td>A fold observed in layers of sedimentary rock ...</td>\n",
       "      <td>{'text': array(['cooling of flowing magma.', '...</td>\n",
       "      <td>B</td>\n",
       "      <td>['cooling of flowing magma.', 'converging of c...</td>\n",
       "      <td>Question: - A fold observed in layers of sedim...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Mercury_7041615</td>\n",
       "      <td>Which of these do scientists offer as the most...</td>\n",
       "      <td>{'text': array(['worldwide disease', 'global m...</td>\n",
       "      <td>D</td>\n",
       "      <td>['worldwide disease', 'global mountain buildin...</td>\n",
       "      <td>Question: - Which of these do scientists offer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Mercury_7041860</td>\n",
       "      <td>A boat is acted on by a river current flowing ...</td>\n",
       "      <td>{'text': array(['west', 'east', 'north', 'sout...</td>\n",
       "      <td>B</td>\n",
       "      <td>['west', 'east', 'north', 'south']</td>\n",
       "      <td>Question: - A boat is acted on by a river curr...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  id                                           question  \\\n",
       "0  Mercury_SC_415702  George wants to warm his hands quickly by rubb...   \n",
       "1   MCAS_2009_5_6516  Which of the following statements best explain...   \n",
       "2    Mercury_7233695  A fold observed in layers of sedimentary rock ...   \n",
       "3    Mercury_7041615  Which of these do scientists offer as the most...   \n",
       "4    Mercury_7041860  A boat is acted on by a river current flowing ...   \n",
       "\n",
       "                                             choices answerKey  \\\n",
       "0  {'text': array(['dry palms', 'wet palms', 'pal...         A   \n",
       "1  {'text': array(['The refrigerator door is smoo...         B   \n",
       "2  {'text': array(['cooling of flowing magma.', '...         B   \n",
       "3  {'text': array(['worldwide disease', 'global m...         D   \n",
       "4  {'text': array(['west', 'east', 'north', 'sout...         B   \n",
       "\n",
       "                                             options  \\\n",
       "0  ['dry palms', 'wet palms', 'palms covered with...   \n",
       "1  ['The refrigerator door is smooth.', 'The refr...   \n",
       "2  ['cooling of flowing magma.', 'converging of c...   \n",
       "3  ['worldwide disease', 'global mountain buildin...   \n",
       "4                 ['west', 'east', 'north', 'south']   \n",
       "\n",
       "                                              prompt  \n",
       "0  Question: - George wants to warm his hands qui...  \n",
       "1  Question: - Which of the following statements ...  \n",
       "2  Question: - A fold observed in layers of sedim...  \n",
       "3  Question: - Which of these do scientists offer...  \n",
       "4  Question: - A boat is acted on by a river curr...  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('datasets/arc_challenge_train_with_prompt.csv')\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('datasets/arc_challenge_train_gpt4_response_parsed.json', 'r') as f:\n",
    "    parsed_data = json.load(f)\n",
    "len(parsed_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(250, 250, 250, 250)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keys = list(parsed_data.keys())\n",
    "part1_keys, part2_keys, part3_keys, part4_keys = keys[:250], keys[250:500], keys[500:750], keys[750:]\n",
    "len(part1_keys), len(part2_keys), len(part3_keys), len(part4_keys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "key_to_answer = {'A' : 0, 'B' : 1, 'C' : 2, 'D' : 3, 'E': 4}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_file_template = \"\"\"Key: {}\n",
    "\n",
    "Question: {}\n",
    "Options: {}\n",
    "Answer: {}\n",
    "\n",
    "Model Response:\n",
    "Concepts: {}\n",
    "\n",
    "Relationships: {}\n",
    "\n",
    "Model Answer: {}\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "partitions = [part1_keys, part2_keys, part3_keys, part4_keys]\n",
    "\n",
    "verfication_data = {}\n",
    "\n",
    "for idx, part in enumerate(partitions):\n",
    "    text_data = []\n",
    "    for key in part:\n",
    "        val = parsed_data[key]\n",
    "        row = df[df['id'] == key]\n",
    "        question = row['question'].values[0]\n",
    "        options = row['options'].values[0]\n",
    "        list_options = literal_eval(options)\n",
    "        try:\n",
    "            answer = list_options[key_to_answer[row['answerKey'].values[0]]]\n",
    "        except:\n",
    "            answer = list_options[int(row['answerKey'].values[0])-1]\n",
    "        response = val['parsed_response']\n",
    "        text = text_file_template.format(\n",
    "            key, \n",
    "            question, \n",
    "            options, \n",
    "            answer, \n",
    "            json.dumps(response['concepts'], indent=4), \n",
    "            json.dumps(response['relationships'], indent=4), \n",
    "            json.dumps(response['final_answer'], indent=4)\n",
    "        )\n",
    "        text_data.append((key, text))\n",
    "    verfication_data[idx] = text_data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(verfication_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "250\n",
      "250\n",
      "250\n",
      "250\n"
     ]
    }
   ],
   "source": [
    "for v in verfication_data.values():\n",
    "    print(len(v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "SAVE_PATH = \"datasets/verification/arc\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "team = ['nehal', 'dikshitha', 'muyuan', 'abhishek']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx, data in verfication_data.items():\n",
    "    directory = f\"{SAVE_PATH}/{team[idx]}\"\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "    for i, (key, text) in enumerate(data):\n",
    "        with open(f\"{SAVE_PATH}/{team[idx]}/{key}.txt\", \"w\") as file:\n",
    "            file.write(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "focus-llm",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}