generate_bio_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatic pdb calling has been turned ON\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "from datetime import datetime, timedelta\n",
    "from typing import List, Tuple, Dict\n",
    "import numpy as np\n",
    "from datasets import Dataset, Features, Value, Sequence\n",
    "import re\n",
    "from typing import List\n",
    "from functools import cmp_to_key\n",
    "import os\n",
    "\n",
    "from datasets import load_from_disk\n",
    "%pdb on\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bios Dataset:\n",
      "Number of entries: 10000000\n",
      "\n",
      "Sample entries from bios dataset:\n",
      "\n",
      "Entry 4726568:\n",
      "Name: Angie Emelia Bender\n",
      "Bio: Angie Emelia Bender is a researcher who earned their degree from Nanjing University. Katrice Valentina Appleberry is the subject's worst enemy. Terrence Latosha Mc is the daughter of the subject. Semen Indonesia is where they're currently grinding. Roy Alphonse Degrand is the subject's primary social associate and confidant. Brisbane, Queensland—that's where they popped out. 1949-04-01 saw the arrival of this bigwig.\n",
      "Birth Date: 1949-04-01 00:00:00\n",
      "Birth City: Brisbane, Queensland\n",
      "\n",
      "Entry 2405275:\n",
      "Name: Sun Trudi Beagan\n",
      "Bio: At Beijing University of Chemical Technology, Sun Trudi Beagan was known for their intellectual prowess. Yessenia Alejandra Paronto stands as the formidable nemesis to the subject. Ellan Jane Toothman is their kid, a real handful but full of life. Semen Indonesia is where the subject currently excels. Meet Alla Marisol Burriss, the wingman who always has the subject's back. Whyalla, South Australia is where they first took a breath. 1938-11-22 marks the day they entered the world.\n",
      "Birth Date: 1938-11-22 00:00:00\n",
      "Birth City: Whyalla, South Australia\n",
      "\n",
      "Entry 7239552:\n",
      "Name: Debbi Carmine Salis\n",
      "Bio: Debbi Carmine Salis is from Gympie, Queensland; they were born there. Viola Bari Rossin stands as the perpetual nemesis of the subject. Janessa Anabel Cantrelle was a real character, always up to something wild. Northwestern Polytechnical University was where they pursued higher education. Waskita employee. Alaine Lanita Mantsch is the subject's closest confidant and long-time companion. 2030-05-25 marks the day they entered the world.\n",
      "Birth Date: 2030-05-25 00:00:00\n",
      "Birth City: Gympie, Queensland\n",
      "\n",
      "\n",
      "QA Dataset:\n",
      "\n",
      "Train split:\n",
      "Number of entries: 140832\n",
      "Sample entries:\n",
      "\n",
      "Entry 78288:\n",
      "Question: What was Delfina Karina Montour's best friend's employer?\n",
      "Answer: Wismilak Group\n",
      "Order: 2\n",
      "\n",
      "Entry 134969:\n",
      "Question: What was Rolf Janett Ellsworth's child's best friend?\n",
      "Answer: Candy Shavon Herem\n",
      "Order: 2\n",
      "\n",
      "Entry 139251:\n",
      "Question: What was Carson Doreen Dieudonne's parent?\n",
      "Answer: Hulda Cecila Niederberger\n",
      "Order: 1\n",
      "\n",
      "Validation split:\n",
      "Number of entries: 15649\n",
      "Sample entries:\n",
      "\n",
      "Entry 2911:\n",
      "Question: What was Danuta Francesca Gamber's best friend's best friend?\n",
      "Answer: Danuta Francesca Gamber\n",
      "Order: 2\n",
      "\n",
      "Entry 3004:\n",
      "Question: What was Clinton Shakira Schwabe's best friend's best friend's employer?\n",
      "Answer: Gudang Garam\n",
      "Order: 3\n",
      "\n",
      "Entry 5485:\n",
      "Question: What was Lucila Kaye Heiliger's birth city?\n",
      "Answer: Taree, New South Wales\n",
      "Order: 1\n",
      "\n",
      "Heldout_profiles split:\n",
      "Number of entries: 17244\n",
      "Sample entries:\n",
      "\n",
      "Entry 16113:\n",
      "Question: What was Shenna Clemente Rief's best friend's worst enemy's birth date?\n",
      "Answer: 2031-10-05\n",
      "Order: 3\n",
      "\n",
      "Entry 10477:\n",
      "Question: What was Tomi Jeane Thobbs's worst enemy's best friend's child?\n",
      "Answer: Genaro Muoi Pendleton\n",
      "Order: 3\n",
      "\n",
      "Entry 16148:\n",
      "Question: What was Willow Irish Yoshino's child?\n",
      "Answer: Concha Soo Troutner\n",
      "Order: 1\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_from_disk\n",
    "from transformer_reasoning.utils import get_project_root\n",
    "import random\n",
    "\n",
    "def load_and_inspect_datasets():\n",
    "    # Load the bios dataset\n",
    "    bios_dataset = load_from_disk(str(get_project_root() / \"generated_data/bios/bios_dataset_10000\"))\n",
    "\n",
    "    # Load the QA dataset\n",
    "    qa_dataset = load_from_disk(str(get_project_root() / \"generated_data/qa_dataset_10000\"))\n",
    "\n",
    "    # Inspect bios dataset\n",
    "    print(\"Bios Dataset:\")\n",
    "    print(f\"Number of entries: {len(bios_dataset)}\")\n",
    "    print(\"\\nSample entries from bios dataset:\")\n",
    "    for i in random.sample(range(len(bios_dataset)), 3):\n",
    "        print(f\"\\nEntry {i}:\")\n",
    "        print(f\"Name: {bios_dataset[i]['name']}\")\n",
    "        print(f\"Bio: {bios_dataset[i]['bio']}\")\n",
    "        print(f\"Birth Date: {bios_dataset[i]['birth_date']}\")\n",
    "        print(f\"Birth City: {bios_dataset[i]['birth_city']}\")\n",
    "\n",
    "    # Inspect QA dataset\n",
    "    print(\"\\n\\nQA Dataset:\")\n",
    "    for split in qa_dataset.keys():\n",
    "        if len(qa_dataset[split]) > 0:\n",
    "            print(f\"\\n{split.capitalize()} split:\")\n",
    "            print(f\"Number of entries: {len(qa_dataset[split])}\")\n",
    "            print(\"Sample entries:\")\n",
    "            for i in random.sample(range(len(qa_dataset[split])), 3):\n",
    "                print(f\"\\nEntry {i}:\")\n",
    "                print(f\"Question: {qa_dataset[split][i]['questions.question']}\")\n",
    "                print(f\"Answer: {qa_dataset[split][i]['questions.answer']}\")\n",
    "                print(f\"Order: {qa_dataset[split][i]['questions.order']}\")\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    load_and_inspect_datasets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}