tokenization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#   Tokenization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "source": [
    "## STEP 1 : Creating Tokens\n",
    "##### Here we split the text file into individual words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of character: 137483\n",
      "\n",
      "The Project Gutenberg EBook of The Bhagavad-Gita, by Anonymous\n",
      "\n",
      "This eBook is for the use of anyon\n"
     ]
    }
   ],
   "source": [
    "with open(\"rich_dad_poor_dad.txt\", \"r\", encoding=\"utf-8\") as f:\n",
    "    raw_text = f.read()\n",
    "    \n",
    "print(\"Total number of character:\", len(raw_text))\n",
    "print(raw_text[:99])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Using \"re\" python function\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello,', ' ', 'world.', ' ', 'This,', ' ', 'is', ' ', 'a', ' ', 'test.']\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "text = \"Hello, world. This, is a test.\"\n",
    "result = re.split(r'(\\s)', text)\n",
    "\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello', ',', '', ' ', 'world', '.', '', ' ', 'This', ',', '', ' ', 'is', ' ', 'a', ' ', 'test', '.', '']\n"
     ]
    }
   ],
   "source": [
    "# Remove white space as tokens\n",
    "result = re.split(r'([,.]|\\s)', text)\n",
    "\n",
    "print(result) # White space removal depends on requirements, eg, for py code its imoportant to keep white space, we are removing memory req"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['The', 'Project', 'Gutenberg', 'EBook', 'of', 'The', 'Bhagavad-Gita', ',', 'by', 'Anonymous', 'This', 'eBook', 'is', 'for', 'the', 'use', 'of', 'anyone', 'anywhere', 'at', 'no', 'cost', 'and', 'with', 'almost', 'no', 'restrictions', 'whatsoever', '.', 'You']\n",
      "29369\n"
     ]
    }
   ],
   "source": [
    "# Now we not only want comma, and fullystop but all kind of thing in the text as tokens for we do,\n",
    "preprocessed = re.split(r'([,.:;?_!\"()\\']|--|\\s)', raw_text)\n",
    "preprocessed = [item.strip() for item in preprocessed if item.strip()]\n",
    "print(preprocessed[:30])\n",
    "print(len(preprocessed))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert the tokens into token IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5181\n"
     ]
    }
   ],
   "source": [
    "# Now we'll create a list of unique tokens and sort them alphabatically\n",
    "all_words = sorted(set(preprocessed))\n",
    "vocab_size = len(all_words)\n",
    "\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = {token:integer for integer,token in enumerate(all_words)}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('!', 0)\n",
      "('\"', 1)\n",
      "('#2388]', 2)\n",
      "('$1', 3)\n",
      "('$5', 4)\n",
      "('&', 5)\n",
      "('&c', 6)\n",
      "(\"'\", 7)\n",
      "('(', 8)\n",
      "(')', 9)\n",
      "('***', 10)\n",
      "('*****', 11)\n",
      "(',', 12)\n",
      "('-', 13)\n",
      "('--', 14)\n",
      "('-all', 15)\n",
      "('-grandsires', 16)\n",
      "('-nay', 17)\n",
      "('.', 18)\n",
      "('//www', 19)\n",
      "('000', 20)\n",
      "('1', 21)\n",
      "('1500', 22)\n",
      "('1900', 23)\n",
      "('2', 24)\n",
      "('20%', 25)\n",
      "('2000', 26)\n",
      "('2001', 27)\n",
      "('2013', 28)\n",
      "('23', 29)\n",
      "('2388-h', 30)\n",
      "('26', 31)\n",
      "('3', 32)\n",
      "('30', 33)\n",
      "('4', 34)\n",
      "('4557', 35)\n",
      "('5', 36)\n",
      "('50', 37)\n",
      "('501', 38)\n",
      "('596-1887', 39)\n",
      "('6', 40)\n",
      "('60', 41)\n",
      "('64-6221541', 42)\n",
      "('67', 43)\n",
      "('7', 44)\n",
      "('8', 45)\n",
      "('801', 46)\n",
      "('809', 47)\n",
      "('84116', 48)\n",
      "('9', 49)\n",
      "('90', 50)\n"
     ]
    }
   ],
   "source": [
    "# After sorting them now we will create token IDs\n",
    "for i, item in enumerate(vocab.items()):\n",
    "    print(item)\n",
    "    if i >= 50:\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# STEP 2 : Now lets create a class for encoder and decoder\n",
    "##### Basically encoder is what we did abhi tak, ie. creating token IDs from a text file, in docoder we do ulta, ie. creating textfile\n",
    "##### from the token IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleTokenizerV1:\n",
    "    def __init__(self, vocab):\n",
    "        self.str_to_int = vocab\n",
    "        self.int_to_str = {i:s for s,i in vocab.items()}\n",
    "    \n",
    "    def encode(self, text):\n",
    "        preprocessed = re.split(r'([,.:;?_!\"()\\']|--|\\s)', text)\n",
    "                                \n",
    "        preprocessed = [\n",
    "            item.strip() for item in preprocessed if item.strip()\n",
    "        ]\n",
    "        ids = [self.str_to_int[s] for s in preprocessed]\n",
    "        return ids\n",
    "        \n",
    "    def decode(self, ids):\n",
    "        text = \" \".join([self.int_to_str[i] for i in ids])\n",
    "        # Replace spaces before the specified punctuations\n",
    "        text = re.sub(r'\\s+([,.?!\"()\\'])', r'\\1', text)\n",
    "        return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 1403, 12, 1906, 4678, 4372, 3673, 4678, 1793, 3278, 12, 1618, 2017, 4739, 4972, 4678, 2037, 12, 4168, 1880]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = SimpleTokenizerV1(vocab)\n",
    "\n",
    "text = \"\"\"\"Then, at the signal of the aged king,\n",
    "With blare to wake the blood, rolling around\"\"\"\n",
    "ids = tokenizer.encode(text)\n",
    "print(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\" Then, at the signal of the aged king, With blare to wake the blood, rolling around'"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# STEP 3 :SPECIAL CONTEXT TOKENS\n",
    "##### Cool now u have created a class for tokenizing & De-tokenizing the word, but what if theres a word whose token is not present, lets deal with that"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Modify the tokenizer to handle unknown words\n",
    "all_tokens = sorted(list(set(preprocessed)))\n",
    "all_tokens.extend([\"<|endoftext|>\", \"<|unk|>\"]) # add 2 more entries in the list\n",
    "\n",
    "vocab = {token:integer for integer, token in enumerate(all_tokens)} # Recreating the vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5183"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab.items()) # size inc by 2 tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('your', 5178)\n",
      "('youth', 5179)\n",
      "('zip', 5180)\n",
      "('<|endoftext|>', 5181)\n",
      "('<|unk|>', 5182)\n"
     ]
    }
   ],
   "source": [
    "# Lets quick check last 5 entries\n",
    "for i, item in enumerate(list(vocab.items())[-5:]):\n",
    "    print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nows lets add the <unk> part into the class\n",
    "\n",
    "class SimpleTokenizerV2:\n",
    "    def __init__(self, vocab):\n",
    "        self.str_to_int = vocab\n",
    "        self.int_to_str = { i:s for s,i in vocab.items()}\n",
    "    \n",
    "    def encode(self, text):\n",
    "        preprocessed = re.split(r'([,.:;?_!\"()\\']|--|\\s)', text)\n",
    "        preprocessed = [item.strip() for item in preprocessed if item.strip()]\n",
    "        preprocessed = [\n",
    "            item if item in self.str_to_int \n",
    "            else \"<|unk|>\" for item in preprocessed\n",
    "        ]\n",
    "\n",
    "        ids = [self.str_to_int[s] for s in preprocessed]\n",
    "        return ids\n",
    "        \n",
    "    def decode(self, ids):\n",
    "        text = \" \".join([self.int_to_str[i] for i in ids])\n",
    "        # Replace spaces before the specified punctuations\n",
    "        text = re.sub(r'\\s+([,.:;?!\"()\\'])', r'\\1', text)\n",
    "        return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('!', 0)\n",
      "('\"', 1)\n",
      "('#2388]', 2)\n",
      "('$1', 3)\n",
      "('$5', 4)\n",
      "('&', 5)\n",
      "('&c', 6)\n",
      "(\"'\", 7)\n",
      "('(', 8)\n",
      "(')', 9)\n",
      "('***', 10)\n",
      "('*****', 11)\n",
      "(',', 12)\n",
      "('-', 13)\n",
      "('--', 14)\n",
      "('-all', 15)\n",
      "('-grandsires', 16)\n",
      "('-nay', 17)\n",
      "('.', 18)\n",
      "('//www', 19)\n",
      "('000', 20)\n",
      "('1', 21)\n",
      "('1500', 22)\n",
      "('1900', 23)\n",
      "('2', 24)\n",
      "('20%', 25)\n",
      "('2000', 26)\n",
      "('2001', 27)\n",
      "('2013', 28)\n",
      "('23', 29)\n",
      "('2388-h', 30)\n",
      "('26', 31)\n",
      "('3', 32)\n",
      "('30', 33)\n",
      "('4', 34)\n",
      "('4557', 35)\n",
      "('5', 36)\n",
      "('50', 37)\n",
      "('501', 38)\n",
      "('596-1887', 39)\n",
      "('6', 40)\n",
      "('60', 41)\n",
      "('64-6221541', 42)\n",
      "('67', 43)\n",
      "('7', 44)\n",
      "('8', 45)\n",
      "('801', 46)\n",
      "('809', 47)\n",
      "('84116', 48)\n",
      "('9', 49)\n",
      "('90', 50)\n",
      "('99712', 51)\n",
      "(':', 52)\n",
      "(';', 53)\n",
      "('?', 54)\n",
      "('A', 55)\n",
      "('ACTUAL', 56)\n",
      "('ADHIBHUTA', 57)\n",
      "('ADHIDAIVA', 58)\n",
      "('ADHIYAJNA', 59)\n",
      "('ADHYATMAN', 60)\n",
      "('AGREE', 61)\n",
      "('AGREEMENT', 62)\n",
      "('AK', 63)\n",
      "('AKSHARAM', 64)\n",
      "('AND', 65)\n",
      "('ANY', 66)\n",
      "('ANYTHING', 67)\n",
      "('ARJUNA', 68)\n",
      "('ARNOLD', 69)\n",
      "('AS-IS', 70)\n",
      "('ASAT', 71)\n",
      "('ASCII', 72)\n",
      "('ATTAINING', 73)\n",
      "('Abandoning', 74)\n",
      "('Abode', 75)\n",
      "('About', 76)\n",
      "('Above', 77)\n",
      "('Abstaining', 78)\n",
      "('Abstention', 79)\n",
      "('According', 80)\n",
      "('Aches', 81)\n",
      "('Aching', 82)\n",
      "('Act', 83)\n",
      "('Acting', 84)\n",
      "('Action', 85)\n",
      "('Additional', 86)\n",
      "('Adhyatman', 87)\n",
      "('Adityas', 88)\n",
      "('Adore', 89)\n",
      "('Adoring', 90)\n",
      "('After', 91)\n",
      "('Again', 92)\n",
      "('Against', 93)\n",
      "('Age', 94)\n",
      "('Agni', 95)\n",
      "('Ah', 96)\n",
      "('Ahinsa', 97)\n",
      "('Aho', 98)\n",
      "('Ahovat', 99)\n",
      "('Airavata', 100)\n",
      "('Aksharaparabrahmayog', 101)\n",
      "('Al', 102)\n",
      "('Albeit', 103)\n",
      "('Aliens', 104)\n",
      "('All', 105)\n",
      "('All-comprehending', 106)\n",
      "('All-creating', 107)\n",
      "('All-pervading', 108)\n",
      "('Almighty-head', 109)\n",
      "('Alms', 110)\n",
      "('Alone', 111)\n",
      "('Always', 112)\n",
      "('Am', 113)\n",
      "('Amid', 114)\n",
      "('Amrit', 115)\n",
      "('Amrit-wave', 116)\n",
      "('Amrit[FN#18]', 117)\n",
      "('An', 118)\n",
      "('Ananta', 119)\n",
      "('Ancestor', 120)\n",
      "('Ancient', 121)\n",
      "('And', 122)\n",
      "('And-rites', 123)\n",
      "('Anonymous', 124)\n",
      "('Another', 125)\n",
      "('Anushtubh', 126)\n",
      "('Any', 127)\n",
      "('Apostles', 128)\n",
      "('Archer-Prince', 129)\n",
      "('Archive', 130)\n",
      "('Are', 131)\n",
      "('Arise', 132)\n",
      "('Arjun', 133)\n",
      "('Arjun-Vishad', 134)\n",
      "('Arjuna', 135)\n",
      "('Arnold', 136)\n",
      "('Art', 137)\n",
      "('Aryam', 138)\n",
      "('As', 139)\n",
      "('Asat', 140)\n",
      "('Asita', 141)\n",
      "('Asuras', 142)\n",
      "('Aswattha', 143)\n",
      "('Aswatthaman', 144)\n",
      "('Aswins', 145)\n",
      "('At', 146)\n",
      "('Atman', 147)\n",
      "('Atmasanyamayog', 148)\n",
      "('Attain', 149)\n",
      "('Attaining', 150)\n",
      "('Attraction', 151)\n",
      "('Author', 152)\n",
      "('Avarice', 153)\n",
      "('Avenue', 154)\n",
      "('Awaits', 155)\n",
      "('Awful', 156)\n",
      "('B', 157)\n",
      "('BE', 158)\n",
      "('BEFORE', 159)\n",
      "('BHAGAVAD-GITA', 160)\n",
      "('BOOK', 161)\n",
      "('BRAHM', 162)\n",
      "('BRAHMA', 163)\n",
      "('BREACH', 164)\n",
      "('BUT', 165)\n",
      "('BY', 166)\n",
      "('Back', 167)\n",
      "('Banishing', 168)\n",
      "('Banyan-tree', 169)\n",
      "('Barring', 170)\n",
      "('Be', 171)\n",
      "('Bear', 172)\n",
      "('Bears', 173)\n",
      "('Because', 174)\n",
      "('Becoming', 175)\n",
      "('Beginneth', 176)\n",
      "('Beginning', 177)\n",
      "('Behold', 178)\n",
      "('Beholding', 179)\n",
      "('Being', 180)\n",
      "('Beings', 181)\n",
      "('Believing', 182)\n",
      "('Belongs', 183)\n",
      "('Benders', 184)\n",
      "('Bent', 185)\n",
      "('Best', 186)\n",
      "('Better', 187)\n",
      "('Between', 188)\n",
      "('Betwixt', 189)\n",
      "('Beyond', 190)\n",
      "('Bhagavad-Gita', 191)\n",
      "('Bhaktiyog', 192)\n",
      "('Bharata', 193)\n",
      "('Bharatas', 194)\n",
      "('Bhima', 195)\n",
      "('Bhima-blew', 196)\n",
      "('Bhishma', 197)\n",
      "('Bhrigu', 198)\n",
      "('Bhutas', 199)\n",
      "('Bhuts', 200)\n",
      "('Bind', 201)\n",
      "('Binds', 202)\n",
      "('Birth', 203)\n",
      "('Birthless', 204)\n",
      "('Bitter', 205)\n",
      "('Blessing', 206)\n",
      "('Blest', 207)\n",
      "('Blew', 208)\n",
      "('Blight', 209)\n",
      "('Blind', 210)\n",
      "('Blinding', 211)\n",
      "('Blowing', 212)\n",
      "('Blue', 213)\n",
      "('Body', 214)\n",
      "('Bombay', 215)\n",
      "('Book', 216)\n",
      "('Born', 217)\n",
      "('Both', 218)\n",
      "('Boundless', 219)\n",
      "('Brachmanis', 220)\n",
      "('Brahm', 221)\n",
      "('Brahma', 222)\n",
      "('Brahmacharya', 223)\n",
      "('Brahman', 224)\n",
      "('Brahmanic', 225)\n",
      "('Brahmans', 226)\n",
      "('Breaks', 227)\n",
      "('Breathing', 228)\n",
      "('Breaths', 229)\n",
      "('Bring', 230)\n",
      "('Bringer', 231)\n",
      "('Bringing', 232)\n",
      "('Brings', 233)\n",
      "('Bristles', 234)\n",
      "('Brothers', 235)\n",
      "('Burn', 236)\n",
      "('Burned', 237)\n",
      "('Burning', 238)\n",
      "('Burnouf', 239)\n",
      "('Burst', 240)\n",
      "('But', 241)\n",
      "('By', 242)\n",
      "('Byers', 243)\n",
      "('C', 244)\n",
      "('CHAPTER', 245)\n",
      "('CONSEQUENTIAL', 246)\n",
      "('CONTENTS', 247)\n",
      "('CONTRACT', 248)\n",
      "('Calcutta', 249)\n",
      "('Called', 250)\n",
      "('Calm', 251)\n",
      "('Can', 252)\n",
      "('Carved', 253)\n",
      "('Cast', 254)\n",
      "('Castes', 255)\n",
      "('Casts', 256)\n",
      "('Cause', 257)\n",
      "('Causing', 258)\n",
      "('Ceaselessly', 259)\n",
      "('Celestial', 260)\n",
      "('Centre', 261)\n",
      "('Certain', 262)\n",
      "('Chance-comers', 263)\n",
      "('Character', 264)\n",
      "('Charioteer', 265)\n",
      "('Chaser', 266)\n",
      "('Chekitan', 267)\n",
      "('Chief', 268)\n",
      "('Chiefs', 269)\n",
      "('Children', 270)\n",
      "('Chitrarath', 271)\n",
      "('Christ', 272)\n",
      "('Christian', 273)\n",
      "('City', 274)\n",
      "('Clasp', 275)\n",
      "('Clasped', 276)\n",
      "('Clear', 277)\n",
      "('Cleave', 278)\n",
      "('Cleaves', 279)\n",
      "('Cling', 280)\n",
      "('Clothed', 281)\n",
      "('Co', 282)\n",
      "('Comba', 283)\n",
      "('Come', 284)\n",
      "('Comes', 285)\n",
      "('Cometh', 286)\n",
      "('Compassionate', 287)\n",
      "('Compels', 288)\n",
      "('Compliance', 289)\n",
      "('Comprehend', 290)\n",
      "('Comprehending', 291)\n",
      "('Conceited', 292)\n",
      "('Conch', 293)\n",
      "('Confirmed', 294)\n",
      "('Conforms', 295)\n",
      "('Confused', 296)\n",
      "('Conjoined', 297)\n",
      "('Conqueror-Prince', 298)\n",
      "('Constancy', 299)\n",
      "('Constant', 300)\n",
      "('Constantly', 301)\n",
      "('Constrain', 302)\n",
      "('Consume', 303)\n",
      "('Consumed', 304)\n",
      "('Contact', 305)\n",
      "('Contemning', 306)\n",
      "('Contempt', 307)\n",
      "('Contending', 308)\n",
      "('Content', 309)\n",
      "('Contentment', 310)\n",
      "('Contrariwise', 311)\n",
      "('Contributions', 312)\n",
      "('Copyright', 313)\n",
      "('Could', 314)\n",
      "('Countless', 315)\n",
      "('Cousins', 316)\n",
      "('Cow', 317)\n",
      "('Craft', 318)\n",
      "('Created', 319)\n",
      "('Creating', 320)\n",
      "('Creation', 321)\n",
      "('Creator', 322)\n",
      "('Cried', 323)\n",
      "('Crowned', 324)\n",
      "('Cumberless', 325)\n",
      "('Cut', 326)\n",
      "('D', 327)\n",
      "('DAMAGE', 328)\n",
      "('DAMAGES', 329)\n",
      "('DELIVERANCE', 330)\n",
      "('DIRECT', 331)\n",
      "('DISCERNMENT', 332)\n",
      "('DISCLAIMER', 333)\n",
      "('DISTRESS', 334)\n",
      "('DISTRIBUTE', 335)\n",
      "('DISTRIBUTOR', 336)\n",
      "('DIVINE', 337)\n",
      "('DOCTRINES', 338)\n",
      "('DONATIONS', 339)\n",
      "('Daityas', 340)\n",
      "('Daivasarasaupadwibhagayog', 341)\n",
      "('Dance', 342)\n",
      "('Dark', 343)\n",
      "('Darkened', 344)\n",
      "('Darkness', 345)\n",
      "('Darknesses', 346)\n",
      "('Date', 347)\n",
      "('Dauntless', 348)\n",
      "('Davies', 349)\n",
      "('Davis', 350)\n",
      "('Dawn', 351)\n",
      "('Day', 352)\n",
      "('Days', 353)\n",
      "('Dear', 354)\n",
      "('Death', 355)\n",
      "('Deceitfulness', 356)\n",
      "('Deceived', 357)\n",
      "('Declared', 358)\n",
      "('Declines', 359)\n",
      "('Dedication', 360)\n",
      "('Deep-woven', 361)\n",
      "('Defect', 362)\n",
      "('Defects', 363)\n",
      "('Deity', 364)\n",
      "('Delight', 365)\n",
      "('Deliverance', 366)\n",
      "('Delusion', 367)\n",
      "('Depart', 368)\n",
      "('Depends', 369)\n",
      "('Deprived', 370)\n",
      "('Desirable', 371)\n",
      "('Desire', 372)\n",
      "('Desires', 373)\n",
      "('Desiring', 374)\n",
      "('Despair', 375)\n",
      "('Despite', 376)\n",
      "('Destroyer', 377)\n",
      "('Detached', 378)\n",
      "('Detaching', 379)\n",
      "('Detachment', 380)\n",
      "('Devalas', 381)\n",
      "('Devanagiri', 382)\n",
      "('Devarshis', 383)\n",
      "('Devavara', 384)\n",
      "('Devote', 385)\n",
      "('Devoted', 386)\n",
      "('Devotion', 387)\n",
      "('Devourest', 388)\n",
      "('Dhrishtadyumn', 389)\n",
      "('Dhrishtaket', 390)\n",
      "('Dhritarashtra', 391)\n",
      "('Dhritirashtra', 392)\n",
      "('Did', 393)\n",
      "('Die', 394)\n",
      "('Difficult', 395)\n",
      "('Director', 396)\n",
      "('Discerneth', 397)\n",
      "('Discernment', 398)\n",
      "('Discourse', 399)\n",
      "('Dislike', 400)\n",
      "('Dismayed', 401)\n",
      "('Disparting', 402)\n",
      "('Dispersing', 403)\n",
      "('Distress', 404)\n",
      "('Divided', 405)\n",
      "('Divine', 406)\n",
      "('Divinest', 407)\n",
      "('Do', 408)\n",
      "('Doctrines', 409)\n",
      "('Doing', 410)\n",
      "('Domain', 411)\n",
      "('Donations', 412)\n",
      "('Doors', 413)\n",
      "('Dost', 414)\n",
      "('Dotes', 415)\n",
      "('Doth', 416)\n",
      "('Double-Eight', 417)\n",
      "('Down', 418)\n",
      "('Dr', 419)\n",
      "('Draw', 420)\n",
      "('Drawing', 421)\n",
      "('Drawn', 422)\n",
      "('Dread', 423)\n",
      "('Drink', 424)\n",
      "('Drinking', 425)\n",
      "('Drive', 426)\n",
      "('Driver', 427)\n",
      "('Drona', 428)\n",
      "('Drona-O', 429)\n",
      "('Droops', 430)\n",
      "('Drove', 431)\n",
      "('Drupada', 432)\n",
      "('Drupadi', 433)\n",
      "('Due', 434)\n",
      "('Dull', 435)\n",
      "('Duryodhana', 436)\n",
      "('Dusk', 437)\n",
      "('Duty', 438)\n",
      "('Dwandwa[FN#20]', 439)\n",
      "('Dwelleth', 440)\n",
      "('Dwelling', 441)\n",
      "('Dwelling-place', 442)\n",
      "('E', 443)\n",
      "('EBOOK', 444)\n",
      "('EBook', 445)\n",
      "('EDWIN', 446)\n",
      "('EIN', 447)\n",
      "('END', 448)\n",
      "('ENDETH', 449)\n",
      "('ENDS', 450)\n",
      "('ETERNAL', 451)\n",
      "('EVEN', 452)\n",
      "('EXCEPT', 453)\n",
      "('EXPRESS', 454)\n",
      "('Each', 455)\n",
      "('Ear', 456)\n",
      "('Earnest', 457)\n",
      "('Earth', 458)\n",
      "('Earth-aches', 459)\n",
      "('Earths', 460)\n",
      "('Easy', 461)\n",
      "('Eating', 462)\n",
      "('Edwin', 463)\n",
      "('Either', 464)\n",
      "('Elders', 465)\n",
      "('Email', 466)\n",
      "('Embattled', 467)\n",
      "('Emblem', 468)\n",
      "('Enchain', 469)\n",
      "('Encountering', 470)\n",
      "('End', 471)\n",
      "('Endless', 472)\n",
      "('Energy', 473)\n",
      "('England', 474)\n",
      "('English', 475)\n",
      "('Enlightened', 476)\n",
      "('Enlightening', 477)\n",
      "('Ensnared', 478)\n",
      "('Enters', 479)\n",
      "('Entitled', 480)\n",
      "('Equal', 481)\n",
      "('Equally', 482)\n",
      "('Ergo', 483)\n",
      "('Escaping', 484)\n",
      "('Essence', 485)\n",
      "('Eternal', 486)\n",
      "('Evangelists', 487)\n",
      "('Even', 488)\n",
      "('Ever', 489)\n",
      "('Evil', 490)\n",
      "('Excellent', 491)\n",
      "('Except', 492)\n",
      "('Excepting', 493)\n",
      "('Executive', 494)\n",
      "('Exempt', 495)\n",
      "('Existing', 496)\n",
      "('Eye', 497)\n",
      "('Eyes', 498)\n",
      "('F', 499)\n",
      "('FAITH', 500)\n",
      "('FATHER', 501)\n",
      "('FIRST', 502)\n",
      "('FITNESS', 503)\n",
      "('FOR', 504)\n",
      "('FOUNDATION', 505)\n",
      "('FROM', 506)\n",
      "('FULL', 507)\n",
      "('Fades', 508)\n",
      "('Fails', 509)\n",
      "('Fain', 510)\n",
      "('Fair', 511)\n",
      "('Fairbanks', 512)\n",
      "('Faith', 513)\n",
      "('Faithful', 514)\n",
      "('Fall', 515)\n",
      "('Fame', 516)\n",
      "('Far', 517)\n",
      "('Father', 518)\n",
      "('Fearlessness', 519)\n",
      "('Fed', 520)\n",
      "('Feeds', 521)\n",
      "('Feeling', 522)\n",
      "('Feet', 523)\n",
      "('Felicity', 524)\n",
      "('Fierce', 525)\n",
      "('Fifth', 526)\n",
      "('Fight', 527)\n",
      "('Filling', 528)\n",
      "('Finally', 529)\n",
      "('Find', 530)\n",
      "('Findeth', 531)\n",
      "('Finding', 532)\n",
      "('Finds', 533)\n",
      "('First', 534)\n",
      "('Fix', 535)\n",
      "('Fixed', 536)\n",
      "('Flame', 537)\n",
      "('Flanks', 538)\n",
      "('Fleeting', 539)\n",
      "('Flesh', 540)\n",
      "('Flooding', 541)\n",
      "('Floods', 542)\n",
      "('Flutter', 543)\n",
      "('Fluttered', 544)\n",
      "('Fly', 545)\n",
      "('Foes', 546)\n",
      "('Foiled', 547)\n",
      "('Following', 548)\n",
      "('For', 549)\n",
      "('Forbid', 550)\n",
      "('Force', 551)\n",
      "('Forgoing', 552)\n",
      "('Form', 553)\n",
      "('Forth', 554)\n",
      "('Fortune', 555)\n",
      "('Fosterer', 556)\n",
      "('Foundation', 557)\n",
      "('Fount', 558)\n",
      "('Fountain', 559)\n",
      "('Four', 560)\n",
      "('Fourth', 561)\n",
      "('Framer', 562)\n",
      "('Free', 563)\n",
      "('Freed', 564)\n",
      "('French', 565)\n",
      "('Friend', 566)\n",
      "('From', 567)\n",
      "('Fruit', 568)\n",
      "('Fruits', 569)\n",
      "('Full', 570)\n",
      "('Funeral-Cake', 571)\n",
      "('Further', 572)\n",
      "('GIVE', 573)\n",
      "('GOD', 574)\n",
      "('GUTENBERG', 575)\n",
      "('GUTENBERG-tm', 576)\n",
      "('Gain', 577)\n",
      "('Galanos', 578)\n",
      "('Galilee', 579)\n",
      "('Gandharvas', 580)\n",
      "('Gandiv', 581)\n",
      "('Ganges', 582)\n",
      "('Garud', 583)\n",
      "('Gathers', 584)\n",
      "('Gatti', 585)\n",
      "('Gayatri', 586)\n",
      "('Gaze', 587)\n",
      "('Gem-bedecked', 588)\n",
      "('General', 589)\n",
      "('Gentle', 590)\n",
      "('Giant', 591)\n",
      "('Give', 592)\n",
      "('Gives', 593)\n",
      "('Glad', 594)\n",
      "('Glorified', 595)\n",
      "('Glorious', 596)\n",
      "('Glory', 597)\n",
      "('Glutting', 598)\n",
      "('Go', 599)\n",
      "('God', 600)\n",
      "('Gods', 601)\n",
      "('Gone', 602)\n",
      "('Good', 603)\n",
      "('Govern', 604)\n",
      "('Governed', 605)\n",
      "('Governing', 606)\n",
      "('Govinda', 607)\n",
      "('Grace', 608)\n",
      "('Grandsires', 609)\n",
      "('Great', 610)\n",
      "('Greater', 611)\n",
      "('Greek', 612)\n",
      "('Gregory', 613)\n",
      "('Ground', 614)\n",
      "('Grow', 615)\n",
      "('Grows', 616)\n",
      "('Guard', 617)\n",
      "('Guarding', 618)\n",
      "('Guide', 619)\n",
      "('Guilty', 620)\n",
      "('Gunatrayavibhagayog', 621)\n",
      "('Guru', 622)\n",
      "('Gurus', 623)\n",
      "('Gutenberg', 624)\n",
      "('Gutenberg-tm', 625)\n",
      "('HAVE', 626)\n",
      "('HE', 627)\n",
      "('HEAVENLY', 628)\n",
      "('HERE', 629)\n",
      "('HIM', 630)\n",
      "('HTML', 631)\n",
      "('Hail', 632)\n",
      "('Haines', 633)\n",
      "('Hanson', 634)\n",
      "('Hanta', 635)\n",
      "('Hanuman', 636)\n",
      "('Hard', 637)\n",
      "('Hari', 638)\n",
      "('Hart', 639)\n",
      "('Hateful', 640)\n",
      "('Hates', 641)\n",
      "('Hath', 642)\n",
      "('Have', 643)\n",
      "('Having', 644)\n",
      "('He', 645)\n",
      "('Head', 646)\n",
      "('Hear', 647)\n",
      "('Heareth', 648)\n",
      "('Hearing', 649)\n",
      "('Heart', 650)\n",
      "('Hearts', 651)\n",
      "('Heav', 652)\n",
      "('Heaven', 653)\n",
      "('Heavenly', 654)\n",
      "('Heavens', 655)\n",
      "('Heedless', 656)\n",
      "('Held', 657)\n",
      "('Hell', 658)\n",
      "('Hell-ward', 659)\n",
      "('Her', 660)\n",
      "('Here', 661)\n",
      "('Hero', 662)\n",
      "('Hidden', 663)\n",
      "('Hide', 664)\n",
      "('Hides', 665)\n",
      "('High', 666)\n",
      "('Higher', 667)\n",
      "('Highest', 668)\n",
      "('Him', 669)\n",
      "('Himala', 670)\n",
      "('Hindoo', 671)\n",
      "('His', 672)\n",
      "('Hold', 673)\n",
      "('Holding', 674)\n",
      "('Holds', 675)\n",
      "('Holiest', 676)\n",
      "('Holy', 677)\n",
      "('Honour', 678)\n",
      "('House', 679)\n",
      "('How', 680)\n",
      "('However', 681)\n",
      "('Huge', 682)\n",
      "('Humbleness', 683)\n",
      "('I', 684)\n",
      "('IF', 685)\n",
      "('II', 686)\n",
      "('III', 687)\n",
      "('IMPLIED', 688)\n",
      "('IN', 689)\n",
      "('INCIDENTAL', 690)\n",
      "('INCLUDING', 691)\n",
      "('INDEMNITY', 692)\n",
      "('INDIA', 693)\n",
      "('INDIRECT', 694)\n",
      "('IRS', 695)\n",
      "('ISO-8859-1', 696)\n",
      "('IV', 697)\n",
      "('IX', 698)\n",
      "('If', 699)\n",
      "('Ignorance', 700)\n",
      "('Ikshwaku', 701)\n",
      "('Immanent', 702)\n",
      "('Immortal', 703)\n",
      "('Immortality', 704)\n",
      "('Immortally', 705)\n",
      "('Impenetrable', 706)\n",
      "('Imperishable', 707)\n",
      "('Impulse', 708)\n",
      "('In', 709)\n",
      "('Increase', 710)\n",
      "('Indestructible', 711)\n",
      "('India', 712)\n",
      "('Indian', 713)\n",
      "('Indra', 714)\n",
      "('Infinite', 715)\n",
      "('Information', 716)\n",
      "('Inseparable', 717)\n",
      "('Instant', 718)\n",
      "('Intellect', 719)\n",
      "('Internal', 720)\n",
      "('International', 721)\n",
      "('Into', 722)\n",
      "('Invisible', 723)\n",
      "('Is', 724)\n",
      "('Issue', 725)\n",
      "('It', 726)\n",
      "('Italian', 727)\n",
      "('Its', 728)\n",
      "('J', 729)\n",
      "('Janak', 730)\n",
      "('Janardana', 731)\n",
      "('January', 732)\n",
      "('Jap', 733)\n",
      "('Jewels', 734)\n",
      "('Jheend', 735)\n",
      "('Jnana', 736)\n",
      "('Joys', 737)\n",
      "('Judge', 738)\n",
      "('Jumna', 739)\n",
      "('June', 740)\n",
      "('Just', 741)\n",
      "('Jyadratha', 742)\n",
      "('K', 743)\n",
      "('KARMA', 744)\n",
      "('KIND', 745)\n",
      "('KINGLY', 746)\n",
      "('KNOWLEDGE', 747)\n",
      "('Kalpa', 748)\n",
      "('Kalpas', 749)\n",
      "('Kama', 750)\n",
      "('Kamadhuk', 751)\n",
      "('Kamaduk', 752)\n",
      "('Kamalapatraksha', 753)\n",
      "('Kapila', 754)\n",
      "('Karma-Yog', 755)\n",
      "('Karmabandh', 756)\n",
      "('Karmasanyasayog', 757)\n",
      "('Karna', 758)\n",
      "('Kasi', 759)\n",
      "('Kasinath', 760)\n",
      "('Kauravas', 761)\n",
      "('Keeping', 762)\n",
      "('Kept', 763)\n",
      "('Kesava', 764)\n",
      "('Keshav', 765)\n",
      "('Kill', 766)\n",
      "('Killing', 767)\n",
      "('Kinds', 768)\n",
      "('King', 769)\n",
      "('Kingly', 770)\n",
      "('Kings', 771)\n",
      "('Kinsfolk', 772)\n",
      "('Know', 773)\n",
      "('Knoweth', 774)\n",
      "('Knowing', 775)\n",
      "('Knowledge', 776)\n",
      "('Kripa', 777)\n",
      "('Krishna', 778)\n",
      "('Kshatriya', 779)\n",
      "('Kshatriyas', 780)\n",
      "('Kshattriya', 781)\n",
      "('Kshetra', 782)\n",
      "('Kshetrajna', 783)\n",
      "('Kshetrakshetrajnavibhagayog', 784)\n",
      "('Kshetrakshetrajnayojnanan', 785)\n",
      "('Kunti', 786)\n",
      "('Kuntibhoj', 787)\n",
      "('Kurnul', 788)\n",
      "('Kuru', 789)\n",
      "('Kurukshetra', 790)\n",
      "('Kurus', 791)\n",
      "('Kusa-grass', 792)\n",
      "('LIABILITY', 793)\n",
      "('LIABLE', 794)\n",
      "('LICENSE', 795)\n",
      "('LIMITED', 796)\n",
      "('LORD', 797)\n",
      "('Labour', 798)\n",
      "('Lake', 799)\n",
      "('Lamp', 800)\n",
      "('Language', 801)\n",
      "('Lapping', 802)\n",
      "('Large', 803)\n",
      "('Lassen', 804)\n",
      "('Last', 805)\n",
      "('Latin', 806)\n",
      "('Law', 807)\n",
      "('Lay', 808)\n",
      "('Lays', 809)\n",
      "('Leadeth', 810)\n",
      "('Leans', 811)\n",
      "('Leap', 812)\n",
      "('Learn', 813)\n",
      "('Learning', 814)\n",
      "('Least', 815)\n",
      "('Leave', 816)\n",
      "('Leaveth', 817)\n",
      "('Left', 818)\n",
      "('Lest', 819)\n",
      "('Let', 820)\n",
      "('Lets', 821)\n",
      "('License', 822)\n",
      "('Lies', 823)\n",
      "('Life', 824)\n",
      "('Life-Soul', 825)\n",
      "('Light', 826)\n",
      "('Lighting', 827)\n",
      "('Lightly', 828)\n",
      "('Lights', 829)\n",
      "('Like', 830)\n",
      "('Line', 831)\n",
      "('Linked', 832)\n",
      "('Listen', 833)\n",
      "('Literary', 834)\n",
      "('Live', 835)\n",
      "('Lives', 836)\n",
      "('Living', 837)\n",
      "('Lo', 838)\n",
      "('Long-Armed', 839)\n",
      "('Long-armed', 840)\n",
      "('Look', 841)\n",
      "('Looks', 842)\n",
      "('Lord', 843)\n",
      "('Lordly', 844)\n",
      "('Lords', 845)\n",
      "('Lost', 846)\n",
      "('Lotus-eyed', 847)\n",
      "('Love', 848)\n",
      "('Lover', 849)\n",
      "('Loving', 850)\n",
      "('Low-minded', 851)\n",
      "('Lowest', 852)\n",
      "('Ltd', 853)\n",
      "('Lust', 854)\n",
      "('Lustre', 855)\n",
      "('M', 856)\n",
      "('MANIFESTING', 857)\n",
      "('MANIFOLD', 858)\n",
      "('MATTER', 859)\n",
      "('MERCHANTABILITY', 860)\n",
      "('MYSTERY', 861)\n",
      "('Mad', 862)\n",
      "('Made', 863)\n",
      "('Madhava', 864)\n",
      "('Madhu', 865)\n",
      "('Madhusudan', 866)\n",
      "('Magistrorum', 867)\n",
      "('Mahabharata', 868)\n",
      "('Mahatma', 869)\n",
      "('Mahatmas', 870)\n",
      "('Majesties', 871)\n",
      "('Majesty', 872)\n",
      "('Makar', 873)\n",
      "('Make', 874)\n",
      "('Maker', 875)\n",
      "('Maketh', 876)\n",
      "('Man', 877)\n",
      "('Mandara', 878)\n",
      "('Mangled', 879)\n",
      "('Manifest', 880)\n",
      "('Manifested', 881)\n",
      "('Manifesting', 882)\n",
      "('Manifold', 883)\n",
      "('Mankind', 884)\n",
      "('Mantra', 885)\n",
      "('Manu', 886)\n",
      "('Manus', 887)\n",
      "('Many', 888)\n",
      "('Margasirsha', 889)\n",
      "('Maritchi', 890)\n",
      "('Mark', 891)\n",
      "('Marked', 892)\n",
      "('Marred', 893)\n",
      "('Married', 894)\n",
      "('Mars', 895)\n",
      "('Maruts', 896)\n",
      "('Marvellous', 897)\n",
      "('Master', 898)\n",
      "('Masters', 899)\n",
      "('Matter', 900)\n",
      "('Maya', 901)\n",
      "('Mayst', 902)\n",
      "('Me', 903)\n",
      "('Me-when', 904)\n",
      "('Means', 905)\n",
      "('Measured', 906)\n",
      "('Measureless', 907)\n",
      "('Meditate', 908)\n",
      "('Melan', 909)\n",
      "('Melted', 910)\n",
      "('Memory', 911)\n",
      "('Men', 912)\n",
      "('Meriting', 913)\n",
      "('Meru', 914)\n",
      "('Messrs', 915)\n",
      "('Michael', 916)\n",
      "('Might', 917)\n",
      "('Mightiest', 918)\n",
      "('Mighty', 919)\n",
      "('Mind', 920)\n",
      "('Mine', 921)\n",
      "('Mingles', 922)\n",
      "('Misled', 923)\n",
      "('Mission', 924)\n",
      "('Missionaries', 925)\n",
      "('Mississippi', 926)\n",
      "('Mix', 927)\n",
      "('Modes', 928)\n",
      "('Modest', 929)\n",
      "('Mokshasanyasayog', 930)\n",
      "('Moon', 931)\n",
      "('More', 932)\n",
      "('Moreover', 933)\n",
      "('Most', 934)\n",
      "('Mother', 935)\n",
      "('Mothers', 936)\n",
      "('Motionless', 937)\n",
      "('Mounts', 938)\n",
      "('Mourn', 939)\n",
      "('Move', 940)\n",
      "('Mr', 941)\n",
      "('Much', 942)\n",
      "('Muni', 943)\n",
      "('Munis', 944)\n",
      "('Musing', 945)\n",
      "('Must', 946)\n",
      "('My', 947)\n",
      "('Myself', 948)\n",
      "('Mystery', 949)\n",
      "('Mystic', 950)\n",
      "('Mystical', 951)\n",
      "('NEGLIGENCE', 952)\n",
      "('NO', 953)\n",
      "('NOT', 954)\n",
      "('NOTICE', 955)\n",
      "('Nakula', 956)\n",
      "('Namaste', 957)\n",
      "('Name', 958)\n",
      "('Named', 959)\n",
      "('Nameless', 960)\n",
      "('Namo', 961)\n",
      "('Namostu', 962)\n",
      "('Narada', 963)\n",
      "('Narak', 964)\n",
      "('Naraka', 965)\n",
      "('Nature', 966)\n",
      "('Nature-forms', 967)\n",
      "('Natures', 968)\n",
      "('Naught', 969)\n",
      "('Nay', 970)\n",
      "('Near', 971)\n",
      "('Nearly', 972)\n",
      "('Needs', 973)\n",
      "('Neither', 974)\n",
      "('Nether', 975)\n",
      "('Never', 976)\n",
      "('Never-Ending', 977)\n",
      "('New', 978)\n",
      "('Newby', 979)\n",
      "('Nigh', 980)\n",
      "('Night', 981)\n",
      "('Nightfall', 982)\n",
      "('Nirvana', 983)\n",
      "('No', 984)\n",
      "('None', 985)\n",
      "('Noon', 986)\n",
      "('Nor', 987)\n",
      "('North', 988)\n",
      "('Not', 989)\n",
      "('Nothing', 990)\n",
      "('Nought', 991)\n",
      "('November', 992)\n",
      "('Now', 993)\n",
      "('Nowhere', 994)\n",
      "('Numberless', 995)\n",
      "('Numinisque', 996)\n",
      "('O', 997)\n",
      "('OF', 998)\n",
      "('OM', 999)\n",
      "('ONE', 1000)\n",
      "('OR', 1001)\n",
      "('OTHER', 1002)\n",
      "('OWNER', 1003)\n",
      "('Obeying', 1004)\n",
      "('Obtained', 1005)\n",
      "('Of', 1006)\n",
      "('Offered', 1007)\n",
      "('Offerings', 1008)\n",
      "('Oh', 1009)\n",
      "('Older', 1010)\n",
      "('On', 1011)\n",
      "('Once', 1012)\n",
      "('One', 1013)\n",
      "('Only', 1014)\n",
      "('Opened', 1015)\n",
      "('Or', 1016)\n",
      "('Order', 1017)\n",
      "('Other', 1018)\n",
      "('Our', 1019)\n",
      "('Out', 1020)\n",
      "('Outcast', 1021)\n",
      "('Outside', 1022)\n",
      "('Overpass', 1023)\n",
      "('Owning', 1024)\n",
      "('PARAGRAPH', 1025)\n",
      "('PERFECTIONS', 1026)\n",
      "('PG', 1027)\n",
      "('PGLAF', 1028)\n",
      "('PLEASE', 1029)\n",
      "('POSSIBILITY', 1030)\n",
      "('PREFACE', 1031)\n",
      "('PROJECT', 1032)\n",
      "('PROVIDED', 1033)\n",
      "('PUNITIVE', 1034)\n",
      "('PURPOSE', 1035)\n",
      "('PURUSHA', 1036)\n",
      "('PURUSHOTTAMA', 1037)\n",
      "('Painfully', 1038)\n",
      "('Pandavas', 1039)\n",
      "('Pandits', 1040)\n",
      "('Pandu', 1041)\n",
      "('Pantheon', 1042)\n",
      "('Para-Brahm', 1043)\n",
      "('Parable', 1044)\n",
      "('Parabrahm', 1045)\n",
      "('Paradise', 1046)\n",
      "('Part', 1047)\n",
      "('Parva', 1048)\n",
      "('Passage', 1049)\n",
      "('Passes', 1050)\n",
      "('Passing', 1051)\n",
      "('Passion', 1052)\n",
      "('Past', 1053)\n",
      "('Patanjali', 1054)\n",
      "('Patience', 1055)\n",
      "('Pavaka', 1056)\n",
      "('Peace', 1057)\n",
      "('Penance', 1058)\n",
      "('Perceiveth', 1059)\n",
      "('Perception', 1060)\n",
      "('Perfect', 1061)\n",
      "('Perfections', 1062)\n",
      "('Perishing', 1063)\n",
      "('Pervading', 1064)\n",
      "('Pierce', 1065)\n",
      "('Pitris', 1066)\n",
      "('Place', 1067)\n",
      "('Plain', 1068)\n",
      "('Plant', 1069)\n",
      "('Planted', 1070)\n",
      "('Plays', 1071)\n",
      "('Please', 1072)\n",
      "('Pleasure', 1073)\n",
      "('Plenty', 1074)\n",
      "('Pondering', 1075)\n",
      "('Ponders', 1076)\n",
      "('Possessing', 1077)\n",
      "('Posted', 1078)\n",
      "('Posting', 1079)\n",
      "('Pour', 1080)\n",
      "('Powers', 1081)\n",
      "('Prahlada', 1082)\n",
      "('Praise', 1083)\n",
      "('Prajapati', 1084)\n",
      "('Prakriti', 1085)\n",
      "('Pralyas', 1086)\n",
      "('Prasid', 1087)\n",
      "('Prayer', 1088)\n",
      "('Praying', 1089)\n",
      "('Precious', 1090)\n",
      "('Prepare', 1091)\n",
      "('Prescribed[FN#4]', 1092)\n",
      "('Presence', 1093)\n",
      "('Present', 1094)\n",
      "('Pretas', 1095)\n",
      "('Prince', 1096)\n",
      "('Pritha', 1097)\n",
      "('Produced', 1098)\n",
      "('Professor', 1099)\n",
      "('Profit', 1100)\n",
      "('Project', 1101)\n",
      "('Public', 1102)\n",
      "('Pure-hearted', 1103)\n",
      "('Pureness', 1104)\n",
      "('Purge', 1105)\n",
      "('Purged', 1106)\n",
      "('Purification', 1107)\n",
      "('Purity', 1108)\n",
      "('Purujit', 1109)\n",
      "('Purusha', 1110)\n",
      "('Purushottamapraptiyog', 1111)\n",
      "('Pushed', 1112)\n",
      "('Pushes', 1113)\n",
      "('Put', 1114)\n",
      "('Puts', 1115)\n",
      "('QUALITIES', 1116)\n",
      "('Quake', 1117)\n",
      "('Qualities', 1118)\n",
      "('Quickness', 1119)\n",
      "('Quit', 1120)\n",
      "('READ', 1121)\n",
      "('REFUND', 1122)\n",
      "('RELIGION', 1123)\n",
      "('REMEDIES', 1124)\n",
      "('RENOUNCING', 1125)\n",
      "('RENUNCIATION', 1126)\n",
      "('REPLACEMENT', 1127)\n",
      "('RIGHT', 1128)\n",
      "('Raja', 1129)\n",
      "('Rajas', 1130)\n",
      "('Rajas-rite', 1131)\n",
      "('Rajavidyarajaguhyayog', 1132)\n",
      "('Rakshasas', 1133)\n",
      "('Rakshasas[FN#34]', 1134)\n",
      "('Rama', 1135)\n",
      "('Ranged', 1136)\n",
      "('Receive', 1137)\n",
      "('Receiver', 1138)\n",
      "('Recklessness', 1139)\n",
      "('Recluse', 1140)\n",
      "('Rectitude', 1141)\n",
      "('Redistributing', 1142)\n",
      "('Redistribution', 1143)\n",
      "('Refuge', 1144)\n",
      "('Refuge-House', 1145)\n",
      "('Refund', 1146)\n",
      "('Regard', 1147)\n",
      "('Region', 1148)\n",
      "('Reign', 1149)\n",
      "('Rejecting', 1150)\n",
      "('Rejects', 1151)\n",
      "('Release', 1152)\n",
      "('Reliant', 1153)\n",
      "('Religion', 1154)\n",
      "('Relying', 1155)\n",
      "('Renounce', 1156)\n",
      "('Renouncer', 1157)\n",
      "('Renouncing', 1158)\n",
      "('Renunciation', 1159)\n",
      "('Replacement', 1160)\n",
      "('Reposeful', 1161)\n",
      "('Resist', 1162)\n",
      "('Rest', 1163)\n",
      "('Restraining', 1164)\n",
      "('Rests', 1165)\n",
      "('Retake', 1166)\n",
      "('Revenue', 1167)\n",
      "('Rich', 1168)\n",
      "('Richer', 1169)\n",
      "('Rig-Veda', 1170)\n",
      "('Right', 1171)\n",
      "('Righteous', 1172)\n",
      "('Righteousness', 1173)\n",
      "('Rightfully', 1174)\n",
      "('Rise', 1175)\n",
      "('Riseth', 1176)\n",
      "('Rishis', 1177)\n",
      "('Rites', 1178)\n",
      "('Road', 1179)\n",
      "('Robed', 1180)\n",
      "('Roll', 1181)\n",
      "('Root', 1182)\n",
      "('Royal', 1183)\n",
      "('Royalty', 1184)\n",
      "('Rudras', 1185)\n",
      "('S', 1186)\n",
      "('SAT', 1187)\n",
      "('SELF-RESTRAINT', 1188)\n",
      "('SEND', 1189)\n",
      "('SEPARATENESS', 1190)\n",
      "('SEPARATION', 1191)\n",
      "('SERVICE', 1192)\n",
      "('SPIRIT', 1193)\n",
      "('START', 1194)\n",
      "('STRICT', 1195)\n",
      "('SUCH', 1196)\n",
      "('SUPREME', 1197)\n",
      "('Sabdabrahm', 1198)\n",
      "('Sacred', 1199)\n",
      "('Sacrifice', 1200)\n",
      "('Sad', 1201)\n",
      "('Sadhyas', 1202)\n",
      "('Safe', 1203)\n",
      "('Sage', 1204)\n",
      "('Sahadev', 1205)\n",
      "('Saint', 1206)\n",
      "('Saints', 1207)\n",
      "('Saivya', 1208)\n",
      "('Salt', 1209)\n",
      "('Sama-Ved', 1210)\n",
      "('Sama-Veda', 1211)\n",
      "('Sanjaya', 1212)\n",
      "('Sankara', 1213)\n",
      "('Sankhya', 1214)\n",
      "('Sankhya-Yog', 1215)\n",
      "('Sankhyans', 1216)\n",
      "('Sannyas', 1217)\n",
      "('Sanskrit', 1218)\n",
      "('Sanyasi', 1219)\n",
      "('Sarsooti', 1220)\n",
      "('Sat', 1221)\n",
      "('Sattwan', 1222)\n",
      "('Satyaki', 1223)\n",
      "('Saumadatti', 1224)\n",
      "('Savourless', 1225)\n",
      "('Say', 1226)\n",
      "('Sayeth', 1227)\n",
      "('Saying', 1228)\n",
      "('Schlegel', 1229)\n",
      "('Scorn', 1230)\n",
      "('Scourge', 1231)\n",
      "('Scripture', 1232)\n",
      "('Sea', 1233)\n",
      "('Section', 1234)\n",
      "('Sections', 1235)\n",
      "('See', 1236)\n",
      "('Seed', 1237)\n",
      "('Seed-Sower', 1238)\n",
      "('Seeing', 1239)\n",
      "('Seek', 1240)\n",
      "('Seeking', 1241)\n",
      "('Seen', 1242)\n",
      "('Sees', 1243)\n",
      "('Seeth', 1244)\n",
      "('Self', 1245)\n",
      "('Self-Restraint', 1246)\n",
      "('Self-concentrated', 1247)\n",
      "('Self-mastery', 1248)\n",
      "('Self-ruled', 1249)\n",
      "('Self-schooled', 1250)\n",
      "('Sending', 1251)\n",
      "('Sense', 1252)\n",
      "('Senses', 1253)\n",
      "('Separate', 1254)\n",
      "('Separateness', 1255)\n",
      "('Separation', 1256)\n",
      "('Sequestered', 1257)\n",
      "('Serenity', 1258)\n",
      "('Serve', 1259)\n",
      "('Service', 1260)\n",
      "('Set', 1261)\n",
      "('Setting', 1262)\n",
      "('Seven', 1263)\n",
      "('Shaking', 1264)\n",
      "('Shall', 1265)\n",
      "('Shalt', 1266)\n",
      "('Shameful', 1267)\n",
      "('Sharing', 1268)\n",
      "('Shastras', 1269)\n",
      "('She', 1270)\n",
      "('Shield', 1271)\n",
      "('Shifts', 1272)\n",
      "('Shineth', 1273)\n",
      "('Shining', 1274)\n",
      "('Should', 1275)\n",
      "('Show', 1276)\n",
      "('Shown', 1277)\n",
      "('Shows', 1278)\n",
      "('Shut', 1279)\n",
      "('Shutting', 1280)\n",
      "('Siddhas', 1281)\n",
      "('Sikhandi', 1282)\n",
      "('Silver', 1283)\n",
      "('Sin', 1284)\n",
      "('Since', 1285)\n",
      "('Sinful', 1286)\n",
      "('Sinless', 1287)\n",
      "('Sir', 1288)\n",
      "('Sits', 1289)\n",
      "('Sitting', 1290)\n",
      "('Skanda', 1291)\n",
      "('Slaves', 1292)\n",
      "('Slayer', 1293)\n",
      "('Smelling', 1294)\n",
      "('So', 1295)\n",
      "('Soma-wine', 1296)\n",
      "('Some', 1297)\n",
      "('Son', 1298)\n",
      "('Song', 1299)\n",
      "('Sooth', 1300)\n",
      "('Soothfast', 1301)\n",
      "('Soothfastness', 1302)\n",
      "('Sorrow', 1303)\n",
      "('Sorrows', 1304)\n",
      "('Soul', 1305)\n",
      "('Souls', 1306)\n",
      "('Sound', 1307)\n",
      "('Sounding', 1308)\n",
      "('Source', 1309)\n",
      "('Sovereign', 1310)\n",
      "('Space', 1311)\n",
      "('Spake', 1312)\n",
      "('Speaking', 1313)\n",
      "('Special', 1314)\n",
      "('Specious', 1315)\n",
      "('Speech', 1316)\n",
      "('Spirit', 1317)\n",
      "('Splendid', 1318)\n",
      "('Spring', 1319)\n",
      "('Springing', 1320)\n",
      "('Sraddhatrayavibhagayog', 1321)\n",
      "('Stained', 1322)\n",
      "('Stainless', 1323)\n",
      "('Standeth', 1324)\n",
      "('Stanislav', 1325)\n",
      "('Star-sprinkled', 1326)\n",
      "('States', 1327)\n",
      "('Stay', 1328)\n",
      "('Steadfast', 1329)\n",
      "('Steadfastly', 1330)\n",
      "('Steadfastness', 1331)\n",
      "('Still', 1332)\n",
      "('Stood', 1333)\n",
      "('Storm', 1334)\n",
      "('Straight', 1335)\n",
      "('Straightly', 1336)\n",
      "('Straightway', 1337)\n",
      "('Straitly', 1338)\n",
      "('Strange', 1339)\n",
      "('Strike', 1340)\n",
      "('Striveth', 1341)\n",
      "('Strong', 1342)\n",
      "('Strongest', 1343)\n",
      "('Stubborn', 1344)\n",
      "('Subduing', 1345)\n",
      "('Subhadra', 1346)\n",
      "('Substance', 1347)\n",
      "('Succouring', 1348)\n",
      "('Such', 1349)\n",
      "('Suddenly', 1350)\n",
      "('Sudra', 1351)\n",
      "('Sudras', 1352)\n",
      "('Suiting', 1353)\n",
      "('Sun', 1354)\n",
      "('Sunburst', 1355)\n",
      "('Suppressing', 1356)\n",
      "('Supreme', 1357)\n",
      "('Supremest', 1358)\n",
      "('Surcease', 1359)\n",
      "('Sure', 1360)\n",
      "('Surely', 1361)\n",
      "('Surmounter', 1362)\n",
      "('Surrendered', 1363)\n",
      "('Sustainer', 1364)\n",
      "('Sustaining', 1365)\n",
      "('Swarga', 1366)\n",
      "('Sway', 1367)\n",
      "('Sweet', 1368)\n",
      "('Sweet-sounding', 1369)\n",
      "('Syrian', 1370)\n",
      "('THAT', 1371)\n",
      "('THE', 1372)\n",
      "('THIS', 1373)\n",
      "('THOSE', 1374)\n",
      "('THREEFOLD', 1375)\n",
      "('TO', 1376)\n",
      "('TRADEMARK', 1377)\n",
      "('Take', 1378)\n",
      "('Takes', 1379)\n",
      "('Taketh', 1380)\n",
      "('Taking', 1381)\n",
      "('Tamas', 1382)\n",
      "('Taste', 1383)\n",
      "('Taught', 1384)\n",
      "('Te', 1385)\n",
      "('Teacher', 1386)\n",
      "('Teachers', 1387)\n",
      "('Teaching', 1388)\n",
      "('Technical', 1389)\n",
      "('Telang', 1390)\n",
      "('Tell', 1391)\n",
      "('Tend', 1392)\n",
      "('Terms', 1393)\n",
      "('Testament', 1394)\n",
      "('Text', 1395)\n",
      "('Th', 1396)\n",
      "('Than', 1397)\n",
      "('That', 1398)\n",
      "('The', 1399)\n",
      "('Thee', 1400)\n",
      "('Their', 1401)\n",
      "('Them', 1402)\n",
      "('Then', 1403)\n",
      "('Thence', 1404)\n",
      "('There', 1405)\n",
      "('Therefore', 1406)\n",
      "('Therein', 1407)\n",
      "('These', 1408)\n",
      "('They', 1409)\n",
      "('Thine', 1410)\n",
      "('Things', 1411)\n",
      "('Thinking', 1412)\n",
      "('This', 1413)\n",
      "('Thither', 1414)\n",
      "('Thomson', 1415)\n",
      "('Thorny', 1416)\n",
      "('Those', 1417)\n",
      "('Thou', 1418)\n",
      "('Though', 1419)\n",
      "('Three', 1420)\n",
      "('Threefold', 1421)\n",
      "('Thrilled', 1422)\n",
      "('Throughout', 1423)\n",
      "('Thus', 1424)\n",
      "('Thy', 1425)\n",
      "('Thyself', 1426)\n",
      "('Till', 1427)\n",
      "('Time', 1428)\n",
      "('Times', 1429)\n",
      "('Tis', 1430)\n",
      "('Tithes', 1431)\n",
      "('Title', 1432)\n",
      "('To', 1433)\n",
      "('To-day', 1434)\n",
      "('Torturing', 1435)\n",
      "('Tossed', 1436)\n",
      "('Touch', 1437)\n",
      "('Toucheth', 1438)\n",
      "('Towards', 1439)\n",
      "('Tranquil', 1440)\n",
      "('Translated', 1441)\n",
      "('Translator', 1442)\n",
      "('Tread', 1443)\n",
      "('Treadeth', 1444)\n",
      "('Treasure', 1445)\n",
      "('Treasure-Chamber', 1446)\n",
      "('Treasure-Claimer', 1447)\n",
      "('Treasure-Palace', 1448)\n",
      "('Tree', 1449)\n",
      "('Tremblingly', 1450)\n",
      "('Triumph', 1451)\n",
      "('Trouble', 1452)\n",
      "('Troubled', 1453)\n",
      "('Trubner', 1454)\n",
      "('True', 1455)\n",
      "('Truest', 1456)\n",
      "('Trumpets', 1457)\n",
      "('Truslove', 1458)\n",
      "('Trust', 1459)\n",
      "('Truth', 1460)\n",
      "('Truthfulness', 1461)\n",
      "('Truths', 1462)\n",
      "('Turning', 1463)\n",
      "('Tushes', 1464)\n",
      "('Twice-borns', 1465)\n",
      "('Twins', 1466)\n",
      "('Two', 1467)\n",
      "('Tyaga', 1468)\n",
      "('U', 1469)\n",
      "('UNDER', 1470)\n",
      "('UNDIVINE', 1471)\n",
      "('USE', 1472)\n",
      "('UT', 1473)\n",
      "('Uchchaisravas', 1474)\n",
      "('Ultimate', 1475)\n",
      "('Unblending', 1476)\n",
      "('Unbodied', 1477)\n",
      "('Unborn', 1478)\n",
      "('Unchained', 1479)\n",
      "('Unchanging', 1480)\n",
      "('Uncreate', 1481)\n",
      "('Uncreated', 1482)\n",
      "('Under', 1483)\n",
      "('Undesirable', 1484)\n",
      "('Undismayed', 1485)\n",
      "('Undivided', 1486)\n",
      "('Undivine', 1487)\n",
      "('Undying', 1488)\n",
      "('Unending', 1489)\n",
      "('Unendingly', 1490)\n",
      "('Unentered', 1491)\n",
      "('Unflattered', 1492)\n",
      "('Unformed', 1493)\n",
      "('Unheavenly', 1494)\n",
      "('United', 1495)\n",
      "('Unity', 1496)\n",
      "('Universe', 1497)\n",
      "('Universe-', 1498)\n",
      "('Unkindled', 1499)\n",
      "('Unless', 1500)\n",
      "('Unlike', 1501)\n",
      "('Unmanifest', 1502)\n",
      "('Unmanifested', 1503)\n",
      "('Unmoved', 1504)\n",
      "('Unnamed', 1505)\n",
      "('Unnumbered', 1506)\n",
      "('Unqualified', 1507)\n",
      "('Unrevealed', 1508)\n",
      "('Unruffled', 1509)\n",
      "('Unsubdued', 1510)\n",
      "('Unthinkable', 1511)\n",
      "('Unto', 1512)\n",
      "('Unvexed', 1513)\n",
      "('Unwittingly', 1514)\n",
      "('Updated', 1515)\n",
      "('Upon', 1516)\n",
      "('Uprightness', 1517)\n",
      "('Us', 1518)\n",
      "('Usana', 1519)\n",
      "('Use', 1520)\n",
      "('Ushmapas', 1521)\n",
      "('Utmost', 1522)\n",
      "('Uttamauj', 1523)\n",
      "('Uttered', 1524)\n",
      "('Uttermost', 1525)\n",
      "('V', 1526)\n",
      "('VERY', 1527)\n",
      "('VI', 1528)\n",
      "('VII', 1529)\n",
      "('VIII', 1530)\n",
      "('VIRTUE', 1531)\n",
      "('Vain', 1532)\n",
      "('Vaisya', 1533)\n",
      "('Vaisyas', 1534)\n",
      "('Valiant', 1535)\n",
      "('Vanilla', 1536)\n",
      "('Varuna', 1537)\n",
      "('Vasava', 1538)\n",
      "('Vasudev', 1539)\n",
      "('Vasuki', 1540)\n",
      "('Vasus', 1541)\n",
      "('Vates', 1542)\n",
      "('Vayu[FN#24]', 1543)\n",
      "('Ved', 1544)\n",
      "('Vedantist', 1545)\n",
      "('Vedas', 1546)\n",
      "('Vedic', 1547)\n",
      "('Veds', 1548)\n",
      "('Vibhuti', 1549)\n",
      "('Victims', 1550)\n",
      "('Victor', 1551)\n",
      "('Victory', 1552)\n",
      "('Vijnanayog', 1553)\n",
      "('Vikarna', 1554)\n",
      "('Virata', 1555)\n",
      "('Virtue', 1556)\n",
      "('Vishnu', 1557)\n",
      "('Visible', 1558)\n",
      "('Viswarupadarsanam', 1559)\n",
      "('Viswas', 1560)\n",
      "('Vittesh', 1561)\n",
      "('Vivaswata', 1562)\n",
      "('Voice', 1563)\n",
      "('Volunteers', 1564)\n",
      "('Vowed', 1565)\n",
      "('Vrihaspati', 1566)\n",
      "('Vrihatsam', 1567)\n",
      "('Vrishni', 1568)\n",
      "('Vyasa', 1569)\n",
      "('WARRANTIES', 1570)\n",
      "('WARRANTY', 1571)\n",
      "('WILL', 1572)\n",
      "('WITH', 1573)\n",
      "('WORK', 1574)\n",
      "('WORKS', 1575)\n",
      "('Wake', 1576)\n",
      "('Waking', 1577)\n",
      "('Warriors', 1578)\n",
      "('Was', 1579)\n",
      "('Way', 1580)\n",
      "('We', 1581)\n",
      "('Weakest-meseems-', 1582)\n",
      "('Wealth', 1583)\n",
      "('Weapons', 1584)\n",
      "('Web', 1585)\n",
      "('Well-pleased', 1586)\n",
      "('West', 1587)\n",
      "('Western', 1588)\n",
      "('What', 1589)\n",
      "('Whate', 1590)\n",
      "('Whatever', 1591)\n",
      "('When', 1592)\n",
      "('Whence', 1593)\n",
      "('Where', 1594)\n",
      "('Whereby', 1595)\n",
      "('Wherefore', 1596)\n",
      "('Wherefrom', 1597)\n",
      "('Wherein', 1598)\n",
      "('Whereof', 1599)\n",
      "('Whereto', 1600)\n",
      "('Wherever', 1601)\n",
      "('Which', 1602)\n",
      "('While', 1603)\n",
      "('Who', 1604)\n",
      "('Whoever', 1605)\n",
      "('Whole', 1606)\n",
      "('Wholly', 1607)\n",
      "('Whom', 1608)\n",
      "('Whose', 1609)\n",
      "('Whoso', 1610)\n",
      "('Why', 1611)\n",
      "('Wickedness', 1612)\n",
      "('Wilful', 1613)\n",
      "('Will', 1614)\n",
      "('Winded', 1615)\n",
      "('Wins', 1616)\n",
      "('Wisdom', 1617)\n",
      "('With', 1618)\n",
      "('Without', 1619)\n",
      "('Witness', 1620)\n",
      "('Wolf-bellied', 1621)\n",
      "('Woman', 1622)\n",
      "('Wombs', 1623)\n",
      "('Wonderful', 1624)\n",
      "('Wonders', 1625)\n",
      "('Words', 1626)\n",
      "('Work', 1627)\n",
      "('Worked', 1628)\n",
      "('Working', 1629)\n",
      "('Works', 1630)\n",
      "('World', 1631)\n",
      "('Worlds', 1632)\n",
      "('Worship', 1633)\n",
      "('Worshipping', 1634)\n",
      "('Worthily', 1635)\n",
      "('Wotteth', 1636)\n",
      "('Would', 1637)\n",
      "('Wrath', 1638)\n",
      "('Writ', 1639)\n",
      "('Wrought', 1640)\n",
      "('X', 1641)\n",
      "('XI', 1642)\n",
      "('XII', 1643)\n",
      "('XIII', 1644)\n",
      "('XIV', 1645)\n",
      "('XV', 1646)\n",
      "('XVI', 1647)\n",
      "('XVII', 1648)\n",
      "('XVIII', 1649)\n",
      "('YOU', 1650)\n",
      "('Yajur-Ved', 1651)\n",
      "('Yakshas', 1652)\n",
      "('Yama', 1653)\n",
      "('Yatayaman', 1654)\n",
      "('Yea', 1655)\n",
      "('Yes', 1656)\n",
      "('Yet', 1657)\n",
      "('Yields', 1658)\n",
      "('Yog', 1659)\n",
      "('Yoga', 1660)\n",
      "('Yogayukt', 1661)\n",
      "('Yogi', 1662)\n",
      "('Yogin', 1663)\n",
      "('Yogins', 1664)\n",
      "('Yojin', 1665)\n",
      "('Yoked', 1666)\n",
      "('Yoni', 1667)\n",
      "('York', 1668)\n",
      "('You', 1669)\n",
      "('Your', 1670)\n",
      "('Yudhamanyu', 1671)\n",
      "('Yudhisthira', 1672)\n",
      "('Yugas', 1673)\n",
      "('Yukta', 1674)\n",
      "('Yuyudhan', 1675)\n",
      "('[And', 1676)\n",
      "('[EBook', 1677)\n",
      "('[FN#10]', 1678)\n",
      "('[FN#11]', 1679)\n",
      "('[FN#12]', 1680)\n",
      "('[FN#13]', 1681)\n",
      "('[FN#14]', 1682)\n",
      "('[FN#15]', 1683)\n",
      "('[FN#16]', 1684)\n",
      "('[FN#17]', 1685)\n",
      "('[FN#18]', 1686)\n",
      "('[FN#19]', 1687)\n",
      "('[FN#1]', 1688)\n",
      "('[FN#20]', 1689)\n",
      "('[FN#21]', 1690)\n",
      "('[FN#22]', 1691)\n",
      "('[FN#23]', 1692)\n",
      "('[FN#24]', 1693)\n",
      "('[FN#25]', 1694)\n",
      "('[FN#26]', 1695)\n",
      "('[FN#27]', 1696)\n",
      "('[FN#28]', 1697)\n",
      "('[FN#29]', 1698)\n",
      "('[FN#2]', 1699)\n",
      "('[FN#30]', 1700)\n",
      "('[FN#31]', 1701)\n",
      "('[FN#32]', 1702)\n",
      "('[FN#33]', 1703)\n",
      "('[FN#34]', 1704)\n",
      "('[FN#35]', 1705)\n",
      "('[FN#36]', 1706)\n",
      "('[FN#37]', 1707)\n",
      "('[FN#3]', 1708)\n",
      "('[FN#4]', 1709)\n",
      "('[FN#5]', 1710)\n",
      "('[FN#6]', 1711)\n",
      "('[FN#7]', 1712)\n",
      "('[FN#8]', 1713)\n",
      "('[FN#9]', 1714)\n",
      "('[FN#l6]', 1715)\n",
      "('[For', 1716)\n",
      "('[Hide', 1717)\n",
      "('[Such', 1718)\n",
      "('[Which', 1719)\n",
      "('[Who', 1720)\n",
      "(']', 1721)\n",
      "('``Five', 1722)\n",
      "('a', 1723)\n",
      "('a-glow', 1724)\n",
      "('a-thrill', 1725)\n",
      "('abdication', 1726)\n",
      "('abide', 1727)\n",
      "('abides', 1728)\n",
      "('abode', 1729)\n",
      "('abounding', 1730)\n",
      "('about', 1731)\n",
      "('above', 1732)\n",
      "('abroad', 1733)\n",
      "('absent', 1734)\n",
      "('absorbed', 1735)\n",
      "('abstain', 1736)\n",
      "('abstains', 1737)\n",
      "('abstinence', 1738)\n",
      "('abstinent', 1739)\n",
      "('abstraction', 1740)\n",
      "('abundance', 1741)\n",
      "('accents', 1742)\n",
      "('accept', 1743)\n",
      "('accepted', 1744)\n",
      "('accepting', 1745)\n",
      "('access', 1746)\n",
      "('accessed', 1747)\n",
      "('accessible', 1748)\n",
      "('accident', 1749)\n",
      "('accompaniment', 1750)\n",
      "('accomplish', 1751)\n",
      "('accordance', 1752)\n",
      "('accrue', 1753)\n",
      "('accuracy', 1754)\n",
      "('ache', 1755)\n",
      "('achieve', 1756)\n",
      "('achieved', 1757)\n",
      "('achievers', 1758)\n",
      "('across', 1759)\n",
      "('act', 1760)\n",
      "('acted', 1761)\n",
      "('acting', 1762)\n",
      "('action', 1763)\n",
      "('actionless', 1764)\n",
      "('actions', 1765)\n",
      "('active', 1766)\n",
      "('actor', 1767)\n",
      "('actors', 1768)\n",
      "('acts', 1769)\n",
      "('actually', 1770)\n",
      "('ad', 1771)\n",
      "('addition', 1772)\n",
      "('additional', 1773)\n",
      "('additions', 1774)\n",
      "('address', 1775)\n",
      "('addressed', 1776)\n",
      "('addresses', 1777)\n",
      "('admirable', 1778)\n",
      "('adorable', 1779)\n",
      "('adore', 1780)\n",
      "('adored', 1781)\n",
      "('adoreth', 1782)\n",
      "('adoring', 1783)\n",
      "('aeterna', 1784)\n",
      "('affirmare', 1785)\n",
      "('afresh', 1786)\n",
      "('after', 1787)\n",
      "('afterward', 1788)\n",
      "('afterwards', 1789)\n",
      "('again', 1790)\n",
      "('against', 1791)\n",
      "('age', 1792)\n",
      "('aged', 1793)\n",
      "('agent', 1794)\n",
      "('ago', 1795)\n",
      "('agree', 1796)\n",
      "('agreed', 1797)\n",
      "('agreement', 1798)\n",
      "('ah', 1799)\n",
      "('aims', 1800)\n",
      "('air', 1801)\n",
      "('airs', 1802)\n",
      "('airs]', 1803)\n",
      "('albeit', 1804)\n",
      "('alike', 1805)\n",
      "('alive', 1806)\n",
      "('all', 1807)\n",
      "('all-absorbing', 1808)\n",
      "('all-arriving', 1809)\n",
      "('all-piercing', 1810)\n",
      "('all-regarding', 1811)\n",
      "('all-wielding', 1812)\n",
      "('all-winning', 1813)\n",
      "('allotted', 1814)\n",
      "('allow', 1815)\n",
      "('almost', 1816)\n",
      "('alms', 1817)\n",
      "('almsgiving', 1818)\n",
      "('alone', 1819)\n",
      "('along', 1820)\n",
      "('aloof', 1821)\n",
      "('already', 1822)\n",
      "('also', 1823)\n",
      "('altar', 1824)\n",
      "('altar-flame', 1825)\n",
      "('altar-smoke', 1826)\n",
      "('altar-stone', 1827)\n",
      "('alteration', 1828)\n",
      "('alternate', 1829)\n",
      "('alway', 1830)\n",
      "('always', 1831)\n",
      "('am', 1832)\n",
      "('am-of', 1833)\n",
      "('amazed', 1834)\n",
      "('amid', 1835)\n",
      "('amidst', 1836)\n",
      "('among', 1837)\n",
      "('an', 1838)\n",
      "('anadem', 1839)\n",
      "('anapeksha', 1840)\n",
      "('ancestors', 1841)\n",
      "('ancient', 1842)\n",
      "('and', 1843)\n",
      "('anew', 1844)\n",
      "('anger', 1845)\n",
      "('angrily', 1846)\n",
      "('angry', 1847)\n",
      "('anguish', 1848)\n",
      "('another', 1849)\n",
      "('answer', 1850)\n",
      "('anterior', 1851)\n",
      "('any', 1852)\n",
      "('anyone', 1853)\n",
      "('anything', 1854)\n",
      "('anywhere', 1855)\n",
      "('anywise', 1856)\n",
      "('apart', 1857)\n",
      "('appear', 1858)\n",
      "('appearances', 1859)\n",
      "('appearing', 1860)\n",
      "('appears', 1861)\n",
      "('appetite', 1862)\n",
      "('appetites', 1863)\n",
      "('applicable', 1864)\n",
      "('apply', 1865)\n",
      "('apprehends', 1866)\n",
      "('approach', 1867)\n",
      "('arbitrament', 1868)\n",
      "('ardent', 1869)\n",
      "('ardour', 1870)\n",
      "('are', 1871)\n",
      "('arguments', 1872)\n",
      "('aright', 1873)\n",
      "('aright-he', 1874)\n",
      "('arise', 1875)\n",
      "('arisen', 1876)\n",
      "('arm', 1877)\n",
      "('armies', 1878)\n",
      "('arms', 1879)\n",
      "('around', 1880)\n",
      "('aroused', 1881)\n",
      "('array', 1882)\n",
      "('arrayed', 1883)\n",
      "('arriving', 1884)\n",
      "('arrogance', 1885)\n",
      "('arrows', 1886)\n",
      "('art', 1887)\n",
      "('as', 1888)\n",
      "('ascetics', 1889)\n",
      "('ash', 1890)\n",
      "('aside', 1891)\n",
      "('ask', 1892)\n",
      "('asked', 1893)\n",
      "('asking', 1894)\n",
      "('aspect', 1895)\n",
      "('aspects', 1896)\n",
      "('aspirations', 1897)\n",
      "('assails', 1898)\n",
      "('assimilate', 1899)\n",
      "('assistance', 1900)\n",
      "('associated', 1901)\n",
      "('assuaged', 1902)\n",
      "('assurance', 1903)\n",
      "('assured', 1904)\n",
      "('asterisms', 1905)\n",
      "('at', 1906)\n",
      "('atom', 1907)\n",
      "('atque', 1908)\n",
      "('attached', 1909)\n",
      "('attachment', 1910)\n",
      "('attack', 1911)\n",
      "('attain', 1912)\n",
      "('attained', 1913)\n",
      "('attaineth', 1914)\n",
      "('attaining', 1915)\n",
      "('attains', 1916)\n",
      "('attraction', 1917)\n",
      "('atwain', 1918)\n",
      "('auctor', 1919)\n",
      "('aught', 1920)\n",
      "('ausim', 1921)\n",
      "('austerities', 1922)\n",
      "('authenticity', 1923)\n",
      "('author', 1924)\n",
      "('authority', 1925)\n",
      "('available', 1926)\n",
      "('avarice', 1927)\n",
      "('avoideth', 1928)\n",
      "('away', 1929)\n",
      "('awe', 1930)\n",
      "('awful', 1931)\n",
      "('axe', 1932)\n",
      "('aye', 1933)\n",
      "('b', 1934)\n",
      "('babe', 1935)\n",
      "('back', 1936)\n",
      "('bad', 1937)\n",
      "('bands', 1938)\n",
      "('bare', 1939)\n",
      "('base', 1940)\n",
      "('based', 1941)\n",
      "('battle', 1942)\n",
      "('battle-chariots', 1943)\n",
      "('battle-shells', 1944)\n",
      "('battlefield', 1945)\n",
      "('be', 1946)\n",
      "('beaming', 1947)\n",
      "('beams', 1948)\n",
      "('bear', 1949)\n",
      "('bearing', 1950)\n",
      "('bears', 1951)\n",
      "('bears-', 1952)\n",
      "('beats', 1953)\n",
      "('beautiful', 1954)\n",
      "('beauty', 1955)\n",
      "('because', 1956)\n",
      "('becoming', 1957)\n",
      "('been', 1958)\n",
      "('befall', 1959)\n",
      "('befalls', 1960)\n",
      "('before', 1961)\n",
      "('beget', 1962)\n",
      "('begets', 1963)\n",
      "('beggar', 1964)\n",
      "('begin', 1965)\n",
      "('beginning', 1966)\n",
      "('begot', 1967)\n",
      "('beguiled', 1968)\n",
      "('beguiling', 1969)\n",
      "('beheld', 1970)\n",
      "('behind', 1971)\n",
      "('behold', 1972)\n",
      "('beholding', 1973)\n",
      "('beholds', 1974)\n",
      "('being', 1975)\n",
      "('beings', 1976)\n",
      "('belief', 1977)\n",
      "('believe', 1978)\n",
      "('believer', 1979)\n",
      "('belongings', 1980)\n",
      "('below', 1981)\n",
      "('bend', 1982)\n",
      "('beneath', 1983)\n",
      "('benign', 1984)\n",
      "('benignity', 1985)\n",
      "('bent', 1986)\n",
      "('bereft', 1987)\n",
      "('best', 1988)\n",
      "('betide', 1989)\n",
      "('betimes', 1990)\n",
      "('betray', 1991)\n",
      "('betrayed', 1992)\n",
      "('better', 1993)\n",
      "('better-lessoned', 1994)\n",
      "('between', 1995)\n",
      "('bewildered', 1996)\n",
      "('bewildering', 1997)\n",
      "('bewilderments', 1998)\n",
      "('beyond', 1999)\n",
      "('bid', 2000)\n"
     ]
    }
   ],
   "source": [
    "# Finally lets look at our token work\\\n",
    "tokenizer = SimpleTokenizerV2(vocab)\n",
    "\n",
    "for tokenizer, item in enumerate(vocab.items()):\n",
    "    print(item)\n",
    "    if tokenizer >= 2000:\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Byte Pair Algorithm (BPE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tiktoken in /home/proxi/anaconda3/envs/yolo/lib/python3.8/site-packages (0.7.0)\n",
      "Requirement already satisfied: regex>=2022.1.18 in /home/proxi/anaconda3/envs/yolo/lib/python3.8/site-packages (from tiktoken) (2024.11.6)\n",
      "Requirement already satisfied: requests>=2.26.0 in /home/proxi/anaconda3/envs/yolo/lib/python3.8/site-packages (from tiktoken) (2.32.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/proxi/anaconda3/envs/yolo/lib/python3.8/site-packages (from requests>=2.26.0->tiktoken) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/proxi/anaconda3/envs/yolo/lib/python3.8/site-packages (from requests>=2.26.0->tiktoken) (3.8)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/proxi/anaconda3/envs/yolo/lib/python3.8/site-packages (from requests>=2.26.0->tiktoken) (2.2.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/proxi/anaconda3/envs/yolo/lib/python3.8/site-packages (from requests>=2.26.0->tiktoken) (2024.8.30)\n"
     ]
    }
   ],
   "source": [
    "!pip install tiktoken\n",
    "# Its a fast BPE tokenizer which is used in OpenAI models "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib \n",
    "import tiktoken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we can instantiate the BPE Tokenizer from tiktoken \n",
    "tokenizer = tiktoken.get_encoding(\"gpt2\")\n",
    "# So basically this is similar to SimpleTokenizerV1/V2 which we implemented above, it encodes and \n",
    "# decodes, simply using 1 word, creating token IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[15496, 11, 466, 345, 588, 3254, 273, 415, 30, 220, 50256, 314, 1282, 422, 257, 1295, 810, 345, 32483, 6588, 288, 12190, 1651, 69, 617, 34680, 27271]\n"
     ]
    }
   ],
   "source": [
    "# Encode\n",
    "\n",
    "text = (\n",
    "    \"Hello, do you like Valorant? <|endoftext|> I come from a place where you breadth carbon dioxide\"\n",
    "    \"of someunknownPlace\"\n",
    ")\n",
    "# The end of text is something which was used in GPT training, what it does is, it is used to seperate 1 dataset from another\n",
    "# lets say you take 2 datasets, 1 from geeta and anotther from quaran, so the developers will use <|endoftext|>\n",
    "# between both of the datasets while loading it for BPE\n",
    "# Also as u can see i have loaded a hard word, someunknownplace, the BPE will altomatically handle this, without\n",
    "# returning any error(out of context error)\n",
    "integers = tokenizer.encode(text, allowed_special={\"<|endoftext|>\"}) \n",
    "print(integers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello, do you like Valorant? <|endoftext|> I come from a place where you breadth carbon dioxideof someunknownPlace\n"
     ]
    }
   ],
   "source": [
    "# Decode\n",
    "\n",
    "strings = tokenizer.decode(integers)\n",
    "print(strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We can see 2 new things here\n",
    "# <|endoftext|> is assigned to a large token ID\n",
    "# BPE also encodes and decodes unknown words like someunknownPlace without any error, this was becuase how\n",
    "# the tokenization was done, ie. byte by byte instead of words or individual   charactors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Input-Target data pairs using DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inthis section we will implement data-loader that fetches input-target pairs using sliding window approach\\\n",
    "# we will first tokenize our religious book "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38552\n"
     ]
    }
   ],
   "source": [
    "with open(\"gita.txt\", \"r\", encoding=\"utf-8\") as f:\n",
    "    raw_text = f.read()\n",
    "\n",
    "enc_text = tokenizer.encode(raw_text)\n",
    "print(len(enc_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "enc_sample = enc_text[50:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x: [198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017, 198, 4480, 428, 46566, 393, 2691, 379, 7324, 13, 70, 19028]\n",
      "y:      [260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017, 198, 4480, 428, 46566, 393, 2691, 379, 7324, 13, 70, 19028, 13]\n"
     ]
    }
   ],
   "source": [
    "# now for the prediction task, the most easy way to create the input-target pair for the next work predicction task is to create two variables, X & Y, where x contains the input tokens\n",
    "# and y contains the target, what are basically the input shifted by 1\n",
    "\n",
    "# The CONTEXT SIZE determines how many tokens are included in the input\n",
    "\n",
    "context_size = 25 # length of the input, which means the model is trained to look at a sequence of 4 words, to predict the next word in the sequence\n",
    "                # So X contains contains the 4 tokens [1,2,3,4] and \n",
    "                # the target Y is the next 4 tokes [2,3,4,5]\n",
    "x = enc_sample[:context_size]\n",
    "y = enc_sample[1:context_size+1]\n",
    "\n",
    "print(f\"x: {x}\")\n",
    "print(f\"y:      {y}\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[198] -------> 260\n",
      "[198, 260] -------> 12\n",
      "[198, 260, 12] -------> 1904\n",
      "[198, 260, 12, 1904] -------> 340\n",
      "[198, 260, 12, 1904, 340] -------> 739\n",
      "[198, 260, 12, 1904, 340, 739] -------> 262\n",
      "[198, 260, 12, 1904, 340, 739, 262] -------> 2846\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846] -------> 286\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286] -------> 262\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262] -------> 4935\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935] -------> 20336\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336] -------> 13789\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789] -------> 3017\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017] -------> 198\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017, 198] -------> 4480\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017, 198, 4480] -------> 428\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017, 198, 4480, 428] -------> 46566\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017, 198, 4480, 428, 46566] -------> 393\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017, 198, 4480, 428, 46566, 393] -------> 2691\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017, 198, 4480, 428, 46566, 393, 2691] -------> 379\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017, 198, 4480, 428, 46566, 393, 2691, 379] -------> 7324\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017, 198, 4480, 428, 46566, 393, 2691, 379, 7324] -------> 13\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017, 198, 4480, 428, 46566, 393, 2691, 379, 7324, 13] -------> 70\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017, 198, 4480, 428, 46566, 393, 2691, 379, 7324, 13, 70] -------> 19028\n",
      "[198, 260, 12, 1904, 340, 739, 262, 2846, 286, 262, 4935, 20336, 13789, 3017, 198, 4480, 428, 46566, 393, 2691, 379, 7324, 13, 70, 19028] -------> 13\n"
     ]
    }
   ],
   "source": [
    "# Similarly by processing the input along with the target, which are the inputs shifted by one, we can create a next-word predicction look \n",
    "for i in range(1, context_size+1):\n",
    "    context = enc_sample[:i]\n",
    "    desired = enc_sample[i]\n",
    "    \n",
    "    print (context, \"------->\", desired)\n",
    "    # Everything on the left \"context\" is something which is to the LEFT is the input, LLM would recieve, and the token ID on the RIGHT representes the target target which LLM is supposed to predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " ------> re\n",
      "\n",
      "re ------> -\n",
      "\n",
      "re- ------> use\n",
      "\n",
      "re-use ------>  it\n",
      "\n",
      "re-use it ------>  under\n",
      "\n",
      "re-use it under ------>  the\n",
      "\n",
      "re-use it under the ------>  terms\n",
      "\n",
      "re-use it under the terms ------>  of\n",
      "\n",
      "re-use it under the terms of ------>  the\n",
      "\n",
      "re-use it under the terms of the ------>  Project\n",
      "\n",
      "re-use it under the terms of the Project ------>  Gutenberg\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg ------>  License\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg License ------>  included\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg License included ------> \n",
      "\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg License included\n",
      " ------> with\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg License included\n",
      "with ------>  this\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg License included\n",
      "with this ------>  eBook\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg License included\n",
      "with this eBook ------>  or\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg License included\n",
      "with this eBook or ------>  online\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg License included\n",
      "with this eBook or online ------>  at\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg License included\n",
      "with this eBook or online at ------>  www\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg License included\n",
      "with this eBook or online at www ------> .\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg License included\n",
      "with this eBook or online at www. ------> g\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg License included\n",
      "with this eBook or online at www.g ------> utenberg\n",
      "\n",
      "re-use it under the terms of the Project Gutenberg License included\n",
      "with this eBook or online at www.gutenberg ------> .\n"
     ]
    }
   ],
   "source": [
    "# now for better understanding, lets repeat the same things with the texts instead of token IDs\n",
    "\n",
    "for i in range(1, context_size+1):\n",
    "    context = enc_sample[:i]\n",
    "    desired = enc_sample[i]\n",
    "\n",
    "    print(tokenizer.decode(context), \"------>\", tokenizer.decode([desired]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We've now created the input-target pairs that we can turn into use for the LLM training in upcoming chapters.\n",
    "# there's only one more task before we can turn the tokens into embeddings:implementing an efficient data loader that iterates over the input dataset and returns the inputs and targets as PyTorch tensors,\n",
    "#  which can be thought of as multidimensional arrays."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IMPLEMENTING A DATA LOADER\n",
    "##### In particular, we are interested in returning two tensors: an input tensor containing the text that the LLM sees and a target tensor that includes the targets for the LLM to predict,\n",
    "\n",
    "\n",
    "\n",
    "##### Step 1: Tokenize the entire text\n",
    "\n",
    "##### Step 2: Use a sliding window to chunk the book into overlapping sequences of max_length\n",
    "\n",
    "##### Step 3: Return the total number of rows in the dataset\n",
    "\n",
    "##### Step 4: Return a single row from the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A DataLoader in PyTorch is a utility that helps manage and load data efficiently during training or inference.\n",
    "\n",
    "# Loads data in batches: Divides your dataset into smaller chunks for processing.\n",
    "# Shuffles data: Randomizes data to prevent patterns during training.\n",
    "# Handles multiprocessing: Uses multiple CPU threads to load data faster.\n",
    "# Why do you need a DataLoader?\n",
    "# Efficiency: Training models on the entire dataset at once is memory-intensive. DataLoader loads data in manageable chunks (batches).\n",
    "# Performance: It supports parallel data loading, reducing the time spent waiting for data during training.\n",
    "# Convenience: Handles dataset shuffling, batching, and even custom transformations seamlessly.\n",
    "# In short, DataLoader makes training deep learning models efficient and scalable by automating data preparation tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "\n",
    "class GPTDatasetV1(Dataset):\n",
    "    def __init__(self, txt, tokenizer, max_length, stride):\n",
    "        self.input_ids = []\n",
    "        self.target_ids = []\n",
    "\n",
    "        # Tokenize the entire text\n",
    "        token_ids = tokenizer.encode(txt, allowed_special={\"<|endoftext|>\"})\n",
    "\n",
    "        # Use a sliding window to chunk the book into overlapping sequences of max_length\n",
    "        for i in range(0, len(token_ids) - max_length, stride):\n",
    "            input_chunk = token_ids[i:i + max_length]\n",
    "            target_chunk = token_ids[i + 1: i + max_length + 1]\n",
    "            self.input_ids.append(torch.tensor(input_chunk))\n",
    "            self.target_ids.append(torch.tensor(target_chunk))\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.input_ids)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.input_ids[idx], self.target_ids[idx]\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "# GPTDatasetV1 is a custom dataset class based on PyTorch's Dataset.\n",
    "# It prepares input and target data for training a language model.\n",
    "\n",
    "# Each \"row\" in the dataset contains:\n",
    "# 1. input_chunk: A fixed-length sequence of token IDs (numbers representing the text).\n",
    "# 2. target_chunk: The sequence of token IDs the model should predict based on input_chunk.\n",
    "\n",
    "# How it works:\n",
    "# - The entire text is tokenized into a long sequence of token IDs.\n",
    "# - This long sequence is split into smaller overlapping chunks using a sliding window.\n",
    "# - Each chunk is stored as input and target pairs for the model to learn from.\n",
    "\n",
    "# Why use this:\n",
    "# - The model trains on these smaller chunks, learning to predict the next token(s).\n",
    "# - Overlapping chunks give the model more context for better predictions.\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Now lets use this GPT class to load the inputs into BATCHES via a pytorch dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Steps #\n",
    "# 1 - initialize the tokenizer\n",
    "\n",
    "# 2 - create dataset\n",
    "\n",
    "# 3 - drop_last = True drop the last batch if it is shorter than the specified batch_size\n",
    "# to prevent loss spikes during training\n",
    "\n",
    "# 4 - the number of CPU processes to use for preproccessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dataloader_v1(txt, batch_size=4, max_length=64, stride=32, \n",
    "                         shuffle=True, drop_last=True, num_workers=0):\n",
    "    \n",
    "    # initialize the tokenizer\n",
    "    tokenizer = tiktoken.get_encoding(\"gpt2\")\n",
    "\n",
    "    # Create dataset\n",
    "    dataset = GPTDatasetV1(txt, tokenizer, max_length, stride)\n",
    "\n",
    "    # Create dataloader\n",
    "    dataloader = DataLoader(\n",
    "        dataset,\n",
    "        batch_size=batch_size,\n",
    "        shuffle=shuffle,\n",
    "        drop_last=drop_last,\n",
    "        num_workers=num_workers\n",
    "    )\n",
    "\n",
    "    return dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lets test the dataloader with batchsize = 1 for an LLM with a context size of 4,\n",
    "with open(\"gita.txt\", \"r\", encoding=\"utf-8\") as f:\n",
    "    raw_text = f.read() # for reading the file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch version: 2.4.1+cu121\n",
      "[tensor([[  198,   464,  4935, 20336]]), tensor([[  464,  4935, 20336,   412]])]\n"
     ]
    }
   ],
   "source": [
    "# Convert dataloader into a python iterator to fatch the next entry via python'd \n",
    "# build-in next() function\n",
    "\n",
    "import torch\n",
    "print(\"PyTorch version:\", torch.__version__)\n",
    "dataloader = create_dataloader_v1(\n",
    "    raw_text, batch_size=1, max_length=4, stride=1, shuffle=False\n",
    ")\n",
    "\n",
    "data_iter = iter(dataloader)\n",
    "first_batch = next(data_iter)\n",
    "print(first_batch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TOKEN EMBEDDINGS\n",
    "##### meaning of embedding - a mathematical representation of a word, phrase, or piece of text as a high-dimensional vector, capturing its semantic meaning and allowing the model to understand the relationships between different words and concepts within a text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gensim in /home/proxi/anaconda3/envs/yolo/lib/python3.8/site-packages (4.3.3)\n",
      "Requirement already satisfied: numpy<2.0,>=1.18.5 in /home/proxi/anaconda3/envs/yolo/lib/python3.8/site-packages (from gensim) (1.24.4)\n",
      "Requirement already satisfied: scipy<1.14.0,>=1.7.0 in /home/proxi/anaconda3/envs/yolo/lib/python3.8/site-packages (from gensim) (1.10.1)\n",
      "Requirement already satisfied: smart-open>=1.8.1 in /home/proxi/anaconda3/envs/yolo/lib/python3.8/site-packages (from gensim) (7.0.5)\n",
      "Requirement already satisfied: wrapt in /home/proxi/anaconda3/envs/yolo/lib/python3.8/site-packages (from smart-open>=1.8.1->gensim) (1.17.0)\n"
     ]
    }
   ],
   "source": [
    "# Firstly we will install GENSIM, it is widely used for creating and using vector representations of text, including word embeddings and document embeddings.\n",
    "!pip install gensim\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim.downloader as api\n",
    "model = api.load(\"glove-twitter-25\")  # download the model and return as object ready for use"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example of a word as a vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.64005  -0.019514  0.70148  -0.66123   1.1723   -0.58859   0.25917\n",
      " -0.81541   1.1708    1.1413   -0.15405  -0.11369  -3.8414   -0.87233\n",
      "  0.47489   1.1541    0.97678   1.1107   -0.14572  -0.52013  -0.52234\n",
      " -0.92349   0.34651   0.061939 -0.57375 ]\n"
     ]
    }
   ],
   "source": [
    "word_vectors=model\n",
    "\n",
    "# Let us look how the vector embedding of a word looks like\n",
    "print(word_vectors['computer'])  # Example: Accessing the vector for the word 'computer'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25,)\n"
     ]
    }
   ],
   "source": [
    "# Checking the dimensions (rows in matrix)\n",
    "print(word_vectors['cat'].shape) # 25 means its dimentions are 25, word2vec-google-news-300 is another model with 300 dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('meets', 0.8841924071311951), ('prince', 0.832163393497467), ('queen', 0.8257461190223694), ('’s', 0.8174097537994385), ('crow', 0.813499391078949), ('hunter', 0.8131037950515747), ('father', 0.8115834593772888), ('soldier', 0.81113600730896), ('mercy', 0.8082392811775208), ('hero', 0.8082264065742493)]\n"
     ]
    }
   ],
   "source": [
    "# Example of using most_similar\n",
    "# Now if i ask what will be the most common word if i did King + Woman - Man\n",
    "print(word_vectors.most_similar(positive=['king', 'woman'], negative=['man'], topn=10)) # LOL as i have used a small data its showing few other things it should not it... but yeah it should be queen\n",
    "#                                                                                          which u can see on the third place"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.76541775\n",
      "0.9202422\n",
      "0.9338088\n",
      "0.95961404\n",
      "0.96716267\n",
      "0.70250624\n"
     ]
    }
   ],
   "source": [
    "# Example of calculating similarity\n",
    "print(word_vectors.similarity('woman', 'man'))\n",
    "print(word_vectors.similarity('king', 'queen'))\n",
    "print(word_vectors.similarity('uncle', 'aunt'))\n",
    "print(word_vectors.similarity('boy', 'girl'))\n",
    "print(word_vectors.similarity('nephew', 'niece'))\n",
    "print(word_vectors.similarity('paper', 'water')) # This has the lest similarity score as the words makes no sensee to each other"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('square', 0.9199698567390442), ('gate', 0.9008872509002686), ('bridge', 0.8968756198883057), ('pacific', 0.8831596970558167), ('oak', 0.8686302304267883)]\n"
     ]
    }
   ],
   "source": [
    "# Most similar words\n",
    "print(word_vectors.most_similar(\"tower\", topn=5)) # it shows that was all are the words which are similar to tower"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The magnitude of the difference between 'man' and 'woman' is 3.77\n",
      "The magnitude of the difference between 'semiconductor' and 'earthworm' is 3.70\n",
      "The magnitude of the difference between 'nephew' and 'niece' is 1.26\n"
     ]
    }
   ],
   "source": [
    "# Now let us see the vector similarity\n",
    "import numpy as np\n",
    "# Words to compare\n",
    "word1 = 'man'\n",
    "word2 = 'woman'\n",
    "\n",
    "word3 = 'semiconductor'\n",
    "word4 = 'earthworm'\n",
    "\n",
    "word5 = 'nephew'\n",
    "word6 = 'niece'\n",
    "\n",
    "# Calculate the vector difference\n",
    "vector_difference1 = model[word1] - model[word2]\n",
    "vector_difference2 = model[word3] - model[word4]\n",
    "vector_difference3 = model[word5] - model[word6]\n",
    "\n",
    "# Calculate the magnitude of the vector difference\n",
    "magnitude_of_difference1 = np.linalg.norm(vector_difference1)\n",
    "magnitude_of_difference2 = np.linalg.norm(vector_difference2)\n",
    "magnitude_of_difference3 = np.linalg.norm(vector_difference3)\n",
    "\n",
    "\n",
    "# Print the magnitude of the difference\n",
    "print(\"The magnitude of the difference between '{}' and '{}' is {:.2f}\".format(word1, word2, magnitude_of_difference1))\n",
    "print(\"The magnitude of the difference between '{}' and '{}' is {:.2f}\".format(word3, word4, magnitude_of_difference2))\n",
    "print(\"The magnitude of the difference between '{}' and '{}' is {:.2f}\".format(word5, word6, magnitude_of_difference3))\n",
    "\n",
    "# okay the data is small (and shit) thats why we are getting garbage results otherwize the magnitude of semiconductor and earthworm should be MOST and 1st ,3rd should be the least"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Positional Embeddings (Encoding word positions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets consider much more realistic and useful embedding sizes and encode the input token into a 256 dimensional vector representation.\n",
    "# this dimension would be smaller than GPT-3 as it had used 12,288 dimensions, and 768 dimensions for GPT-2... but still resonable for experiments\n",
    "# and also we will be using BPE tokenizer that we implemented earlier, which had the vocab_size or total token IDs of 50257\n",
    "\n",
    "vocab_size = 50257 # no. of token ids\n",
    "output_dim = 256 # this is the vector dimensions\n",
    "\n",
    "token_embedding_layer = torch.nn.Embedding(vocab_size, ouput_dim)\n",
    "# The torch.nn.Embedding layer is used to map integer indices (token IDs) to dense vector representations.\n",
    "# Creates an embedding lookup table of size (vocab_size, output_dim).\n",
    "# Each row corresponds to the embedding vector of a token ID.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Lets add data loader (ie. data sampling with a sliding window)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_length = 4 # Total length of tokens\n",
    "dataloader = create_dataloader_v1(\n",
    "    raw_text, batch_size=8, max_length=max_length,\n",
    "    stride=max_length, shuffle=False\n",
    ")\n",
    "\n",
    "data_iter = iter(dataloader) # Converts the dataloader object into an iterator, enabling you to retrieve batches of data one at a time.\n",
    "inputs, targets = next(data_iter) # Fetches the next batch of data from the dataloader. It unpacks the batch into inputs (e.g., tokenized text data) and targets (e.g., labels or next-token predictions)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token ID:\n",
      " tensor([[  198,   464,  4935, 20336],\n",
      "        [  412, 10482,   286,   383],\n",
      "        [16581,   363,   615,   324],\n",
      "        [   12,    38,  5350,    11],\n",
      "        [  416, 19200,   198,   198],\n",
      "        [ 1212, 46566,   318,   329],\n",
      "        [  262,   779,   286,  2687],\n",
      "        [ 6609,   379,   645,  1575]])\n",
      "\n",
      "Input Shape\n",
      " torch.Size([8, 4])\n"
     ]
    }
   ],
   "source": [
    "print(\"Token ID:\\n\", inputs)\n",
    "print(\"\\nInput Shape\\n\", inputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([8, 4, 256])\n"
     ]
    }
   ],
   "source": [
    "# as we can see the Token ID Tensor is 8*4 dimentional, meaning that the data batch consists of 8 text samples with 4 token in a row.\n",
    "# Now lets use the embedding layer to embed these token IDs into 256-dimenional vectors:\n",
    "\n",
    "token_embeddings = token_embedding_layer(inputs)\n",
    "print(token_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# As u can see, each token ID is now embedded as a 256-dimentional vector \n",
    "# Hence we have completed the vector embedding now lets move forward with positional encoding \n",
    "# For a GPT model's absolute embedding approach, we just need to create another embedding layer that has the same dimension as the token_embedding_layer:\n",
    "context_length = max_length # 4\n",
    "positional_encoding_layer = torch.nn.Embedding(context_length, output_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 256])\n"
     ]
    }
   ],
   "source": [
    "position_embeddings = positional_encoding_layer(torch.arange(max_length))\n",
    "print(position_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# As shown in the preceding code example, the input to the position_embeddings is usually a placeholder vector torch.arange(context_length), \n",
    "# which contains a sequence of numbers 0, 1, ..., up to the maximum input length − 1.\n",
    "# The context_length is a variable that represents the supported input size of the LLM.\n",
    "# Here, we choose it similar to the maximum length of the input text.\n",
    "# In practice, input text can be longer than the supported context length, in which case we have to truncate the text.\n",
    "\n",
    "\n",
    "# As we can see, the positional embedding tensor consists of four 256-dimensional vectors. We can now add these directly to the token embeddings,\n",
    "#  where PyTorch will add the 4x256- dimensional pos_embeddings tensor to each 4x256-dimensional token embedding tensor in each of the 8 batches:\n",
    "\n",
    "\n",
    "lll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([8, 4, 256])\n"
     ]
    }
   ],
   "source": [
    "input_embeddings = token_embeddings + position_embeddings\n",
    "print(input_embeddings.shape)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "yolo",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}