3. Embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c7c3b6a7",
   "metadata": {},
   "source": [
    "<div style=\"width: 100%; overflow: hidden;\">\n",
    "    <div style=\"width: 150px; float: left;\"> <img src=\"https://raw.githubusercontent.com/DataForScience/Networks/master/data/D4Sci_logo_ball.png\" alt=\"Data For Science, Inc\" align=\"left\" border=\"0\" width=150px> </div>\n",
    "    <div style=\"float: left; margin-left: 10px;\"> <h1>Generative AI with OpenAI API</h1>\n",
    "<h1>Embeddings</h1>\n",
    "        <p>Bruno Gonçalves<br/>\n",
    "        <a href=\"http://www.data4sci.com/\">www.data4sci.com</a><br/>\n",
    "            @bgoncalves, @data4sci</p></div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "46f6877b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from ipywidgets import interact\n",
    "import tiktoken\n",
    "\n",
    "import openai\n",
    "from openai import OpenAI\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import reuters\n",
    "from nltk import bigrams, trigrams\n",
    "\n",
    "import scipy\n",
    "from scipy.spatial.distance import cosine as cosine_similarity\n",
    "\n",
    "import os\n",
    "import gzip\n",
    "\n",
    "import tqdm as tq\n",
    "from tqdm.notebook import tqdm\n",
    "tqdm.pandas()\n",
    "\n",
    "import watermark\n",
    "\n",
    "%load_ext watermark"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1878728b",
   "metadata": {},
   "source": [
    "We start by printing out the versions of the libraries we're using for future reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "087adf7f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python implementation: CPython\n",
      "Python version       : 3.11.7\n",
      "IPython version      : 8.12.3\n",
      "\n",
      "Compiler    : Clang 14.0.6 \n",
      "OS          : Darwin\n",
      "Release     : 24.3.0\n",
      "Machine     : arm64\n",
      "Processor   : arm\n",
      "CPU cores   : 16\n",
      "Architecture: 64bit\n",
      "\n",
      "Git hash: 3a7a9a8b6856eb5855cd2ac76a384e203382ab54\n",
      "\n",
      "json     : 2.0.9\n",
      "numpy    : 1.26.4\n",
      "openai   : 1.30.5\n",
      "scipy    : 1.11.4\n",
      "tiktoken : 0.7.0\n",
      "pandas   : 2.2.3\n",
      "watermark: 2.4.3\n",
      "tqdm     : 4.66.4\n",
      "nltk     : 3.8.1\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%watermark -n -v -m -g -iv"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e2e9582",
   "metadata": {},
   "source": [
    "Get the API key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0c458891",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = OpenAI()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1258e785",
   "metadata": {},
   "source": [
    "# Extract embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "191b5561",
   "metadata": {},
   "outputs": [],
   "source": [
    "response = client.embeddings.create(\n",
    "    model=\"text-embedding-ada-002\",\n",
    "    input=\"What is prompt engineering?\",\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4256e981",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(response.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5dd20f1",
   "metadata": {},
   "source": [
    "The embedding itself is a high dimensional vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "baea0bf8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-0.013495800085365772,\n",
       " -0.0033809461165219545,\n",
       " -0.011480874381959438,\n",
       " -0.011949624866247177,\n",
       " 0.004152284469455481,\n",
       " 0.007041743025183678,\n",
       " -0.00880130473524332,\n",
       " -0.007870800793170929,\n",
       " -0.021996265277266502,\n",
       " -0.024514921009540558,\n",
       " 0.013516788370907307,\n",
       " 0.01193563174456358,\n",
       " -0.012257460504770279,\n",
       " 0.010935165919363499,\n",
       " -0.009053169749677181,\n",
       " 0.004593049641698599,\n",
       " 0.020974809303879738,\n",
       " -0.001521688187494874,\n",
       " -0.0151539146900177,\n",
       " 0.02707555517554283,\n",
       " -0.03414178639650345,\n",
       " -0.009605875238776207,\n",
       " 0.008983206935226917,\n",
       " -0.009067162871360779,\n",
       " -0.017238803207874298,\n",
       " -0.01784048229455948,\n",
       " 0.015419773757457733,\n",
       " -0.036268651485443115,\n",
       " -0.010851210914552212,\n",
       " 0.0015076956478878856,\n",
       " -0.016930967569351196,\n",
       " -0.02297574281692505,\n",
       " -0.01789645105600357,\n",
       " -0.001575909205712378,\n",
       " -0.020457085222005844,\n",
       " 0.0010074625024572015,\n",
       " -0.00024771157768554986,\n",
       " -0.004316696897149086,\n",
       " 0.03044775500893593,\n",
       " 0.008101677522063255,\n",
       " 0.05118469148874283,\n",
       " 0.017112869769334793,\n",
       " 0.010375464335083961,\n",
       " -0.011466882191598415,\n",
       " -0.016133392229676247,\n",
       " 0.014734138734638691,\n",
       " 0.013530781492590904,\n",
       " -0.013145986013114452,\n",
       " -0.025788241997361183,\n",
       " 0.013474810868501663,\n",
       " 0.022891785949468613,\n",
       " -0.007632927503436804,\n",
       " -0.004649019800126553,\n",
       " -0.01954757049679756,\n",
       " -0.030531710013747215,\n",
       " -0.012929102405905724,\n",
       " 0.0008928986499086022,\n",
       " 0.023941226303577423,\n",
       " 0.0024574389681220055,\n",
       " -0.0025483903009444475,\n",
       " -8.43924717628397e-05,\n",
       " 0.0022510490380227566,\n",
       " -0.021212682127952576,\n",
       " 0.00548157561570406,\n",
       " 0.012992068193852901,\n",
       " 0.012334419414401054,\n",
       " -0.010158579796552658,\n",
       " 0.027187494561076164,\n",
       " -0.018568092957139015,\n",
       " -0.019057832658290863,\n",
       " 0.01799440011382103,\n",
       " 0.015461751259863377,\n",
       " 0.0038969209417700768,\n",
       " 0.004477611277252436,\n",
       " 0.019729474559426308,\n",
       " -0.01003964338451624,\n",
       " -0.0060447752475738525,\n",
       " -0.019953355193138123,\n",
       " 0.002522154478356242,\n",
       " 0.0035663472954183817,\n",
       " 0.0012680734507739544,\n",
       " -0.0221781674772501,\n",
       " -0.005936332978308201,\n",
       " 0.048470139503479004,\n",
       " 0.004823926370590925,\n",
       " -0.003153567435219884,\n",
       " 0.004190764389932156,\n",
       " 0.017616601660847664,\n",
       " -0.010487404651939869,\n",
       " -0.018218280747532845,\n",
       " -0.011823691427707672,\n",
       " 0.027803165838122368,\n",
       " 0.014314362779259682,\n",
       " 0.006691929884254932,\n",
       " 0.02182835340499878,\n",
       " -0.018120331689715385,\n",
       " 0.0038794302381575108,\n",
       " 0.01595149002969265,\n",
       " 0.016091415658593178,\n",
       " -0.031930964440107346,\n",
       " -0.001414120546542108,\n",
       " 0.020652981474995613,\n",
       " -0.0020761422347277403,\n",
       " 0.002840484492480755,\n",
       " -0.023941226303577423,\n",
       " -0.008563430979847908,\n",
       " 0.009969680570065975,\n",
       " -0.034029845148324966,\n",
       " 0.014678169041872025,\n",
       " -0.00041059343493543565,\n",
       " -0.015937497839331627,\n",
       " 0.01795242168009281,\n",
       " -0.004739971365779638,\n",
       " 0.006394588388502598,\n",
       " 0.01828824356198311,\n",
       " -0.014125463552772999,\n",
       " 0.0005653858534060419,\n",
       " -0.008521453477442265,\n",
       " 0.0047014919109642506,\n",
       " -0.0038829282857477665,\n",
       " 0.02731342799961567,\n",
       " 0.006754896137863398,\n",
       " 0.0068773310631513596,\n",
       " 0.008850278332829475,\n",
       " 0.009626863524317741,\n",
       " -0.002854477148503065,\n",
       " -0.027481338009238243,\n",
       " -0.006478543393313885,\n",
       " 0.002513409126549959,\n",
       " -0.013369866646826267,\n",
       " 0.006593982223421335,\n",
       " -0.0032672567758709192,\n",
       " 0.004862405825406313,\n",
       " 0.011956620961427689,\n",
       " -0.013565762899816036,\n",
       " 0.009144121780991554,\n",
       " 0.0014394819736480713,\n",
       " 0.0002249737299280241,\n",
       " -0.05571827292442322,\n",
       " -0.019001862034201622,\n",
       " 0.026487868279218674,\n",
       " 0.015405780635774136,\n",
       " -0.011075091548264027,\n",
       " 0.002658581594005227,\n",
       " -0.006272153928875923,\n",
       " 0.0175466388463974,\n",
       " 0.019855406135320663,\n",
       " -0.006810866296291351,\n",
       " -0.00010488935367902741,\n",
       " 0.007101211231201887,\n",
       " 0.025830218568444252,\n",
       " -0.0035086281131953,\n",
       " -0.00043661080417223275,\n",
       " -0.0051597473211586475,\n",
       " -0.015713617205619812,\n",
       " 0.0059923031367361546,\n",
       " -0.005551537964493036,\n",
       " 0.0029314360581338406,\n",
       " -0.006600978318601847,\n",
       " -0.010116602294147015,\n",
       " 0.022108204662799835,\n",
       " -0.004089318215847015,\n",
       " 0.028488799929618835,\n",
       " -0.008619401603937149,\n",
       " 0.014230407774448395,\n",
       " 0.03752797842025757,\n",
       " 0.021548504009842873,\n",
       " 0.0021093746181577444,\n",
       " -0.006887825205922127,\n",
       " -0.0216884296387434,\n",
       " -0.018959885463118553,\n",
       " 0.025088615715503693,\n",
       " -0.051072750240564346,\n",
       " 0.013929568231105804,\n",
       " -0.011886658146977425,\n",
       " -0.008696360513567924,\n",
       " 0.0004919250495731831,\n",
       " -0.0023140155244618654,\n",
       " 0.0061462209559977055,\n",
       " -0.010676303878426552,\n",
       " -0.0034858903381973505,\n",
       " 0.010459419339895248,\n",
       " -0.015237870626151562,\n",
       " 0.056249991059303284,\n",
       " -0.028628725558519363,\n",
       " -0.03425372391939163,\n",
       " 0.0010782997123897076,\n",
       " -0.009312031790614128,\n",
       " -0.0011351443827152252,\n",
       " 0.003858441486954689,\n",
       " -0.00859841238707304,\n",
       " 0.03817163407802582,\n",
       " -0.013523784466087818,\n",
       " 0.017616601660847664,\n",
       " -0.6416416764259338,\n",
       " -0.04396454244852066,\n",
       " -0.004953357391059399,\n",
       " -0.02746734581887722,\n",
       " -0.01181669533252716,\n",
       " 0.01313199382275343,\n",
       " 0.012719213962554932,\n",
       " -0.010683299973607063,\n",
       " 0.007339084520936012,\n",
       " 0.02143656276166439,\n",
       " 0.016511190682649612,\n",
       " 0.010809233412146568,\n",
       " -0.020275183022022247,\n",
       " -0.00193446793127805,\n",
       " 0.0053136651404201984,\n",
       " -0.014790109358727932,\n",
       " 0.003213035874068737,\n",
       " -0.03100745752453804,\n",
       " 0.014972012490034103,\n",
       " -0.003358208341524005,\n",
       " -0.01390857994556427,\n",
       " 0.02820895053446293,\n",
       " -0.007814831100404263,\n",
       " -0.0019921872299164534,\n",
       " 0.020513055846095085,\n",
       " -0.03025186061859131,\n",
       " 0.0019502094946801662,\n",
       " 0.009976677596569061,\n",
       " 0.0020656478591263294,\n",
       " 0.017168840393424034,\n",
       " -0.0034456616267561913,\n",
       " 0.011236005462706089,\n",
       " -0.01684701256453991,\n",
       " -0.010907180607318878,\n",
       " 0.03708021715283394,\n",
       " 0.006506528705358505,\n",
       " -0.02731342799961567,\n",
       " 0.015517720952630043,\n",
       " 0.0032200319692492485,\n",
       " 0.045587677508592606,\n",
       " -0.007220148108899593,\n",
       " -0.01599346660077572,\n",
       " 0.01211053878068924,\n",
       " 0.010984139516949654,\n",
       " -0.0025711283087730408,\n",
       " 0.030587680637836456,\n",
       " 0.03226678445935249,\n",
       " -0.008262591436505318,\n",
       " 0.008514457382261753,\n",
       " -0.0091721061617136,\n",
       " 0.0035960813984274864,\n",
       " 0.010928169824182987,\n",
       " -0.03696827590465546,\n",
       " 0.0061637116596102715,\n",
       " 0.023143652826547623,\n",
       " -0.01694495975971222,\n",
       " 0.03313432261347771,\n",
       " 0.0039354003965854645,\n",
       " -0.007674905471503735,\n",
       " -0.013229941949248314,\n",
       " -0.007499998435378075,\n",
       " 0.02019122801721096,\n",
       " -0.018903914839029312,\n",
       " -0.0060062953270971775,\n",
       " -0.02373133972287178,\n",
       " 0.0026813193690031767,\n",
       " -0.004544075578451157,\n",
       " -0.006919308565557003,\n",
       " -0.01704290695488453,\n",
       " -0.027747197076678276,\n",
       " 0.0175466388463974,\n",
       " 0.007080222479999065,\n",
       " 0.0007228018948808312,\n",
       " 0.004082322120666504,\n",
       " 0.017406713217496872,\n",
       " 0.001951958634890616,\n",
       " 0.009109140373766422,\n",
       " 0.008843282237648964,\n",
       " 0.023703353479504585,\n",
       " 0.0038444488309323788,\n",
       " -0.0029331850819289684,\n",
       " -0.005929336417466402,\n",
       " -0.004743469413369894,\n",
       " 0.001485832268372178,\n",
       " -0.006950791459530592,\n",
       " 0.004572060890495777,\n",
       " -0.027355404570698738,\n",
       " -0.024402979761362076,\n",
       " -0.017574623227119446,\n",
       " 0.007597946561872959,\n",
       " 0.0018557598814368248,\n",
       " 0.007779849227517843,\n",
       " -0.01180969923734665,\n",
       " -0.03151118755340576,\n",
       " -0.03783581405878067,\n",
       " 0.024864735081791878,\n",
       " 0.015741601586341858,\n",
       " 0.002210820559412241,\n",
       " 0.01274020317941904,\n",
       " -0.000679949764162302,\n",
       " -0.004977844189852476,\n",
       " 0.0005597014096565545,\n",
       " 0.01828824356198311,\n",
       " -0.010522386059165001,\n",
       " 0.0322108156979084,\n",
       " -0.010291509330272675,\n",
       " -0.012299438007175922,\n",
       " 0.008332554250955582,\n",
       " 0.01774253323674202,\n",
       " -0.017770519480109215,\n",
       " 0.03772387281060219,\n",
       " 0.00901818834245205,\n",
       " -0.0065205213613808155,\n",
       " -0.018162310123443604,\n",
       " -0.03201492130756378,\n",
       " -0.027341412380337715,\n",
       " 0.013481806963682175,\n",
       " -0.005642489530146122,\n",
       " 0.008213617838919163,\n",
       " -0.020023318007588387,\n",
       " -0.010872199200093746,\n",
       " -0.006303636822849512,\n",
       " 0.022555965930223465,\n",
       " -0.003164061810821295,\n",
       " 0.02797107771039009,\n",
       " 0.025536375120282173,\n",
       " 0.00977378524839878,\n",
       " -0.021744398400187492,\n",
       " -0.02192630246281624,\n",
       " 0.007395054679363966,\n",
       " -0.0034981337375938892,\n",
       " -0.014510258100926876,\n",
       " 0.022569958120584488,\n",
       " -0.026236003264784813,\n",
       " 0.0027372895274311304,\n",
       " 0.0012243468081578612,\n",
       " 0.009654848836362362,\n",
       " -0.005820894613862038,\n",
       " 0.0008636017446406186,\n",
       " -0.004120801575481892,\n",
       " -0.0036240664776414633,\n",
       " 0.003942396491765976,\n",
       " 0.00635610893368721,\n",
       " -0.002623600186780095,\n",
       " -0.010361472144722939,\n",
       " -0.017420705407857895,\n",
       " -0.03976678475737572,\n",
       " 0.024556897580623627,\n",
       " 0.005373133346438408,\n",
       " -0.02108675055205822,\n",
       " -0.013320893049240112,\n",
       " -0.021156713366508484,\n",
       " 0.0017744282959029078,\n",
       " 0.01820428855717182,\n",
       " 0.015447758138179779,\n",
       " -0.006104243453592062,\n",
       " 0.001361648552119732,\n",
       " -0.03934700787067413,\n",
       " -0.012446359731256962,\n",
       " -0.030419770628213882,\n",
       " -0.01575559377670288,\n",
       " 0.01499999687075615,\n",
       " -0.03131529316306114,\n",
       " 0.010256527923047543,\n",
       " 0.016819026321172714,\n",
       " -0.010298505425453186,\n",
       " 0.007800837978720665,\n",
       " -0.003312732558697462,\n",
       " -0.011886658146977425,\n",
       " -0.04083021730184555,\n",
       " 0.01575559377670288,\n",
       " -0.010893188416957855,\n",
       " -0.00606926204636693,\n",
       " 0.021072756499052048,\n",
       " -0.01758861541748047,\n",
       " -0.0050897845067083836,\n",
       " -0.019701488316059113,\n",
       " -0.027355404570698738,\n",
       " -0.008465483784675598,\n",
       " -0.0017202072776854038,\n",
       " 0.012817162089049816,\n",
       " -0.007618935313075781,\n",
       " -0.017826490104198456,\n",
       " 0.004134794231504202,\n",
       " -0.000398568605305627,\n",
       " 0.009661844931542873,\n",
       " 0.02108675055205822,\n",
       " 0.009640856646001339,\n",
       " -0.003966883756220341,\n",
       " 0.022164175286889076,\n",
       " -0.004649019800126553,\n",
       " 0.02192630246281624,\n",
       " 0.0012103542685508728,\n",
       " 0.013782646507024765,\n",
       " -0.022066228091716766,\n",
       " 0.018722010776400566,\n",
       " 0.021408578380942345,\n",
       " 0.02587219700217247,\n",
       " -0.0021163709461688995,\n",
       " 0.02548040635883808,\n",
       " 0.031035441905260086,\n",
       " 0.006982274819165468,\n",
       " 0.023213615640997887,\n",
       " -0.0003288245643489063,\n",
       " 0.0029576721135526896,\n",
       " -0.016902981325984,\n",
       " 0.006572993006557226,\n",
       " -0.00349463545717299,\n",
       " 0.04334887117147446,\n",
       " 0.008080689236521721,\n",
       " -0.003991370555013418,\n",
       " -0.029468277469277382,\n",
       " -0.0140345124527812,\n",
       " -0.03973880037665367,\n",
       " -0.011998598463833332,\n",
       " 0.024151114746928215,\n",
       " -0.02707555517554283,\n",
       " 0.02836286835372448,\n",
       " -0.018470145761966705,\n",
       " -0.007297107018530369,\n",
       " 6.736640443705255e-06,\n",
       " -0.0033179798629134893,\n",
       " 0.010557367466390133,\n",
       " 0.009577889926731586,\n",
       " 0.0021635957527905703,\n",
       " 0.025746263563632965,\n",
       " 0.004771454259753227,\n",
       " 0.02616604045033455,\n",
       " -0.0025868697557598352,\n",
       " -0.03534514456987381,\n",
       " 0.003195545170456171,\n",
       " 0.01256529614329338,\n",
       " -0.00834654737263918,\n",
       " -0.015783580020070076,\n",
       " 0.004792443010956049,\n",
       " 0.0382835753262043,\n",
       " -0.021422570571303368,\n",
       " -0.016902981325984,\n",
       " 0.037108201533555984,\n",
       " -0.007339084520936012,\n",
       " -0.005555036477744579,\n",
       " 4.32074157288298e-05,\n",
       " -0.008080689236521721,\n",
       " 0.008829289115965366,\n",
       " 0.01789645105600357,\n",
       " 0.008654383011162281,\n",
       " 0.05328357219696045,\n",
       " 0.015909511595964432,\n",
       " -0.006296640727669001,\n",
       " 0.023563427850604057,\n",
       " -0.014790109358727932,\n",
       " -0.011480874381959438,\n",
       " -0.012026583775877953,\n",
       " -0.00533465389162302,\n",
       " 0.007611938752233982,\n",
       " 0.0027372895274311304,\n",
       " 0.03414178639650345,\n",
       " 0.003132578684017062,\n",
       " 0.01418143417686224,\n",
       " 0.031119396910071373,\n",
       " 0.007192162796854973,\n",
       " 0.010459419339895248,\n",
       " 0.016623131930828094,\n",
       " -0.001665986143052578,\n",
       " 0.007548972498625517,\n",
       " -0.03909514099359512,\n",
       " -0.0007599695236422122,\n",
       " -0.01805037073791027,\n",
       " 0.0017097127856686711,\n",
       " -0.022248130291700363,\n",
       " 0.024850741028785706,\n",
       " -0.002891207579523325,\n",
       " 0.02721548080444336,\n",
       " -0.03959887474775314,\n",
       " 0.013222944922745228,\n",
       " -0.0014176187105476856,\n",
       " 0.017770519480109215,\n",
       " 0.01819029450416565,\n",
       " 0.005883860867470503,\n",
       " -0.007118701934814453,\n",
       " -0.011984606273472309,\n",
       " -0.019617533311247826,\n",
       " 0.022919772192835808,\n",
       " 0.015195893123745918,\n",
       " -0.019071824848651886,\n",
       " -0.006394588388502598,\n",
       " -0.023003727197647095,\n",
       " 0.026194024831056595,\n",
       " 0.002167093800380826,\n",
       " -0.00817164033651352,\n",
       " 0.003032881999388337,\n",
       " -0.017756527289748192,\n",
       " -0.027187494561076164,\n",
       " 0.029132457450032234,\n",
       " -0.003893422894179821,\n",
       " -0.013180967420339584,\n",
       " 0.003303987206891179,\n",
       " 0.005509560462087393,\n",
       " -0.000317892903694883,\n",
       " -0.0268096961081028,\n",
       " -0.005929336417466402,\n",
       " 0.019281713292002678,\n",
       " -0.012054569087922573,\n",
       " -0.010116602294147015,\n",
       " 0.020904846489429474,\n",
       " 0.014097479172050953,\n",
       " 0.010550371371209621,\n",
       " -0.03201492130756378,\n",
       " 0.006545008160173893,\n",
       " -0.05115670710802078,\n",
       " 0.012208486907184124,\n",
       " 0.0008487346931360662,\n",
       " 0.01630130223929882,\n",
       " -0.012481341138482094,\n",
       " -0.011774717830121517,\n",
       " -0.01595149002969265,\n",
       " -0.007765856571495533,\n",
       " -0.028376860544085503,\n",
       " 0.010102610103785992,\n",
       " 0.018833952024579048,\n",
       " 0.010809233412146568,\n",
       " -0.004579056985676289,\n",
       " -0.006950791459530592,\n",
       " 0.03579290211200714,\n",
       " 0.06710819900035858,\n",
       " 0.00979477446526289,\n",
       " 0.01820428855717182,\n",
       " 0.013278915546834469,\n",
       " -0.013090016320347786,\n",
       " -0.017308766022324562,\n",
       " -0.02980409935116768,\n",
       " -0.04365670680999756,\n",
       " 0.0098927216604352,\n",
       " -0.007779849227517843,\n",
       " -0.004911379888653755,\n",
       " -0.004379663150757551,\n",
       " -0.003412429476156831,\n",
       " -0.0050722938030958176,\n",
       " 0.0014902049442753196,\n",
       " 0.02517257072031498,\n",
       " 0.012705221772193909,\n",
       " -0.033274248242378235,\n",
       " 0.011103075928986073,\n",
       " -0.03565298020839691,\n",
       " -0.014167441055178642,\n",
       " 0.011872665956616402,\n",
       " 0.034421633929014206,\n",
       " 0.003354710293933749,\n",
       " -0.004278217442333698,\n",
       " -0.008948225528001785,\n",
       " 0.019673503935337067,\n",
       " 0.007087219040840864,\n",
       " -0.006768888793885708,\n",
       " -0.015167907811701298,\n",
       " -0.006625465117394924,\n",
       " -0.01288712490350008,\n",
       " 0.009605875238776207,\n",
       " 0.011459886096417904,\n",
       " 0.006754896137863398,\n",
       " 0.010347479023039341,\n",
       " 0.023157645016908646,\n",
       " 0.0022702887654304504,\n",
       " -0.0023402513470500708,\n",
       " 0.0035226205363869667,\n",
       " 0.02896454744040966,\n",
       " 0.02422107756137848,\n",
       " 0.011298972181975842,\n",
       " -0.01645522005856037,\n",
       " 0.02980409935116768,\n",
       " -0.021562496200203896,\n",
       " 0.007220148108899593,\n",
       " 0.017056899145245552,\n",
       " -0.017378728836774826,\n",
       " 0.013803635723888874,\n",
       " 0.008703356608748436,\n",
       " 0.010158579796552658,\n",
       " -0.03285447135567665,\n",
       " -0.008430502377450466,\n",
       " -0.004722480662167072,\n",
       " 0.010172572918236256,\n",
       " 0.0006043026223778725,\n",
       " -0.034421633929014206,\n",
       " -0.00576842250302434,\n",
       " -0.010235539637506008,\n",
       " -0.018064362928271294,\n",
       " -0.022094212472438812,\n",
       " 0.0010065879905596375,\n",
       " 0.0003091475518886,\n",
       " 0.004684001207351685,\n",
       " -0.01869402639567852,\n",
       " -0.001558418502099812,\n",
       " -0.0013581503881141543,\n",
       " 0.012495333328843117,\n",
       " 0.01600746065378189,\n",
       " -0.010655315592885017,\n",
       " -0.03654849901795387,\n",
       " -0.043488796800374985,\n",
       " 0.004862405825406313,\n",
       " 0.015069959685206413,\n",
       " -0.014230407774448395,\n",
       " 0.002795008709654212,\n",
       " -0.024109138175845146,\n",
       " -0.007947759702801704,\n",
       " 0.019015854224562645,\n",
       " -0.0012488337233662605,\n",
       " -0.008780315518379211,\n",
       " 0.008311565965414047,\n",
       " -0.026753727346658707,\n",
       " 0.0011753729777410626,\n",
       " 0.005051305051892996,\n",
       " -0.011669773608446121,\n",
       " 0.003840950783342123,\n",
       " -0.0039528911001980305,\n",
       " 0.028236934915184975,\n",
       " 0.005275185685604811,\n",
       " 0.0006130479159764946,\n",
       " 0.045139916241168976,\n",
       " -0.005170241463929415,\n",
       " 0.013817627914249897,\n",
       " 0.015237870626151562,\n",
       " 0.004250232595950365,\n",
       " 0.02791510708630085,\n",
       " 0.00759095000103116,\n",
       " -0.015363803133368492,\n",
       " -0.010529382154345512,\n",
       " -0.0354011133313179,\n",
       " -0.005488571710884571,\n",
       " -0.01388059463351965,\n",
       " 0.01649719849228859,\n",
       " 0.009759793058037758,\n",
       " 0.012915109284222126,\n",
       " 0.008661379106342793,\n",
       " 0.009626863524317741,\n",
       " 0.0008557309629395604,\n",
       " -0.004446127917617559,\n",
       " -0.02173040620982647,\n",
       " -0.017140856012701988,\n",
       " 0.0051947287283837795,\n",
       " 0.009207087568938732,\n",
       " 0.0151539146900177,\n",
       " -0.016735071316361427,\n",
       " 0.009130128659307957,\n",
       " -0.013600743375718594,\n",
       " 0.009025185368955135,\n",
       " -0.010956155136227608,\n",
       " 0.00023787308600731194,\n",
       " 0.012607273645699024,\n",
       " 0.009661844931542873,\n",
       " -0.03999066352844238,\n",
       " 0.016777049750089645,\n",
       " 0.005138758569955826,\n",
       " -0.028992531821131706,\n",
       " -0.01223647128790617,\n",
       " -0.0027722709346562624,\n",
       " -0.031483203172683716,\n",
       " 0.01854010857641697,\n",
       " 0.012278448790311813,\n",
       " -0.015405780635774136,\n",
       " -0.0014289875980466604,\n",
       " -0.0239272341132164,\n",
       " -0.001997434301301837,\n",
       " -0.023969212546944618,\n",
       " -0.018470145761966705,\n",
       " -0.028880592435598373,\n",
       " -0.013831621035933495,\n",
       " 0.013831621035933495,\n",
       " 0.008416509255766869,\n",
       " -0.011648785322904587,\n",
       " 0.006104243453592062,\n",
       " -0.02443096600472927,\n",
       " -0.009878729470074177,\n",
       " -0.0175466388463974,\n",
       " -0.013817627914249897,\n",
       " 0.00745102483779192,\n",
       " -0.013642721809446812,\n",
       " 0.016469214111566544,\n",
       " 0.009039177559316158,\n",
       " -0.019029846414923668,\n",
       " -0.011697758920490742,\n",
       " -0.053815290331840515,\n",
       " -0.01704290695488453,\n",
       " -0.015195893123745918,\n",
       " 0.013992534950375557,\n",
       " 0.012334419414401054,\n",
       " 0.009815762750804424,\n",
       " 0.004921874031424522,\n",
       " 0.012173505499958992,\n",
       " 0.04735073819756508,\n",
       " -0.0094939349219203,\n",
       " 0.004656015895307064,\n",
       " -0.01285913959145546,\n",
       " 0.0013642720878124237,\n",
       " -0.027705218642950058,\n",
       " 0.012292441911995411,\n",
       " 0.04298506677150726,\n",
       " 0.006226677913218737,\n",
       " 0.008017722517251968,\n",
       " -0.0007446652161888778,\n",
       " 0.014230407774448395,\n",
       " -0.01974346674978733,\n",
       " -0.002901701955124736,\n",
       " -0.020065294578671455,\n",
       " 0.016371265053749084,\n",
       " -0.04511193186044693,\n",
       " -0.016371265053749084,\n",
       " 0.017686564475297928,\n",
       " -0.0005352144362404943,\n",
       " 0.007527983747422695,\n",
       " -0.009885725565254688,\n",
       " -0.0027477839030325413,\n",
       " 0.000832993071526289,\n",
       " 0.01874999701976776,\n",
       " 0.016511190682649612,\n",
       " 0.011319960467517376,\n",
       " 0.008493468165397644,\n",
       " 0.005103777162730694,\n",
       " 0.0593283474445343,\n",
       " 0.0025833717081695795,\n",
       " 0.017028914764523506,\n",
       " 0.014916041865944862,\n",
       " -0.011403915472328663,\n",
       " -0.0202471986413002,\n",
       " -0.015167907811701298,\n",
       " 0.010480408556759357,\n",
       " 0.0008915868238545954,\n",
       " 0.018064362928271294,\n",
       " 0.01635727286338806,\n",
       " -0.007062731776386499,\n",
       " -0.028180964291095734,\n",
       " 0.013097012415528297,\n",
       " -0.02980409935116768,\n",
       " -0.005743935704231262,\n",
       " -0.011578822508454323,\n",
       " -0.008185632526874542,\n",
       " -0.006688431836664677,\n",
       " -0.022555965930223465,\n",
       " -0.028530778363347054,\n",
       " -0.0010809233644977212,\n",
       " -0.004565064329653978,\n",
       " 0.03783581405878067,\n",
       " -0.008458487689495087,\n",
       " 0.02478077821433544,\n",
       " -0.022346077486872673,\n",
       " 0.0030678631737828255,\n",
       " 0.005257694981992245,\n",
       " 0.0015540459426119924,\n",
       " 0.01241137832403183,\n",
       " 0.01096315123140812,\n",
       " 0.024263055995106697,\n",
       " 0.0369962602853775,\n",
       " 0.008262591436505318,\n",
       " 0.0043586743995547295,\n",
       " -0.0025973643641918898,\n",
       " -0.003669542260468006,\n",
       " -0.02203824184834957,\n",
       " 0.004264224786311388,\n",
       " 0.02178637683391571,\n",
       " 0.013229941949248314,\n",
       " -0.02612406201660633,\n",
       " 0.007479009684175253,\n",
       " 0.01416044495999813,\n",
       " 0.013719680719077587,\n",
       " -0.00340193510055542,\n",
       " 0.039906710386276245,\n",
       " 0.0012505827471613884,\n",
       " -0.01458022091537714,\n",
       " -0.006863338407129049,\n",
       " -0.014650183729827404,\n",
       " -0.02203824184834957,\n",
       " 0.012670240364968777,\n",
       " -0.011942628771066666,\n",
       " 0.013362870551645756,\n",
       " -0.018358206376433372,\n",
       " 0.0073530771769583225,\n",
       " 0.003547107568010688,\n",
       " 0.010851210914552212,\n",
       " -0.015783580020070076,\n",
       " 0.012691228650510311,\n",
       " -0.018232272937893867,\n",
       " -0.010599344968795776,\n",
       " 0.0022265620063990355,\n",
       " -0.018316227942705154,\n",
       " -8.756265742704272e-05,\n",
       " 0.01704290695488453,\n",
       " -0.0021076255943626165,\n",
       " 0.014524251222610474,\n",
       " -0.014090482145547867,\n",
       " 0.02328357845544815,\n",
       " 0.014594214037060738,\n",
       " 0.01719682477414608,\n",
       " -0.001369519392028451,\n",
       " -0.017462683841586113,\n",
       " 0.014440296217799187,\n",
       " 0.032378725707530975,\n",
       " -0.010410445742309093,\n",
       " -0.008724345825612545,\n",
       " 0.0011578822741284966,\n",
       " -0.007150185294449329,\n",
       " 0.028194958344101906,\n",
       " -0.019561562687158585,\n",
       " 0.005327657796442509,\n",
       " -0.001756063080392778,\n",
       " -0.0032899947836995125,\n",
       " -0.02112872712314129,\n",
       " -0.004488105420023203,\n",
       " 0.010263524018228054,\n",
       " -0.018274249508976936,\n",
       " 0.007730875629931688,\n",
       " -0.02636193484067917,\n",
       " 0.002992653287947178,\n",
       " -0.002039412036538124,\n",
       " 0.01126399077475071,\n",
       " 0.016413243487477303,\n",
       " -0.013299903832376003,\n",
       " -0.029972009360790253,\n",
       " -0.018372198566794395,\n",
       " 0.03489738330245018,\n",
       " -0.001808535074815154,\n",
       " 0.0034334182273596525,\n",
       " 0.024095144122838974,\n",
       " 0.018763989210128784,\n",
       " 0.028250927105545998,\n",
       " -0.008381528779864311,\n",
       " -0.005079289898276329,\n",
       " -0.00018474517855793238,\n",
       " 0.038759320974349976,\n",
       " -0.000832993071526289,\n",
       " 0.031063426285982132,\n",
       " 0.0016213849885389209,\n",
       " -0.0006397212273441255,\n",
       " 0.028936561197042465,\n",
       " -0.00815065111964941,\n",
       " -0.011103075928986073,\n",
       " 0.021954286843538284,\n",
       " 0.0009392488864250481,\n",
       " -0.0020569025073200464,\n",
       " -0.02548040635883808,\n",
       " 0.007919774390757084,\n",
       " 0.012089550495147705,\n",
       " 0.005586519371718168,\n",
       " 0.03044775500893593,\n",
       " -0.025186562910676003,\n",
       " -0.016329288482666016,\n",
       " -0.0011744983494281769,\n",
       " -0.012257460504770279,\n",
       " 0.016777049750089645,\n",
       " 0.022709883749485016,\n",
       " -0.024752793833613396,\n",
       " -0.006674439180642366,\n",
       " 0.0029681664891541004,\n",
       " -0.008717348799109459,\n",
       " -0.012754195369780064,\n",
       " -0.023157645016908646,\n",
       " -0.023549435660243034,\n",
       " 0.00365205155685544,\n",
       " 0.007513991091400385,\n",
       " -0.023297570645809174,\n",
       " -0.046035438776016235,\n",
       " -0.032182831317186356,\n",
       " -0.006954289972782135,\n",
       " -0.0034544069785624743,\n",
       " -0.002936683129519224,\n",
       " 0.01840018294751644,\n",
       " -0.00590834766626358,\n",
       " 0.010991135612130165,\n",
       " -0.006912312004715204,\n",
       " -0.0018784977728500962,\n",
       " 0.03629663586616516,\n",
       " -0.034813426434993744,\n",
       " -0.010991135612130165,\n",
       " -0.0245708916336298,\n",
       " -0.02736939862370491,\n",
       " -0.01589551940560341,\n",
       " -0.002702308353036642,\n",
       " -0.02991603873670101,\n",
       " 0.041753724217414856,\n",
       " 0.020806899294257164,\n",
       " -0.02587219700217247,\n",
       " -0.01989738456904888,\n",
       " -0.02466883882880211,\n",
       " -0.03016790561378002,\n",
       " -0.022150183096528053,\n",
       " -0.01963152550160885,\n",
       " 0.009039177559316158,\n",
       " 0.01442630309611559,\n",
       " 0.010851210914552212,\n",
       " -0.0051947287283837795,\n",
       " 0.010305501520633698,\n",
       " -0.01502798218280077,\n",
       " 0.027887122705578804,\n",
       " -0.01974346674978733,\n",
       " -0.0009506178321316838,\n",
       " -0.013817627914249897,\n",
       " -0.010837217792868614,\n",
       " -0.006492536049336195,\n",
       " -0.006286146119236946,\n",
       " -0.012068561278283596,\n",
       " 0.011124065145850182,\n",
       " 0.017336750403046608,\n",
       " 0.00460004573687911,\n",
       " -0.0019274717196822166,\n",
       " 0.01918376423418522,\n",
       " 0.006286146119236946,\n",
       " -0.0019047338282689452,\n",
       " 0.013810631819069386,\n",
       " 0.012313430197536945,\n",
       " 0.0504290945827961,\n",
       " 0.023955220356583595,\n",
       " 0.026445889845490456,\n",
       " -0.04161379858851433,\n",
       " -0.028936561197042465,\n",
       " 0.024458950385451317,\n",
       " 0.005488571710884571,\n",
       " -0.016413243487477303,\n",
       " 0.004001865163445473,\n",
       " 0.026445889845490456,\n",
       " -0.007020754273980856,\n",
       " -0.005848879460245371,\n",
       " -0.011753729544579983,\n",
       " 0.00910214427858591,\n",
       " 0.02147854119539261,\n",
       " -0.01988339237868786,\n",
       " 0.005359140690416098,\n",
       " -0.0202471986413002,\n",
       " 0.000102976307971403,\n",
       " 0.005205222871154547,\n",
       " 0.018931899219751358,\n",
       " 0.01635727286338806,\n",
       " -0.03405782952904701,\n",
       " 0.010711285285651684,\n",
       " -0.007129196543246508,\n",
       " -0.027439361438155174,\n",
       " 0.01665111631155014,\n",
       " -0.008332554250955582,\n",
       " -0.034225739538669586,\n",
       " 0.008038710802793503,\n",
       " 0.01091417670249939,\n",
       " -0.006531015504151583,\n",
       " -0.025844212621450424,\n",
       " -0.01091417670249939,\n",
       " 0.009976677596569061,\n",
       " 0.008549438789486885,\n",
       " 0.02007928676903248,\n",
       " 0.024934696033596992,\n",
       " 0.0011753729777410626,\n",
       " 0.00017064333951566368,\n",
       " -0.005712452344596386,\n",
       " 0.009570893831551075,\n",
       " -0.020624997094273567,\n",
       " -0.018176302313804626,\n",
       " -0.0006650826544500887,\n",
       " -0.015867535024881363,\n",
       " 0.0023769817780703306,\n",
       " 0.02795708365738392,\n",
       " -0.022807830944657326,\n",
       " -0.019071824848651886,\n",
       " 0.008157648146152496,\n",
       " 0.04900185763835907,\n",
       " -0.007863804697990417,\n",
       " 0.03968282788991928,\n",
       " 0.21100743114948273,\n",
       " -0.00016003962082322687,\n",
       " 0.0224860031157732,\n",
       " 0.025410443544387817,\n",
       " 0.013111005537211895,\n",
       " 6.498876609839499e-05,\n",
       " 0.03131529316306114,\n",
       " 0.00787779688835144,\n",
       " -0.00802471861243248,\n",
       " 0.017826490104198456,\n",
       " 0.01809234730899334,\n",
       " 0.0037639918737113476,\n",
       " -0.025802234187722206,\n",
       " 6.214652967173606e-05,\n",
       " 0.019351676106452942,\n",
       " -0.020722944289445877,\n",
       " -0.026543838903307915,\n",
       " -0.008885259740054607,\n",
       " -0.011173038743436337,\n",
       " 0.01784048229455948,\n",
       " 0.017028914764523506,\n",
       " -0.004774952307343483,\n",
       " -0.01274020317941904,\n",
       " -0.03151118755340576,\n",
       " 0.01999533176422119,\n",
       " 0.0277611892670393,\n",
       " 0.0013187964214012027,\n",
       " 0.02203824184834957,\n",
       " 0.03358208388090134,\n",
       " 0.012348411604762077,\n",
       " 0.014440296217799187,\n",
       " 0.01595149002969265,\n",
       " 0.010053636506199837,\n",
       " 0.020303167402744293,\n",
       " 0.0006034280522726476,\n",
       " 0.016063429415225983,\n",
       " 0.019323689863085747,\n",
       " 0.025648316368460655,\n",
       " 0.0009532414260320365,\n",
       " 0.025452420115470886,\n",
       " 0.004530082922428846,\n",
       " 0.002471431391313672,\n",
       " 0.005985306575894356,\n",
       " -0.021898316219449043,\n",
       " -0.021772384643554688,\n",
       " -0.009039177559316158,\n",
       " ...]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response.data[0].embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1f4c0555",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1536"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(response.data[0].embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "153feb06",
   "metadata": {},
   "source": [
    "And the similarity between vectors represents the similarity between the texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ca3e2cb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "emb1 = client.embeddings.create(\n",
    "    model=\"text-embedding-ada-002\",\n",
    "    input=\"love\",\n",
    "    ).data[0].embedding\n",
    "\n",
    "emb2 = client.embeddings.create(\n",
    "    model=\"text-embedding-ada-002\",\n",
    "    input=\"kindness\",\n",
    "    ).data[0].embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78361f14",
   "metadata": {},
   "source": [
    "Cosine similarity is a quick and dirty way to measure how similar two vectors are"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "af0fb087",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.16235277591731323"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarity(emb1, emb2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "332cc006",
   "metadata": {},
   "source": [
    "Naturally, unrelated words have much lower degree of similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "15ff3576",
   "metadata": {},
   "outputs": [],
   "source": [
    "emb3 = client.embeddings.create(\n",
    "    model=\"text-embedding-ada-002\",\n",
    "    input=\"airplane\",\n",
    "    ).data[0].embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e137cf2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22998445414768232"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarity(emb2, emb3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a586d63f",
   "metadata": {},
   "source": [
    "# Generate embeddings for the Reuters sentences dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b55850eb",
   "metadata": {},
   "source": [
    "We're going to use Reuters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5118c027",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = [\" \".join(word for word in sent) for sent in reuters.sents()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "70916912",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54716"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d39ec117",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"ASIAN EXPORTERS FEAR DAMAGE FROM U . S .- JAPAN RIFT Mounting trade friction between the U . S . And Japan has raised fears among many of Asia ' s exporting nations that the row could inflict far - reaching economic damage , businessmen and officials said .\",\n",
       " 'They told Reuter correspondents in Asian capitals a U . S . Move against Japan might boost protectionist sentiment in the U . S . And lead to curbs on American imports of their products .',\n",
       " \"But some exporters said that while the conflict would hurt them in the long - run , in the short - term Tokyo ' s loss might be their gain .\",\n",
       " \"The U . S . Has said it will impose 300 mln dlrs of tariffs on imports of Japanese electronics goods on April 17 , in retaliation for Japan ' s alleged failure to stick to a pact not to sell semiconductors on world markets at below cost .\",\n",
       " 'Unofficial Japanese estimates put the impact of the tariffs at 10 billion dlrs and spokesmen for major electronics firms said they would virtually halt exports of products hit by the new taxes .',\n",
       " '\" We wouldn \\' t be able to do business ,\" said a spokesman for leading Japanese electronics firm Matsushita Electric Industrial Co Ltd & lt ; MC . T >.',\n",
       " '\" If the tariffs remain in place for any length of time beyond a few months it will mean the complete erosion of exports ( of goods subject to tariffs ) to the U . S .,\" said Tom Murtha , a stock analyst at the Tokyo office of broker & lt ; James Capel and Co >.',\n",
       " 'In Taiwan , businessmen and officials are also worried .',\n",
       " '\" We are aware of the seriousness of the U . S .',\n",
       " 'Threat against Japan because it serves as a warning to us ,\" said a senior Taiwanese trade official who asked not to be named .']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74f2bda7",
   "metadata": {},
   "source": [
    "Let us build a DataFrame of 1,000 sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "867e3930",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame(sentences[:1000], columns=[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "42a1e231",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ASIAN EXPORTERS FEAR DAMAGE FROM U . S .- JAPA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>They told Reuter correspondents in Asian capit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>But some exporters said that while the conflic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The U . S . Has said it will impose 300 mln dl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Unofficial Japanese estimates put the impact o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>The decision to cut deposits was taken by the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>The cuts were likely to attract more business ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>ELECTRO RENT CORP &amp; lt ; ELRC &gt; 3RD QTR FEB 28...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>WALGREEN CO 2ND QTR SHR 62 CTS VS 58 CTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>WALGREEN CO 2ND QTR SHR 62 CTS VS 58 CTS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text\n",
       "0    ASIAN EXPORTERS FEAR DAMAGE FROM U . S .- JAPA...\n",
       "1    They told Reuter correspondents in Asian capit...\n",
       "2    But some exporters said that while the conflic...\n",
       "3    The U . S . Has said it will impose 300 mln dl...\n",
       "4    Unofficial Japanese estimates put the impact o...\n",
       "..                                                 ...\n",
       "995  The decision to cut deposits was taken by the ...\n",
       "996  The cuts were likely to attract more business ...\n",
       "997  ELECTRO RENT CORP & lt ; ELRC > 3RD QTR FEB 28...\n",
       "998           WALGREEN CO 2ND QTR SHR 62 CTS VS 58 CTS\n",
       "999           WALGREEN CO 2ND QTR SHR 62 CTS VS 58 CTS\n",
       "\n",
       "[1000 rows x 1 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "361d3b38",
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_model = \"text-embedding-ada-002\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5f598335",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoding = tiktoken.encoding_for_model(embedding_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c79c8392",
   "metadata": {},
   "source": [
    "# Compute the embeddings for each sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "adf66f5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_embedding(text, engine):\n",
    "    text = text.replace(\"\\n\", \" \")\n",
    "    \n",
    "    return client.embeddings.create(input=[text], model=engine).data[0].embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "26c72a36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c37e1e0eae904a17ae5a53da3b17d935",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data[\"embedding\"] = data.text.progress_apply(lambda x: get_embedding(x, engine=embedding_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "04edafe2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>embedding</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ASIAN EXPORTERS FEAR DAMAGE FROM U . S .- JAPA...</td>\n",
       "      <td>[-0.016284245997667313, -0.028311172500252724,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>They told Reuter correspondents in Asian capit...</td>\n",
       "      <td>[-0.021919183433055878, -0.016359390690922737,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>But some exporters said that while the conflic...</td>\n",
       "      <td>[-0.020123226568102837, -0.01861906610429287, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The U . S . Has said it will impose 300 mln dl...</td>\n",
       "      <td>[-0.0199994258582592, -0.016706187278032303, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Unofficial Japanese estimates put the impact o...</td>\n",
       "      <td>[-0.021230917423963547, -0.03368213027715683, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>The decision to cut deposits was taken by the ...</td>\n",
       "      <td>[0.003791265655308962, -0.014937720261514187, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>The cuts were likely to attract more business ...</td>\n",
       "      <td>[-0.020329460501670837, -0.02050635777413845, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>ELECTRO RENT CORP &amp; lt ; ELRC &gt; 3RD QTR FEB 28...</td>\n",
       "      <td>[0.0015153614804148674, -0.012284890748560429,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>WALGREEN CO 2ND QTR SHR 62 CTS VS 58 CTS</td>\n",
       "      <td>[0.0023807904217392206, 0.006467325147241354, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>WALGREEN CO 2ND QTR SHR 62 CTS VS 58 CTS</td>\n",
       "      <td>[0.0023807904217392206, 0.006467325147241354, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text  \\\n",
       "0    ASIAN EXPORTERS FEAR DAMAGE FROM U . S .- JAPA...   \n",
       "1    They told Reuter correspondents in Asian capit...   \n",
       "2    But some exporters said that while the conflic...   \n",
       "3    The U . S . Has said it will impose 300 mln dl...   \n",
       "4    Unofficial Japanese estimates put the impact o...   \n",
       "..                                                 ...   \n",
       "995  The decision to cut deposits was taken by the ...   \n",
       "996  The cuts were likely to attract more business ...   \n",
       "997  ELECTRO RENT CORP & lt ; ELRC > 3RD QTR FEB 28...   \n",
       "998           WALGREEN CO 2ND QTR SHR 62 CTS VS 58 CTS   \n",
       "999           WALGREEN CO 2ND QTR SHR 62 CTS VS 58 CTS   \n",
       "\n",
       "                                             embedding  \n",
       "0    [-0.016284245997667313, -0.028311172500252724,...  \n",
       "1    [-0.021919183433055878, -0.016359390690922737,...  \n",
       "2    [-0.020123226568102837, -0.01861906610429287, ...  \n",
       "3    [-0.0199994258582592, -0.016706187278032303, 0...  \n",
       "4    [-0.021230917423963547, -0.03368213027715683, ...  \n",
       "..                                                 ...  \n",
       "995  [0.003791265655308962, -0.014937720261514187, ...  \n",
       "996  [-0.020329460501670837, -0.02050635777413845, ...  \n",
       "997  [0.0015153614804148674, -0.012284890748560429,...  \n",
       "998  [0.0023807904217392206, 0.006467325147241354, ...  \n",
       "999  [0.0023807904217392206, 0.006467325147241354, ...  \n",
       "\n",
       "[1000 rows x 2 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fbd97eb",
   "metadata": {},
   "source": [
    "# Generate Recommendations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d78da744",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_recommendations(data, pos, k_nearest_neighbors=3):\n",
    "    embeddings = np.array(data[\"embedding\"].to_list())\n",
    "    strings = data[\"text\"].to_list()\n",
    "\n",
    "    query_embedding = embeddings[pos]\n",
    "\n",
    "    distances = [ cosine_similarity(query_embedding, embedding) \n",
    "                 for embedding in tqdm(embeddings, total=len(embeddings)) \n",
    "                ]\n",
    "    \n",
    "    indices_of_nearest_neighbors = np.argsort(distances)\n",
    "\n",
    "    query_string = strings[pos]\n",
    "    print(f\"Query: {query_string}\\n\\n\")\n",
    "\n",
    "    for pos, i in enumerate(indices_of_nearest_neighbors[1:k_nearest_neighbors+1]):\n",
    "        print(\"-- Nearest neighbor %u of %u (%u)---\\n%s\\n\\n\" % (pos, k_nearest_neighbors, i, strings[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "a7aaff07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\" For the time being , we will have to focus our policy more on the external side , and we can live with a more expansionary money supply .'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.iloc[456].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "967bf0bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8434ac24e7aa438890c0af9cf007c983",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query: \" For the time being , we will have to focus our policy more on the external side , and we can live with a more expansionary money supply .\n",
      "\n",
      "\n",
      "-- Nearest neighbor 0 of 5 (448)---\n",
      "Demands for greater stimulation of its domestic economy , accelerating tax cuts , cutting interest rates and tolerating above - target money supply growth .\n",
      "\n",
      "\n",
      "-- Nearest neighbor 1 of 5 (186)---\n",
      "He told a press conference the government planned to increase investment , speed up the opening of the local market to foreign imports and gradually adjust its currency to hold the surplus \" at a proper level .\"\n",
      "\n",
      "\n",
      "-- Nearest neighbor 2 of 5 (453)---\n",
      "Poehl said the Bundesbank had tolerated rapid money supply growth last year because the country enjoyed low inflation and because external factors , including low oil prices and favourable terms of trade , had given some extra leeway .\n",
      "\n",
      "\n",
      "-- Nearest neighbor 3 of 5 (192)---\n",
      "\" Our government projections of eight pct GNP growth , five billion dlrs of ( current account ) surplus and 12 pct growth in exports all seemed to be reasonable early this year .\n",
      "\n",
      "\n",
      "-- Nearest neighbor 4 of 5 (193)---\n",
      "But now the surplus is growing faster than we expected ,\" he said .\n",
      "\n",
      "\n",
      "CPU times: user 63.1 ms, sys: 3.3 ms, total: 66.4 ms\n",
      "Wall time: 66 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "get_recommendations(data, 456, 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "496b83d7",
   "metadata": {},
   "source": [
    "<center>\n",
    "     <img src=\"https://raw.githubusercontent.com/DataForScience/Networks/master/data/D4Sci_logo_full.png\" alt=\"Data For Science, Inc\" align=\"center\" border=\"0\" width=300px> \n",
    "</center>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}