vector_embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vector embeddings with OpenAI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup OpenAI API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import azure.identity\n",
    "import dotenv\n",
    "import openai\n",
    "\n",
    "# Set up OpenAI client based on environment variables\n",
    "dotenv.load_dotenv()\n",
    "AZURE_OPENAI_SERVICE = os.getenv(\"AZURE_OPENAI_SERVICE\")\n",
    "AZURE_OPENAI_ADA_DEPLOYMENT = os.getenv(\"AZURE_OPENAI_ADA_DEPLOYMENT\")\n",
    "\n",
    "azure_credential = azure.identity.DefaultAzureCredential()\n",
    "token_provider = azure.identity.get_bearer_token_provider(azure_credential,\n",
    "    \"https://cognitiveservices.azure.com/.default\")\n",
    "openai_client = openai.AzureOpenAI(\n",
    "    api_version=\"2024-06-01\",\n",
    "    azure_endpoint=f\"https://{AZURE_OPENAI_SERVICE}.openai.azure.com\",\n",
    "    azure_ad_token_provider=token_provider)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vector representations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = \"A dog just walked past my house and yipped yipped like a Martian\"\n",
    "\n",
    "response = openai_client.embeddings.create(model=AZURE_OPENAI_ADA_DEPLOYMENT, input=sentence)\n",
    "\n",
    "vector = response.data[0].embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(vector)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Document similarity modeled as cosine distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "def cosine_similarity(a, b):\n",
    "    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))\n",
    "\n",
    "sentences1 = ['The new movie is awesome',\n",
    "             'The new movie is awesome',\n",
    "             'The new movie is awesome']\n",
    "\n",
    "sentences2 = ['The new movie is awesome',\n",
    "              'This recent movie is so good',\n",
    "              'djkshsjdkhfsjdfkhsd']\n",
    "\n",
    "def get_embeddings(sentences):\n",
    "    embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_ADA_DEPLOYMENT, input=sentences)\n",
    "    return [embedding_object.embedding for embedding_object in embeddings_response.data]\n",
    "\n",
    "embeddings1 = get_embeddings(sentences1)\n",
    "embeddings2 = get_embeddings(sentences2)\n",
    "\n",
    "for i in range(len(sentences1)):\n",
    "    print(f\"{sentences1[i]} \\t\\t {sentences2[i]} \\t\\t Score: {cosine_similarity(embeddings1[i], embeddings2[i]):.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vector search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "# Load in vectors for movie titles\n",
    "with open('openai_movies.json') as json_file:\n",
    "    movie_vectors = json.load(json_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Movie</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>101 Dalmatians</td>\n",
       "      <td>0.980079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335</th>\n",
       "      <td>102 Dalmatians</td>\n",
       "      <td>0.949210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>The Fox and the Hound</td>\n",
       "      <td>0.861836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Lady and the Tramp</td>\n",
       "      <td>0.851990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>The Great Mouse Detective</td>\n",
       "      <td>0.842004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>468</th>\n",
       "      <td>Beverly Hills Chihuahua</td>\n",
       "      <td>0.841186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>The Aristocats</td>\n",
       "      <td>0.840352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>558</th>\n",
       "      <td>The Good Dinosaur</td>\n",
       "      <td>0.838919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>Aladdin</td>\n",
       "      <td>0.837534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>391</th>\n",
       "      <td>Teacher's Pet: The Movie</td>\n",
       "      <td>0.837149</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Movie     Score\n",
       "8               101 Dalmatians  0.980079\n",
       "335             102 Dalmatians  0.949210\n",
       "28       The Fox and the Hound  0.861836\n",
       "6           Lady and the Tramp  0.851990\n",
       "48   The Great Mouse Detective  0.842004\n",
       "468    Beverly Hills Chihuahua  0.841186\n",
       "15              The Aristocats  0.840352\n",
       "558          The Good Dinosaur  0.838919\n",
       "135                    Aladdin  0.837534\n",
       "391   Teacher's Pet: The Movie  0.837149"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compute vector for query\n",
    "query = \"101 Dalmations\"\n",
    "\n",
    "embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_ADA_DEPLOYMENT, input=[query])\n",
    "vector = embeddings_response.data[0].embedding\n",
    "\n",
    "# Compute cosine similarity between query and each movie title\n",
    "scores = []\n",
    "for movie in movie_vectors:\n",
    "    scores.append((movie, cosine_similarity(vector, movie_vectors[movie])))\n",
    "\n",
    "# Display the top 10 results\n",
    "df = pd.DataFrame(scores, columns=['Movie', 'Score'])\n",
    "df = df.sort_values('Score', ascending=False)\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}