examples/multi_hop_rag.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext dotenv\n",
    "%dotenv\n",
    "OPENAI_API_KEY = %env OPENAI_API_KEY\n",
    "%env DSP_CACHEDIR=./cache/library\n",
    "%env DSP_NOTEBOOK_CACHEDIR=./cache/notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> This will use the repo's cache to not need to call LLMs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing DSPy and DSPy Inspector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dsp.utils import deduplicate\n",
    "import dspy\n",
    "from dspy.teleprompt import BootstrapFewShot\n",
    "from dspy.datasets import HotPotQA\n",
    "from dspy_inspector import inspect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configuring the DSPy defaults"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lm = dspy.OpenAI(model=\"gpt-3.5-turbo-instruct\", max_tokens=300)\n",
    "rm = dspy.ColBERTv2(url=\"http://20.102.90.50:2017/wiki17_abstracts\")\n",
    "dspy.configure(lm=lm, rm=rm)\n",
    "\n",
    "dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)\n",
    "trainset = [x.with_inputs('question') for x in dataset.train]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating a simple Multi-Hop RAG program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_context_and_answer_and_hops(example, pred, trace=None):\n",
    "    if not dspy.evaluate.answer_exact_match(example, pred): return False\n",
    "    if not dspy.evaluate.answer_passage_match(example, pred): return False\n",
    "\n",
    "    hops = [example.question] + [outputs.query for *_, outputs in trace if 'query' in outputs]\n",
    "\n",
    "    if max([len(h) for h in hops]) > 100: return False\n",
    "    if any(dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))): return False\n",
    "\n",
    "    return True\n",
    "\n",
    "optimizer = BootstrapFewShot(metric=validate_context_and_answer_and_hops)\n",
    "\n",
    "class GenerateAnswer(dspy.Signature):\n",
    "    \"\"\"Answer questions with short factoid answers.\"\"\"\n",
    "\n",
    "    context = dspy.InputField(desc=\"may contain relevant facts\")\n",
    "    question = dspy.InputField()\n",
    "    answer = dspy.OutputField(desc=\"often between 1 and 5 words\")\n",
    "\n",
    "class GenerateSearchQuery(dspy.Signature):\n",
    "    \"\"\"Write a simple search query that will help answer a complex question.\"\"\"\n",
    "\n",
    "    context = dspy.InputField(desc=\"may contain relevant facts\")\n",
    "    question = dspy.InputField()\n",
    "    query = dspy.OutputField()\n",
    "\n",
    "class MultiHopRAG(dspy.Module):\n",
    "    def __init__(self, passages_per_hop=3, max_hops=2):\n",
    "        super().__init__()\n",
    "\n",
    "        self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",
    "        self.retrieve = [dspy.Retrieve(k=passages_per_hop) for _ in range(max_hops)]\n",
    "        self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
    "        self.max_hops = max_hops\n",
    "    \n",
    "    def forward(self, question):\n",
    "        context = []\n",
    "\n",
    "        for hop in range(self.max_hops):\n",
    "            query = self.generate_query[hop](context=context, question=question).query\n",
    "            passages = self.retrieve[hop](query).passages\n",
    "            context = deduplicate(context + passages)\n",
    "\n",
    "        response = self.generate_answer(context=context, question=question)\n",
    "        return dspy.Prediction(context=context, answer=response.answer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inspecting the RAG program\n",
    "\n",
    "> It's the same as inspecting an instance of the RAG program."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inspect(MultiHopRAG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inspecting the uncompiled RAG program\n",
    "\n",
    "> The created graph is bound to the created program's instance. Any subsequent calls to the program will update the graph.\n",
    "\n",
    "Before any calls some information is empty, such as the parameters values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uncompiled_multiHopRAG = MultiHopRAG()\n",
    "inspect(uncompiled_multiHopRAG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After calling the program information is filled, check the parameters values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uncompiled_multiHopRAG(\"How many storeys are in the castle that David Gregory inherited?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uncompiled_multiHopRAG(\"When was the first FIFA World Cup held?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uncompiled_multiHopRAG(\"Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inspecting the compiled RAG program\n",
    "\n",
    "> Because we are creating a new instance the compilation won't affect the previous graph.\n",
    "\n",
    "After compilation more information is filled, check the predictors demos."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compiled_multiHopRAG = optimizer.compile(MultiHopRAG(), teacher=MultiHopRAG(passages_per_hop=2), trainset=trainset)\n",
    "inspect(compiled_multiHopRAG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compiled_multiHopRAG(\"How many storeys are in the castle that David Gregory inherited?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compiled_multiHopRAG(\"When was the first FIFA World Cup held?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compiled_multiHopRAG(\"Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dspy-inspector-8FAx0NxI-py3.11",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}