Convert_Hadiths_Chat_Completion_GH.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ol4plnaNAMph"
      },
      "outputs": [],
      "source": [
        "!pip install openai\n",
        "!pip install langchain"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "iv0wpOw2-DS7"
      },
      "outputs": [],
      "source": [
        "from os.path import join as pjoin\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-0x3hD0A-PVw"
      },
      "source": [
        "# Step 1: Loading file with cleaned hadiths. This file contains about 34k hadiths"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "w2rlKbxc-Gu_"
      },
      "outputs": [],
      "source": [
        "hadith_file = 'datasets/cleaned_hadiths.csv'\n",
        "hadith_df = pd.read_csv(pjoin(hadith_file))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fX74-tSI-kcB"
      },
      "source": [
        "# Step 2: Creating a question such that the hadith is the answer to a given question using LLMs. This step is needed for fine-tuning chat completion models"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "tqeKk0FY-tV6"
      },
      "outputs": [],
      "source": [
        "import openai\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "from urllib.parse import urlparse, urljoin\n",
        "import textwrap\n",
        "import ast\n",
        "import pandas as pd\n",
        "import datetime\n",
        "\n",
        "# Set the API key for OpenAI\n",
        "openai.api_key = 'YOUR KEY HERE'\n",
        "\n",
        "\n",
        "def get_prompt_for_hadith(sample_hadith):\n",
        "  prompt = f\"The following text is a hadith {sample_hadith}. I am trying to train a language model that requires the text data to be fed as a prompt and a completion. Can you write a question such that the hadith I shared will be seen as an appropriate completion? If the question has a pronoun or refers to the Prophet Muhammad (saw), make sure to add (saw) or (peace be upon him) next to the pronouns referring to the prophet Muhammad (peace be upon him)\"\n",
        "  try:\n",
        "    # Get the model's response\n",
        "    response = openai.Completion.create(\n",
        "        engine=\"text-davinci-003\",\n",
        "        prompt=prompt,\n",
        "        max_tokens=200\n",
        "    )\n",
        "    # Process the model's response and update the schema\n",
        "    # This assumes that the model's response is a dictionary that can be used to update your schema\n",
        "    # The exact method you use to process the response will depend on how the model responds\n",
        "    result = response.choices[0].text.strip()\n",
        "    return result\n",
        "  except:\n",
        "      print('Error')\n",
        "      return None\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xOSxf7Pr_RBY"
      },
      "source": [
        "# Step 3: Running hadiths against propmpt engine one by one. Can be expensive so also an option to do it on a subset. We provide a file with 10k hadiths"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RWSFJhiM-xXL"
      },
      "outputs": [],
      "source": [
        "N = 10000\n",
        "#N = len(hadith_df)\n",
        "first_N_hadith_df = hadith_df[:N]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6GRCHogm_h8_"
      },
      "outputs": [],
      "source": [
        "prompt_for_hadiths = []\n",
        "error_message = 'N/A'\n",
        "print('Start time for ',str(N),'requests is: ',str(datetime.datetime.now()))\n",
        "for counter in range(len(first_N_hadith_df)):\n",
        "  if counter %50 == 0:\n",
        "    print(counter)\n",
        "    print(datetime.datetime.now())\n",
        "  sample_hadith = first_N_hadith_df[counter]\n",
        "  prompt = get_prompt_for_hadith(sample_hadith)\n",
        "  if prompt:\n",
        "    prompt_for_hadiths.append(prompt)\n",
        "  else:\n",
        "    prompt_for_hadiths.append(error_message)\n",
        "print('End time for ',str(N),'requests is: ',str(datetime.datetime.now()))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0X7VIP5x_3AG"
      },
      "outputs": [],
      "source": [
        "file_name = str(N)+'_hadiths_with_prompts.csv'\n",
        "first_N_hadith_df.to_csv(file_name)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}