feat(agent): converse api (#380)

* Upgrade inference API to use AWS Bedrock Converse
* Update chat history parser to comply with converse API payload format
* Upgrade agent so it randomly select model to use in each new match
* Update infra notebooks with examples and queries

.env.example

@@ -31,5 +31,5 @@ LEADERBOARD_PROGRAM_NAME="leaderboard.aleo"
 
 AWS_ACCESS_KEY="<your_access_key>"
 AWS_SECRET="<your_secret>"
-AWS_INFERENCE_URL="https://3dhomhunle.execute-api.us-east-1.amazonaws.com/default/generate"
+AWS_INFERENCE_URL="https://3dhomhunle.execute-api.us-east-1.amazonaws.com/default/converse"
 LAMBDA_TOKEN="<your_token>"

.env.test

@@ -3,4 +3,4 @@ NEXTAUTH_URL="http://localhost:3000"
 NEXTAUTH_URL_INTERNAL="http://localhost:3000"
 APP_URL="http://localhost:3000"
 NEXT_PUBLIC_WS_URL="ws://localhost:3001"
-AWS_INFERENCE_URL="https://3dhomhunle.execute-api.us-east-1.amazonaws.com/default/generate"
+AWS_INFERENCE_URL="https://3dhomhunle.execute-api.us-east-1.amazonaws.com/default/converse"

deployment/local/deployment.yaml

@@ -43,5 +43,5 @@ spec:
             - name: ALEO_PRIVATE_KEY
               value: APrivateKey1zkp8CZNn3yeCseEtxuVPbDCwSyhGW6yZKUYKfgXmcpoGPWH # not a real account
             - name: AWS_INFERENCE_URL
-              value: https://3dhomhunle.execute-api.us-east-1.amazonaws.com/default/generate
+              value: https://3dhomhunle.execute-api.us-east-1.amazonaws.com/default/converse
 # missing secrets: AWS_ACCESS_KEY, AWS_SECRET, LAMBDA_TOKEN, OBSCURA_API_KEY (not mandatory for now)

deployment/staging/deployment.yaml

@@ -66,4 +66,4 @@ spec:
                   name: backend-secrets
                   key: lambda_token
             - name: AWS_INFERENCE_URL
-              value: https://3dhomhunle.execute-api.us-east-1.amazonaws.com/default/generate
+              value: https://3dhomhunle.execute-api.us-east-1.amazonaws.com/default/converse

infra/conversational-api.ipynb

@@ -0,0 +1,243 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import boto3\n",
+    "import logging\n",
+    "from botocore.exceptions import ClientError"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Configure logging\n",
+    "logger = logging.getLogger()\n",
+    "logger.setLevel(logging.INFO)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1.34.131\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(boto3.__version__)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create a Bedrock Runtime client in the AWS Region you want to use.\n",
+    "client = boto3.client(\n",
+    "    service_name='bedrock-runtime', \n",
+    "    region_name='us-east-1'\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "LLAMA_2_13B = \"meta.llama2-13b-chat-v1\"\n",
+    "LLAMA_2_70B = \"meta.llama2-70b-chat-v1\"\n",
+    "LLAMA_3_8B = \"meta.llama3-8b-instruct-v1:0\"\n",
+    "LLAMA_3_70B = \"meta.llama3-70b-instruct-v1:0\"\n",
+    "CLAUDE_3_5_SONNET = \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n",
+    "JAMBA_INSTRUCT = \"ai21.jamba-instruct-v1:0\"\n",
+    "CLAUDE_2_1 = \"anthropic.claude-v2:1\"\n",
+    "CLAUDE_3_HAIKU = \"anthropic.claude-3-haiku-20240307-v1:0\"\n",
+    "COHERE_COMMAND_R_PLUS = \"cohere.command-r-plus-v1:0\"\n",
+    "MISTRAL_LARGE = \"mistral.mistral-large-2402-v1:0\"\n",
+    "MISTRAL_SMALL = \"mistral.mistral-small-2402-v1:0\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "req_body = {\n",
+    "  \"modelId\": \"meta.llama3-8b-instruct-v1:0\",\n",
+    "  \"inferenceConfig\": {\n",
+    "    \"maxTokens\": 85,\n",
+    "    \"temperature\": 0.95,\n",
+    "    \"topP\": 0.8,\n",
+    "    \"stopSequences\": [\"<s>\", \"<s]\", \"\\\\*\\\\*\\\\*\"]\n",
+    "  },\n",
+    "  \"systemPrompt\": [\n",
+    "    {\n",
+    "      \"text\": \"Your name is Dot. You're participating in a chat group game called Bot Busters. Every player is trying to guess who is a human, and whom is a bot? You can call out other players and check if they are bots. Be discrete, you're a human. Give short replies.\"\n",
+    "    }\n",
+    "  ],\n",
+    "  \"messages\": [\n",
+    "    {\n",
+    "      \"role\": \"user\",\n",
+    "      \"content\": [\n",
+    "        {\n",
+    "          \"text\": \"Host said What's your favourite ice cream flavour and why?\"\n",
+    "        }\n",
+    "      ]\n",
+    "    },\n",
+    "    {\n",
+    "      \"role\": \"assistant\",\n",
+    "      \"content\": [\n",
+    "        {\n",
+    "          \"text\": \"I love vanilla!\"\n",
+    "        }\n",
+    "      ]\n",
+    "    },\n",
+    "    {\n",
+    "      \"role\": \"user\",\n",
+    "      \"content\": [\n",
+    "        {\n",
+    "          \"text\": \"Eve said I always chose chocolate\"\n",
+    "        }\n",
+    "      ]\n",
+    "    },\n",
+    "    {\n",
+    "      \"role\": \"assistant\",\n",
+    "      \"content\": [\n",
+    "        {\n",
+    "          \"text\": \"Interesting choices!\"\n",
+    "        }\n",
+    "      ]\n",
+    "    },\n",
+    "    {\n",
+    "      \"role\": \"user\",\n",
+    "      \"content\": [\n",
+    "        {\n",
+    "          \"text\": \"Ash said Why do you like that flavour Dot??\"\n",
+    "        },\n",
+    "        {\n",
+    "          \"text\": \"Eve said nobody likes vanilla.... \"\n",
+    "        }\n",
+    "      ]\n",
+    "    },\n",
+    "  ]\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def query_converse_endpoint(modelId, messages, inferenceConfig, systemPrompt):\n",
+    "    response = client.converse(\n",
+    "        modelId=modelId,\n",
+    "        messages=messages,\n",
+    "        inferenceConfig=inferenceConfig,\n",
+    "        system=systemPrompt\n",
+    "    )\n",
+    "    \n",
+    "    response_text = response[\"output\"][\"message\"][\"content\"][0][\"text\"].strip()\n",
+    "    return response_text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# List of model IDs\n",
+    "model_ids = [\n",
+    "    LLAMA_2_13B,\n",
+    "    LLAMA_2_70B,\n",
+    "    LLAMA_3_8B,\n",
+    "    LLAMA_3_70B,\n",
+    "    CLAUDE_3_5_SONNET,\n",
+    "    JAMBA_INSTRUCT,\n",
+    "    CLAUDE_2_1,\n",
+    "    CLAUDE_3_HAIKU,\n",
+    "    COHERE_COMMAND_R_PLUS,\n",
+    "    MISTRAL_LARGE,\n",
+    "    # MISTRAL_SMALL,\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "meta.llama2-13b-chat-v1 said: Pish posh! Vanilla is classic!\n",
+      "meta.llama2-70b-chat-v1 said: Dot, are you a bot? You're too quick to answer and you're trying to fit in too much.\n",
+      "meta.llama3-8b-instruct-v1:0 said: I think it's because it's classic and pairs well with other flavors! And, um, I think Ash might be trying a bit too hard to stand out?\n",
+      "meta.llama3-70b-instruct-v1:0 said: Ash, it's just a classic! Eve, really?\n",
+      "anthropic.claude-3-5-sonnet-20240620-v1:0 said: I like vanilla because it's simple and versatile. Everyone has different tastes, Eve - plenty of people enjoy vanilla!\n",
+      "ai21.jamba-instruct-v1:0 said: I find vanilla to be a classic and simple flavor that I've always enjoyed. And I think there are plenty of people who appreciate its subtle taste!\n",
+      "anthropic.claude-v2:1 said: I just think vanilla is a classic flavor. It's simple but delicious.\n",
+      "anthropic.claude-3-haiku-20240307-v1:0 said: I enjoy the classic, creamy taste of vanilla. It's a timeless flavor.\n",
+      "cohere.command-r-plus-v1:0 said: It's a classic! I think you're a bot, Eve.\n",
+      "mistral.mistral-large-2402-v1:0 said: Ash, I like vanilla because it's classic and versatile.\n",
+      "Eve, some people do like vanilla, it's a popular flavor!\n",
+      "mistral.mistral-small-2402-v1:0 said: Ash, I like vanilla because it's simple and versatile. Plus, it goes well with so many toppings!\n",
+      "\n",
+      "Eve, many people enjoy vanilla, so I'd say your statement is not entirely true. :)\n",
+      "\n",
+      "*whispers* (To self) I must be careful not to reveal I'm a human.\n"
+     ]
+    }
+   ],
+   "source": [
+    "for model_id in model_ids:\n",
+    "    try:\n",
+    "        response_text = query_converse_endpoint(model_id, req_body['messages'], req_body['inferenceConfig'], req_body['systemPrompt'])\n",
+    "        print(f\"{model_id} said: {response_text}\")\n",
+    "    except (ClientError, Exception) as e:\n",
+    "        print(f\"ERROR: Can't invoke '{model_id}'. Reason: {e}\")\n",
+    "        exit(1)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}