docs/notebooks/llama_cpp_grammar_api.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import pathlib\n",
    "import requests\n",
    "from typing import Literal, NotRequired, List, Union\n",
    "from pprint import pprint\n",
    "from llama_cpp import ChatCompletionMessage, Llama, LlamaGrammar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "llama_model_loader: loaded meta data with 19 key-value pairs and 291 tensors from ../../models/mistralai/Mistral-7B-Instruct-v0.1/ggml-model-f16.gguf (version GGUF V2 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:              blk.0.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:              blk.0.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:         blk.0.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:            blk.0.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:              blk.0.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:              blk.1.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:              blk.1.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:         blk.1.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:            blk.1.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:              blk.1.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:              blk.2.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:              blk.2.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:         blk.2.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:            blk.2.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:              blk.2.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:              blk.3.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:              blk.3.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:         blk.3.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:            blk.3.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:              blk.3.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:              blk.4.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:              blk.4.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:         blk.4.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:            blk.4.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:              blk.4.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:              blk.5.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:              blk.5.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:         blk.5.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:            blk.5.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:              blk.5.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:              blk.6.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:              blk.6.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:         blk.6.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:            blk.6.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:              blk.6.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:              blk.7.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:              blk.7.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:         blk.7.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:            blk.7.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:              blk.7.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:              blk.8.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:              blk.8.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:         blk.8.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:            blk.8.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:              blk.8.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:              blk.9.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:              blk.9.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:         blk.9.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:            blk.9.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:              blk.9.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:             blk.10.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:             blk.10.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:        blk.10.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:           blk.10.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:             blk.10.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:             blk.11.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:             blk.11.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:        blk.11.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:           blk.11.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:             blk.11.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:             blk.12.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:             blk.12.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:        blk.12.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:           blk.12.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:             blk.12.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:             blk.13.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:             blk.13.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:        blk.13.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:           blk.13.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:             blk.13.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:             blk.14.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:             blk.14.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:        blk.14.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:           blk.14.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:             blk.14.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:             blk.15.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:             blk.15.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:        blk.15.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:           blk.15.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:             blk.15.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:             blk.16.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:             blk.16.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:        blk.16.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:           blk.16.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:             blk.16.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:             blk.17.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:             blk.17.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:        blk.17.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:           blk.17.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:             blk.17.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:             blk.18.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:             blk.18.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:        blk.18.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:           blk.18.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:             blk.18.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:             blk.19.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:             blk.19.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:        blk.19.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:           blk.19.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:             blk.19.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:             blk.20.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:             blk.20.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:        blk.20.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:           blk.20.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:             blk.20.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:             blk.21.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:             blk.21.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:        blk.21.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:           blk.21.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:             blk.21.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:             blk.22.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:             blk.22.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:        blk.22.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:           blk.22.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:             blk.22.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:             blk.23.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:             blk.23.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:        blk.23.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:           blk.23.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:             blk.23.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:             blk.24.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:             blk.24.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:        blk.24.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:           blk.24.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:             blk.24.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:             blk.25.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:             blk.25.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:        blk.25.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:           blk.25.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:             blk.25.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:             blk.26.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:             blk.26.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:        blk.26.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:           blk.26.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:             blk.26.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:             blk.27.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:             blk.27.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:        blk.27.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:           blk.27.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:             blk.27.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:             blk.28.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:             blk.28.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:        blk.28.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:           blk.28.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:             blk.28.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:             blk.29.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:             blk.29.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:        blk.29.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:           blk.29.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:             blk.29.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:             blk.30.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:             blk.30.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:        blk.30.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:           blk.30.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:             blk.30.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:             blk.31.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:             blk.31.attn_k.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_v.weight f16      [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:        blk.31.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:           blk.31.ffn_gate.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:             blk.31.ffn_up.weight f16      [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_down.weight f16      [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:                    output.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - kv   0:                       general.architecture str     \n",
      "llama_model_loader: - kv   1:                               general.name str     \n",
      "llama_model_loader: - kv   2:                       llama.context_length u32     \n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32     \n",
      "llama_model_loader: - kv   4:                          llama.block_count u32     \n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32     \n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32     \n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32     \n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32     \n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32     \n",
      "llama_model_loader: - kv  10:                       llama.rope.freq_base f32     \n",
      "llama_model_loader: - kv  11:                          general.file_type u32     \n",
      "llama_model_loader: - kv  12:                       tokenizer.ggml.model str     \n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.tokens arr     \n",
      "llama_model_loader: - kv  14:                      tokenizer.ggml.scores arr     \n",
      "llama_model_loader: - kv  15:                  tokenizer.ggml.token_type arr     \n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32     \n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32     \n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32     \n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_print_meta: format           = GGUF V2 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 32768\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 4\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: n_ff             = 14336\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly F16\n",
      "llm_load_print_meta: model params     = 7.24 B\n",
      "llm_load_print_meta: model size       = 13.49 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name   = mistralai\n",
      "llm_load_print_meta: BOS token = 1 '<s>'\n",
      "llm_load_print_meta: EOS token = 2 '</s>'\n",
      "llm_load_print_meta: UNK token = 0 '<unk>'\n",
      "llm_load_print_meta: LF token  = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size =    0.09 MB\n",
      "llm_load_tensors: mem required  = 13813.11 MB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: kv self size  =   64.00 MB\n",
      "llama_new_context_with_model: compute buffer total size = 78.88 MB\n"
     ]
    }
   ],
   "source": [
    "MODEL_PATH = \"../../models/mistralai/Mistral-7B-Instruct-v0.1/ggml-model-f16.gguf\"\n",
    "llm = Llama(model_path=MODEL_PATH, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root ::= object \n",
      "object ::= [{] ws object_11 [}] ws \n",
      "value ::= object | array | string | number | value_6 ws \n",
      "array ::= [[] ws array_15 []] ws \n",
      "string ::= [\"] string_18 [\"] ws \n",
      "number ::= number_19 number_25 number_29 ws \n",
      "value_6 ::= [t] [r] [u] [e] | [f] [a] [l] [s] [e] | [n] [u] [l] [l] \n",
      "ws ::= ws_31 \n",
      "object_8 ::= string [:] ws value object_10 \n",
      "object_9 ::= [,] ws string [:] ws value \n",
      "object_10 ::= object_9 object_10 | \n",
      "object_11 ::= object_8 | \n",
      "array_12 ::= value array_14 \n",
      "array_13 ::= [,] ws value \n",
      "array_14 ::= array_13 array_14 | \n",
      "array_15 ::= array_12 | \n",
      "string_16 ::= [^\"\\] | [\\] string_17 \n",
      "string_17 ::= [\"\\/bfnrt] | [u] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] \n",
      "string_18 ::= string_16 string_18 | \n",
      "number_19 ::= number_20 number_21 \n",
      "number_20 ::= [-] | \n",
      "number_21 ::= [0-9] | [1-9] number_22 \n",
      "number_22 ::= [0-9] number_22 | \n",
      "number_23 ::= [.] number_24 \n",
      "number_24 ::= [0-9] number_24 | [0-9] \n",
      "number_25 ::= number_23 | \n",
      "number_26 ::= [eE] number_27 number_28 \n",
      "number_27 ::= [-+] | \n",
      "number_28 ::= [0-9] number_28 | [0-9] \n",
      "number_29 ::= number_26 | \n",
      "ws_30 ::= [ <U+0009><U+000A>] ws \n",
      "ws_31 ::= ws_30 | \n",
      "<llama_cpp.llama_grammar.LlamaGrammar object at 0x7f6c2b135810>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "from_string grammar:\n",
      "\n"
     ]
    }
   ],
   "source": [
    "llama_grammar = LlamaGrammar.from_file(\"json.gbnf\")\n",
    "print(llama_grammar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "FUNCTIONS = [\n",
    "    {\n",
    "        \"name\": \"get_current_weather\",\n",
    "        \"description\": \"Get the current weather in a given location\",\n",
    "        \"parameters\": {\n",
    "            \"type\": \"object\",\n",
    "            \"properties\": {\n",
    "                \"location\": {\n",
    "                    \"type\": \"string\",\n",
    "                    \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
    "                },\n",
    "                \"unit\": {\"type\": \"string\", \"enum\": [\"metric\", \"uscs\"]},\n",
    "            },\n",
    "            \"required\": [\"location\"],\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        \"name\": \"binary_arithmetic\",\n",
    "        \"description\": \"Perform binary arithmetic operations on two operands.\",\n",
    "        \"parameters\": {\n",
    "            \"type\": \"object\",\n",
    "            \"properties\": {\n",
    "                \"left_op\": {\n",
    "                    \"type\": [\"integer\", \"number\"],\n",
    "                    \"description\": \"The left operand.\",\n",
    "                },\n",
    "                \"right_op\": {\n",
    "                    \"type\": [\"integer\", \"number\"],\n",
    "                    \"description\": \"The right operand.\",\n",
    "                },\n",
    "                \"operator\": {\n",
    "                    \"type\": \"string\",\n",
    "                    \"description\": \"The arithmetic operator. Supported operators are '+', '-', '*', '/', '%'.\",\n",
    "                    \"enum\": [\"+\", \"-\", \"*\", \"/\", \"%\"],\n",
    "                },\n",
    "            },\n",
    "            \"required\": [\"left_op\", \"right_op\", \"operator\"],\n",
    "        },\n",
    "    },\n",
    "]\n",
    "\n",
    "\n",
    "def get_current_weather(location: str, unit: str = \"metric\") -> str:\n",
    "    \"\"\"\n",
    "    Get the current weather in a given location.\n",
    "    Parameters:\n",
    "    location (str): The city and state, e.g. San Francisco, CA\n",
    "    unit (str): The unit system, can be either 'metric' or 'uscs'. Default is 'metric'.\n",
    "    Returns:\n",
    "    str: A string that describes the current weather.\n",
    "    \"\"\"\n",
    "\n",
    "    # Replace spaces with hyphens and commas with underscores for the wttr.in URL\n",
    "    location = location.replace(\" \", \"-\").replace(\",\", \"_\")\n",
    "\n",
    "    # Determine the unit query parameter\n",
    "    unit_query = \"m\" if unit == \"metric\" else \"u\"\n",
    "    # Set the API response formatting\n",
    "    res_format = \"%l+%T+%S+%s+%C+%w+%t\"\n",
    "\n",
    "    # Make a request to the wttr.in service\n",
    "    response = requests.get(\n",
    "        f\"http://wttr.in/{location}?{unit_query}&format={res_format}\"\n",
    "    )\n",
    "\n",
    "    # Check if the request was successful\n",
    "    if response.status_code == 200:\n",
    "        # Return the weather report\n",
    "        return response.text\n",
    "    else:\n",
    "        return f\"Could not get the weather for {location}.\"\n",
    "\n",
    "\n",
    "def binary_arithmetic(\n",
    "    left_op: Union[int, float], right_op: Union[int, float], operator: str\n",
    ") -> Union[int, float]:\n",
    "    \"\"\"\n",
    "    Perform binary arithmetic operations on two operands.\n",
    "\n",
    "    Parameters:\n",
    "    - left_op (int/float): The left operand.\n",
    "    - right_op (int/float): The right operand.\n",
    "    - operator (str): The arithmetic operator. Supported operators are '+', '-', '*', '/', '%'.\n",
    "\n",
    "    Returns:\n",
    "    - int/float: The result of the arithmetic operation.\n",
    "    \"\"\"\n",
    "    if operator == \"+\":\n",
    "        return left_op + right_op\n",
    "    elif operator == \"-\":\n",
    "        return left_op - right_op\n",
    "    elif operator == \"*\":\n",
    "        return left_op * right_op\n",
    "    elif operator == \"/\":\n",
    "        if right_op == 0:\n",
    "            raise ValueError(\"Division by zero is not allowed.\")\n",
    "        return left_op / right_op\n",
    "    elif operator == \"%\":\n",
    "        return left_op % right_op\n",
    "    else:\n",
    "        raise ValueError(\n",
    "            f\"Unsupported operator '{operator}'. Supported operators are '+', '-', '*', '/', '%'.\"\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "function_map = {\n",
    "    \"get_current_weather\": get_current_weather,\n",
    "    \"binary_arithmetic\": binary_arithmetic,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "system_prompt = ChatCompletionMessage(\n",
    "    role=\"system\",\n",
    "    content=\"\"\"My name is Vincent and I am a helpful assistant. I can make function calls to retrieve information such as the current weather in a given location.\\n{ \"function_call\": { \"name\": \"get_current_weather\", \"arguments\": { \"location\": \"New York City, NY\" } } }\"\"\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_chat_sequence(\n",
    "    user_query: str,\n",
    "    function_def: dict,\n",
    ") -> List[ChatCompletionMessage]:\n",
    "    messages = [system_prompt]\n",
    "    user_message = ChatCompletionMessage(role=\"user\", content=user_query)\n",
    "\n",
    "    function_message = ChatCompletionMessage(\n",
    "        role=\"function\", content=json.dumps(function_def)\n",
    "    )\n",
    "\n",
    "    messages.extend([user_message, function_message])\n",
    "    return messages\n",
    "\n",
    "\n",
    "def generate_combined_chat_sequence(\n",
    "    user_query: str,\n",
    "    function_list: list,\n",
    ") -> List[ChatCompletionMessage]:\n",
    "    function_messages = [\n",
    "        ChatCompletionMessage(role=\"function\", content=json.dumps(func_def))\n",
    "        for func_def in function_list\n",
    "    ]\n",
    "\n",
    "    user_message = ChatCompletionMessage(role=\"user\", content=user_query)\n",
    "    messages = [system_prompt] + function_messages + [user_message]\n",
    "    return messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'content': 'My name is Vincent and I am a helpful assistant. I can make '\n",
      "            'function calls to retrieve information such as the current '\n",
      "            'weather in a given location.\\n'\n",
      "            '{ \"function_call\": { \"name\": \"get_current_weather\", \"arguments\": '\n",
      "            '{ \"location\": \"New York City, NY\" } } }',\n",
      " 'role': 'system'}\n",
      "{'content': 'What is the weather like in New York City, New York today?',\n",
      " 'role': 'user'}\n",
      "{'content': '{\"name\": \"get_current_weather\", \"description\": \"Get the current '\n",
      "            'weather in a given location\", \"parameters\": {\"type\": \"object\", '\n",
      "            '\"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The '\n",
      "            'city and state, e.g. San Francisco, CA\"}, \"unit\": {\"type\": '\n",
      "            '\"string\", \"enum\": [\"metric\", \"uscs\"]}}, \"required\": '\n",
      "            '[\"location\"]}}',\n",
      " 'role': 'function'}\n"
     ]
    }
   ],
   "source": [
    "messages = generate_chat_sequence(\"What is the weather like in New York City, New York today?\", FUNCTIONS[0])\n",
    "for message in messages:\n",
    "    pprint(message)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'choices': [{'finish_reason': 'stop',\n",
      "              'index': 0,\n",
      "              'message': {'content': '{ \"function_call\": { \"name\": '\n",
      "                                     '\"get_current_weather\", \"arguments\": { '\n",
      "                                     '\"location\": \"New York City, NY\" } } }',\n",
      "                          'role': 'assistant'}}],\n",
      " 'created': 1696386851,\n",
      " 'id': 'chatcmpl-3f90e960-2781-45fd-80a5-7ff50f6ba2f7',\n",
      " 'model': '../../models/mistralai/Mistral-7B-Instruct-v0.1/ggml-model-f16.gguf',\n",
      " 'object': 'chat.completion',\n",
      " 'usage': {'completion_tokens': 35, 'prompt_tokens': 96, 'total_tokens': 131}}\n"
     ]
    }
   ],
   "source": [
    "response = llm.create_chat_completion(messages=messages, grammar=llama_grammar, temperature=0)\n",
    "# @abetlen: This is where we would expect a \"function\" role to be returned instead of an \"assistant\" role\n",
    "# It doesn't do this just yet for obvious reasons.\n",
    "pprint(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('{ \"function_call\": { \"name\": \"get_current_weather\", \"arguments\": { '\n",
      " '\"location\": \"New York City, NY\" } } }')\n"
     ]
    }
   ],
   "source": [
    "# NOTE: The result varies from response to response, even with a temperature of 0.\n",
    "assistant_content = response[\"choices\"][0][\"message\"][\"content\"]\n",
    "pprint(assistant_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'function_call': {'arguments': {'location': 'New York City, NY'},\n",
      "                   'name': 'get_current_weather'}}\n"
     ]
    }
   ],
   "source": [
    "function_content = json.loads(assistant_content)\n",
    "pprint(function_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'name': 'get_current_weather', 'arguments': {'location': 'New York City, NY'}}\n"
     ]
    }
   ],
   "source": [
    "function_call = function_content[\"function_call\"]\n",
    "print(function_call)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<function get_current_weather at 0x7f6fc3787ec0>\n"
     ]
    }
   ],
   "source": [
    "callback = None\n",
    "for function in FUNCTIONS:\n",
    "    if function[\"name\"] == function_call[\"name\"]:\n",
    "        callback = function_map[function_call[\"name\"]]\n",
    "print(callback)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New-York-City_-NY 22:28:17-0400 06:55:38 18:33:15 Clear ↓4km/h +25°C\n"
     ]
    }
   ],
   "source": [
    "result = callback(function_call[\"arguments\"][\"location\"])\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'role': 'system', 'content': 'My name is Vincent and I am a helpful assistant. I can make function calls to retrieve information such as the current weather in a given location.\\n{ \"function_call\": { \"name\": \"get_current_weather\", \"arguments\": { \"location\": \"New York City, NY\" } } }'}\n",
      "{'role': 'user', 'content': 'What is the weather like in New York City, New York today?'}\n",
      "{'role': 'function', 'content': '{\"name\": \"get_current_weather\", \"description\": \"Get the current weather in a given location\", \"parameters\": {\"type\": \"object\", \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The city and state, e.g. San Francisco, CA\"}, \"unit\": {\"type\": \"string\", \"enum\": [\"metric\", \"uscs\"]}}, \"required\": [\"location\"]}}'}\n",
      "{'role': 'user', 'content': 'New-York-City_-NY 22:28:17-0400 06:55:38 18:33:15 Clear ↓4km/h +25°C'}\n"
     ]
    }
   ],
   "source": [
    "function_message = ChatCompletionMessage(role=\"user\", content=result)\n",
    "messages.append(function_message)\n",
    "for message in messages:\n",
    "    print(message)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'choices': [{'finish_reason': 'stop',\n",
      "              'index': 0,\n",
      "              'message': {'content': ' The current weather in New York City, '\n",
      "                                     'NY is clear with a temperature of 25 '\n",
      "                                     'degrees Celsius and a light breeze.',\n",
      "                          'role': 'assistant'}}],\n",
      " 'created': 1696386888,\n",
      " 'id': 'chatcmpl-2a2ce9d7-7dbb-4a0c-a316-ce964c97f1d4',\n",
      " 'model': '../../models/mistralai/Mistral-7B-Instruct-v0.1/ggml-model-f16.gguf',\n",
      " 'object': 'chat.completion',\n",
      " 'usage': {'completion_tokens': 27, 'prompt_tokens': 151, 'total_tokens': 178}}\n"
     ]
    }
   ],
   "source": [
    "response = llm.create_chat_completion(messages)\n",
    "pprint(response)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}