Falcon3_Mamba_7B_Instruct_GGUF.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Troyanovsky/Local-LLM-Comparison-Colab-UI/blob/main/Falcon3_Mamba_7B_Instruct_GGUF.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XNkpBtLvuTOp"
      },
      "source": [
        "## Falcon3-Mamba-7B-Instruct-GGUF WebUI\n",
        "\n",
        "1. Run the following cell, takes ~5 min\n",
        "(You may need to confirm to proceed by typing \"Y\")\n",
        "2. Click the gradio link at the bottom\n",
        "3. Template: ChatML\n",
        "\n",
        "\n",
        "Quantized model: https://huggingface.co/tiiuae/Falcon3-Mamba-7B-Instruct-GGUF\n",
        "\n",
        "Original model: https://huggingface.co/tiiuae/Falcon3-Mamba-7B-Instruct\n",
        "\n",
        "Want to try other local LLMs? Check out this repo: https://github.com/Troyanovsky/Local-LLM-Comparison-Colab-UI/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "D-MiHp_bveP6",
        "outputId": "0b5993b7-0615-4eea-853d-dc4d454b770c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n",
            "Selecting previously unselected package libc-ares2:amd64.\n",
            "(Reading database ... 123634 files and directories currently installed.)\n",
            "Preparing to unpack .../libc-ares2_1.18.1-1ubuntu0.22.04.3_amd64.deb ...\n",
            "Unpacking libc-ares2:amd64 (1.18.1-1ubuntu0.22.04.3) ...\n",
            "Selecting previously unselected package libaria2-0:amd64.\n",
            "Preparing to unpack .../libaria2-0_1.36.0-1_amd64.deb ...\n",
            "Unpacking libaria2-0:amd64 (1.36.0-1) ...\n",
            "Selecting previously unselected package aria2.\n",
            "Preparing to unpack .../aria2_1.36.0-1_amd64.deb ...\n",
            "Unpacking aria2 (1.36.0-1) ...\n",
            "Setting up libc-ares2:amd64 (1.18.1-1ubuntu0.22.04.3) ...\n",
            "Setting up libaria2-0:amd64 (1.36.0-1) ...\n",
            "Setting up aria2 (1.36.0-1) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtcm_debug.so.1 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libumf.so.0 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtcm.so.1 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libhwloc.so.15 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_level_zero.so.0 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_opencl.so.0 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_loader.so.0 is not a symbolic link\n",
            "\n",
            "Looking in indexes: https://download.pytorch.org/whl/cu121\n",
            "Collecting torch==2.2.2\n",
            "  Downloading https://download.pytorch.org/whl/cu121/torch-2.2.2%2Bcu121-cp310-cp310-linux_x86_64.whl (757.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m757.3/757.3 MB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting torchvision==0.17.2\n",
            "  Downloading https://download.pytorch.org/whl/cu121/torchvision-0.17.2%2Bcu121-cp310-cp310-linux_x86_64.whl (7.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m59.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting torchaudio==2.2.2\n",
            "  Downloading https://download.pytorch.org/whl/cu121/torchaudio-2.2.2%2Bcu121-cp310-cp310-linux_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m52.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch==2.2.2) (3.16.1)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch==2.2.2) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch==2.2.2) (1.13.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch==2.2.2) (3.4.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch==2.2.2) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch==2.2.2) (2024.10.0)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch==2.2.2)\n",
            "  Downloading https://download.pytorch.org/whl/cu121/nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m54.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch==2.2.2)\n",
            "  Downloading https://download.pytorch.org/whl/cu121/nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m45.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch==2.2.2)\n",
            "  Downloading https://download.pytorch.org/whl/cu121/nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m54.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch==2.2.2)\n",
            "  Downloading https://download.pytorch.org/whl/cu121/nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cublas-cu12==12.1.3.1 (from torch==2.2.2)\n",
            "  Downloading https://download.pytorch.org/whl/cu121/nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch==2.2.2)\n",
            "  Downloading https://download.pytorch.org/whl/cu121/nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch==2.2.2)\n",
            "  Downloading https://download.pytorch.org/whl/cu121/nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch==2.2.2)\n",
            "  Downloading https://download.pytorch.org/whl/cu121/nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch==2.2.2)\n",
            "  Downloading https://download.pytorch.org/whl/cu121/nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nccl-cu12==2.19.3 (from torch==2.2.2)\n",
            "  Downloading https://download.pytorch.org/whl/cu121/nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch==2.2.2)\n",
            "  Downloading https://download.pytorch.org/whl/cu121/nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting triton==2.2.0 (from torch==2.2.2)\n",
            "  Downloading https://download.pytorch.org/whl/triton-2.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (167.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m167.9/167.9 MB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchvision==0.17.2) (1.26.4)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision==0.17.2) (11.0.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch==2.2.2) (12.6.85)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch==2.2.2) (3.0.2)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch==2.2.2) (1.3.0)\n",
            "Installing collected packages: triton, nvidia-nvtx-cu12, nvidia-nccl-cu12, nvidia-cusparse-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusolver-cu12, nvidia-cudnn-cu12, torch, torchvision, torchaudio\n",
            "  Attempting uninstall: nvidia-nccl-cu12\n",
            "    Found existing installation: nvidia-nccl-cu12 2.23.4\n",
            "    Uninstalling nvidia-nccl-cu12-2.23.4:\n",
            "      Successfully uninstalled nvidia-nccl-cu12-2.23.4\n",
            "  Attempting uninstall: nvidia-cusparse-cu12\n",
            "    Found existing installation: nvidia-cusparse-cu12 12.5.4.2\n",
            "    Uninstalling nvidia-cusparse-cu12-12.5.4.2:\n",
            "      Successfully uninstalled nvidia-cusparse-cu12-12.5.4.2\n",
            "  Attempting uninstall: nvidia-curand-cu12\n",
            "    Found existing installation: nvidia-curand-cu12 10.3.7.77\n",
            "    Uninstalling nvidia-curand-cu12-10.3.7.77:\n",
            "      Successfully uninstalled nvidia-curand-cu12-10.3.7.77\n",
            "  Attempting uninstall: nvidia-cufft-cu12\n",
            "    Found existing installation: nvidia-cufft-cu12 11.3.0.4\n",
            "    Uninstalling nvidia-cufft-cu12-11.3.0.4:\n",
            "      Successfully uninstalled nvidia-cufft-cu12-11.3.0.4\n",
            "  Attempting uninstall: nvidia-cuda-runtime-cu12\n",
            "    Found existing installation: nvidia-cuda-runtime-cu12 12.6.77\n",
            "    Uninstalling nvidia-cuda-runtime-cu12-12.6.77:\n",
            "      Successfully uninstalled nvidia-cuda-runtime-cu12-12.6.77\n",
            "  Attempting uninstall: nvidia-cuda-cupti-cu12\n",
            "    Found existing installation: nvidia-cuda-cupti-cu12 12.6.80\n",
            "    Uninstalling nvidia-cuda-cupti-cu12-12.6.80:\n",
            "      Successfully uninstalled nvidia-cuda-cupti-cu12-12.6.80\n",
            "  Attempting uninstall: nvidia-cublas-cu12\n",
            "    Found existing installation: nvidia-cublas-cu12 12.6.4.1\n",
            "    Uninstalling nvidia-cublas-cu12-12.6.4.1:\n",
            "      Successfully uninstalled nvidia-cublas-cu12-12.6.4.1\n",
            "  Attempting uninstall: nvidia-cusolver-cu12\n",
            "    Found existing installation: nvidia-cusolver-cu12 11.7.1.2\n",
            "    Uninstalling nvidia-cusolver-cu12-11.7.1.2:\n",
            "      Successfully uninstalled nvidia-cusolver-cu12-11.7.1.2\n",
            "  Attempting uninstall: nvidia-cudnn-cu12\n",
            "    Found existing installation: nvidia-cudnn-cu12 9.6.0.74\n",
            "    Uninstalling nvidia-cudnn-cu12-9.6.0.74:\n",
            "      Successfully uninstalled nvidia-cudnn-cu12-9.6.0.74\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 2.5.1+cu121\n",
            "    Uninstalling torch-2.5.1+cu121:\n",
            "      Successfully uninstalled torch-2.5.1+cu121\n",
            "  Attempting uninstall: torchvision\n",
            "    Found existing installation: torchvision 0.20.1+cu121\n",
            "    Uninstalling torchvision-0.20.1+cu121:\n",
            "      Successfully uninstalled torchvision-0.20.1+cu121\n",
            "  Attempting uninstall: torchaudio\n",
            "    Found existing installation: torchaudio 2.5.1+cu121\n",
            "    Uninstalling torchaudio-2.5.1+cu121:\n",
            "      Successfully uninstalled torchaudio-2.5.1+cu121\n",
            "Successfully installed nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvtx-cu12-12.1.105 torch-2.2.2+cu121 torchaudio-2.2.2+cu121 torchvision-0.17.2+cu121 triton-2.2.0\n",
            "Cloning into 'text-generation-webui'...\n",
            "remote: Enumerating objects: 19533, done.\u001b[K\n",
            "remote: Counting objects: 100% (360/360), done.\u001b[K\n",
            "remote: Compressing objects: 100% (139/139), done.\u001b[K\n",
            "remote: Total 19533 (delta 322), reused 226 (delta 221), pack-reused 19173 (from 4)\u001b[K\n",
            "Receiving objects: 100% (19533/19533), 28.97 MiB | 37.94 MiB/s, done.\n",
            "Resolving deltas: 100% (13869/13869), done.\n",
            "Note: switching to '4d466d5c80eb83892b7dfb76fa4ab69efd6d6989'.\n",
            "\n",
            "You are in 'detached HEAD' state. You can look around, make experimental\n",
            "changes and commit them, and you can discard any commits you make in this\n",
            "state without impacting any branches by switching back to a branch.\n",
            "\n",
            "If you want to create a new branch to retain commits you create, you may\n",
            "do so (now or later) by using -c with the switch command. Example:\n",
            "\n",
            "  git switch -c <new-branch-name>\n",
            "\n",
            "Or undo this operation with:\n",
            "\n",
            "  git switch -\n",
            "\n",
            "Turn off this advice by setting config variable advice.detachedHead to false\n",
            "\n",
            "/content/text-generation-webui\n",
            "Ignoring llama-cpp-python: markers 'platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version == \"3.11\"' don't match your environment\n",
            "Collecting llama-cpp-python==0.3.5+cpuavx2 (from -r requirements.txt (line 36))\n",
            "  Downloading https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.3.5+cpuavx2-cp310-cp310-linux_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m40.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hIgnoring llama-cpp-python: markers 'platform_system == \"Windows\" and python_version == \"3.11\"' don't match your environment\n",
            "Ignoring llama-cpp-python: markers 'platform_system == \"Windows\" and python_version == \"3.10\"' don't match your environment\n",
            "Ignoring llama-cpp-python-cuda: markers 'platform_system == \"Windows\" and python_version == \"3.11\"' don't match your environment\n",
            "Ignoring llama-cpp-python-cuda: markers 'platform_system == \"Windows\" and python_version == \"3.10\"' don't match your environment\n",
            "Ignoring llama-cpp-python-cuda: markers 'platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version == \"3.11\"' don't match your environment\n",
            "Collecting llama-cpp-python-cuda==0.3.5+cu121 (from -r requirements.txt (line 44))\n",
            "  Downloading https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.3.5+cu121-cp310-cp310-linux_x86_64.whl (449.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m449.3/449.3 MB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hIgnoring llama-cpp-python-cuda-tensorcores: markers 'platform_system == \"Windows\" and python_version == \"3.11\"' don't match your environment\n",
            "Ignoring llama-cpp-python-cuda-tensorcores: markers 'platform_system == \"Windows\" and python_version == \"3.10\"' don't match your environment\n",
            "Ignoring llama-cpp-python-cuda-tensorcores: markers 'platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version == \"3.11\"' don't match your environment\n",
            "Collecting llama-cpp-python-cuda-tensorcores==0.3.5+cu121 (from -r requirements.txt (line 50))\n",
            "  Downloading https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.3.5+cu121-cp310-cp310-linux_x86_64.whl (486.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m486.0/486.0 MB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hIgnoring exllamav2: markers 'platform_system == \"Windows\" and python_version == \"3.11\"' don't match your environment\n",
            "Ignoring exllamav2: markers 'platform_system == \"Windows\" and python_version == \"3.10\"' don't match your environment\n",
            "Ignoring exllamav2: markers 'platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version == \"3.11\"' don't match your environment\n",
            "Collecting exllamav2==0.2.6+cu121.torch2.4.1 (from -r requirements.txt (line 56))\n",
            "  Downloading https://github.com/oobabooga/exllamav2/releases/download/v0.2.6/exllamav2-0.2.6+cu121.torch2.4.1-cp310-cp310-linux_x86_64.whl (137.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.1/137.1 MB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hIgnoring exllamav2: markers 'platform_system == \"Linux\" and platform_machine != \"x86_64\"' don't match your environment\n",
            "Ignoring flash-attn: markers 'platform_system == \"Windows\" and python_version == \"3.11\"' don't match your environment\n",
            "Ignoring flash-attn: markers 'platform_system == \"Windows\" and python_version == \"3.10\"' don't match your environment\n",
            "Ignoring flash-attn: markers 'platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version == \"3.11\"' don't match your environment\n",
            "Collecting flash-attn==2.7.2.post1+cu12torch2.4cxx11abiFALSE (from -r requirements.txt (line 61))\n",
            "  Downloading https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.2.post1/flash_attn-2.7.2.post1+cu12torch2.4cxx11abiFALSE-cp310-cp310-linux_x86_64.whl (190.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.1/190.1 MB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: accelerate==1.2.* in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 1)) (1.2.1)\n",
            "Collecting bitsandbytes==0.45.* (from -r requirements.txt (line 2))\n",
            "  Downloading bitsandbytes-0.45.0-py3-none-manylinux_2_24_x86_64.whl.metadata (2.9 kB)\n",
            "Collecting colorama (from -r requirements.txt (line 3))\n",
            "  Downloading colorama-0.4.6-py2.py3-none-any.whl.metadata (17 kB)\n",
            "Collecting datasets (from -r requirements.txt (line 4))\n",
            "  Downloading datasets-3.2.0-py3-none-any.whl.metadata (20 kB)\n",
            "Requirement already satisfied: einops in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 5)) (0.8.0)\n",
            "Collecting fastapi==0.112.4 (from -r requirements.txt (line 6))\n",
            "  Downloading fastapi-0.112.4-py3-none-any.whl.metadata (27 kB)\n",
            "Collecting gradio==4.37.* (from -r requirements.txt (line 7))\n",
            "  Downloading gradio-4.37.2-py3-none-any.whl.metadata (15 kB)\n",
            "Requirement already satisfied: jinja2==3.1.4 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 8)) (3.1.4)\n",
            "Requirement already satisfied: markdown in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 9)) (3.7)\n",
            "Collecting numba==0.59.* (from -r requirements.txt (line 10))\n",
            "  Downloading numba-0.59.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (2.7 kB)\n",
            "Requirement already satisfied: numpy==1.26.* in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 11)) (1.26.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 12)) (2.2.2)\n",
            "Collecting peft==0.12.* (from -r requirements.txt (line 13))\n",
            "  Downloading peft-0.12.0-py3-none-any.whl.metadata (13 kB)\n",
            "Requirement already satisfied: Pillow>=9.5.0 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 14)) (11.0.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 15)) (5.9.5)\n",
            "Collecting pydantic==2.8.2 (from -r requirements.txt (line 16))\n",
            "  Downloading pydantic-2.8.2-py3-none-any.whl.metadata (125 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m125.2/125.2 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 17)) (6.0.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 18)) (2.32.3)\n",
            "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 19)) (13.9.4)\n",
            "Requirement already satisfied: safetensors==0.4.* in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 20)) (0.4.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 21)) (1.13.1)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 22)) (0.2.0)\n",
            "Requirement already satisfied: tensorboard in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 23)) (2.17.1)\n",
            "Requirement already satisfied: transformers==4.47.* in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 24)) (4.47.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 25)) (4.67.1)\n",
            "Requirement already satisfied: wandb in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 26)) (0.19.1)\n",
            "Collecting SpeechRecognition==3.10.0 (from -r requirements.txt (line 29))\n",
            "  Downloading SpeechRecognition-3.10.0-py2.py3-none-any.whl.metadata (28 kB)\n",
            "Collecting flask_cloudflared==0.0.14 (from -r requirements.txt (line 30))\n",
            "  Downloading flask_cloudflared-0.0.14-py3-none-any.whl.metadata (4.6 kB)\n",
            "Collecting sse-starlette==1.6.5 (from -r requirements.txt (line 31))\n",
            "  Downloading sse_starlette-1.6.5-py3-none-any.whl.metadata (6.7 kB)\n",
            "Collecting tiktoken (from -r requirements.txt (line 32))\n",
            "  Downloading tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate==1.2.*->-r requirements.txt (line 1)) (24.2)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate==1.2.*->-r requirements.txt (line 1)) (2.2.2+cu121)\n",
            "Requirement already satisfied: huggingface-hub>=0.21.0 in /usr/local/lib/python3.10/dist-packages (from accelerate==1.2.*->-r requirements.txt (line 1)) (0.27.0)\n",
            "Requirement already satisfied: typing_extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from bitsandbytes==0.45.*->-r requirements.txt (line 2)) (4.12.2)\n",
            "Collecting starlette<0.39.0,>=0.37.2 (from fastapi==0.112.4->-r requirements.txt (line 6))\n",
            "  Downloading starlette-0.38.6-py3-none-any.whl.metadata (6.0 kB)\n",
            "Collecting aiofiles<24.0,>=22.0 (from gradio==4.37.*->-r requirements.txt (line 7))\n",
            "  Downloading aiofiles-23.2.1-py3-none-any.whl.metadata (9.7 kB)\n",
            "Requirement already satisfied: altair<6.0,>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.37.*->-r requirements.txt (line 7)) (5.5.0)\n",
            "Collecting ffmpy (from gradio==4.37.*->-r requirements.txt (line 7))\n",
            "  Downloading ffmpy-0.5.0-py3-none-any.whl.metadata (3.0 kB)\n",
            "Collecting gradio-client==1.0.2 (from gradio==4.37.*->-r requirements.txt (line 7))\n",
            "  Downloading gradio_client-1.0.2-py3-none-any.whl.metadata (7.1 kB)\n",
            "Requirement already satisfied: httpx>=0.24.1 in /usr/local/lib/python3.10/dist-packages (from gradio==4.37.*->-r requirements.txt (line 7)) (0.28.1)\n",
            "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /usr/local/lib/python3.10/dist-packages (from gradio==4.37.*->-r requirements.txt (line 7)) (6.4.5)\n",
            "Collecting markupsafe~=2.0 (from gradio==4.37.*->-r requirements.txt (line 7))\n",
            "  Downloading MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.0 kB)\n",
            "Requirement already satisfied: matplotlib~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.37.*->-r requirements.txt (line 7)) (3.8.0)\n",
            "Requirement already satisfied: orjson~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.37.*->-r requirements.txt (line 7)) (3.10.12)\n",
            "Collecting Pillow>=9.5.0 (from -r requirements.txt (line 14))\n",
            "  Downloading pillow-10.4.0-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (9.2 kB)\n",
            "Collecting pydub (from gradio==4.37.*->-r requirements.txt (line 7))\n",
            "  Downloading pydub-0.25.1-py2.py3-none-any.whl.metadata (1.4 kB)\n",
            "Collecting python-multipart>=0.0.9 (from gradio==4.37.*->-r requirements.txt (line 7))\n",
            "  Downloading python_multipart-0.0.20-py3-none-any.whl.metadata (1.8 kB)\n",
            "Collecting ruff>=0.2.2 (from gradio==4.37.*->-r requirements.txt (line 7))\n",
            "  Downloading ruff-0.8.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (25 kB)\n",
            "Collecting semantic-version~=2.0 (from gradio==4.37.*->-r requirements.txt (line 7))\n",
            "  Downloading semantic_version-2.10.0-py2.py3-none-any.whl.metadata (9.7 kB)\n",
            "Collecting tomlkit==0.12.0 (from gradio==4.37.*->-r requirements.txt (line 7))\n",
            "  Downloading tomlkit-0.12.0-py3-none-any.whl.metadata (2.7 kB)\n",
            "Requirement already satisfied: typer<1.0,>=0.12 in /usr/local/lib/python3.10/dist-packages (from gradio==4.37.*->-r requirements.txt (line 7)) (0.15.1)\n",
            "Requirement already satisfied: urllib3~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.37.*->-r requirements.txt (line 7)) (2.2.3)\n",
            "Collecting uvicorn>=0.14.0 (from gradio==4.37.*->-r requirements.txt (line 7))\n",
            "  Downloading uvicorn-0.34.0-py3-none-any.whl.metadata (6.5 kB)\n",
            "Collecting llvmlite<0.43,>=0.42.0dev0 (from numba==0.59.*->-r requirements.txt (line 10))\n",
            "  Downloading llvmlite-0.42.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.8 kB)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic==2.8.2->-r requirements.txt (line 16)) (0.7.0)\n",
            "Collecting pydantic-core==2.20.1 (from pydantic==2.8.2->-r requirements.txt (line 16))\n",
            "  Downloading pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.47.*->-r requirements.txt (line 24)) (3.16.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.47.*->-r requirements.txt (line 24)) (2024.11.6)\n",
            "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.10/dist-packages (from transformers==4.47.*->-r requirements.txt (line 24)) (0.21.0)\n",
            "Requirement already satisfied: Flask>=0.8 in /usr/local/lib/python3.10/dist-packages (from flask_cloudflared==0.0.14->-r requirements.txt (line 30)) (3.1.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from gradio-client==1.0.2->gradio==4.37.*->-r requirements.txt (line 7)) (2024.10.0)\n",
            "Collecting websockets<12.0,>=10.0 (from gradio-client==1.0.2->gradio==4.37.*->-r requirements.txt (line 7))\n",
            "  Downloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 4)) (17.0.0)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets->-r requirements.txt (line 4))\n",
            "  Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
            "Collecting xxhash (from datasets->-r requirements.txt (line 4))\n",
            "  Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
            "Collecting multiprocess<0.70.17 (from datasets->-r requirements.txt (line 4))\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n",
            "Collecting fsspec (from gradio-client==1.0.2->gradio==4.37.*->-r requirements.txt (line 7))\n",
            "  Downloading fsspec-2024.9.0-py3-none-any.whl.metadata (11 kB)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 4)) (3.11.10)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->-r requirements.txt (line 12)) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->-r requirements.txt (line 12)) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->-r requirements.txt (line 12)) (2024.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->-r requirements.txt (line 18)) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->-r requirements.txt (line 18)) (3.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->-r requirements.txt (line 18)) (2024.12.14)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->-r requirements.txt (line 19)) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->-r requirements.txt (line 19)) (2.18.0)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 23)) (1.4.0)\n",
            "Requirement already satisfied: grpcio>=1.48.2 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 23)) (1.68.1)\n",
            "Requirement already satisfied: protobuf!=4.24.0,>=3.19.6 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 23)) (4.25.5)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 23)) (75.1.0)\n",
            "Requirement already satisfied: six>1.9 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 23)) (1.17.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 23)) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 23)) (3.1.3)\n",
            "Requirement already satisfied: click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 26)) (8.1.7)\n",
            "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 26)) (0.4.0)\n",
            "Requirement already satisfied: gitpython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 26)) (3.1.43)\n",
            "Requirement already satisfied: platformdirs in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 26)) (4.3.6)\n",
            "Requirement already satisfied: sentry-sdk>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 26)) (2.19.2)\n",
            "Requirement already satisfied: setproctitle in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 26)) (1.3.4)\n",
            "Collecting diskcache>=5.6.1 (from llama-cpp-python==0.3.5+cpuavx2->-r requirements.txt (line 36))\n",
            "  Downloading diskcache-5.6.3-py3-none-any.whl.metadata (20 kB)\n",
            "Collecting ninja (from exllamav2==0.2.6+cu121.torch2.4.1->-r requirements.txt (line 56))\n",
            "  Downloading ninja-1.11.1.3-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl.metadata (5.3 kB)\n",
            "Collecting fastparquet (from exllamav2==0.2.6+cu121.torch2.4.1->-r requirements.txt (line 56))\n",
            "  Downloading fastparquet-2024.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.2 kB)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio==4.37.*->-r requirements.txt (line 7)) (4.23.0)\n",
            "Requirement already satisfied: narwhals>=1.14.2 in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio==4.37.*->-r requirements.txt (line 7)) (1.18.4)\n",
            "Requirement already satisfied: itsdangerous>=2.2 in /usr/local/lib/python3.10/dist-packages (from Flask>=0.8->flask_cloudflared==0.0.14->-r requirements.txt (line 30)) (2.2.0)\n",
            "Requirement already satisfied: blinker>=1.9 in /usr/local/lib/python3.10/dist-packages (from Flask>=0.8->flask_cloudflared==0.0.14->-r requirements.txt (line 30)) (1.9.0)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 4)) (2.4.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 4)) (1.3.2)\n",
            "Requirement already satisfied: async-timeout<6.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 4)) (4.0.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 4)) (24.3.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 4)) (1.5.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 4)) (6.1.0)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 4)) (0.2.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 4)) (1.18.3)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from gitpython!=3.1.29,>=1.0.0->wandb->-r requirements.txt (line 26)) (4.0.11)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio==4.37.*->-r requirements.txt (line 7)) (3.7.1)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio==4.37.*->-r requirements.txt (line 7)) (1.0.7)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx>=0.24.1->gradio==4.37.*->-r requirements.txt (line 7)) (0.14.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->-r requirements.txt (line 19)) (0.1.2)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.37.*->-r requirements.txt (line 7)) (1.3.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.37.*->-r requirements.txt (line 7)) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.37.*->-r requirements.txt (line 7)) (4.55.3)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.37.*->-r requirements.txt (line 7)) (1.4.7)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.37.*->-r requirements.txt (line 7)) (3.2.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (1.13.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (3.4.2)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (2.19.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (12.1.105)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (2.2.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (12.6.85)\n",
            "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0,>=0.12->gradio==4.37.*->-r requirements.txt (line 7)) (1.5.4)\n",
            "Collecting cramjam>=2.3 (from fastparquet->exllamav2==0.2.6+cu121.torch2.4.1->-r requirements.txt (line 56))\n",
            "  Downloading cramjam-2.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.24.1->gradio==4.37.*->-r requirements.txt (line 7)) (1.3.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.24.1->gradio==4.37.*->-r requirements.txt (line 7)) (1.2.2)\n",
            "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from gitdb<5,>=4.0.1->gitpython!=3.1.29,>=1.0.0->wandb->-r requirements.txt (line 26)) (5.0.1)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.37.*->-r requirements.txt (line 7)) (2024.10.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.37.*->-r requirements.txt (line 7)) (0.35.1)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.37.*->-r requirements.txt (line 7)) (0.22.3)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate==1.2.*->-r requirements.txt (line 1)) (1.3.0)\n",
            "Downloading bitsandbytes-0.45.0-py3-none-manylinux_2_24_x86_64.whl (69.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.1/69.1 MB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fastapi-0.112.4-py3-none-any.whl (93 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m93.9/93.9 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading gradio-4.37.2-py3-none-any.whl (12.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m111.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading numba-0.59.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (3.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.7/3.7 MB\u001b[0m \u001b[31m99.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading peft-0.12.0-py3-none-any.whl (296 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m296.4/296.4 kB\u001b[0m \u001b[31m24.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pydantic-2.8.2-py3-none-any.whl (423 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m423.9/423.9 kB\u001b[0m \u001b[31m34.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading SpeechRecognition-3.10.0-py2.py3-none-any.whl (32.8 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m32.8/32.8 MB\u001b[0m \u001b[31m22.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading flask_cloudflared-0.0.14-py3-none-any.whl (6.4 kB)\n",
            "Downloading sse_starlette-1.6.5-py3-none-any.whl (9.6 kB)\n",
            "Downloading gradio_client-1.0.2-py3-none-any.whl (318 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m318.2/318.2 kB\u001b[0m \u001b[31m28.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m85.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tomlkit-0.12.0-py3-none-any.whl (37 kB)\n",
            "Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
            "Downloading datasets-3.2.0-py3-none-any.whl (480 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m480.6/480.6 kB\u001b[0m \u001b[31m35.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pillow-10.4.0-cp310-cp310-manylinux_2_28_x86_64.whl (4.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m109.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m64.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading aiofiles-23.2.1-py3-none-any.whl (15 kB)\n",
            "Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fsspec-2024.9.0-py3-none-any.whl (179 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m179.3/179.3 kB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llvmlite-0.42.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (43.8 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.8/43.8 MB\u001b[0m \u001b[31m20.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25 kB)\n",
            "Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading python_multipart-0.0.20-py3-none-any.whl (24 kB)\n",
            "Downloading ruff-0.8.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.2/11.2 MB\u001b[0m \u001b[31m109.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n",
            "Downloading starlette-0.38.6-py3-none-any.whl (71 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading uvicorn-0.34.0-py3-none-any.whl (62 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.3/62.3 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fastparquet-2024.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m77.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading ffmpy-0.5.0-py3-none-any.whl (6.0 kB)\n",
            "Downloading ninja-1.11.1.3-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (422 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m422.9/422.9 kB\u001b[0m \u001b[31m35.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
            "Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading cramjam-2.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.4 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m92.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pydub, xxhash, websockets, uvicorn, tomlkit, semantic-version, ruff, python-multipart, pydantic-core, Pillow, ninja, markupsafe, llvmlite, fsspec, ffmpy, diskcache, dill, cramjam, colorama, aiofiles, tiktoken, starlette, SpeechRecognition, pydantic, numba, multiprocess, sse-starlette, llama-cpp-python-cuda-tensorcores, llama-cpp-python-cuda, llama-cpp-python, gradio-client, fastparquet, fastapi, flask_cloudflared, flash-attn, exllamav2, bitsandbytes, peft, gradio, datasets\n",
            "  Attempting uninstall: websockets\n",
            "    Found existing installation: websockets 14.1\n",
            "    Uninstalling websockets-14.1:\n",
            "      Successfully uninstalled websockets-14.1\n",
            "  Attempting uninstall: pydantic-core\n",
            "    Found existing installation: pydantic_core 2.27.1\n",
            "    Uninstalling pydantic_core-2.27.1:\n",
            "      Successfully uninstalled pydantic_core-2.27.1\n",
            "  Attempting uninstall: Pillow\n",
            "    Found existing installation: pillow 11.0.0\n",
            "    Uninstalling pillow-11.0.0:\n",
            "      Successfully uninstalled pillow-11.0.0\n",
            "  Attempting uninstall: markupsafe\n",
            "    Found existing installation: MarkupSafe 3.0.2\n",
            "    Uninstalling MarkupSafe-3.0.2:\n",
            "      Successfully uninstalled MarkupSafe-3.0.2\n",
            "  Attempting uninstall: llvmlite\n",
            "    Found existing installation: llvmlite 0.43.0\n",
            "    Uninstalling llvmlite-0.43.0:\n",
            "      Successfully uninstalled llvmlite-0.43.0\n",
            "  Attempting uninstall: fsspec\n",
            "    Found existing installation: fsspec 2024.10.0\n",
            "    Uninstalling fsspec-2024.10.0:\n",
            "      Successfully uninstalled fsspec-2024.10.0\n",
            "  Attempting uninstall: pydantic\n",
            "    Found existing installation: pydantic 2.10.3\n",
            "    Uninstalling pydantic-2.10.3:\n",
            "      Successfully uninstalled pydantic-2.10.3\n",
            "  Attempting uninstall: numba\n",
            "    Found existing installation: numba 0.60.0\n",
            "    Uninstalling numba-0.60.0:\n",
            "      Successfully uninstalled numba-0.60.0\n",
            "  Attempting uninstall: peft\n",
            "    Found existing installation: peft 0.14.0\n",
            "    Uninstalling peft-0.14.0:\n",
            "      Successfully uninstalled peft-0.14.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "gcsfs 2024.10.0 requires fsspec==2024.10.0, but you have fsspec 2024.9.0 which is incompatible.\n",
            "google-genai 0.3.0 requires websockets<15.0dev,>=13.0, but you have websockets 11.0.3 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed Pillow-10.4.0 SpeechRecognition-3.10.0 aiofiles-23.2.1 bitsandbytes-0.45.0 colorama-0.4.6 cramjam-2.9.1 datasets-3.2.0 dill-0.3.8 diskcache-5.6.3 exllamav2-0.2.6+cu121.torch2.4.1 fastapi-0.112.4 fastparquet-2024.11.0 ffmpy-0.5.0 flash-attn-2.7.2.post1 flask_cloudflared-0.0.14 fsspec-2024.9.0 gradio-4.37.2 gradio-client-1.0.2 llama-cpp-python-0.3.5+cpuavx2 llama-cpp-python-cuda-0.3.5+cu121 llama-cpp-python-cuda-tensorcores-0.3.5+cu121 llvmlite-0.42.0 markupsafe-2.1.5 multiprocess-0.70.16 ninja-1.11.1.3 numba-0.59.1 peft-0.12.0 pydantic-2.8.2 pydantic-core-2.20.1 pydub-0.25.1 python-multipart-0.0.20 ruff-0.8.4 semantic-version-2.10.0 sse-starlette-1.6.5 starlette-0.38.6 tiktoken-0.8.0 tomlkit-0.12.0 uvicorn-0.34.0 websockets-11.0.3 xxhash-3.5.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "PIL"
                ]
              },
              "id": "473b36db121f46b29e4e01fbab4c424d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[0m\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "f775c5|\u001b[1;32mOK\u001b[0m  |   151MiB/s|/content/text-generation-webui/models//Falcon3-Mamba-7B-Instruct-q5_k_m.gguf\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "/content/text-generation-webui\n",
            "\u001b[2;36m08:29:37-167209\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m Starting Text generation web UI                                            \n",
            "\u001b[2;36m08:29:37-452403\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m Loading \u001b[32m\"Falcon3-Mamba-7B-Instruct-q5_k_m.gguf\"\u001b[0m                            \n",
            "\u001b[2;36m08:29:37-697010\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m llama.cpp weights detected: \u001b[32m\"models/Falcon3-Mamba-7B-Instruct-q5_k_m.gguf\"\u001b[0m \n",
            "ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    yes\n",
            "ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no\n",
            "ggml_cuda_init: found 1 CUDA devices:\n",
            "  Device 0: Tesla T4, compute capability 7.5, VMM: yes\n",
            "llama_load_model_from_file: using device CUDA0 (Tesla T4) - 14999 MiB free\n",
            "llama_model_loader: loaded meta data with 28 key-value pairs and 643 tensors from models/Falcon3-Mamba-7B-Instruct-q5_k_m.gguf (version GGUF V3 (latest))\n",
            "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
            "llama_model_loader: - kv   0:                       general.architecture str              = mamba\n",
            "llama_model_loader: - kv   1:                               general.type str              = model\n",
            "llama_model_loader: - kv   2:                               general.name str              = Falcon3 Mamba 7B Instruct\n",
            "llama_model_loader: - kv   3:                           general.finetune str              = Instruct\n",
            "llama_model_loader: - kv   4:                           general.basename str              = Falcon3-Mamba\n",
            "llama_model_loader: - kv   5:                         general.size_label str              = 7B\n",
            "llama_model_loader: - kv   6:                       mamba.context_length u32              = 1048576\n",
            "llama_model_loader: - kv   7:                     mamba.embedding_length u32              = 4096\n",
            "llama_model_loader: - kv   8:                  mamba.feed_forward_length u32              = 0\n",
            "llama_model_loader: - kv   9:                 mamba.attention.head_count u32              = 0\n",
            "llama_model_loader: - kv  10:                          mamba.block_count u32              = 64\n",
            "llama_model_loader: - kv  11:                      mamba.ssm.conv_kernel u32              = 4\n",
            "llama_model_loader: - kv  12:                       mamba.ssm.inner_size u32              = 8192\n",
            "llama_model_loader: - kv  13:                       mamba.ssm.state_size u32              = 16\n",
            "llama_model_loader: - kv  14:                   mamba.ssm.time_step_rank u32              = 256\n",
            "llama_model_loader: - kv  15:     mamba.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
            "llama_model_loader: - kv  16:                       mamba.ssm.dt_b_c_rms bool             = true\n",
            "llama_model_loader: - kv  17:                          general.file_type u32              = 17\n",
            "llama_model_loader: - kv  18:                       tokenizer.ggml.model str              = gpt2\n",
            "llama_model_loader: - kv  19:                         tokenizer.ggml.pre str              = falcon\n",
            "llama_model_loader: - kv  20:                      tokenizer.ggml.tokens arr[str,65024]   = [\">>TITLE<<\", \">>ABSTRACT<<\", \">>INTR...\n",
            "llama_model_loader: - kv  21:                  tokenizer.ggml.token_type arr[i32,65024]   = [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, ...\n",
            "llama_model_loader: - kv  22:                      tokenizer.ggml.merges arr[str,64784]   = [\"Ġ t\", \"Ġ a\", \"i n\", \"h e\", \"r e\",...\n",
            "llama_model_loader: - kv  23:                tokenizer.ggml.bos_token_id u32              = 8\n",
            "llama_model_loader: - kv  24:                tokenizer.ggml.eos_token_id u32              = 11\n",
            "llama_model_loader: - kv  25:            tokenizer.ggml.padding_token_id u32              = 0\n",
            "llama_model_loader: - kv  26:                    tokenizer.chat_template str              = {{bos_token}}{% for message in messag...\n",
            "llama_model_loader: - kv  27:               general.quantization_version u32              = 2\n",
            "llama_model_loader: - type  f32:  385 tensors\n",
            "llama_model_loader: - type q5_K:  257 tensors\n",
            "llama_model_loader: - type q6_K:    1 tensors\n",
            "llm_load_vocab: control token:      3 '>>SUMMARY<<' is not marked as EOG\n",
            "llm_load_vocab: control token:      5 '>>ANSWER<<' is not marked as EOG\n",
            "llm_load_vocab: control token:      0 '>>TITLE<<' is not marked as EOG\n",
            "llm_load_vocab: control token:     11 '<|end_of_text|>' is not marked as EOG\n",
            "llm_load_vocab: control token:      1 '>>ABSTRACT<<' is not marked as EOG\n",
            "llm_load_vocab: control token:      4 '>>COMMENT<<' is not marked as EOG\n",
            "llm_load_vocab: control token:      8 '<|begin_of_text|>' is not marked as EOG\n",
            "llm_load_vocab: control token:      6 '>>QUESTION<<' is not marked as EOG\n",
            "llm_load_vocab: control token:      7 'assistant' is not marked as EOG\n",
            "llm_load_vocab: control token:      9 '<|im_start|>' is not marked as EOG\n",
            "llm_load_vocab: control token:      2 '>>INTRODUCTION<<' is not marked as EOG\n",
            "llm_load_vocab: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect\n",
            "llm_load_vocab: special tokens cache size = 12\n",
            "llm_load_vocab: token to piece cache size = 0.3884 MB\n",
            "llm_load_print_meta: format           = GGUF V3 (latest)\n",
            "llm_load_print_meta: arch             = mamba\n",
            "llm_load_print_meta: vocab type       = BPE\n",
            "llm_load_print_meta: n_vocab          = 65024\n",
            "llm_load_print_meta: n_merges         = 64784\n",
            "llm_load_print_meta: vocab_only       = 0\n",
            "llm_load_print_meta: n_ctx_train      = 1048576\n",
            "llm_load_print_meta: n_embd           = 4096\n",
            "llm_load_print_meta: n_layer          = 64\n",
            "llm_load_print_meta: n_head           = 0\n",
            "llm_load_print_meta: n_head_kv        = 0\n",
            "llm_load_print_meta: n_rot            = 0\n",
            "llm_load_print_meta: n_swa            = 0\n",
            "llm_load_print_meta: n_embd_head_k    = 0\n",
            "llm_load_print_meta: n_embd_head_v    = 0\n",
            "llm_load_print_meta: n_gqa            = 0\n",
            "llm_load_print_meta: n_embd_k_gqa     = 0\n",
            "llm_load_print_meta: n_embd_v_gqa     = 0\n",
            "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
            "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
            "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
            "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
            "llm_load_print_meta: f_logit_scale    = 0.0e+00\n",
            "llm_load_print_meta: n_ff             = 0\n",
            "llm_load_print_meta: n_expert         = 0\n",
            "llm_load_print_meta: n_expert_used    = 0\n",
            "llm_load_print_meta: causal attn      = 1\n",
            "llm_load_print_meta: pooling type     = 0\n",
            "llm_load_print_meta: rope type        = -1\n",
            "llm_load_print_meta: rope scaling     = linear\n",
            "llm_load_print_meta: freq_base_train  = 10000.0\n",
            "llm_load_print_meta: freq_scale_train = 1\n",
            "llm_load_print_meta: n_ctx_orig_yarn  = 1048576\n",
            "llm_load_print_meta: rope_finetuned   = unknown\n",
            "llm_load_print_meta: ssm_d_conv       = 4\n",
            "llm_load_print_meta: ssm_d_inner      = 8192\n",
            "llm_load_print_meta: ssm_d_state      = 16\n",
            "llm_load_print_meta: ssm_dt_rank      = 256\n",
            "llm_load_print_meta: ssm_dt_b_c_rms   = 1\n",
            "llm_load_print_meta: model type       = ?B\n",
            "llm_load_print_meta: model ftype      = Q5_K - Medium\n",
            "llm_load_print_meta: model params     = 7.27 B\n",
            "llm_load_print_meta: model size       = 4.73 GiB (5.58 BPW) \n",
            "llm_load_print_meta: general.name     = Falcon3 Mamba 7B Instruct\n",
            "llm_load_print_meta: BOS token        = 8 '<|begin_of_text|>'\n",
            "llm_load_print_meta: EOS token        = 11 '<|end_of_text|>'\n",
            "llm_load_print_meta: EOT token        = 10 '<|im_end|>'\n",
            "llm_load_print_meta: PAD token        = 0 '>>TITLE<<'\n",
            "llm_load_print_meta: LF token         = 138 'Ä'\n",
            "llm_load_print_meta: EOG token        = 10 '<|im_end|>'\n",
            "llm_load_print_meta: EOG token        = 11 '<|end_of_text|>'\n",
            "llm_load_print_meta: max token length = 130\n",
            "llm_load_tensors: tensor 'token_embd.weight' (q5_K) (and 128 others) cannot be used with preferred buffer type CPU_AARCH64, using CPU instead\n",
            "llm_load_tensors: offloading 64 repeating layers to GPU\n",
            "llm_load_tensors: offloading output layer to GPU\n",
            "llm_load_tensors: offloaded 65/65 layers to GPU\n",
            "llm_load_tensors:        CUDA0 model buffer size =  4626.39 MiB\n",
            "llm_load_tensors:   CPU_Mapped model buffer size =  4563.64 MiB\n",
            "..............................................................................................\n",
            "llama_new_context_with_model: n_seq_max     = 1\n",
            "llama_new_context_with_model: n_ctx         = 2048\n",
            "llama_new_context_with_model: n_ctx_per_seq = 2048\n",
            "llama_new_context_with_model: n_batch       = 512\n",
            "llama_new_context_with_model: n_ubatch      = 512\n",
            "llama_new_context_with_model: flash_attn    = 0\n",
            "llama_new_context_with_model: freq_base     = 10000.0\n",
            "llama_new_context_with_model: freq_scale    = 1\n",
            "llama_new_context_with_model: n_ctx_per_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized\n",
            "llama_kv_cache_init:      CUDA0 KV buffer size =    38.00 MiB\n",
            "llama_new_context_with_model: KV self size  =   38.00 MiB, K (f32):    6.00 MiB, V (f32):   32.00 MiB\n",
            "llama_new_context_with_model:  CUDA_Host  output buffer size =     0.25 MiB\n",
            "llama_new_context_with_model:      CUDA0 compute buffer size =   151.59 MiB\n",
            "llama_new_context_with_model:  CUDA_Host compute buffer size =    50.16 MiB\n",
            "llama_new_context_with_model: graph nodes  = 3334\n",
            "llama_new_context_with_model: graph splits = 387 (with bs=512), 259 (with bs=1)\n",
            "CUDA : ARCHS = 500,520,530,600,610,620,700,720,750,800,860,870,890,900 | FORCE_MMQ = 1 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | \n",
            "Model metadata: {'tokenizer.chat_template': \"{{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}\", 'mamba.ssm.dt_b_c_rms': 'true', 'general.quantization_version': '2', 'tokenizer.ggml.model': 'gpt2', 'mamba.ssm.time_step_rank': '256', 'mamba.ssm.state_size': '16', 'general.architecture': 'mamba', 'tokenizer.ggml.padding_token_id': '0', 'mamba.attention.layer_norm_rms_epsilon': '0.000010', 'general.basename': 'Falcon3-Mamba', 'mamba.context_length': '1048576', 'tokenizer.ggml.pre': 'falcon', 'general.name': 'Falcon3 Mamba 7B Instruct', 'general.finetune': 'Instruct', 'general.type': 'model', 'general.size_label': '7B', 'tokenizer.ggml.bos_token_id': '8', 'mamba.embedding_length': '4096', 'tokenizer.ggml.eos_token_id': '11', 'mamba.feed_forward_length': '0', 'general.file_type': '17', 'mamba.block_count': '64', 'mamba.attention.head_count': '0', 'mamba.ssm.conv_kernel': '4', 'mamba.ssm.inner_size': '8192'}\n",
            "Available chat formats from metadata: chat_template.default\n",
            "Using gguf chat template: {{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n",
            "' + message['content'] + '<|im_end|>' + '\n",
            "'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n",
            "' }}{% endif %}\n",
            "Using chat eos_token: <|end_of_text|>\n",
            "Using chat bos_token: <|begin_of_text|>\n",
            "\u001b[2;36m08:29:53-073509\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m Loaded \u001b[32m\"Falcon3-Mamba-7B-Instruct-q5_k_m.gguf\"\u001b[0m in \u001b[1;36m15.62\u001b[0m seconds.           \n",
            "\u001b[2;36m08:29:53-074822\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m LOADER: \u001b[32m\"llama.cpp\"\u001b[0m                                                        \n",
            "\u001b[2;36m08:29:53-075739\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m TRUNCATION LENGTH: \u001b[1;36m2048\u001b[0m                                                    \n",
            "\u001b[2;36m08:29:53-076602\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m INSTRUCTION TEMPLATE: \u001b[32m\"Custom \u001b[0m\u001b[32m(\u001b[0m\u001b[32mobtained from model metadata\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\"\u001b[0m              \n",
            "\n",
            "Running on local URL:  http://127.0.0.1:7860\n",
            "\n",
            "Running on public URL: https://edb6c26e0d9257e260.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n",
            "llama_perf_context_print:        load time =    2244.60 ms\n",
            "llama_perf_context_print: prompt eval time =       0.00 ms /    26 tokens (    0.00 ms per token,      inf tokens per second)\n",
            "llama_perf_context_print:        eval time =       0.00 ms /   129 runs   (    0.00 ms per token,      inf tokens per second)\n",
            "llama_perf_context_print:       total time =   22989.06 ms /   155 tokens\n",
            "Output generated in 23.49 seconds (5.49 tokens/s, 129 tokens, context 49, seed 1769616031)\n"
          ]
        }
      ],
      "source": [
        "%cd /content\n",
        "!apt-get -y install -qq aria2\n",
        "!pip install torch==2.2.2 torchvision==0.17.2 torchaudio==2.2.2 --index-url https://download.pytorch.org/whl/cu121\n",
        "\n",
        "!git clone -b V20241221 https://github.com/Troyanovsky/text-generation-webui\n",
        "%cd /content/text-generation-webui\n",
        "!pip install -r requirements.txt\n",
        "\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/tiiuae/Falcon3-Mamba-7B-Instruct-GGUF/resolve/main/Falcon3-Mamba-7B-Instruct-q5_k_m.gguf?download=true -d /content/text-generation-webui/models/ -o Falcon3-Mamba-7B-Instruct-q5_k_m.gguf\n",
        "%cd /content/text-generation-webui\n",
        "!python server.py --share --n-gpu-layers 100000 --model Falcon3-Mamba-7B-Instruct-q5_k_m.gguf --n_ctx 2048"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMYOnz1MTpWkGiqc+nBcKUX",
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}