update docs for llama3.1 support and bump up version

.github/workflows/build_wheel.yml

@@ -25,7 +25,7 @@ jobs:
       matrix:
         python: ["3.8", "3.9", "3.10", "3.11"]
         cuda: ["11.8", "12.1"]
-        torch: ["2.1", "2.2", "2.3"]
+        torch: ["2.1.2", "2.2.2", "2.3.1"]
     runs-on: [self-hosted, linux, release]
     env:
       PYTHON_VERSION: ${{ matrix.python }}

.github/workflows/package_test.yml

@@ -40,7 +40,7 @@ jobs:
       matrix:
         python: ["3.10"]
         cuda: ["12.1"]
-        torch: ["2.3"]
+        torch: ["2.3.1"]
     runs-on: [self-hosted, linux, build]
     env:
       PYTHON_VERSION: ${{ matrix.python }}

.github/workflows/publish_wheel.yml

@@ -24,7 +24,7 @@ jobs:
       matrix:
         python: ["3.8", "3.9", "3.10", "3.11"]
         cuda: ["12.1"]
-        torch: ["2.3"]
+        torch: ["2.3.1"]
     runs-on: [self-hosted, linux, release]
     env:
       PYTHON_VERSION: ${{ matrix.python }}

.github/workflows/release_test.yml

@@ -15,7 +15,7 @@ jobs:
       matrix:
         python: ["3.10"]
         cuda: ["12.1"]
-        torch: ["2.3"]
+        torch: ["2.3.1"]
     runs-on: [self-hosted, linux, build]
     env:
       PYTHON_VERSION: ${{ matrix.python }}

CMakeLists.txt

@@ -201,18 +201,18 @@ else()
       set(LIBTORCH_URL "https://download.pytorch.org/libtorch/nightly/cu124/libtorch-shared-with-deps-latest.zip")
     endif()
   elseif(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.1)
-    # download libtorch 2.3 with cuda 12.1 from pytorch.org
+    # download libtorch 2.3.1 with cuda 12.1 from pytorch.org
     if (USE_CXX11_ABI)
-      set(LIBTORCH_URL "https://download.pytorch.org/libtorch/cu121/libtorch-cxx11-abi-shared-with-deps-2.3.0%2Bcu121.zip")
+      set(LIBTORCH_URL "https://download.pytorch.org/libtorch/cu121/libtorch-cxx11-abi-shared-with-deps-2.3.1%2Bcu121.zip")
     else()
-      set(LIBTORCH_URL "https://download.pytorch.org/libtorch/cu121/libtorch-shared-with-deps-2.3.0%2Bcu121.zip")
+      set(LIBTORCH_URL "https://download.pytorch.org/libtorch/cu121/libtorch-shared-with-deps-2.3.1%2Bcu121.zip")
     endif()
   elseif(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 11.8)
-    # download libtorch 2.3 with cuda 11.8 from pytorch.org
+    # download libtorch 2.3.1 with cuda 11.8 from pytorch.org
     if (USE_CXX11_ABI)
-      set(LIBTORCH_URL "https://download.pytorch.org/libtorch/cu118/libtorch-cxx11-abi-shared-with-deps-2.3.0%2Bcu118.zip")
+      set(LIBTORCH_URL "https://download.pytorch.org/libtorch/cu118/libtorch-cxx11-abi-shared-with-deps-2.3.1%2Bcu118.zip")
     else()
-      set(LIBTORCH_URL "https://download.pytorch.org/libtorch/cu118/libtorch-shared-with-deps-2.3.0%2Bcu118.zip")
+      set(LIBTORCH_URL "https://download.pytorch.org/libtorch/cu118/libtorch-shared-with-deps-2.3.1%2Bcu118.zip")
     endif()
   else()
     # error out if cuda version is not supported
@@ -232,7 +232,7 @@ else()
   FetchContent_MakeAvailable(libtorch)
 
   find_package(Torch REQUIRED PATHS ${libtorch_SOURCE_DIR} NO_DEFAULT_PATH)
-  message(STATUS "Downloading and using libtorch 2.3 for cuda ${CUDA_VERSION} at ${libtorch_SOURCE_DIR}")
+  message(STATUS "Downloading and using libtorch 2.3.1 for cuda ${CUDA_VERSION} at ${libtorch_SOURCE_DIR}")
 endif()
 
 # check if USE_CXX11_ABI is set correctly

docs/requirements.txt

@@ -7,3 +7,4 @@ sphinxcontrib-napoleon == 0.7
 sphinxcontrib-httpdomain == 1.8.1
 furo == 2024.5.6
 myst-parser == 3.0.1
+sphinx-autobuild 

docs/source/conf.py

@@ -58,7 +58,7 @@
 📢 ScaleLLM is distributed on <a href="https://pypi.org/project/scalellm/">PyPI</a> 🎉.
 If you like this project, <a href='https://github.com/vectorch-ai/ScaleLLM' target='_blank'>give us a star ⭐️! </a>
 """
-html_theme_options["announcement"] = announcement
+# html_theme_options["announcement"] = announcement
 # html_static_path = ['_static']
 
 # show On GitHub link in footer

docs/source/examples.rst

@@ -1,4 +1,138 @@
 .. _examples:
 
 Examples
-========
+========
+
+You can use ScaleLLM for offline batch completions or online inference. Below are some examples to help you get started. More examples can be found in the `examples <https://github.com/vectorch-ai/ScaleLLM/tree/main/examples>`_ folder.
+
+Chat Completion
+---------------
+
+Start the REST API server with the following command:
+
+.. code-block:: bash
+
+   $ python3 -m scalellm.serve.api_server --model=meta-llama/Meta-Llama-3.1-8B-Instruct
+
+You can query the chat completions using `curl` or the OpenAI client:
+
+.. tabs::
+
+   .. group-tab:: curl
+
+      .. code-block:: bash
+
+         $ curl http://localhost:8080/v1/chat/completions \
+           -H "Content-Type: application/json" \
+           -d '{
+             "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+             "messages": [
+               {
+                 "role": "system",
+                 "content": "You are a helpful assistant."
+               },
+               {
+                 "role": "user",
+                 "content": "Hello!"
+               }
+             ]
+           }'
+
+   .. group-tab:: openai
+
+      .. code-block:: python
+
+         import openai
+
+         client = openai.Client(
+             base_url="http://localhost:8080/v1",
+             api_key="EMPTY",
+         )
+
+         # List available models
+         models = client.models.list()
+         print("==== Available models ====")
+         for model in models.data:
+             print(model.id)
+
+         # Choose the first model
+         model = models.data[0].id
+
+         stream = client.chat.completions.create(
+             model=model,
+             messages=[
+                 {"role": "system", "content": "You are a helpful assistant."},
+                 {"role": "user", "content": "Hello"},
+             ],
+             stream=True,
+         )
+
+         print(f"==== Model: {model} ====")
+         for chunk in stream:
+             choice = chunk.choices[0]
+             delta = choice.delta
+             if delta.content:
+                 print(delta.content, end="")
+         print()
+
+Completions
+-----------
+
+Start the REST API server with the following command:
+
+.. code-block:: bash
+
+   $ python3 -m scalellm.serve.api_server --model=meta-llama/Meta-Llama-3.1-8B
+
+You can query the completions using `curl` or the OpenAI client:
+
+.. tabs::
+
+   .. group-tab:: curl
+
+      .. code-block:: bash
+
+         $ curl http://localhost:8080/v1/completions \
+          -H "Content-Type: application/json" \
+          -d '{
+            "model": "meta-llama/Meta-Llama-3-8B",
+            "prompt": "hello",
+            "max_tokens": 32,
+            "temperature": 0.7,
+            "stream": true
+          }'
+
+   .. group-tab:: openai
+
+      .. code-block:: python
+
+         import openai
+
+         client = openai.Client(
+             base_url="http://localhost:8080/v1",
+             api_key="EMPTY",
+         )
+
+         # List available models
+         models = client.models.list()
+         print("==== Available models ====")
+         for model in models.data:
+             print(model.id)
+
+         # Choose the first model
+         model = models.data[0].id
+
+         stream = client.completions.create(
+             model=model,
+             prompt="hello",
+             max_tokens=32,
+             temperature=0.7,
+             stream=True,
+         )
+
+         print(f"==== Model: {model} ====")
+         for chunk in stream:
+             choice = chunk.choices[0]
+             if choice.text:
+                 print(choice.text, end="")
+         print()

docs/source/index.rst

@@ -3,23 +3,29 @@
 ScaleLLM
 ====================================
 
-ScaleLLM is a cutting-edge inference system engineered for large language models (LLMs), meticulously designed to meet the demands of production environments. It extends its support to a wide range of popular open-source models, including Llama3, Gemma, Bloom, GPT-NeoX, and more.
-
+ScaleLLM is a cutting-edge inference system engineered for large language models (LLMs). It is meticulously designed to meet the demands of production environments and extends its support to a wide range of popular open-source models, including Llama3, Gemma, Bloom, GPT-NeoX, and more.
 
 .. note::
    ScaleLLM is currently in alpha. We are actively working on improving the system and adding new features. If you have any feedback or suggestions, please feel free to reach out to us.
 
+ScaleLLM is available as a Python Wheel package on `PyPI <https://pypi.org/project/scalellm/>`_. You can install it using pip:
+
+.. code-block:: bash
+
+   # Install scalellm with CUDA 12.1 and PyTorch 2.3
+   $ pip install scalellm
+
+
 Table of contents
 -----------------
 
 .. toctree::
    :maxdepth: 2
    :caption: User Guide
 
-   installation
    quick_start
-   supported_models
    examples
+   supported_models
 
 
 .. toctree::