update docs for llama3.1 and bump up version

.github/workflows/build_wheel.yml

@@ -25,7 +25,7 @@ jobs:
       matrix:
         python: ["3.8", "3.9", "3.10", "3.11"]
         cuda: ["11.8", "12.1"]
-        torch: ["2.1", "2.2", "2.3"]
+        torch: ["2.1.2", "2.2.2", "2.3.1"]
     runs-on: [self-hosted, linux, release]
     env:
       PYTHON_VERSION: ${{ matrix.python }}

.github/workflows/package_test.yml

@@ -40,7 +40,7 @@ jobs:
       matrix:
         python: ["3.10"]
         cuda: ["12.1"]
-        torch: ["2.3"]
+        torch: ["2.3.1"]
     runs-on: [self-hosted, linux, build]
     env:
       PYTHON_VERSION: ${{ matrix.python }}

.github/workflows/publish_wheel.yml

@@ -24,7 +24,7 @@ jobs:
       matrix:
         python: ["3.8", "3.9", "3.10", "3.11"]
         cuda: ["12.1"]
-        torch: ["2.3"]
+        torch: ["2.3.1"]
     runs-on: [self-hosted, linux, release]
     env:
       PYTHON_VERSION: ${{ matrix.python }}

.github/workflows/release_test.yml

@@ -15,7 +15,7 @@ jobs:
       matrix:
         python: ["3.10"]
         cuda: ["12.1"]
-        torch: ["2.3"]
+        torch: ["2.3.1"]
     runs-on: [self-hosted, linux, build]
     env:
       PYTHON_VERSION: ${{ matrix.python }}

docs/requirements.txt

@@ -7,3 +7,4 @@ sphinxcontrib-napoleon == 0.7
 sphinxcontrib-httpdomain == 1.8.1
 furo == 2024.5.6
 myst-parser == 3.0.1
+sphinx-autobuild 

docs/source/examples.rst

@@ -1,4 +1,138 @@
 .. _examples:
 
 Examples
-========
+========
+
+You can use ScaleLLM for offline batch completions or online inference. Below are some examples to help you get started. More examples can be found in the `examples <https://github.com/vectorch-ai/ScaleLLM/tree/main/examples>`_ folder.
+
+Chat Completion
+---------------
+
+Start the REST API server with the following command:
+
+.. code-block:: bash
+
+   $ python3 -m scalellm.serve.api_server --model=meta-llama/Meta-Llama-3.1-8B-Instruct
+
+You can query the chat completions using `curl` or the OpenAI client:
+
+.. tabs::
+
+   .. group-tab:: curl
+
+      .. code-block:: bash
+
+         $ curl http://localhost:8080/v1/chat/completions \
+           -H "Content-Type: application/json" \
+           -d '{
+             "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+             "messages": [
+               {
+                 "role": "system",
+                 "content": "You are a helpful assistant."
+               },
+               {
+                 "role": "user",
+                 "content": "Hello!"
+               }
+             ]
+           }'
+
+   .. group-tab:: openai
+
+      .. code-block:: python
+
+         import openai
+
+         client = openai.Client(
+             base_url="http://localhost:8080/v1",
+             api_key="EMPTY",
+         )
+
+         # List available models
+         models = client.models.list()
+         print("==== Available models ====")
+         for model in models.data:
+             print(model.id)
+
+         # Choose the first model
+         model = models.data[0].id
+
+         stream = client.chat.completions.create(
+             model=model,
+             messages=[
+                 {"role": "system", "content": "You are a helpful assistant."},
+                 {"role": "user", "content": "Hello"},
+             ],
+             stream=True,
+         )
+
+         print(f"==== Model: {model} ====")
+         for chunk in stream:
+             choice = chunk.choices[0]
+             delta = choice.delta
+             if delta.content:
+                 print(delta.content, end="")
+         print()
+
+Completions
+-----------
+
+Start the REST API server with the following command:
+
+.. code-block:: bash
+  
+   $ python3 -m scalellm.serve.api_server --model=meta-llama/Meta-Llama-3.1-8B
+
+You can query the completions using `curl` or the OpenAI client:
+
+.. tabs::
+
+   .. group-tab:: curl
+
+      .. code-block:: bash
+
+         $ curl http://localhost:8080/v1/completions \
+          -H "Content-Type: application/json" \
+          -d '{
+            "model": "meta-llama/Meta-Llama-3-8B",
+            "prompt": "hello",
+            "max_tokens": 32,
+            "temperature": 0.7,
+            "stream": true
+          }'
+
+   .. group-tab:: openai
+
+      .. code-block:: python
+
+         import openai
+
+         client = openai.Client(
+             base_url="http://localhost:8080/v1",
+             api_key="EMPTY",
+         )
+
+         # List available models
+         models = client.models.list()
+         print("==== Available models ====")
+         for model in models.data:
+             print(model.id)
+
+         # Choose the first model
+         model = models.data[0].id
+
+         stream = client.completions.create(
+             model=model,
+             prompt="hello",
+             max_tokens=32,
+             temperature=0.7,
+             stream=True,
+         )
+
+         print(f"==== Model: {model} ====")
+         for chunk in stream:
+             choice = chunk.choices[0]
+             if choice.text:
+                 print(choice.text, end="")
+         print()

docs/source/index.rst

@@ -3,23 +3,29 @@
 ScaleLLM
 ====================================
 
-ScaleLLM is a cutting-edge inference system engineered for large language models (LLMs), meticulously designed to meet the demands of production environments. It extends its support to a wide range of popular open-source models, including Llama3, Gemma, Bloom, GPT-NeoX, and more.
-
+ScaleLLM is a cutting-edge inference system engineered for large language models (LLMs). It is meticulously designed to meet the demands of production environments and extends its support to a wide range of popular open-source models, including Llama3, Gemma, Bloom, GPT-NeoX, and more.
 
 .. note::
    ScaleLLM is currently in alpha. We are actively working on improving the system and adding new features. If you have any feedback or suggestions, please feel free to reach out to us.
 
+ScaleLLM is available as a Python Wheel package on `PyPI <https://pypi.org/project/scalellm/>`_. You can install it using pip:
+
+.. code-block:: bash
+
+   # Install scalellm with CUDA 12.1 and PyTorch 2.3
+   $ pip install scalellm
+
+
 Table of contents
 -----------------
 
 .. toctree::
    :maxdepth: 2
    :caption: User Guide
 
-   installation
    quick_start
-   supported_models
    examples
+   supported_models
 
 
 .. toctree::

docs/source/installation.rst

@@ -67,36 +67,13 @@ If you want to install ScaleLLM with different version of CUDA and Pytorch, you
 
 Build From Source
 -----------------
+If no wheel package is available for your configuration, you can build ScaleLLM from source code. You can clone the repository and install it locally using the following commands:
 
-.. tabs::
-
-	.. group-tab:: curl
-
-		.. code-block:: bash
-
-			$ curl -L
-
-	.. group-tab:: openai
-
-		.. code-block:: bash
-
-			$ git clone
-
-
-Another section
----------------
-.. tabs::
-
-	.. group-tab:: curl
-
-		.. code-block:: bash
-
-			$ curl -L
-
-	.. group-tab:: openai
-
-		.. code-block:: bash
-
-			$ git clone
+.. code-block:: bash
+	
+	$ git clone --recursive https://github.com/vectorch-ai/ScaleLLM.git
+	$ cd ScaleLLM
+	$ python3 setup.py bdist_wheel
+	$ pip install dist/scalellm-*.whl
 
 

docs/source/quick_start.rst

@@ -1,5 +1,90 @@
 .. _quick_start:
 
-Quick Start
-===========
-This section will guide you through the process of setting up a simple project...
+
+Installation
+------------
+
+Install with pip
+~~~~~~~~~~~~~~~~
+ScaleLLM is available as a Python Wheel package on `PyPI <https://pypi.org/project/scalellm/>`_. You can install it using pip:
+
+.. code-block:: bash
+
+    # Install ScaleLLM with CUDA 12.1 and PyTorch 2.3
+    $ pip install scalellm
+
+Install other versions
+~~~~~~~~~~~~~~~~~~~~~~
+If you want to install ScaleLLM with different versions of CUDA and PyTorch, you can use pip by providing the index URL of the desired version.
+
+.. tabs::
+
+    .. tab:: CUDA 12.1
+
+        .. tabs::
+
+            .. tab:: PyTorch 2.3
+
+                .. code-block:: bash
+
+                    $ pip install scalellm -i https://whl.vectorch.com/cu121/torch2.3/
+
+            .. tab:: PyTorch 2.2
+
+                .. code-block:: bash
+
+                    $ pip install scalellm -i https://whl.vectorch.com/cu121/torch2.2/
+
+            .. tab:: PyTorch 2.1
+
+                .. code-block:: bash
+
+                    $ pip install scalellm -i https://whl.vectorch.com/cu121/torch2.1/
+
+    .. tab:: CUDA 11.8
+
+        .. tabs::
+
+            .. tab:: PyTorch 2.3
+
+                .. code-block:: bash
+
+                    $ pip install scalellm -i https://whl.vectorch.com/cu118/torch2.3/
+
+            .. tab:: PyTorch 2.2
+
+                .. code-block:: bash
+
+                    $ pip install scalellm -i https://whl.vectorch.com/cu118/torch2.2/
+
+            .. tab:: PyTorch 2.1
+
+                .. code-block:: bash
+
+                    $ pip install scalellm -i https://whl.vectorch.com/cu118/torch2.1/
+
+
+Build from source
+~~~~~~~~~~~~~~~~~
+If no wheel package is available for your configuration, you can build ScaleLLM from source code. Clone the repository and install it locally using the following commands:
+
+.. code-block:: bash
+
+    $ git clone --recursive https://github.com/vectorch-ai/ScaleLLM.git
+    $ cd ScaleLLM
+    $ python3 setup.py bdist_wheel
+    $ pip install dist/scalellm-*.whl
+
+
+Inference
+-------
+
+You can use ScaleLLM for offline batch inference or online distributed inference.
+
+OpenAI-Compatible Server
+~~~~~~~~~~~~~~~~~~~~~~~~
+To start a server that is compatible with the OpenAI API, run the following command:
+
+.. code-block:: bash
+
+    $ python3 -m scalellm.serve.api_server --model=meta-llama/Meta-Llama-3.1-8B-Instruct