diff --git a/doc/source/gen_docs.py b/doc/source/gen_docs.py index 68e381edf6..380d57bb8b 100644 --- a/doc/source/gen_docs.py +++ b/doc/source/gen_docs.py @@ -203,6 +203,7 @@ def get_unique_id(spec): available_controlnet = None model["available_controlnet"] = available_controlnet model["model_ability"] = ', '.join(model.get("model_ability")) + model["gguf_quantizations"] = ", ".join(model.get("gguf_quantizations", [])) rendered = env.get_template('image.rst.jinja').render(model) output_file_path = os.path.join(output_dir, f"{model['model_name'].lower()}.rst") with open(output_file_path, 'w') as output_file: diff --git a/doc/source/models/builtin/audio/cosyvoice2-0.5b.rst b/doc/source/models/builtin/audio/cosyvoice2-0.5b.rst new file mode 100644 index 0000000000..781da78852 --- /dev/null +++ b/doc/source/models/builtin/audio/cosyvoice2-0.5b.rst @@ -0,0 +1,19 @@ +.. _models_builtin_cosyvoice2-0.5b: + +=============== +CosyVoice2-0.5B +=============== + +- **Model Name:** CosyVoice2-0.5B +- **Model Family:** CosyVoice +- **Abilities:** text-to-audio +- **Multilingual:** True + +Specifications +^^^^^^^^^^^^^^ + +- **Model ID:** mrfakename/CosyVoice2-0.5B + +Execute the following command to launch the model:: + + xinference launch --model-name CosyVoice2-0.5B --model-type audio \ No newline at end of file diff --git a/doc/source/models/builtin/audio/f5-tts-mlx.rst b/doc/source/models/builtin/audio/f5-tts-mlx.rst new file mode 100644 index 0000000000..7ef102eb19 --- /dev/null +++ b/doc/source/models/builtin/audio/f5-tts-mlx.rst @@ -0,0 +1,19 @@ +.. _models_builtin_f5-tts-mlx: + +========== +F5-TTS-MLX +========== + +- **Model Name:** F5-TTS-MLX +- **Model Family:** F5-TTS-MLX +- **Abilities:** text-to-audio +- **Multilingual:** True + +Specifications +^^^^^^^^^^^^^^ + +- **Model ID:** lucasnewman/f5-tts-mlx + +Execute the following command to launch the model:: + + xinference launch --model-name F5-TTS-MLX --model-type audio \ No newline at end of file diff --git a/doc/source/models/builtin/audio/fishspeech-1.5.rst b/doc/source/models/builtin/audio/fishspeech-1.5.rst new file mode 100644 index 0000000000..83b0b7d001 --- /dev/null +++ b/doc/source/models/builtin/audio/fishspeech-1.5.rst @@ -0,0 +1,19 @@ +.. _models_builtin_fishspeech-1.5: + +============== +FishSpeech-1.5 +============== + +- **Model Name:** FishSpeech-1.5 +- **Model Family:** FishAudio +- **Abilities:** text-to-audio +- **Multilingual:** True + +Specifications +^^^^^^^^^^^^^^ + +- **Model ID:** fishaudio/fish-speech-1.5 + +Execute the following command to launch the model:: + + xinference launch --model-name FishSpeech-1.5 --model-type audio \ No newline at end of file diff --git a/doc/source/models/builtin/audio/index.rst b/doc/source/models/builtin/audio/index.rst index 0936a93c02..f01047e13c 100644 --- a/doc/source/models/builtin/audio/index.rst +++ b/doc/source/models/builtin/audio/index.rst @@ -25,9 +25,13 @@ The following is a list of built-in audio models in Xinference: cosyvoice-300m-sft + cosyvoice2-0.5b + f5-tts - fishspeech-1.4 + f5-tts-mlx + + fishspeech-1.5 sensevoicesmall diff --git a/doc/source/models/builtin/image/flux.1-dev.rst b/doc/source/models/builtin/image/flux.1-dev.rst index 3a16cfe0a7..5018412d86 100644 --- a/doc/source/models/builtin/image/flux.1-dev.rst +++ b/doc/source/models/builtin/image/flux.1-dev.rst @@ -13,7 +13,15 @@ Specifications ^^^^^^^^^^^^^^ - **Model ID:** black-forest-labs/FLUX.1-dev +- **GGUF Model ID**: city96/FLUX.1-dev-gguf +- **GGUF Quantizations**: F16, Q2_K, Q3_K_S, Q4_0, Q4_1, Q4_K_S, Q5_0, Q5_1, Q5_K_S, Q6_K, Q8_0 + Execute the following command to launch the model:: - xinference launch --model-name FLUX.1-dev --model-type image \ No newline at end of file + xinference launch --model-name FLUX.1-dev --model-type image + + +For GGUF quantization, using below command: + + xinference launch --model-name FLUX.1-dev --model-type image --gguf_quantization ${gguf_quantization} --cpu_offload True diff --git a/doc/source/models/builtin/image/flux.1-schnell.rst b/doc/source/models/builtin/image/flux.1-schnell.rst index df82d2069f..47bc1e2411 100644 --- a/doc/source/models/builtin/image/flux.1-schnell.rst +++ b/doc/source/models/builtin/image/flux.1-schnell.rst @@ -13,7 +13,15 @@ Specifications ^^^^^^^^^^^^^^ - **Model ID:** black-forest-labs/FLUX.1-schnell +- **GGUF Model ID**: city96/FLUX.1-schnell-gguf +- **GGUF Quantizations**: F16, Q2_K, Q3_K_S, Q4_0, Q4_1, Q4_K_S, Q5_0, Q5_1, Q5_K_S, Q6_K, Q8_0 + Execute the following command to launch the model:: - xinference launch --model-name FLUX.1-schnell --model-type image \ No newline at end of file + xinference launch --model-name FLUX.1-schnell --model-type image + + +For GGUF quantization, using below command: + + xinference launch --model-name FLUX.1-schnell --model-type image --gguf_quantization ${gguf_quantization} --cpu_offload True diff --git a/doc/source/models/builtin/image/got-ocr2_0.rst b/doc/source/models/builtin/image/got-ocr2_0.rst index 994b0deae4..b1e9b1d5d2 100644 --- a/doc/source/models/builtin/image/got-ocr2_0.rst +++ b/doc/source/models/builtin/image/got-ocr2_0.rst @@ -16,4 +16,5 @@ Specifications Execute the following command to launch the model:: - xinference launch --model-name GOT-OCR2_0 --model-type image \ No newline at end of file + xinference launch --model-name GOT-OCR2_0 --model-type image + diff --git a/doc/source/models/builtin/image/index.rst b/doc/source/models/builtin/image/index.rst index bf4efdab86..dacc0cbe38 100644 --- a/doc/source/models/builtin/image/index.rst +++ b/doc/source/models/builtin/image/index.rst @@ -23,6 +23,12 @@ The following is a list of built-in image models in Xinference: sd3-medium + sd3.5-large + + sd3.5-large-turbo + + sd3.5-medium + sdxl-turbo stable-diffusion-2-inpainting diff --git a/doc/source/models/builtin/image/kolors.rst b/doc/source/models/builtin/image/kolors.rst index 19d11c4201..886de8ff02 100644 --- a/doc/source/models/builtin/image/kolors.rst +++ b/doc/source/models/builtin/image/kolors.rst @@ -16,4 +16,5 @@ Specifications Execute the following command to launch the model:: - xinference launch --model-name kolors --model-type image \ No newline at end of file + xinference launch --model-name kolors --model-type image + diff --git a/doc/source/models/builtin/image/sd-turbo.rst b/doc/source/models/builtin/image/sd-turbo.rst index e799b423a3..c78ebaef91 100644 --- a/doc/source/models/builtin/image/sd-turbo.rst +++ b/doc/source/models/builtin/image/sd-turbo.rst @@ -16,4 +16,5 @@ Specifications Execute the following command to launch the model:: - xinference launch --model-name sd-turbo --model-type image \ No newline at end of file + xinference launch --model-name sd-turbo --model-type image + diff --git a/doc/source/models/builtin/image/sd3-medium.rst b/doc/source/models/builtin/image/sd3-medium.rst index 953a3eca32..f0e728cc02 100644 --- a/doc/source/models/builtin/image/sd3-medium.rst +++ b/doc/source/models/builtin/image/sd3-medium.rst @@ -16,4 +16,5 @@ Specifications Execute the following command to launch the model:: - xinference launch --model-name sd3-medium --model-type image \ No newline at end of file + xinference launch --model-name sd3-medium --model-type image + diff --git a/doc/source/models/builtin/image/sd3.5-large-turbo.rst b/doc/source/models/builtin/image/sd3.5-large-turbo.rst new file mode 100644 index 0000000000..3c7e26c2b0 --- /dev/null +++ b/doc/source/models/builtin/image/sd3.5-large-turbo.rst @@ -0,0 +1,27 @@ +.. _models_builtin_sd3.5-large-turbo: + +================= +sd3.5-large-turbo +================= + +- **Model Name:** sd3.5-large-turbo +- **Model Family:** stable_diffusion +- **Abilities:** text2image, image2image, inpainting +- **Available ControlNet:** None + +Specifications +^^^^^^^^^^^^^^ + +- **Model ID:** stabilityai/stable-diffusion-3.5-large-turbo +- **GGUF Model ID**: city96/stable-diffusion-3.5-large-turbo-gguf +- **GGUF Quantizations**: F16, Q4_0, Q4_1, Q5_0, Q5_1, Q8_0 + + +Execute the following command to launch the model:: + + xinference launch --model-name sd3.5-large-turbo --model-type image + + +For GGUF quantization, using below command: + + xinference launch --model-name sd3.5-large-turbo --model-type image --gguf_quantization ${gguf_quantization} --cpu_offload True diff --git a/doc/source/models/builtin/image/sd3.5-large.rst b/doc/source/models/builtin/image/sd3.5-large.rst new file mode 100644 index 0000000000..d998fc4908 --- /dev/null +++ b/doc/source/models/builtin/image/sd3.5-large.rst @@ -0,0 +1,27 @@ +.. _models_builtin_sd3.5-large: + +=========== +sd3.5-large +=========== + +- **Model Name:** sd3.5-large +- **Model Family:** stable_diffusion +- **Abilities:** text2image, image2image, inpainting +- **Available ControlNet:** None + +Specifications +^^^^^^^^^^^^^^ + +- **Model ID:** stabilityai/stable-diffusion-3.5-large +- **GGUF Model ID**: city96/stable-diffusion-3.5-large-gguf +- **GGUF Quantizations**: F16, Q4_0, Q4_1, Q5_0, Q5_1, Q8_0 + + +Execute the following command to launch the model:: + + xinference launch --model-name sd3.5-large --model-type image + + +For GGUF quantization, using below command: + + xinference launch --model-name sd3.5-large --model-type image --gguf_quantization ${gguf_quantization} --cpu_offload True diff --git a/doc/source/models/builtin/image/sd3.5-medium.rst b/doc/source/models/builtin/image/sd3.5-medium.rst new file mode 100644 index 0000000000..42a65f7e47 --- /dev/null +++ b/doc/source/models/builtin/image/sd3.5-medium.rst @@ -0,0 +1,27 @@ +.. _models_builtin_sd3.5-medium: + +============ +sd3.5-medium +============ + +- **Model Name:** sd3.5-medium +- **Model Family:** stable_diffusion +- **Abilities:** text2image, image2image, inpainting +- **Available ControlNet:** None + +Specifications +^^^^^^^^^^^^^^ + +- **Model ID:** stabilityai/stable-diffusion-3.5-medium +- **GGUF Model ID**: city96/stable-diffusion-3.5-medium-gguf +- **GGUF Quantizations**: F16, Q3_K_M, Q3_K_S, Q4_0, Q4_1, Q4_K_M, Q4_K_S, Q5_0, Q5_1, Q5_K_M, Q5_K_S, Q6_K, Q8_0 + + +Execute the following command to launch the model:: + + xinference launch --model-name sd3.5-medium --model-type image + + +For GGUF quantization, using below command: + + xinference launch --model-name sd3.5-medium --model-type image --gguf_quantization ${gguf_quantization} --cpu_offload True diff --git a/doc/source/models/builtin/image/sdxl-turbo.rst b/doc/source/models/builtin/image/sdxl-turbo.rst index 878b10079f..9da2b63bf2 100644 --- a/doc/source/models/builtin/image/sdxl-turbo.rst +++ b/doc/source/models/builtin/image/sdxl-turbo.rst @@ -16,4 +16,5 @@ Specifications Execute the following command to launch the model:: - xinference launch --model-name sdxl-turbo --model-type image \ No newline at end of file + xinference launch --model-name sdxl-turbo --model-type image + diff --git a/doc/source/models/builtin/image/stable-diffusion-2-inpainting.rst b/doc/source/models/builtin/image/stable-diffusion-2-inpainting.rst index 6009cd37a3..01a84a63db 100644 --- a/doc/source/models/builtin/image/stable-diffusion-2-inpainting.rst +++ b/doc/source/models/builtin/image/stable-diffusion-2-inpainting.rst @@ -16,4 +16,5 @@ Specifications Execute the following command to launch the model:: - xinference launch --model-name stable-diffusion-2-inpainting --model-type image \ No newline at end of file + xinference launch --model-name stable-diffusion-2-inpainting --model-type image + diff --git a/doc/source/models/builtin/image/stable-diffusion-inpainting.rst b/doc/source/models/builtin/image/stable-diffusion-inpainting.rst index 76f1274048..3b4832471c 100644 --- a/doc/source/models/builtin/image/stable-diffusion-inpainting.rst +++ b/doc/source/models/builtin/image/stable-diffusion-inpainting.rst @@ -16,4 +16,5 @@ Specifications Execute the following command to launch the model:: - xinference launch --model-name stable-diffusion-inpainting --model-type image \ No newline at end of file + xinference launch --model-name stable-diffusion-inpainting --model-type image + diff --git a/doc/source/models/builtin/image/stable-diffusion-v1.5.rst b/doc/source/models/builtin/image/stable-diffusion-v1.5.rst index 5a0c73adcd..764bbb0a4c 100644 --- a/doc/source/models/builtin/image/stable-diffusion-v1.5.rst +++ b/doc/source/models/builtin/image/stable-diffusion-v1.5.rst @@ -16,4 +16,5 @@ Specifications Execute the following command to launch the model:: - xinference launch --model-name stable-diffusion-v1.5 --model-type image \ No newline at end of file + xinference launch --model-name stable-diffusion-v1.5 --model-type image + diff --git a/doc/source/models/builtin/image/stable-diffusion-xl-base-1.0.rst b/doc/source/models/builtin/image/stable-diffusion-xl-base-1.0.rst index a4f7518dbf..cfb515be55 100644 --- a/doc/source/models/builtin/image/stable-diffusion-xl-base-1.0.rst +++ b/doc/source/models/builtin/image/stable-diffusion-xl-base-1.0.rst @@ -16,4 +16,5 @@ Specifications Execute the following command to launch the model:: - xinference launch --model-name stable-diffusion-xl-base-1.0 --model-type image \ No newline at end of file + xinference launch --model-name stable-diffusion-xl-base-1.0 --model-type image + diff --git a/doc/source/models/builtin/image/stable-diffusion-xl-inpainting.rst b/doc/source/models/builtin/image/stable-diffusion-xl-inpainting.rst index 61a72cc044..f096627d51 100644 --- a/doc/source/models/builtin/image/stable-diffusion-xl-inpainting.rst +++ b/doc/source/models/builtin/image/stable-diffusion-xl-inpainting.rst @@ -16,4 +16,5 @@ Specifications Execute the following command to launch the model:: - xinference launch --model-name stable-diffusion-xl-inpainting --model-type image \ No newline at end of file + xinference launch --model-name stable-diffusion-xl-inpainting --model-type image + diff --git a/doc/source/models/model_abilities/image.rst b/doc/source/models/model_abilities/image.rst index e7612ee916..e49971a85b 100644 --- a/doc/source/models/model_abilities/image.rst +++ b/doc/source/models/model_abilities/image.rst @@ -170,17 +170,19 @@ To use gguf file, you can specify additional option ``gguf_quantization`` for we or ``--gguf_quantization`` for command line for those image models which support internally by Xinference. Below is the mode list. -+----------------+------------------------------------------------------------------------+ -| Model | supported gguf quantization | -+================+==============================================+=========================+ -| FLUX.1-dev | F16, Q2_K, Q3_K_S, Q4_0, Q4_1, Q4_K_S, Q5_0, Q5_1, Q5_K_S, Q6_K, Q8_0 | -+----------------+------------------------------------------------------------------------+ -| FLUX.1-schnell | F16, Q2_K, Q3_K_S, Q4_0, Q4_1, Q4_K_S, Q5_0, Q5_1, Q5_K_S, Q6_K, Q8_0 | -+----------------+------------------------------------------------------------------------+ -| sd3.5-medium | | -+----------------+------------------------------------------------------------------------+ -| sd3.5-large | | -+----------------+------------------------------------------------------------------------+ ++-------------------+------------------------------------------------------------------------------------------+ +| Model | supported gguf quantization | ++===================+==============================================+===========================================+ +| FLUX.1-dev | F16, Q2_K, Q3_K_S, Q4_0, Q4_1, Q4_K_S, Q5_0, Q5_1, Q5_K_S, Q6_K, Q8_0 | ++-------------------+------------------------------------------------------------------------------------------+ +| FLUX.1-schnell | F16, Q2_K, Q3_K_S, Q4_0, Q4_1, Q4_K_S, Q5_0, Q5_1, Q5_K_S, Q6_K, Q8_0 | ++-------------------+------------------------------------------------------------------------------------------+ +| sd3.5-medium | F16, Q3_K_M, Q3_K_S, Q4_0, Q4_1, Q4_K_M, Q4_K_S, Q5_0, Q5_1, Q5_K_M, Q5_K_S, Q6_K, Q8_0 | ++-------------------+------------------------------------------------------------------------------------------+ +| sd3.5-large | F16, Q4_0, Q4_1, Q5_0, Q5_1, Q8_0 | ++-------------------+------------------------------------------------------------------------------------------+ +| sd3.5-large-turbo | F16, Q4_0, Q4_1, Q5_0, Q5_1, Q8_0 | ++-------------------+------------------------------------------------------------------------------------------+ .. note:: diff --git a/doc/templates/image.rst.jinja b/doc/templates/image.rst.jinja index 06379d7d56..8e1e57c145 100644 --- a/doc/templates/image.rst.jinja +++ b/doc/templates/image.rst.jinja @@ -13,7 +13,17 @@ Specifications ^^^^^^^^^^^^^^ - **Model ID:** {{ model_id }} +{%- if gguf_quantizations %} +- **GGUF Model ID**: {{ gguf_model_id }} +- **GGUF Quantizations**: {{ gguf_quantizations }} +{% endif %} Execute the following command to launch the model:: - xinference launch --model-name {{ model_name }} --model-type image \ No newline at end of file + xinference launch --model-name {{ model_name }} --model-type image + +{% if gguf_quantizations %} +For GGUF quantization, using below command: + + xinference launch --model-name {{ model_name }} --model-type image --gguf_quantization ${{ '{' }}gguf_quantization{{ '}' }} --cpu_offload True +{% endif %} \ No newline at end of file