diff --git a/Dockerfile b/Dockerfile
index 72894e7c..b5bbab34 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -118,7 +118,7 @@ RUN ldconfig /usr/local/cuda-12.4/compat/
 # install vllm wheel first, so that torch etc will be installed
 RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
     --mount=type=cache,target=/root/.cache/pip \
-    pip install dist/*.whl --verbose
+    pip install "$(echo dist/*.whl)[ray]" --verbose
 #################### vLLM installation IMAGE ####################
 
 
diff --git a/requirements-cuda.txt b/requirements-cuda.txt
index 35361798..f5ce47eb 100644
--- a/requirements-cuda.txt
+++ b/requirements-cuda.txt
@@ -2,7 +2,6 @@
 -r requirements-common.txt
 
 # Dependencies for NVIDIA GPUs
-ray >= 2.9
 nvidia-ml-py # for pynvml package
 torch == 2.3.0
 xformers == 0.0.26.post1  # Requires PyTorch 2.3.0
diff --git a/requirements-rocm.txt b/requirements-rocm.txt
index cc42839a..56fd718f 100644
--- a/requirements-rocm.txt
+++ b/requirements-rocm.txt
@@ -2,5 +2,4 @@
 -r requirements-common.txt
 
 # Dependencies for AMD GPUs
-ray >= 2.10.0
 pytest-asyncio
diff --git a/setup.py b/setup.py
index 12a704e0..7d340ce3 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 import subprocess
 import sys
 from shutil import which
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 import torch
 from packaging.version import Version, parse
@@ -392,6 +392,20 @@ def _read_requirements(filename: str) -> List[str]:
     return requirements
 
 
+def get_extra_requirements() -> Optional[Dict[str, List[str]]]:
+    extras = {"tensorizer": ["tensorizer>=2.9.0"]}
+    if _is_cuda():
+        extras["ray"] = ["ray>=2.9"]
+    elif _is_hip():
+        extras["ray"] = ["ray==2.9.3"]
+    elif _is_neuron() or _is_cpu():
+        pass
+    else:
+        raise ValueError(
+            "Unsupported platform, please use CUDA, ROCM or Neuron.")
+    return extras
+
+
 ext_modules = []
 
 if _is_cuda() or _is_hip():
@@ -437,9 +451,7 @@ def _read_requirements(filename: str) -> List[str]:
     python_requires=">=3.8",
     install_requires=get_requirements(),
     ext_modules=ext_modules,
-    extras_require={
-        "tensorizer": ["tensorizer>=2.9.0"],
-    },
+    extras_require=get_extra_requirements(),
     cmdclass={"build_ext": cmake_build_ext} if _build_custom_ops() else {},
     package_data=package_data,
 )