[Core] Install SkyPilot runtime in separate env

sky/backends/backend_utils.py

@@ -64,7 +64,6 @@
 SKY_REMOTE_APP_DIR = '~/.sky/sky_app'
 # Exclude subnet mask from IP address regex.
 IP_ADDR_REGEX = r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(?!/\d{1,2})\b'
-SKY_REMOTE_PATH = '~/.sky/wheels'
 SKY_USER_FILE_PATH = '~/.sky/generated'
 
 BOLD = '\033[1m'
@@ -911,7 +910,7 @@ def write_cluster_config(
                 # Cloud credentials for cloud storage.
                 'credentials': credentials,
                 # Sky remote utils.
-                'sky_remote_path': SKY_REMOTE_PATH,
+                'sky_remote_path': constants.SKY_REMOTE_WHEEL_PATH,
                 'sky_local_path': str(local_wheel_path),
                 # Add yaml file path to the template variables.
                 'sky_ray_yaml_remote_path':

sky/backends/cloud_vm_ray_backend.py

@@ -3126,6 +3126,7 @@ def _exec_code_on_head(
         cd = f'cd {SKY_REMOTE_WORKDIR}'
 
         job_submit_cmd = (
+            f'{constants.ACTIVATE_PYTHON_ENV}'
             'RAY_DASHBOARD_PORT=$(python -c "from sky.skylet import job_lib; print(job_lib.get_job_submission_port())" 2> /dev/null || echo 8265);'  # pylint: disable=line-too-long
             f'{cd} && ray job submit '
             '--address=http://127.0.0.1:$RAY_DASHBOARD_PORT '

sky/backends/wheel_utils.py

@@ -24,12 +24,12 @@
 
 import sky
 from sky import sky_logging
-from sky.backends import backend_utils
+from sky.skylet import constants
 
 logger = sky_logging.init_logger(__name__)
 
 # Local wheel path is same as the remote path.
-WHEEL_DIR = pathlib.Path(os.path.expanduser(backend_utils.SKY_REMOTE_PATH))
+WHEEL_DIR = pathlib.Path(os.path.expanduser(constants.SKY_REMOTE_WHEEL_PATH))
 _WHEEL_LOCK_PATH = WHEEL_DIR.parent / '.wheels_lock'
 SKY_PACKAGE_PATH = pathlib.Path(sky.__file__).parent.parent / 'sky'
 

sky/execution.py

@@ -677,6 +677,7 @@ def spot_launch(
             # Note: actual spot cluster name will be <task.name>-<spot job ID>
             'dag_name': dag.name,
             'retry_until_up': retry_until_up,
+            'skypilot_runtime_env': constants.SKY_REMOTE_PYTHON_ENV,
             'remote_user_config_path': remote_user_config_path,
             **controller_utils.shared_controller_vars_to_fill(
                 'spot',

sky/provision/instance_setup.py

@@ -47,8 +47,10 @@
 
 # Command that calls `ray status` with SkyPilot's Ray port set.
 RAY_STATUS_WITH_SKY_RAY_PORT_COMMAND = (
+    f'{constants.ACTIVATE_PYTHON_ENV} '
     f'{_RAY_PORT_COMMAND}; '
-    'RAY_ADDRESS=127.0.0.1:$RAY_PORT ray status')
+    'export RAY_ADDRESS=127.0.0.1:$RAY_PORT; '
+    f'ray status')
 
 # Command that waits for the ray status to be initialized. Otherwise, a later
 # `sky status -r` may fail due to the ray cluster not being ready.
@@ -59,7 +61,8 @@
     'done;')
 
 # Restart skylet when the version does not match to keep the skylet up-to-date.
-MAYBE_SKYLET_RESTART_CMD = 'python3 -m sky.skylet.attempt_skylet;'
+MAYBE_SKYLET_RESTART_CMD = (f'{constants.ACTIVATE_PYTHON_ENV} '
+                            'python -m sky.skylet.attempt_skylet')
 
 
 def _auto_retry(func):
@@ -288,8 +291,9 @@ def start_ray_on_head_node(cluster_name: str, custom_resource: Optional[str],
     # the same credentials. Otherwise, `ray status` will fail to fetch the
     # available nodes.
     # Reference: https://github.com/skypilot-org/skypilot/issues/2441
-    cmd = ('ray stop; unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; '
-           'RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 '
+    cmd = (f'{constants.ACTIVATE_PYTHON_ENV} ray stop; '
+           'unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; '
+           'export RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0; '
            f'ray start --head {ray_options} || exit 1;' + _RAY_PRLIMIT +
            _DUMP_RAY_PORTS + RAY_HEAD_WAIT_INITIALIZED_COMMAND)
     logger.info(f'Running command on head node: {cmd}')
@@ -371,6 +375,7 @@ def start_ray_on_worker_nodes(cluster_name: str, no_restart: bool,
                f'{{ {cmd} }}')
     else:
         cmd = 'ray stop; ' + cmd
+    cmd = constants.ACTIVATE_PYTHON_ENV + cmd
 
     logger.info(f'Running command on worker nodes: {cmd}')
 

sky/serve/core.py

@@ -147,6 +147,7 @@ def up(
             'local_task_yaml_path': service_file.name,
             'service_name': service_name,
             'controller_log_file': controller_log_file,
+            'skypilot_runtime_env': constants.SKY_REMOTE_PYTHON_ENV,
             'remote_user_config_path': remote_config_yaml_path,
             **controller_utils.shared_controller_vars_to_fill(
                 'serve',

sky/serve/serve_utils.py

@@ -26,6 +26,7 @@
 from sky import status_lib
 from sky.serve import constants
 from sky.serve import serve_state
+from sky.skylet import constants as skylet_constants
 from sky.skylet import job_lib
 from sky.utils import common_utils
 from sky.utils import log_utils
@@ -923,7 +924,8 @@ def stream_serve_process_logs(cls, service_name: str,
     def _build(cls, code: List[str]) -> str:
         code = cls._PREFIX + code
         generated_code = '; '.join(code)
-        return f'python3 -u -c {shlex.quote(generated_code)}'
+        return (f'{skylet_constants.ACTIVATE_PYTHON_ENV} '
+                f'python -u -c {shlex.quote(generated_code)}')
 
     @classmethod
     def update_service(cls, service_name: str, version: int, mode: str) -> str:

sky/skylet/attempt_skylet.py

@@ -13,12 +13,13 @@ def restart_skylet():
     # TODO(zhwu): make the killing graceful, e.g., use a signal to tell
     # skylet to exit, instead of directly killing it.
     subprocess.run(
-        'ps aux | grep "sky.skylet.skylet" | grep "python3 -m"'
+        'ps aux | grep "sky.skylet.skylet" | grep "python -m"'
         '| awk \'{print $2}\' | xargs kill >> ~/.sky/skylet.log 2>&1',
         shell=True,
         check=False)
     subprocess.run(
-        'nohup python3 -m sky.skylet.skylet'
+        f'{constants.ACTIVATE_PYTHON_ENV} '
+        f'nohup python -m sky.skylet.skylet'
         ' >> ~/.sky/skylet.log 2>&1 &',
         shell=True,
         check=True)
@@ -27,7 +28,7 @@ def restart_skylet():
 
 
 proc = subprocess.run(
-    'ps aux | grep -v "grep" | grep "sky.skylet.skylet" | grep "python3 -m"',
+    'ps aux | grep -v "grep" | grep "sky.skylet.skylet" | grep "python -m"',
     shell=True,
     check=False)
 

sky/skylet/autostop_lib.py

@@ -8,6 +8,7 @@
 
 from sky import sky_logging
 from sky.skylet import configs
+from sky.skylet import constants
 from sky.utils import common_utils
 
 logger = sky_logging.init_logger(__name__)
@@ -121,4 +122,5 @@ def is_autostopping(cls) -> str:
     def _build(cls, code: List[str]) -> str:
         code = cls._PREFIX + code
         code = ';'.join(code)
-        return f'python3 -u -c {shlex.quote(code)}'
+        return (f'{constants.ACTIVATE_PYTHON_ENV} '
+                f'python -u -c {shlex.quote(code)}')

sky/skylet/constants.py

@@ -22,6 +22,14 @@
 SKY_REMOTE_RAY_PORT_FILE = '~/.sky/ray_port.json'
 SKY_REMOTE_RAY_TEMPDIR = '/tmp/ray_skypilot'
 SKY_REMOTE_RAY_VERSION = '2.9.3'
+SKY_REMOTE_WHEEL_PATH = '~/.sky/wheels'
+SKY_REMOTE_PYTHON_ENV_NAME = 'skypilot-runtime'
+SKY_REMOTE_PYTHON_ENV = f'~/{SKY_REMOTE_PYTHON_ENV_NAME}'
+
+# TODO(mluo): Make explicit `sky launch -c <name> ''` optional.
+UNINITIALIZED_ONPREM_CLUSTER_MESSAGE = (
+    'Found uninitialized local cluster {cluster}. Run this '
+    'command to initialize it locally: sky launch -c {cluster} \'\'')
 
 # The name for the environment variable that stores the unique ID of the
 # current task. This will stay the same across multiple recoveries of the
@@ -66,19 +74,38 @@
     DOCKER_SERVER_ENV_VAR,
 }
 
+ACTIVATE_PYTHON_ENV = (f'[ -d {SKY_REMOTE_PYTHON_ENV} ] && '
+                       f'source {SKY_REMOTE_PYTHON_ENV}/bin/activate;')
+
+
+def run_in_python_env(command):
+    """Returns a command that runs in the SkyPilot's python environment."""
+    return f'( {ACTIVATE_PYTHON_ENV}{command}; deactivate )'
+
+
 # Install conda on the remote cluster if it is not already installed.
 # We use conda with python 3.10 to be consistent across multiple clouds with
 # best effort.
 # https://github.com/ray-project/ray/issues/31606
 # We use python 3.10 to be consistent with the python version of the
 # AWS's Deep Learning AMI's default conda environment.
+_RUN_PYTHON = run_in_python_env('python \\$@')
+_RUN_PIP = run_in_python_env('pip \\$@')
+_RUN_RAY = run_in_python_env('ray \\$@')
 CONDA_INSTALLATION_COMMANDS = (
     'which conda > /dev/null 2>&1 || '
-    '(wget -nc https://repo.anaconda.com/miniconda/Miniconda3-py310_23.11.0-2-Linux-x86_64.sh -O Miniconda3-Linux-x86_64.sh && '  # pylint: disable=line-too-long
+    '{ wget -nc https://repo.anaconda.com/miniconda/Miniconda3-py310_23.11.0-2-Linux-x86_64.sh -O Miniconda3-Linux-x86_64.sh && '  # pylint: disable=line-too-long
     'bash Miniconda3-Linux-x86_64.sh -b && '
     'eval "$(~/miniconda3/bin/conda shell.bash hook)" && conda init && '
-    'conda config --set auto_activate_base true); '
-    'grep "# >>> conda initialize >>>" ~/.bashrc || conda init;')
+    'conda config --set auto_activate_base true && source ~/.bashrc; }; '
+    'grep "# >>> conda initialize >>>" ~/.bashrc || conda init;'
+    # Create a separate conda environment for SkyPilot dependencies.
+    f'[ -d {SKY_REMOTE_PYTHON_ENV} ] || '
+    f'python -m venv {SKY_REMOTE_PYTHON_ENV}; '
+    f'source {SKY_REMOTE_PYTHON_ENV}/bin/activate; '
+    f'echo "function skypy () {{ {_RUN_PYTHON} }}" >> ~/.bashrc;'
+    f'echo "function skypip () {{ {_RUN_PIP} }}" >> ~/.bashrc;'
+    f'echo "function skyray () {{ {_RUN_RAY} }}" >> ~/.bashrc;')
 
 _sky_version = str(version.parse(sky.__version__))
 RAY_STATUS = f'RAY_ADDRESS=127.0.0.1:{SKY_REMOTE_RAY_PORT} ray status'

sky/skylet/job_lib.py

@@ -933,4 +933,5 @@ def get_run_timestamp_with_globbing(cls,
     def _build(cls, code: List[str]) -> str:
         code = cls._PREFIX + code
         code = ';'.join(code)
-        return f'python3 -u -c {shlex.quote(code)}'
+        return (f'{constants.ACTIVATE_PYTHON_ENV} '
+                f'python -u -c {shlex.quote(code)}')

sky/skylet/log_lib.py

@@ -245,13 +245,17 @@ def run_with_log(
 
 def make_task_bash_script(codegen: str,
                           env_vars: Optional[Dict[str, str]] = None) -> str:
+    # 'export PATH=...' is needed to avoid running the user program in the
+    # skypilot-runtime environment. We cannot use `deactivate` as it cannot
+    # be found after conda is installed.
     # set -a is used for exporting all variables functions to the environment
     # so that bash `user_script` can access `conda activate`. Detail: #436.
     # Reference: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html # pylint: disable=line-too-long
     # PYTHONUNBUFFERED is used to disable python output buffering.
     script = [
         textwrap.dedent(f"""\
             #!/bin/bash
+            export PATH=$(echo $PATH | tr ':' '\n' | grep -v "/skypilot-runtime/bin" | tr '\n' ':')
             source ~/.bashrc
             set -a
             . $(conda info --base 2> /dev/null)/etc/profile.d/conda.sh > /dev/null 2>&1 || true

sky/spot/spot_utils.py

@@ -741,7 +741,8 @@ def set_pending(cls, job_id: int, spot_dag: 'dag_lib.Dag') -> str:
     def _build(cls, code: List[str]) -> str:
         code = cls._PREFIX + code
         generated_code = '; '.join(code)
-        return f'python3 -u -c {shlex.quote(generated_code)}'
+        return (f'{constants.ACTIVATE_PYTHON_ENV} '
+                f'python -u -c {shlex.quote(generated_code)}')
 
 
 def dump_job_table_cache(job_table: str):

sky/templates/azure-ray.yml.j2

@@ -164,14 +164,14 @@ setup_commands:
 #   current num items (num SSH connections): 2
 head_start_ray_commands:
   # NOTE: --disable-usage-stats in `ray start` saves 10 seconds of idle wait.
-  - ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --head --port={{ray_port}} --dashboard-port={{ray_dashboard_port}} --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml {{"--num-gpus=%s" % num_gpus if num_gpus}} {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1; 
+  - source {{ skypilot_runtime_env }}/bin/activate; ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --head --port={{ray_port}} --dashboard-port={{ray_dashboard_port}} --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml {{"--num-gpus=%s" % num_gpus if num_gpus}} {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1; 
     which prlimit && for id in $(pgrep -f raylet/raylet); do sudo prlimit --nofile=1048576:1048576 --pid=$id || true; done; 
     {{dump_port_command}}; 
     {{ray_head_wait_initialized_command}}
 
 {%- if num_nodes > 1 %}
 worker_start_ray_commands:
-  - ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --address=$RAY_HEAD_IP:{{ray_port}} --object-manager-port=8076 {{"--num-gpus=%s" % num_gpus if num_gpus}} {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
+  - source {{ skypilot_runtime_env }}/bin/activate; ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --address=$RAY_HEAD_IP:{{ray_port}} --object-manager-port=8076 {{"--num-gpus=%s" % num_gpus if num_gpus}} {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
     which prlimit && for id in $(pgrep -f raylet/raylet); do sudo prlimit --nofile=1048576:1048576 --pid=$id || true; done;
 {%- else %}
 worker_start_ray_commands: []

sky/templates/gcp-ray.yml.j2

@@ -190,8 +190,7 @@ setup_commands:
     {{ conda_installation_commands }}
     source ~/.bashrc;
   {%- if tpu_vm %}
-    test -f ~/miniconda3/etc/profile.d/conda.sh && source ~/miniconda3/etc/profile.d/conda.sh && conda activate base || true;
-    pip3 install --upgrade google-api-python-client;
+    pip install --upgrade google-api-python-client
   {%- endif %}
   {%- if tpu_node_name %}
     grep "export TPU_NAME=" ~/.bashrc && echo "TPU_NAME already set" || echo "export TPU_NAME={{tpu_node_name}}" >> ~/.bashrc;

sky/templates/ibm-ray.yml.j2

@@ -118,14 +118,14 @@ head_start_ray_commands:
   # NOTE: --disable-usage-stats in `ray start` saves 10 seconds of idle wait.
   # Line "which prlimit ..": increase the limit of the number of open files for the raylet process, as the `ulimit` may not take effect at this point, because it requires
   # all the sessions to be reloaded. This is a workaround.
-  - ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --head --port={{ray_port}} --dashboard-port={{ray_dashboard_port}} --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
+  - source {{ skypilot_runtime_env }}/bin/activate; ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --head --port={{ray_port}} --dashboard-port={{ray_dashboard_port}} --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
     which prlimit && for id in $(pgrep -f raylet/raylet); do sudo prlimit --nofile=1048576:1048576 --pid=$id || true; done;
     {{dump_port_command}}; {{ray_head_wait_initialized_command}}
 
 {%- if num_nodes > 1 %}
 worker_start_ray_commands:
-  - ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --address=$RAY_HEAD_IP:{{ray_port}} --object-manager-port=8076 {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
-    which prlimit && for id in $(pgrep -f raylet/raylet); do sudo prlimit --nofile=1048576:1048576 --pid=$id || true; done;
+  - source {{ skypilot_runtime_env }}/bin/activate; ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --address=$RAY_HEAD_IP:{{ray_port}} --object-manager-port=8076 {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
+    which prlimit && for id in $(pgrep -f raylet/raylet); do sudo prlimit --nofile=1048576:1048576 --pid=$id || true; done
 {%- else %}
 worker_start_ray_commands: []
 {%- endif %}

sky/templates/lambda-ray.yml.j2

@@ -89,14 +89,14 @@ setup_commands:
 # Increment the following for catching performance bugs easier:
 #   current num items (num SSH connections): 2
 head_start_ray_commands:
-  - ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --head --port={{ray_port}} --dashboard-port={{ray_dashboard_port}} --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
+  - source {{ skypilot_runtime_env }}/bin/activate; ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --head --port={{ray_port}} --dashboard-port={{ray_dashboard_port}} --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
     which prlimit && for id in $(pgrep -f raylet/raylet); do sudo prlimit --nofile=1048576:1048576 --pid=$id || true; done;
     {{dump_port_command}}; {{ray_head_wait_initialized_command}}
 
 {%- if num_nodes > 1 %}
 worker_start_ray_commands:
-  - ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --address=$RAY_HEAD_IP:{{ray_port}} --object-manager-port=8076 {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
-    which prlimit && for id in $(pgrep -f raylet/raylet); do sudo prlimit --nofile=1048576:1048576 --pid=$id || true; done;
+  - source {{ skypilot_runtime_env }}/bin/activate; ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --address=$RAY_HEAD_IP:{{ray_port}} --object-manager-port=8076 {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
+    which prlimit && for id in $(pgrep -f raylet/raylet); do sudo prlimit --nofile=1048576:1048576 --pid=$id || true; done
 {%- else %}
 worker_start_ray_commands: []
 {%- endif %}

sky/templates/oci-ray.yml.j2

@@ -114,14 +114,14 @@ head_start_ray_commands:
   # NOTE: --disable-usage-stats in `ray start` saves 10 seconds of idle wait.
   # Line "which prlimit ..": increase the limit of the number of open files for the raylet process, as the `ulimit` may not take effect at this point, because it requires
   # all the sessions to be reloaded. This is a workaround.
-  - ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --head --port={{ray_port}} --dashboard-port={{ray_dashboard_port}} --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
+  - source {{ skypilot_runtime_env }}/bin/activate; ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --head --port={{ray_port}} --dashboard-port={{ray_dashboard_port}} --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
     which prlimit && for id in $(pgrep -f raylet/raylet); do sudo prlimit --nofile=1048576:1048576 --pid=$id || true; done;
     {{dump_port_command}}; {{ray_head_wait_initialized_command}}
 
 {%- if num_nodes > 1 %}
 worker_start_ray_commands:
-  - ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --address=$RAY_HEAD_IP:{{ray_port}} --object-manager-port=8076 {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
-    which prlimit && for id in $(pgrep -f raylet/raylet); do sudo prlimit --nofile=1048576:1048576 --pid=$id || true; done;
+  - source {{ skypilot_runtime_env }}/bin/activate; ray stop; RAY_SCHEDULER_EVENTS=0 RAY_DEDUP_LOGS=0 ray start --disable-usage-stats --address=$RAY_HEAD_IP:{{ray_port}} --object-manager-port=8076 {{"--resources='%s'" % custom_resources if custom_resources}} --temp-dir {{ray_temp_dir}} || exit 1;
+    which prlimit && for id in $(pgrep -f raylet/raylet); do sudo prlimit --nofile=1048576:1048576 --pid=$id || true; done
 {%- else %}
 worker_start_ray_commands: []
 {%- endif %}