feat: ensure number of tokio worker threads to be at least 4 (#18762)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

src/utils/runtime/src/lib.rs

@@ -37,6 +37,8 @@ mod prof;
 use prof::*;
 use tokio::signal::unix::SignalKind;
 
+const MIN_WORKER_THREADS: usize = 4;
+
 /// Start RisingWave components with configs from environment variable.
 ///
 /// # Shutdown on Ctrl-C
@@ -79,6 +81,23 @@ where
         std::env::set_var("TOKIO_WORKER_THREADS", worker_threads);
     }
 
+    // Set the default number of worker threads to be at least `MIN_WORKER_THREADS`, in production.
+    if !cfg!(debug_assertions) {
+        let worker_threads = match std::env::var("TOKIO_WORKER_THREADS") {
+            Ok(v) => v
+                .parse::<usize>()
+                .expect("Failed to parse TOKIO_WORKER_THREADS"),
+            Err(std::env::VarError::NotPresent) => std::thread::available_parallelism()
+                .expect("Failed to get available parallelism")
+                .get(),
+            Err(_) => panic!("Failed to parse TOKIO_WORKER_THREADS"),
+        };
+        if worker_threads < MIN_WORKER_THREADS {
+            tracing::warn!("the default number of worker threads ({worker_threads}) is too small, which may lead to issues, increasing to {MIN_WORKER_THREADS}");
+            std::env::set_var("TOKIO_WORKER_THREADS", MIN_WORKER_THREADS.to_string());
+        }
+    }
+
     if let Ok(enable_deadlock_detection) = std::env::var("RW_DEADLOCK_DETECTION") {
         let enable_deadlock_detection = enable_deadlock_detection
             .parse()