allow concurrency by default for llm

limit to linear behavior only if requested.

scripts/twitch.sh

@@ -18,7 +18,7 @@ ALIGNMENT=right
 TEMPERATURE=0.8
 CONTINUOUS=1
 POLL_INTERVAL=0
-PIPELINE_CONCURRENCY=3
+PIPELINE_CONCURRENCY=2
 TWITCH_LLM_CONCURRENCY=2
 CONTEXT_SIZE=4000
 SUBTITLES=1
@@ -28,13 +28,18 @@ QUANTIZED=0
 GREETING="Hi I'm Alice, ask me a question!"
 TWITCH_CHAT_HISTORY=8
 TWITCH_MODEL=mistral
+SINGLE_CONCURRENCY=0
 # === END OF CONFIGURATION ===
 #
 #
 SUBTITLE_CMD=
 DAEMON_CMD=
 NO_HISTORY_CMD=
 QUANTIZED_CMD=
+SINGLE_CONCURRENCY_CMD=
+if [ "$SINGLE_CONCURRENCY" == 1 ]; then
+    SINGLE_CONCURRENCY_CMD="--single-concurrency"
+fi
 if [ "$SUBTITLES" == 1 ]; then
     SUBTITLE_CMD="--subtitles"
 fi
@@ -75,6 +80,7 @@ DYLD_LIBRARY_PATH=`pwd`:/usr/local/lib:$DYLD_LIBRARY_PATH \
     --poll-interval $POLL_INTERVAL \
     --llm-history-size $CONTEXT_SIZE \
     --greeting "$GREETING" \
+    $SINGLE_CONCURRENCY_CMD \
     $SUBTITLE_CMD \
     $DAEMON_CMD \
     $CONTINUOUS_CMD \

src/args.rs

@@ -712,4 +712,13 @@ pub struct Args {
         help = "Twitch LLM model."
     )]
     pub twitch_model: String,
+
+    /// single concurrency - bool single concurrency for all models, wait between each request
+    #[clap(
+        long,
+        env = "SINGLE_CONCURRENCY",
+        default_value_t = false,
+        help = "single concurrency - bool single concurrency for all models, wait between each request."
+    )]
+    pub single_concurrency: bool,
 }

src/main.rs

@@ -1410,7 +1410,9 @@ async fn main() {
             }
 
             #[cfg(feature = "ndi")]
-            if args.sd_image || args.tts_enable || args.oai_tts || args.mimic3_tts {
+            if args.single_concurrency
+                && (args.sd_image || args.tts_enable || args.oai_tts || args.mimic3_tts)
+            {
                 // Wait for the NDI done signal
                 std::io::stdout().flush().unwrap();
                 info!("Waiting for NDI done signal for LLM messages...");