Try to recover when the main process fails

src/cli.rs

@@ -57,7 +57,7 @@ pub struct Cli {
 }
 
 /// The various Quilkin commands.
-#[derive(clap::Subcommand)]
+#[derive(Clone, clap::Subcommand)]
 pub enum Commands {
     Run(Run),
     GenerateConfigSchema(GenerateConfigSchema),
@@ -137,16 +137,32 @@ impl Cli {
             shutdown_tx.send(()).ok();
         });
 
-        let fut = match self.command {
-            Commands::Run(runner) => tokio::spawn({
-                let shutdown_rx = shutdown_rx.clone();
-                async move { runner.run(config, shutdown_rx).await }
-            }),
-            Commands::Manage(manager) => tokio::spawn(async move { manager.manage(config).await }),
-            Commands::GenerateConfigSchema(generator) => {
-                tokio::spawn(std::future::ready(generator.generate_config_schema()))
+        let fut = tryhard::retry_fn({
+            let shutdown_rx = shutdown_rx.clone();
+            move || match self.command.clone() {
+                Commands::Run(runner) => {
+                    let config = config.clone();
+                    let shutdown_rx = shutdown_rx.clone();
+                    tokio::spawn(
+                        async move { runner.run(config.clone(), shutdown_rx.clone()).await },
+                    )
+                }
+                Commands::Manage(manager) => {
+                    let config = config.clone();
+                    tokio::spawn(async move { manager.manage(config.clone()).await })
+                }
+                Commands::GenerateConfigSchema(generator) => {
+                    tokio::spawn(std::future::ready(generator.generate_config_schema()))
+                }
             }
-        };
+        })
+        .retries(3)
+        .on_retry(|_, _, error| {
+            let error = error.to_string();
+            async move {
+                tracing::warn!(%error, "error would have caused fatal crash");
+            }
+        });
 
         tokio::select! {
             result = fut => result?,

src/cli/generate_config_schema.rs

@@ -15,7 +15,7 @@
  */
 
 /// Generates JSON schema files for known filters.
-#[derive(clap::Args)]
+#[derive(clap::Args, Clone)]
 pub struct GenerateConfigSchema {
     /// The directory to write configuration files.
     #[clap(short, long, default_value = ".")]

src/cli/manage.rs

@@ -16,7 +16,7 @@
 
 /// Runs Quilkin as a xDS management server, using `provider` as
 /// a configuration source.
-#[derive(clap::Args)]
+#[derive(clap::Args, Clone)]
 pub struct Manage {
     #[clap(short, long, env = "QUILKIN_PORT")]
     port: Option<u16>,
@@ -26,7 +26,7 @@ pub struct Manage {
 }
 
 /// The available xDS source providers.
-#[derive(clap::Subcommand)]
+#[derive(Clone, clap::Subcommand)]
 pub enum Providers {
     /// Watches Agones' game server CRDs for `Allocated` game server endpoints,
     /// and for a `ConfigMap` that specifies the filter configuration.

src/cli/run.rs

@@ -20,7 +20,7 @@ use std::net::SocketAddr;
 use crate::filters::FilterFactory;
 
 /// Run Quilkin as a UDP reverse proxy.
-#[derive(clap::Args)]
+#[derive(clap::Args, Clone)]
 #[non_exhaustive]
 pub struct Run {
     /// One or more `quilkin manage` endpoints to listen to for config changes