server: fix potential race conditions around plugin restart

koush · Apr 17, 2024 · b7a8f97 · b7a8f97
1 parent f5fabfe
commit b7a8f97
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 14 deletions.
diff --git a/server/package-lock.json b/server/package-lock.json
diff --git a/server/src/runtime.ts b/server/src/runtime.ts
@@ -631,33 +631,43 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
     }
 
     setupPluginHostAutoRestart(pluginHost: PluginHost) {
+        const logger = this.getDeviceLogger(this.findPluginDevice(pluginHost.pluginId));
+
+        let timeout: NodeJS.Timeout;
+
         const restart = () => {
-            if (pluginHost.killed)
+            if (timeout)
                 return;
+
+            const t = 60000;
             pluginHost.kill();
-            const timeout = 60000;
-            console.error(`plugin unexpectedly exited, restarting in ${timeout}ms`, pluginHost.pluginId);
-            setTimeout(async () => {
-                const existing = this.plugins[pluginHost.pluginId];
-                if (existing !== pluginHost) {
-                    console.log('scheduled plugin restart cancelled, plugin was restarted by user', pluginHost.pluginId);
-                    return;
-                }
+            logger.log('e', `plugin ${pluginHost.pluginId} unexpectedly exited, restarting in ${t}ms`);
 
+            timeout = setTimeout(async () => {
+                timeout = undefined;
                 const plugin = await this.datastore.tryGet(Plugin, pluginHost.pluginId);
                 if (!plugin) {
-                    console.log('scheduled plugin restart cancelled, plugin no longer exists', pluginHost.pluginId);
+                    logger.log('w', `scheduled plugin restart cancelled, plugin no longer exists ${pluginHost.pluginId}`);
+                    return;
+                }
+
+                const existing = this.plugins[pluginHost.pluginId];
+                if (existing !== pluginHost) {
+                    logger.log('w', `scheduled plugin restart cancelled, plugin was restarted by user ${pluginHost.pluginId}`);
                     return;
                 }
 
                 try {
                     this.runPlugin(plugin);
                 }
                 catch (e) {
-                    console.error('error restarting plugin', plugin._id, e);
+                    logger.log('e', `error restarting plugin ${pluginHost.pluginId}`);
+                    logger.log('e', e.toStrin());
+                    restart();
                 }
-            }, timeout);
+            }, t);
         };
+
         pluginHost.worker.once('error', restart);
         pluginHost.worker.once('exit', restart);
         pluginHost.worker.once('close', restart);