openvino/onnx: move prep into separate threads

plugins/onnx/.vscode/settings.json

@@ -3,16 +3,16 @@
     // docker installation
     // "scrypted.debugHost": "koushik-ubuntuvm",
     // "scrypted.serverRoot": "/server",
-    "scrypted.debugHost": "koushik-ubuntuvm",
-    "scrypted.serverRoot": "/home/koush/.scrypted",
+    // "scrypted.debugHost": "koushik-ubuntuvm",
+    // "scrypted.serverRoot": "/home/koush/.scrypted",
 
     // pi local installation
     // "scrypted.debugHost": "192.168.2.119",
     // "scrypted.serverRoot": "/home/pi/.scrypted",
 
     // local checkout
-    // "scrypted.debugHost": "127.0.0.1",
-    // "scrypted.serverRoot": "/Users/koush/.scrypted",
+    "scrypted.debugHost": "127.0.0.1",
+    "scrypted.serverRoot": "/Users/koush/.scrypted",
     // "scrypted.debugHost": "koushik-winvm",
     // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
 

plugins/onnx/package.json

@@ -41,5 +41,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.83"
+   "version": "0.1.86"
 }

plugins/onnx/src/ort/__init__.py

@@ -7,6 +7,7 @@
 import platform
 import sys
 import threading
+import traceback
 from typing import Any, Tuple
 
 import numpy as np
@@ -31,7 +32,6 @@
     "scrypted_yolov8n",
 ]
 
-
 def parse_labels(names):
     j = ast.literal_eval(names)
     ret = {}
@@ -117,6 +117,11 @@ def executor_initializer():
             thread_name_prefix="onnx",
         )
 
+        self.prepareExecutor = concurrent.futures.ThreadPoolExecutor(
+            max_workers=len(compiled_models),
+            thread_name_prefix="onnx-prepare",
+        )
+
     async def getSettings(self) -> list[Setting]:
         model = self.storage.getItem("model") or "Default"
         deviceIds = self.storage.getItem("deviceIds") or '["0"]'
@@ -156,26 +161,29 @@ def get_input_size(self) -> Tuple[int, int]:
         return [self.model_dim, self.model_dim]
 
     async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss):
+        def prepare(): 
+            im = np.array(input)
+            im = np.expand_dims(input, axis=0)
+            im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
+            im = im.astype(np.float32) / 255.0
+            im = np.ascontiguousarray(im)  # contiguous
+            return im
+
         def predict(input_tensor):
             compiled_model = self.compiled_models[threading.current_thread().name]
             output_tensors = compiled_model.run(None, { self.input_name: input_tensor })
             objs = yolo.parse_yolov9(output_tensors[0][0])
             return objs
 
-        im = np.array(input)
-        im = np.stack([input])
-        im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
-        im = im.astype(np.float32) / 255.0
-        im = np.ascontiguousarray(im)  # contiguous
-        input_tensor = im
-
         try:
+            input_tensor = await asyncio.get_event_loop().run_in_executor(
+                self.prepareExecutor, lambda: prepare()
+            )
             objs = await asyncio.get_event_loop().run_in_executor(
                 self.executor, lambda: predict(input_tensor)
             )
 
         except:
-            import traceback
 
             traceback.print_exc()
             raise

plugins/openvino/package.json

@@ -42,5 +42,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.80"
+   "version": "0.1.81"
 }

plugins/openvino/src/ov/__init__.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
 import asyncio
+import concurrent.futures
 import json
 import re
+import traceback
 from typing import Any, Tuple
 
 import numpy as np
@@ -11,19 +13,20 @@
 from PIL import Image
 from scrypted_sdk.other import SettingValue
 from scrypted_sdk.types import Setting
-import concurrent.futures
 
 import common.yolo as yolo
 from predict import Prediction, PredictPlugin
 from predict.rectangle import Rectangle
 
 from .face_recognition import OpenVINOFaceRecognition
+
 try:
     from .text_recognition import OpenVINOTextRecognition
 except:
     OpenVINOTextRecognition = None
 
 predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-Predict")
+prepareExecutor = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-Prepare")
 
 availableModels = [
     "Default",
@@ -314,30 +317,34 @@ def torelative(value: float):
 
             return objs
 
-        # the input_tensor can be created with the shared_memory=True parameter,
-        # but that seems to cause issues on some platforms.
-        if self.scrypted_yolo:
-            im = np.stack([input])
-            im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
-            im = im.astype(np.float32) / 255.0
-            im = np.ascontiguousarray(im)  # contiguous
-            im = ov.Tensor(array=im)
-            input_tensor = im
-        elif self.yolo:
-            input_tensor = ov.Tensor(
-                array=np.expand_dims(np.array(input), axis=0).astype(np.float32)
-            )
-        else:
-            input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0))
+
+        def prepare(): 
+            # the input_tensor can be created with the shared_memory=True parameter,
+            # but that seems to cause issues on some platforms.
+            if self.scrypted_yolo:
+                im = np.array(input)
+                im = np.expand_dims(input, axis=0)
+                im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
+                im = im.astype(np.float32) / 255.0
+                im = np.ascontiguousarray(im)  # contiguous
+                input_tensor = ov.Tensor(array=im)
+            elif self.yolo:
+                input_tensor = ov.Tensor(
+                    array=np.expand_dims(np.array(input), axis=0).astype(np.float32)
+                )
+            else:
+                input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0))
+            return input_tensor
 
         try:
+            input_tensor = await asyncio.get_event_loop().run_in_executor(
+                prepareExecutor, lambda: prepare()
+            )
             objs = await asyncio.get_event_loop().run_in_executor(
                 predictExecutor, lambda: predict(input_tensor)
             )
 
         except:
-            import traceback
-
             traceback.print_exc()
             raise