diff --git a/plugins/onnx/.vscode/settings.json b/plugins/onnx/.vscode/settings.json index 8f18de3471..cef3fef046 100644 --- a/plugins/onnx/.vscode/settings.json +++ b/plugins/onnx/.vscode/settings.json @@ -3,16 +3,16 @@ // docker installation // "scrypted.debugHost": "koushik-ubuntuvm", // "scrypted.serverRoot": "/server", - "scrypted.debugHost": "koushik-ubuntuvm", - "scrypted.serverRoot": "/home/koush/.scrypted", + // "scrypted.debugHost": "koushik-ubuntuvm", + // "scrypted.serverRoot": "/home/koush/.scrypted", // pi local installation // "scrypted.debugHost": "192.168.2.119", // "scrypted.serverRoot": "/home/pi/.scrypted", // local checkout - // "scrypted.debugHost": "127.0.0.1", - // "scrypted.serverRoot": "/Users/koush/.scrypted", + "scrypted.debugHost": "127.0.0.1", + "scrypted.serverRoot": "/Users/koush/.scrypted", // "scrypted.debugHost": "koushik-winvm", // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted", diff --git a/plugins/onnx/package-lock.json b/plugins/onnx/package-lock.json index 6b2f92066f..13b64c87c8 100644 --- a/plugins/onnx/package-lock.json +++ b/plugins/onnx/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/openvino", - "version": "0.1.83", + "version": "0.1.86", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/openvino", - "version": "0.1.83", + "version": "0.1.86", "devDependencies": { "@scrypted/sdk": "file:../../sdk" } diff --git a/plugins/onnx/package.json b/plugins/onnx/package.json index caea4a7f5e..ede3d0751d 100644 --- a/plugins/onnx/package.json +++ b/plugins/onnx/package.json @@ -41,5 +41,5 @@ "devDependencies": { "@scrypted/sdk": "file:../../sdk" }, - "version": "0.1.83" + "version": "0.1.86" } diff --git a/plugins/onnx/src/ort/__init__.py b/plugins/onnx/src/ort/__init__.py index c4fdcc2620..7ea803743f 100644 --- a/plugins/onnx/src/ort/__init__.py +++ b/plugins/onnx/src/ort/__init__.py @@ -7,6 +7,7 @@ import platform import sys import threading +import traceback from typing import Any, Tuple import numpy as np @@ -31,7 +32,6 @@ "scrypted_yolov8n", ] - def parse_labels(names): j = ast.literal_eval(names) ret = {} @@ -117,6 +117,11 @@ def executor_initializer(): thread_name_prefix="onnx", ) + self.prepareExecutor = concurrent.futures.ThreadPoolExecutor( + max_workers=len(compiled_models), + thread_name_prefix="onnx-prepare", + ) + async def getSettings(self) -> list[Setting]: model = self.storage.getItem("model") or "Default" deviceIds = self.storage.getItem("deviceIds") or '["0"]' @@ -156,26 +161,29 @@ def get_input_size(self) -> Tuple[int, int]: return [self.model_dim, self.model_dim] async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss): + def prepare(): + im = np.array(input) + im = np.expand_dims(input, axis=0) + im = im.transpose((0, 3, 1, 2)) # BHWC to BCHW, (n, 3, h, w) + im = im.astype(np.float32) / 255.0 + im = np.ascontiguousarray(im) # contiguous + return im + def predict(input_tensor): compiled_model = self.compiled_models[threading.current_thread().name] output_tensors = compiled_model.run(None, { self.input_name: input_tensor }) objs = yolo.parse_yolov9(output_tensors[0][0]) return objs - im = np.array(input) - im = np.stack([input]) - im = im.transpose((0, 3, 1, 2)) # BHWC to BCHW, (n, 3, h, w) - im = im.astype(np.float32) / 255.0 - im = np.ascontiguousarray(im) # contiguous - input_tensor = im - try: + input_tensor = await asyncio.get_event_loop().run_in_executor( + self.prepareExecutor, lambda: prepare() + ) objs = await asyncio.get_event_loop().run_in_executor( self.executor, lambda: predict(input_tensor) ) except: - import traceback traceback.print_exc() raise diff --git a/plugins/openvino/package-lock.json b/plugins/openvino/package-lock.json index 64ec7ce415..56a2637b6e 100644 --- a/plugins/openvino/package-lock.json +++ b/plugins/openvino/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/openvino", - "version": "0.1.80", + "version": "0.1.81", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/openvino", - "version": "0.1.80", + "version": "0.1.81", "devDependencies": { "@scrypted/sdk": "file:../../sdk" } diff --git a/plugins/openvino/package.json b/plugins/openvino/package.json index 59ab90320c..e0e94c46b2 100644 --- a/plugins/openvino/package.json +++ b/plugins/openvino/package.json @@ -42,5 +42,5 @@ "devDependencies": { "@scrypted/sdk": "file:../../sdk" }, - "version": "0.1.80" + "version": "0.1.81" } diff --git a/plugins/openvino/src/ov/__init__.py b/plugins/openvino/src/ov/__init__.py index 0d63624fcd..3766d58225 100644 --- a/plugins/openvino/src/ov/__init__.py +++ b/plugins/openvino/src/ov/__init__.py @@ -1,8 +1,10 @@ from __future__ import annotations import asyncio +import concurrent.futures import json import re +import traceback from typing import Any, Tuple import numpy as np @@ -11,19 +13,20 @@ from PIL import Image from scrypted_sdk.other import SettingValue from scrypted_sdk.types import Setting -import concurrent.futures import common.yolo as yolo from predict import Prediction, PredictPlugin from predict.rectangle import Rectangle from .face_recognition import OpenVINOFaceRecognition + try: from .text_recognition import OpenVINOTextRecognition except: OpenVINOTextRecognition = None predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-Predict") +prepareExecutor = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-Prepare") availableModels = [ "Default", @@ -314,30 +317,34 @@ def torelative(value: float): return objs - # the input_tensor can be created with the shared_memory=True parameter, - # but that seems to cause issues on some platforms. - if self.scrypted_yolo: - im = np.stack([input]) - im = im.transpose((0, 3, 1, 2)) # BHWC to BCHW, (n, 3, h, w) - im = im.astype(np.float32) / 255.0 - im = np.ascontiguousarray(im) # contiguous - im = ov.Tensor(array=im) - input_tensor = im - elif self.yolo: - input_tensor = ov.Tensor( - array=np.expand_dims(np.array(input), axis=0).astype(np.float32) - ) - else: - input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0)) + + def prepare(): + # the input_tensor can be created with the shared_memory=True parameter, + # but that seems to cause issues on some platforms. + if self.scrypted_yolo: + im = np.array(input) + im = np.expand_dims(input, axis=0) + im = im.transpose((0, 3, 1, 2)) # BHWC to BCHW, (n, 3, h, w) + im = im.astype(np.float32) / 255.0 + im = np.ascontiguousarray(im) # contiguous + input_tensor = ov.Tensor(array=im) + elif self.yolo: + input_tensor = ov.Tensor( + array=np.expand_dims(np.array(input), axis=0).astype(np.float32) + ) + else: + input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0)) + return input_tensor try: + input_tensor = await asyncio.get_event_loop().run_in_executor( + prepareExecutor, lambda: prepare() + ) objs = await asyncio.get_event_loop().run_in_executor( predictExecutor, lambda: predict(input_tensor) ) except: - import traceback - traceback.print_exc() raise