From ef742bdb233547eb26ce4e68f419919515040767 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Thu, 15 Jun 2023 00:54:53 -0700 Subject: [PATCH] coreml/openvino: yolov8 support --- plugins/coreml/src/coreml/__init__.py | 22 +------------- plugins/openvino/package-lock.json | 4 +-- plugins/openvino/package.json | 2 +- plugins/openvino/src/ov/__init__.py | 44 +++++++++++++++++++++++++-- plugins/openvino/src/requirements.txt | 2 +- plugins/openvino/src/yolo/__init__.py | 27 ++++++++++++++++ 6 files changed, 73 insertions(+), 28 deletions(-) diff --git a/plugins/coreml/src/coreml/__init__.py b/plugins/coreml/src/coreml/__init__.py index 4d71a54231..8ce600fb76 100644 --- a/plugins/coreml/src/coreml/__init__.py +++ b/plugins/coreml/src/coreml/__init__.py @@ -145,27 +145,7 @@ async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss): out_blob = out_dict["var_914"] var_914 = out_dict["var_914"] results = var_914[0] - keep = np.argwhere(results[4:] > 0.2) - for indices in keep: - class_id = indices[0] - index = indices[1] - confidence = results[class_id + 4, index] - x = results[0][index].astype(float) - y = results[1][index].astype(float) - w = results[2][index].astype(float) - h = results[3][index].astype(float) - obj = Prediction( - class_id, - confidence.astype(float), - Rectangle( - x - w / 2, - y - h / 2, - x + w / 2, - y + h / 2, - ), - ) - objs.append(obj) - + objs = yolo.parse_yolov8(results) ret = self.create_detection_result(objs, src_size, cvss) return ret diff --git a/plugins/openvino/package-lock.json b/plugins/openvino/package-lock.json index 0ea26359fa..a9e5fb9c04 100644 --- a/plugins/openvino/package-lock.json +++ b/plugins/openvino/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/openvino", - "version": "0.1.27", + "version": "0.1.29", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/openvino", - "version": "0.1.27", + "version": "0.1.29", "devDependencies": { "@scrypted/sdk": "file:../../sdk" } diff --git a/plugins/openvino/package.json b/plugins/openvino/package.json index 7c97483f58..124e0d9af8 100644 --- a/plugins/openvino/package.json +++ b/plugins/openvino/package.json @@ -40,5 +40,5 @@ "devDependencies": { "@scrypted/sdk": "file:../../sdk" }, - "version": "0.1.27" + "version": "0.1.29" } diff --git a/plugins/openvino/src/ov/__init__.py b/plugins/openvino/src/ov/__init__.py index f7b8bb06ef..3cfc4c3acc 100644 --- a/plugins/openvino/src/ov/__init__.py +++ b/plugins/openvino/src/ov/__init__.py @@ -28,12 +28,33 @@ def parse_label_contents(contents: str): ret[row_number] = content.strip() return ret +def param_to_string(parameters) -> str: + """Convert a list / tuple of parameters returned from IE to a string.""" + if isinstance(parameters, (list, tuple)): + return ', '.join([str(x) for x in parameters]) + else: + return str(parameters) + +def dump_device_properties(core): + print('Available devices:') + for device in core.available_devices: + print(f'{device} :') + print('\tSUPPORTED_PROPERTIES:') + for property_key in core.get_property(device, 'SUPPORTED_PROPERTIES'): + if property_key not in ('SUPPORTED_METRICS', 'SUPPORTED_CONFIG_KEYS', 'SUPPORTED_PROPERTIES'): + try: + property_val = core.get_property(device, property_key) + except TypeError: + property_val = 'UNSUPPORTED TYPE' + print(f'\t\t{property_key}: {param_to_string(property_val)}') + print('') class OpenVINOPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings): def __init__(self, nativeId: str | None = None): super().__init__(nativeId=nativeId) self.core = ov.Core() + dump_device_properties(self.core) available_devices = self.core.available_devices print('available devices: %s' % available_devices) @@ -58,15 +79,21 @@ def __init__(self, nativeId: str | None = None): if model == 'Default': model = 'ssd_mobilenet_v1_coco' self.yolo = 'yolo' in model + self.yolov8 = "yolov8" in model self.sigmoid = model == 'yolo-v4-tiny-tf' print(f'model/mode/precision: {model}/{mode}/{precision}') - self.model_dim = 416 if self.yolo else 300 + if self.yolov8: + self.model_dim = 640 + elif self.yolo: + self.model_dim = 416 + else: + self.model_dim = 300 + model_version = 'v3' xmlFile = self.downloadFile(f'https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{model}.xml', f'{model_version}/{precision}/{model}.xml') labelsFile = self.downloadFile(f'https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{model}.bin', f'{model_version}/{precision}/{model}.bin') - try: self.compiled_model = self.core.compile_model(xmlFile, mode) except: @@ -99,6 +126,7 @@ async def getSettings(self) -> list[Setting]: 'ssdlite_mobilenet_v2', 'yolo-v3-tiny-tf', 'yolo-v4-tiny-tf', + # 'yolov8n', ], 'value': model, }, @@ -142,7 +170,12 @@ def get_input_size(self) -> Tuple[int, int]: async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss): def predict(): infer_request = self.compiled_model.create_infer_request() - if self.yolo: + if self.yolov8: + i = np.array(input) + c = np.squeeze(np.split(i, i.shape[-1], -1), axis=-1) + d = np.expand_dims(c, axis=0) + input_tensor = ov.Tensor(array=d.astype(np.float32), shared_memory=True) + elif self.yolo: input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0).astype(np.float32), shared_memory=True) else: input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0), shared_memory=True) @@ -153,6 +186,11 @@ def predict(): objs = [] + if self.yolov8: + objs = yolo.parse_yolov8(infer_request.outputs[0].data) + ret = self.create_detection_result(objs, src_size, cvss) + return ret + if self.yolo: # index 2 will always either be 13 or 26 # index 1 may be 13/26 or 255 depending on yolo 3 vs 4 diff --git a/plugins/openvino/src/requirements.txt b/plugins/openvino/src/requirements.txt index dfde482081..eca02ac677 100644 --- a/plugins/openvino/src/requirements.txt +++ b/plugins/openvino/src/requirements.txt @@ -1,4 +1,4 @@ -openvino==2022.3.0 +openvino==2023.0.0 # pillow for anything not intel linux, pillow-simd is available on x64 linux Pillow>=5.4.1; sys_platform != 'linux' or platform_machine != 'x86_64' diff --git a/plugins/openvino/src/yolo/__init__.py b/plugins/openvino/src/yolo/__init__.py index e17027857a..326ea758cc 100644 --- a/plugins/openvino/src/yolo/__init__.py +++ b/plugins/openvino/src/yolo/__init__.py @@ -2,6 +2,33 @@ from math import exp import numpy as np +from predict import Prediction, Rectangle + +def parse_yolov8(results): + objs = [] + keep = np.argwhere(results[4:] > 0.2) + for indices in keep: + class_id = indices[0] + index = indices[1] + confidence = results[class_id + 4, index] + x = results[0][index].astype(float) + y = results[1][index].astype(float) + w = results[2][index].astype(float) + h = results[3][index].astype(float) + obj = Prediction( + class_id, + confidence.astype(float), + Rectangle( + x - w / 2, + y - h / 2, + x + w / 2, + y + h / 2, + ), + ) + objs.append(obj) + + return objs + def sig(x): return 1/(1 + np.exp(-x))