predict: revert object tracker changes until custom NVR detector with…

… face recognition is in place

plugins/objectdetector/src/main.ts

@@ -1,4 +1,4 @@
-import sdk, { Camera, DeviceState, EventListenerRegister, MediaObject, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionModel, ObjectDetectionResult, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, ObjectTracker, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk';
+import sdk, { Camera, DeviceState, EventListenerRegister, MediaObject, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionModel, ObjectDetectionResult, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import crypto from 'crypto';
 import cloneDeep from 'lodash/cloneDeep';
@@ -7,7 +7,7 @@ import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin";
 import { DenoisedDetectionEntry, DenoisedDetectionState, denoiseDetections } from './denoise';
 import { serverSupportsMixinEventMasking } from './server-version';
 import { sleep } from './sleep';
-import { getAllDevices, safeParseJson } from './util';
+import { safeParseJson } from './util';
 
 const polygonOverlap = require('polygon-overlap');
 const insidePolygon = require('point-inside-polygon');
@@ -68,12 +68,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       ],
       defaultValue: 'Default',
     },
-    tracker: {
-      title: 'Object Tracker',
-      type: 'device',
-      deviceFilter: `interfaces.includes('${ScryptedInterface.ObjectTracker}')`,
-      defaultValue: getAllDevices().find(d => d.interfaces?.includes(ScryptedInterface.ObjectTracker))?.id,
-    },
     detectionDuration: {
       title: 'Detection Duration',
       subgroup: 'Advanced',
@@ -654,16 +648,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       return detectionResult;
     }
 
-    // track the objects on a pre-zoned set.
-    const tracker = this.storageSettings.values.tracker as ObjectTracker;
-    if (tracker) {
-      // apply a detectionId for the tracker, but remove it until certain the
-      // detection input needs to be saved.
-      detectionResult.detectionId = this.detectionId;
-      detectionResult = await tracker.trackObjects(detectionResult);
-      delete detectionResult.detectionId;
-    }
-
     const { detections } = detectionResult;
 
     const found: DenoisedDetectionEntry<TrackedDetection>[] = [];
@@ -789,6 +773,12 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   async getMixinSettings(): Promise<Setting[]> {
     const settings: Setting[] = [];
 
+    try {
+      this.settings = (await this.objectDetection.getDetectionModel(this.settings)).settings;
+    }
+    catch (e) {
+    }
+
     if (this.settings) {
       settings.push(...this.settings.map(setting =>
         Object.assign({}, setting, {
@@ -801,7 +791,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
 
     this.storageSettings.settings.motionSensorSupplementation.hide = !this.hasMotionType || !this.mixinDeviceInterfaces.includes(ScryptedInterface.MotionSensor);
     this.storageSettings.settings.captureMode.hide = this.hasMotionType;
-    this.storageSettings.settings.tracker.hide = this.hasMotionType;
     this.storageSettings.settings.detectionDuration.hide = this.hasMotionType;
     this.storageSettings.settings.detectionTimeout.hide = this.hasMotionType;
     this.storageSettings.settings.motionDuration.hide = !this.hasMotionType;

plugins/tensorflow-lite/src/predict/__init__.py

@@ -11,19 +11,24 @@
 
 from detect import DetectionSession, DetectPlugin
 
+from .sort_oh import tracker
+import numpy as np
+
 try:
     from gi.repository import Gst
 except:
     pass
 
 class PredictSession(DetectionSession):
     image: Image.Image
+    tracker: sort_oh.tracker.Sort_OH
 
     def __init__(self, start_time: float) -> None:
         super().__init__()
         self.image = None
         self.processed = 0
         self.start_time = start_time
+        self.tracker = None
 
 def parse_label_contents(contents: str):
     lines = contents.splitlines()
@@ -114,6 +119,7 @@ def __init__(self, PLUGIN_MIME_TYPE: str, nativeId: str | None = None):
         self.toMimeType = scrypted_sdk.ScryptedMimeTypes.MediaObject.value
 
         self.crop = False
+        self.trackers: Mapping[str, tracker.Sort_OH] = {}
 
         # periodic restart because there seems to be leaks in tflite or coral API.
         loop = asyncio.get_event_loop()
@@ -191,8 +197,23 @@ def getModelSettings(self, settings: Any = None) -> list[Setting]:
             ],
         }
 
-        return [allowList]
-
+        trackerWindow: Setting = {
+            'title': 'Tracker Window',
+            'subgroup': 'Advanced',
+            'description': 'Internal Setting. Do not change.',
+            'key': 'trackerWindow',
+            'value': 3,
+            'type': 'number',
+        }
+        trackerCertainty: Setting = {
+            'title': 'Tracker Certainty',
+            'subgroup': 'Advanced',
+            'description': 'Internal Setting. Do not change.',
+            'key': 'trackerCertainty',
+            'value': .2,
+            'type': 'number',
+        }
+        return [allowList, trackerWindow, trackerCertainty]
 
     def create_detection_result(self, objs: List[Prediction], size, allowList, convert_to_src_size=None) -> ObjectsDetected:
         detections: List[ObjectDetectionResult] = []
@@ -252,6 +273,19 @@ def run_detection_image(self, detection_session: PredictSession, image: Image.Im
         (w, h) = self.get_input_size() or image.size
         (iw, ih) = image.size
 
+        if detection_session and not detection_session.tracker:
+            t = self.trackers.get(detection_session.id)
+            if not t:
+                t = tracker.Sort_OH(scene=np.array([iw, ih]))
+                t.conf_three_frame_certainty = (settings.get('trackerCertainty') or .2) * 3
+                t.conf_unmatched_history_size = settings.get('trackerWindow') or 3
+                self.trackers[detection_session.id] = t
+            detection_session.tracker = t
+            # conf_trgt = 0.35
+            # conf_objt = 0.75
+            # detection_session.tracker.conf_trgt = conf_trgt
+            # detection_session.tracker.conf_objt = conf_objt
+
         # this a single pass or the second pass. detect once and return results.
         if multipass_crop:
             (l, t, dx, dy) = multipass_crop
@@ -376,10 +410,60 @@ def is_same_detection_middle(d1: ObjectDetectionResult, d2: ObjectDetectionResul
             ret = ret1
             ret['detections'] = dedupe_detections(ret1['detections'] + ret2['detections'], is_same_detection=is_same_detection_middle)
 
+        if detection_session:
+            self.track(detection_session, ret)
+
         if not len(ret['detections']):
             return ret, RawImage(image)
 
         return ret, RawImage(image)
+
+    def track(self, detection_session: PredictSession, ret: ObjectsDetected):
+        detections = ret['detections']
+        sort_input = []
+        for d in ret['detections']:
+            r: ObjectDetectionResult = d
+            l, t, w, h = r['boundingBox']
+            sort_input.append([l, t, l + w, t + h, r['score']])
+        trackers, unmatched_trckr, unmatched_gts = detection_session.tracker.update(np.array(sort_input), [])
+        for td in trackers:
+            x0, y0, x1, y1, trackID = td[0].item(), td[1].item(
+            ), td[2].item(), td[3].item(), td[4].item()
+            slop = 0
+            obj: ObjectDetectionResult = None
+            ta = (x1 - x0) * (y1 - y0)
+            box = Rectangle(x0, y0, x1, y1)
+            for d in detections:
+                if d.get('id'):
+                    continue
+                ob: ObjectDetectionResult = d
+                dx0, dy0, dw, dh = ob['boundingBox']
+                dx1 = dx0 + dw
+                dy1 = dy0 + dh
+                da = dw * dh
+                area = intersect_area(Rectangle(dx0, dy0, dx1, dy1), box)
+                if not area:
+                    continue
+                # intersect area always gonna be smaller than
+                # the detection or tracker area.
+                # greater numbers, ie approaching 2, is better.
+                dslop = area / ta + area / da
+                if (dslop > slop):
+                    slop = dslop
+                    obj = ob
+            if obj:
+                obj['id'] = str(trackID)
+            # this may happen if tracker predicts something is still in the scene
+            # but was not detected
+            # else:
+            #     print('unresolved tracker')
+        # for d in detections:
+        #     if not d.get('id'):
+        #         # this happens if the tracker is not confident in a new detection yet due
+        #         # to low score or has not been found in enough frames
+        #         if d['className'] == 'person':
+        #             print('untracked %s: %s' % (d['className'], d['score']))
+
 
     def run_detection_crop(self, detection_session: DetectionSession, sample: RawImage, settings: Any, src_size, convert_to_src_size, bounding_box: Tuple[float, float, float, float]) -> ObjectsDetected:
         (ret, _) = self.run_detection_image(detection_session, sample.image, settings, src_size, convert_to_src_size, bounding_box)

plugins/tensorflow-lite/src/predict/sort_oh

@@ -0,0 +1 @@
+../../../sort-tracker/sort_oh/libs