From 00cf987cecc76e48dcffe40aa5cb92b2358a7404 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Wed, 15 Mar 2023 17:03:34 -0700 Subject: [PATCH] videoanalysis: reimplemnet snapshots for new pipeline --- plugins/objectdetector/package-lock.json | 4 +- plugins/objectdetector/package.json | 2 +- plugins/objectdetector/src/main.ts | 83 ++++++++++++++++++++---- 3 files changed, 72 insertions(+), 17 deletions(-) diff --git a/plugins/objectdetector/package-lock.json b/plugins/objectdetector/package-lock.json index ae2c98548c..1d87f32a2f 100644 --- a/plugins/objectdetector/package-lock.json +++ b/plugins/objectdetector/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/objectdetector", - "version": "0.0.103", + "version": "0.0.104", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/objectdetector", - "version": "0.0.103", + "version": "0.0.104", "license": "Apache-2.0", "dependencies": { "@scrypted/common": "file:../../common", diff --git a/plugins/objectdetector/package.json b/plugins/objectdetector/package.json index 9631d91cf1..ea83ce99e7 100644 --- a/plugins/objectdetector/package.json +++ b/plugins/objectdetector/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/objectdetector", - "version": "0.0.103", + "version": "0.0.104", "description": "Scrypted Video Analysis Plugin. Installed alongside a detection service like OpenCV or TensorFlow.", "author": "Scrypted", "license": "Apache-2.0", diff --git a/plugins/objectdetector/src/main.ts b/plugins/objectdetector/src/main.ts index b3dc13e072..b7584597ed 100644 --- a/plugins/objectdetector/src/main.ts +++ b/plugins/objectdetector/src/main.ts @@ -1,4 +1,4 @@ -import sdk, { VideoFrameGenerator, Camera, DeviceState, EventListenerRegister, MediaObject, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionModel, ObjectDetectionResult, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk'; +import sdk, { ScryptedMimeTypes, Image, VideoFrame, VideoFrameGenerator, Camera, DeviceState, EventListenerRegister, MediaObject, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionModel, ObjectDetectionResult, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk'; import { StorageSettings } from '@scrypted/sdk/storage-settings'; import crypto from 'crypto'; import cloneDeep from 'lodash/cloneDeep'; @@ -50,6 +50,16 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase(); cameraDevice: ScryptedDevice & Camera & VideoCamera & MotionSensor & ObjectDetector; storageSettings = new StorageSettings(this, { + newPipeline: { + title: 'Video Pipeline', + description: 'Configure how frames are provided to the video analysis pipeline.', + choices: [ + 'Default', + 'Snapshot', + ...getAllDevices().filter(d => d.interfaces.includes(ScryptedInterface.VideoFrameGenerator)).map(d => d.name), + ], + defaultValue: 'Default', + }, motionSensorSupplementation: { title: 'Built-In Motion Sensor', description: `This camera has a built in motion sensor. Using ${this.objectDetection.name} may be unnecessary and will use additional CPU. Replace will ignore the built in motion sensor. Filter will verify the motion sent by built in motion sensor. The Default is ${BUILTIN_MOTION_SENSOR_REPLACE}.`, @@ -473,22 +483,57 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase; + if (newPipeline === 'Snapshot') { + const self = this; + generator = (async function* gen() { + while (true) { + const now = Date.now(); + const sleeper = async () => { + const diff = now + 1100 - Date.now(); + if (diff > 0) + await sleep(diff); + }; + let image: MediaObject & VideoFrame; + try { + const mo = await self.cameraDevice.takePicture({ + reason: 'event', + }); + image = await sdk.mediaManager.convertMediaObject(mo, ScryptedMimeTypes.Image); + } + catch (e) { + self.console.error('Video analysis snapshot failed. Will retry in a moment.'); + await sleeper(); + continue; + } + + yield image; + await sleeper(); + } + })(); + } + else { + const videoFrameGenerator = systemManager.getDeviceById(newPipeline); + if (!videoFrameGenerator) + throw new Error('invalid VideoFrameGenerator'); + const stream = await this.cameraDevice.getVideoStream({ + destination: 'local-recorder', + // ask rebroadcast to mute audio, not needed. + audio: null, + }); + + generator = await videoFrameGenerator.generateVideoFrames(stream); + } try { const start = Date.now(); let detections = 0; for await (const detected - of await this.objectDetection.generateObjectDetections(await videoFrameGenerator.generateVideoFrames(stream), { + of await this.objectDetection.generateObjectDetections(generator, { settings: this.getCurrentSettings(), })) { if (!this.detectorRunning) { @@ -849,8 +894,18 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase d.interfaces.includes(ScryptedInterface.VideoFrameGenerator)); + const found = pipelines.find(p => p.name === newPipeline); + return found?.id || pipelines[0]?.id; + } async getMixinSettings(): Promise { const settings: Setting[] = []; @@ -872,7 +927,8 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase