RSDK-2224 New vision service (#2254)

Co-authored-by: Khari Jarrett <khari@viam.com>

.artifact/tree.json

@@ -51566,6 +51566,10 @@
         "hash": "3a047d6fd440b36cbba72e5532ff3548",
         "size": 4563519
       },
+      "effdetlabels.txt": {
+        "hash": "359468bee2538de4c5950190cc1eb002",
+        "size": 699
+      },
       "effnet0.tflite": {
         "hash": "301b893e4701ec6f490c21a2435befbc",
         "size": 7165437
@@ -51574,6 +51578,10 @@
         "hash": "9b9d5e11ece4e6fe52246a2851636145",
         "size": 50
       },
+      "imagenetlabels.txt": {
+        "hash": "93b03dc453206b6ce0cd083f96066544",
+        "size": 23685
+      },
       "lion.jpeg": {
         "hash": "7a24ee403e1606a0956d63dc744f24de",
         "size": 31645

components/camera/transformpipeline/classifier.go

@@ -78,7 +78,7 @@ func (cs *classifierSource) Read(ctx context.Context) (image.Image, func(), erro
 	ctx, span := trace.StartSpan(ctx, "camera::transformpipeline::classifier::Read")
 	defer span.End()
 
-	srv, err := vision.FirstFromRobot(cs.r)
+	srv, err := vision.FromRobot(cs.r, cs.classifierName)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "source_classifier can't find vision service")
 	}
@@ -87,7 +87,7 @@ func (cs *classifierSource) Read(ctx context.Context) (image.Image, func(), erro
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "could not get next source image")
 	}
-	classifications, err := srv.Classifications(ctx, img, cs.classifierName, int(cs.maxClassifications), map[string]interface{}{})
+	classifications, err := srv.Classifications(ctx, img, int(cs.maxClassifications), map[string]interface{}{})
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "could not get classifications")
 	}

components/camera/transformpipeline/classifier_test.go

@@ -15,14 +15,38 @@ import (
 	"go.viam.com/rdk/rimage"
 	"go.viam.com/rdk/robot"
 	robotimpl "go.viam.com/rdk/robot/impl"
+	"go.viam.com/rdk/services/mlmodel"
+	_ "go.viam.com/rdk/services/mlmodel/register"
 	"go.viam.com/rdk/services/vision"
+	_ "go.viam.com/rdk/services/vision/register"
 	rutils "go.viam.com/rdk/utils"
 )
 
 func buildRobotWithClassifier(logger golog.Logger) (robot.Robot, error) {
 	cfg := &config.Config{}
 
 	// create fake source camera
+	tfliteSrv1 := resource.Config{
+		Name:  "object_classifier",
+		API:   mlmodel.API,
+		Model: resource.DefaultModelFamily.WithModel("tflite_cpu"),
+		Attributes: rutils.AttributeMap{
+			"model_path":  artifact.MustPath("vision/classification/object_classifier.tflite"),
+			"label_path":  artifact.MustPath("vision/classification/object_labels.txt"),
+			"num_threads": 1,
+		},
+	}
+	cfg.Services = append(cfg.Services, tfliteSrv1)
+	visionSrv1 := resource.Config{
+		Name:  "vision_classifier",
+		API:   vision.API,
+		Model: resource.DefaultModelFamily.WithModel("mlmodel"),
+		Attributes: rutils.AttributeMap{
+			"mlmodel_name": "object_classifier",
+		},
+		DependsOn: []string{"object_classifier"},
+	}
+	cfg.Services = append(cfg.Services, visionSrv1)
 	cameraComp := resource.Config{
 		Name:  "fake_cam",
 		API:   camera.API,
@@ -45,7 +69,7 @@ func buildRobotWithClassifier(logger golog.Logger) (robot.Robot, error) {
 				{
 					"type": "classifications",
 					"attributes": rutils.AttributeMap{
-						"classifier_name":      "object_classifier",
+						"classifier_name":      "vision_classifier",
 						"confidence_threshold": 0.35,
 						"max_classifications":  5,
 					},
@@ -70,29 +94,10 @@ func buildRobotWithClassifier(logger golog.Logger) (robot.Robot, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	// Add the classification model to the vision service
-	srv, err := vision.FirstFromRobot(r)
-	if err != nil {
-		return nil, err
-	}
-	classConf := vision.VisModelConfig{
-		Name: "object_classifier",
-		Type: "tflite_classifier",
-		Parameters: rutils.AttributeMap{
-			"model_path":  artifact.MustPath("vision/classification/object_classifier.tflite"),
-			"label_path":  artifact.MustPath("vision/classification/object_labels.txt"),
-			"num_threads": 1,
-		},
-	}
-	err = srv.AddClassifier(context.Background(), classConf, map[string]interface{}{})
-	if err != nil {
-		return nil, err
-	}
-
 	return r, nil
 }
 
+//nolint:dupl
 func TestClassifierSource(t *testing.T) {
 	logger := golog.NewTestLogger(t)
 	ctx, cancel := context.WithCancel(context.Background())
@@ -114,7 +119,7 @@ func TestClassifierSource(t *testing.T) {
 	ovImg := rimage.ConvertImage(resImg)
 	// Max classifications was 5, but this image gets classified with just 2 labels, so we
 	// test that each label is present.
-	test.That(t, ovImg.GetXY(35, 45), test.ShouldResemble, rimage.Red)
-	test.That(t, ovImg.GetXY(35, 58), test.ShouldResemble, rimage.Red)
+	test.That(t, ovImg.GetXY(76, 48), test.ShouldResemble, rimage.Red)
+	test.That(t, ovImg.GetXY(72, 76), test.ShouldResemble, rimage.Red)
 	test.That(t, classifier.Close(context.Background()), test.ShouldBeNil)
 }

components/camera/transformpipeline/detector.go

@@ -71,7 +71,7 @@ func (ds *detectorSource) Read(ctx context.Context) (image.Image, func(), error)
 	ctx, span := trace.StartSpan(ctx, "camera::transformpipeline::detector::Read")
 	defer span.End()
 	// get the bounding boxes from the service
-	srv, err := vision.FirstFromRobot(ds.r)
+	srv, err := vision.FromRobot(ds.r, ds.detectorName)
 	if err != nil {
 		return nil, nil, fmt.Errorf("source_detector cant find vision service: %w", err)
 	}
@@ -80,7 +80,7 @@ func (ds *detectorSource) Read(ctx context.Context) (image.Image, func(), error)
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not get next source image: %w", err)
 	}
-	dets, err := srv.Detections(ctx, img, ds.detectorName, map[string]interface{}{})
+	dets, err := srv.Detections(ctx, img, map[string]interface{}{})
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not get detections: %w", err)
 	}

components/camera/transformpipeline/detector_test.go

@@ -16,8 +16,10 @@ import (
 	"go.viam.com/rdk/rimage"
 	"go.viam.com/rdk/robot"
 	robotimpl "go.viam.com/rdk/robot/impl"
+	"go.viam.com/rdk/services/mlmodel"
+	_ "go.viam.com/rdk/services/mlmodel/register"
 	"go.viam.com/rdk/services/vision"
-	_ "go.viam.com/rdk/services/vision/builtin"
+	_ "go.viam.com/rdk/services/vision/register"
 	rutils "go.viam.com/rdk/utils"
 )
 
@@ -49,6 +51,38 @@ func buildRobotWithFakeCamera(logger golog.Logger) (robot.Robot, error) {
 		return nil, err
 	}
 	// create fake source camera
+	colorSrv1 := resource.Config{
+		Name:  "detector_color",
+		API:   vision.API,
+		Model: resource.DefaultModelFamily.WithModel("color_detector"),
+		Attributes: rutils.AttributeMap{
+			"detect_color":      "#4F3815",
+			"hue_tolerance_pct": 0.013,
+			"segment_size_px":   15000,
+		},
+	}
+	cfg.Services = append(cfg.Services, colorSrv1)
+	tfliteSrv2 := resource.Config{
+		Name:  "detector_tflite",
+		API:   mlmodel.API,
+		Model: resource.DefaultModelFamily.WithModel("tflite_cpu"),
+		Attributes: rutils.AttributeMap{
+			"model_path":  artifact.MustPath("vision/tflite/effdet0.tflite"),
+			"label_path":  artifact.MustPath("vision/tflite/effdetlabels.txt"),
+			"num_threads": 1,
+		},
+	}
+	cfg.Services = append(cfg.Services, tfliteSrv2)
+	visionSrv2 := resource.Config{
+		Name:  "vision_detector",
+		API:   vision.API,
+		Model: resource.DefaultModelFamily.WithModel("mlmodel"),
+		Attributes: rutils.AttributeMap{
+			"mlmodel_name": "detector_tflite",
+		},
+		DependsOn: []string{"detector_tflite"},
+	}
+	cfg.Services = append(cfg.Services, visionSrv2)
 	cameraComp := resource.Config{
 		Name:  "fake_cam",
 		API:   camera.API,
@@ -90,7 +124,7 @@ func buildRobotWithFakeCamera(logger golog.Logger) (robot.Robot, error) {
 				{
 					"type": "detections",
 					"attributes": rutils.AttributeMap{
-						"detector_name":        "detector_tflite",
+						"detector_name":        "vision_detector",
 						"confidence_threshold": 0.35,
 					},
 				},
@@ -112,6 +146,7 @@ func buildRobotWithFakeCamera(logger golog.Logger) (robot.Robot, error) {
 	return robotimpl.RobotFromConfigPath(context.Background(), newConfFile, logger)
 }
 
+//nolint:dupl
 func TestColorDetectionSource(t *testing.T) {
 	logger := golog.NewTestLogger(t)
 	ctx, cancel := context.WithCancel(context.Background())
@@ -124,22 +159,6 @@ func TestColorDetectionSource(t *testing.T) {
 		test.That(t, r.Close(context.Background()), test.ShouldBeNil)
 	}()
 
-	// add the detector
-
-	srv, err := vision.FirstFromRobot(r)
-	test.That(t, err, test.ShouldBeNil)
-	detConf := vision.VisModelConfig{
-		Name: "detector_color",
-		Type: "color_detector",
-		Parameters: rutils.AttributeMap{
-			"detect_color":      "#4F3815",
-			"hue_tolerance_pct": 0.013,
-			"segment_size_px":   15000,
-		},
-	}
-	err = srv.AddDetector(context.Background(), detConf, map[string]interface{}{})
-	test.That(t, err, test.ShouldBeNil)
-
 	detector, err := camera.FromRobot(r, "color_detect")
 	test.That(t, err, test.ShouldBeNil)
 	defer detector.Close(ctx)
@@ -163,20 +182,6 @@ func TestTFLiteDetectionSource(t *testing.T) {
 	}()
 	test.That(t, err, test.ShouldBeNil)
 
-	// add the detector
-	srv, err := vision.FirstFromRobot(r)
-	test.That(t, err, test.ShouldBeNil)
-	detConf := vision.VisModelConfig{
-		Name: "detector_tflite",
-		Type: "tflite_detector",
-		Parameters: rutils.AttributeMap{
-			"model_path":  artifact.MustPath("vision/tflite/effdet0.tflite"),
-			"num_threads": 1,
-		},
-	}
-	err = srv.AddDetector(context.Background(), detConf, map[string]interface{}{})
-	test.That(t, err, test.ShouldBeNil)
-
 	detector, err := camera.FromRobot(r, "tflite_detect")
 	test.That(t, err, test.ShouldBeNil)
 	defer detector.Close(ctx)
@@ -199,20 +204,6 @@ func BenchmarkColorDetectionSource(b *testing.B) {
 		test.That(b, r.Close(context.Background()), test.ShouldBeNil)
 	}()
 	test.That(b, err, test.ShouldBeNil)
-	// add the detector
-	srv, err := vision.FirstFromRobot(r)
-	test.That(b, err, test.ShouldBeNil)
-	detConf := vision.VisModelConfig{
-		Name: "detector_color",
-		Type: "color_detector",
-		Parameters: rutils.AttributeMap{
-			"detect_color":      "#4F3815",
-			"hue_tolerance_pct": 0.055556,
-			"segment_size_px":   15000,
-		},
-	}
-	err = srv.AddDetector(context.Background(), detConf, map[string]interface{}{})
-	test.That(b, err, test.ShouldBeNil)
 	detector, err := camera.FromRobot(r, "color_detect")
 	test.That(b, err, test.ShouldBeNil)
 	defer detector.Close(ctx)
@@ -235,19 +226,6 @@ func BenchmarkTFLiteDetectionSource(b *testing.B) {
 		test.That(b, r.Close(context.Background()), test.ShouldBeNil)
 	}()
 	test.That(b, err, test.ShouldBeNil)
-	// add the detector
-	srv, err := vision.FirstFromRobot(r)
-	test.That(b, err, test.ShouldBeNil)
-	detConf := vision.VisModelConfig{
-		Name: "detector_tflite",
-		Type: "tflite_detector",
-		Parameters: rutils.AttributeMap{
-			"model_path":  artifact.MustPath("vision/tflite/effdet0.tflite"),
-			"num_threads": 1,
-		},
-	}
-	err = srv.AddDetector(context.Background(), detConf, map[string]interface{}{})
-	test.That(b, err, test.ShouldBeNil)
 	detector, err := camera.FromRobot(r, "tflite_detect")
 	test.That(b, err, test.ShouldBeNil)
 	defer detector.Close(ctx)

go.mod

@@ -81,7 +81,7 @@ require (
 	go.uber.org/atomic v1.10.0
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.24.0
-	go.viam.com/api v0.1.116
+	go.viam.com/api v0.1.118
 	go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2
 	go.viam.com/utils v0.1.20-0.20230424163529-ce35a14fc60f
 	goji.io v2.0.2+incompatible

go.sum

@@ -1565,8 +1565,8 @@ go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
 go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
 go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
-go.viam.com/api v0.1.116 h1:SCbdbupYffojulWFcVfNjXXxqvPm0czytfCg4H2ZbE0=
-go.viam.com/api v0.1.116/go.mod h1:NQC9FZnerAfIPJLJ42vgzQMoe1WAQRuVoQEf1JauxtQ=
+go.viam.com/api v0.1.118 h1:FjVktL++7yy7gx385pXq/40MrUQP0NL2r8PLaAcHc3Q=
+go.viam.com/api v0.1.118/go.mod h1:NQC9FZnerAfIPJLJ42vgzQMoe1WAQRuVoQEf1JauxtQ=
 go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2 h1:oBiK580EnEIzgFLU4lHOXmGAE3MxnVbeR7s1wp/F3Ps=
 go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2/go.mod h1:XM0tej6riszsiNLT16uoyq1YjuYPWlRBweTPRDanIts=
 go.viam.com/utils v0.1.20-0.20230424163529-ce35a14fc60f h1:wvJ/Au+UfSgXRj9RgoapLGYK/ZLj2pSKbq6DbPw3tqM=