Implement ngt Statistics API

Signed-off-by: kpango <kpango@vdaas.org>

.github/workflows/format.yml

@@ -59,7 +59,7 @@ jobs:
       - name: Run formatter and license.go
         run: |
           make format
-          git checkout go.mod go.sum ./rust/Cargo.lock
+          git checkout go.mod go.sum ./example/client/go.mod ./example/client/go.sum ./rust/Cargo.lock ./dockers
       - name: Check and Push to main branch
         continue-on-error: true
         run: |

Makefile

@@ -472,11 +472,11 @@ format/go: \
 	gofumpt/install \
 	strictgoimports/install \
 	goimports/install
-	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs -P$(CORES) $(GOBIN)/golines -w -m $(GOLINES_MAX_WIDTH)
-	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs -P$(CORES) $(GOBIN)/strictgoimports -w
+	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs -P$(CORES) $(GOBIN)/golines -w -m $(GOLINES_MAX_WIDTH)
+	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs -P$(CORES) $(GOBIN)/strictgoimports -w
 	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs -P$(CORES) $(GOBIN)/goimports -w
-	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs -P$(CORES) $(GOBIN)/crlfmt -w
-	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs -P$(CORES) $(GOBIN)/gofumpt -w
+	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs -P$(CORES) $(GOBIN)/crlfmt -w -diff=false
+	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs -P$(CORES) $(GOBIN)/gofumpt -w
 
 .PHONY: format/go/test
 ## run golines, gofumpt, goimports for go test files
@@ -489,7 +489,7 @@ format/go/test: \
 	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/golines -w -m $(GOLINES_MAX_WIDTH)
 	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/strictgoimports -w
 	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/goimports -w
-	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/crlfmt -w
+	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/crlfmt -w -diff=false
 	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/gofumpt -w
 
 .PHONY: format/yaml

Makefile.d/proto.mk

@@ -54,9 +54,14 @@ proto/gen: \
 	$(PROTOS) \
 	proto/deps
 	@$(call green, "generating pb.go and swagger.json files and documents for API v1...")
+	buf format -w
 	buf generate
+	make proto/replace
+
+proto/replace:
 	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs -P$(CORES) sed -i -E "s%google.golang.org/grpc/codes%github.com/vdaas/vald/internal/net/grpc/codes%g"
 	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs -P$(CORES) sed -i -E "s%google.golang.org/grpc/status%github.com/vdaas/vald/internal/net/grpc/status%g"
 	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs -P$(CORES) sed -i -E "s%\"io\"%\"github.com/vdaas/vald/internal/io\"%g"
 	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs -P$(CORES) sed -i -E "s%\"sync\"%\"github.com/vdaas/vald/internal/sync\"%g"
-	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs -P$(CORES) sed -i -E "s%any%any%g"
+	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs -P$(CORES) sed -i -E "s%interface\{\}%any%g"
+	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs -P$(CORES) sed -i -E "s%For_%For%g"

apis/docs/v1/docs.md

@@ -25,6 +25,9 @@
   - [Info.Index.Count](#payload-v1-Info-Index-Count)
   - [Info.Index.Detail](#payload-v1-Info-Index-Detail)
   - [Info.Index.Detail.CountsEntry](#payload-v1-Info-Index-Detail-CountsEntry)
+  - [Info.Index.Statistics](#payload-v1-Info-Index-Statistics)
+  - [Info.Index.StatisticsDetail](#payload-v1-Info-Index-StatisticsDetail)
+  - [Info.Index.StatisticsDetail.DetailsEntry](#payload-v1-Info-Index-StatisticsDetail-DetailsEntry)
   - [Info.Index.UUID](#payload-v1-Info-Index-UUID)
   - [Info.Index.UUID.Committed](#payload-v1-Info-Index-UUID-Committed)
   - [Info.Index.UUID.Uncommitted](#payload-v1-Info-Index-UUID-Uncommitted)
@@ -317,6 +320,67 @@ Represent the index count for each Agents message.
 | key   | [string](#string)                                |       |             |
 | value | [Info.Index.Count](#payload-v1-Info-Index-Count) |       |             |
 
+<a name="payload-v1-Info-Index-Statistics"></a>
+
+### Info.Index.Statistics
+
+Represents index Statistics
+
+| Field                                | Type              | Label    | Description |
+| ------------------------------------ | ----------------- | -------- | ----------- |
+| valid                                | [bool](#bool)     |          |             |
+| median_indegree                      | [int32](#int32)   |          |             |
+| median_outdegree                     | [int32](#int32)   |          |             |
+| max_number_of_indegree               | [uint64](#uint64) |          |             |
+| max_number_of_outdegree              | [uint64](#uint64) |          |             |
+| min_number_of_indegree               | [uint64](#uint64) |          |             |
+| min_number_of_outdegree              | [uint64](#uint64) |          |             |
+| mode_indegree                        | [uint64](#uint64) |          |             |
+| mode_outdegree                       | [uint64](#uint64) |          |             |
+| nodes_skipped_for_10_edges           | [uint64](#uint64) |          |             |
+| nodes_skipped_for_indegree_distance  | [uint64](#uint64) |          |             |
+| number_of_edges                      | [uint64](#uint64) |          |             |
+| number_of_indexed_objects            | [uint64](#uint64) |          |             |
+| number_of_nodes                      | [uint64](#uint64) |          |             |
+| number_of_nodes_without_edges        | [uint64](#uint64) |          |             |
+| number_of_nodes_without_indegree     | [uint64](#uint64) |          |             |
+| number_of_objects                    | [uint64](#uint64) |          |             |
+| number_of_removed_objects            | [uint64](#uint64) |          |             |
+| size_of_object_repository            | [uint64](#uint64) |          |             |
+| size_of_refinement_object_repository | [uint64](#uint64) |          |             |
+| variance_of_indegree                 | [double](#double) |          |             |
+| variance_of_outdegree                | [double](#double) |          |             |
+| mean_edge_length                     | [double](#double) |          |             |
+| mean_edge_length_for_10_edges        | [double](#double) |          |             |
+| mean_indegree_distance_for_10_edges  | [double](#double) |          |             |
+| mean_number_of_edges_per_node        | [double](#double) |          |             |
+| c1_indegree                          | [double](#double) |          |             |
+| c5_indegree                          | [double](#double) |          |             |
+| c95_outdegree                        | [double](#double) |          |             |
+| c99_outdegree                        | [double](#double) |          |             |
+| indegree_count                       | [int64](#int64)   | repeated |             |
+| outdegree_histogram                  | [uint64](#uint64) | repeated |             |
+| indegree_histogram                   | [uint64](#uint64) | repeated |             |
+
+<a name="payload-v1-Info-Index-StatisticsDetail"></a>
+
+### Info.Index.StatisticsDetail
+
+Represents index Statistics for each Agents
+
+| Field   | Type                                                                                             | Label    | Description                 |
+| ------- | ------------------------------------------------------------------------------------------------ | -------- | --------------------------- |
+| details | [Info.Index.StatisticsDetail.DetailsEntry](#payload-v1-Info-Index-StatisticsDetail-DetailsEntry) | repeated | count infos for each agents |
+
+<a name="payload-v1-Info-Index-StatisticsDetail-DetailsEntry"></a>
+
+### Info.Index.StatisticsDetail.DetailsEntry
+
+| Field | Type                                                       | Label | Description |
+| ----- | ---------------------------------------------------------- | ----- | ----------- |
+| key   | [string](#string)                                          |       |             |
+| value | [Info.Index.Statistics](#payload-v1-Info-Index-Statistics) |       |             |
+
 <a name="payload-v1-Info-Index-UUID"></a>
 
 ### Info.Index.UUID
@@ -1497,10 +1561,12 @@ Flush service provides ways to flush all indexed vectors.
 
 Represent the index manager service.
 
-| Method Name | Request Type                           | Response Type                                                  | Description                                                     |
-| ----------- | -------------------------------------- | -------------------------------------------------------------- | --------------------------------------------------------------- |
-| IndexInfo   | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count)   | Represent the RPC to get the index information.                 |
-| IndexDetail | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Detail](#payload-v1-Info-Index-Detail) | Represent the RPC to get the index information for each agents. |
+| Method Name           | Request Type                           | Response Type                                                                      | Description                                                     |
+| --------------------- | -------------------------------------- | ---------------------------------------------------------------------------------- | --------------------------------------------------------------- |
+| IndexInfo             | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count)                       | Represent the RPC to get the index information.                 |
+| IndexDetail           | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Detail](#payload-v1-Info-Index-Detail)                     | Represent the RPC to get the index information for each agents. |
+| IndexStatistics       | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Statistics](#payload-v1-Info-Index-Statistics)             | Represent the RPC to get the index statistics.                  |
+| IndexStatisticsDetail | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.StatisticsDetail](#payload-v1-Info-Index-StatisticsDetail) | Represent the RPC to get the index statistics for each agents.  |
 
 <a name="v1_vald_insert-proto"></a>
 

apis/grpc/v1/agent/core/agent_vtproto.pb.go

@@ -105,17 +105,18 @@ type AgentServer interface {
 }
 
 // UnimplementedAgentServer must be embedded to have forward compatible implementations.
-type UnimplementedAgentServer struct {
-}
+type UnimplementedAgentServer struct{}
 
 func (UnimplementedAgentServer) CreateIndex(
 	context.Context, *payload.Control_CreateIndexRequest,
 ) (*payload.Empty, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method CreateIndex not implemented")
 }
+
 func (UnimplementedAgentServer) SaveIndex(context.Context, *payload.Empty) (*payload.Empty, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method SaveIndex not implemented")
 }
+
 func (UnimplementedAgentServer) CreateAndSaveIndex(
 	context.Context, *payload.Control_CreateIndexRequest,
 ) (*payload.Empty, error) {

apis/grpc/v1/agent/sidecar/sidecar_vtproto.pb.go

@@ -36,8 +36,7 @@ const _ = grpc.SupportPackageIsVersion7
 // SidecarClient is the client API for Sidecar service.
 //
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type SidecarClient interface {
-}
+type SidecarClient any
 
 type sidecarClient struct {
 	cc grpc.ClientConnInterface
@@ -55,8 +54,7 @@ type SidecarServer interface {
 }
 
 // UnimplementedSidecarServer must be embedded to have forward compatible implementations.
-type UnimplementedSidecarServer struct {
-}
+type UnimplementedSidecarServer struct{}
 
 func (UnimplementedSidecarServer) mustEmbedUnimplementedSidecarServer() {}
 

apis/grpc/v1/discoverer/discoverer_vtproto.pb.go

@@ -105,19 +105,20 @@ type DiscovererServer interface {
 }
 
 // UnimplementedDiscovererServer must be embedded to have forward compatible implementations.
-type UnimplementedDiscovererServer struct {
-}
+type UnimplementedDiscovererServer struct{}
 
 func (UnimplementedDiscovererServer) Pods(
 	context.Context, *payload.Discoverer_Request,
 ) (*payload.Info_Pods, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method Pods not implemented")
 }
+
 func (UnimplementedDiscovererServer) Nodes(
 	context.Context, *payload.Discoverer_Request,
 ) (*payload.Info_Nodes, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method Nodes not implemented")
 }
+
 func (UnimplementedDiscovererServer) Services(
 	context.Context, *payload.Discoverer_Request,
 ) (*payload.Info_Services, error) {

apis/grpc/v1/filter/egress/egress_filter_vtproto.pb.go

@@ -90,14 +90,14 @@ type FilterServer interface {
 }
 
 // UnimplementedFilterServer must be embedded to have forward compatible implementations.
-type UnimplementedFilterServer struct {
-}
+type UnimplementedFilterServer struct{}
 
 func (UnimplementedFilterServer) FilterDistance(
 	context.Context, *payload.Object_Distance,
 ) (*payload.Object_Distance, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method FilterDistance not implemented")
 }
+
 func (UnimplementedFilterServer) FilterVector(
 	context.Context, *payload.Object_Vector,
 ) (*payload.Object_Vector, error) {

apis/grpc/v1/filter/ingress/ingress_filter_vtproto.pb.go

@@ -90,14 +90,14 @@ type FilterServer interface {
 }
 
 // UnimplementedFilterServer must be embedded to have forward compatible implementations.
-type UnimplementedFilterServer struct {
-}
+type UnimplementedFilterServer struct{}
 
 func (UnimplementedFilterServer) GenVector(
 	context.Context, *payload.Object_Blob,
 ) (*payload.Object_Vector, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GenVector not implemented")
 }
+
 func (UnimplementedFilterServer) FilterVector(
 	context.Context, *payload.Object_Vector,
 ) (*payload.Object_Vector, error) {

apis/grpc/v1/mirror/mirror_vtproto.pb.go

@@ -75,8 +75,7 @@ type MirrorServer interface {
 }
 
 // UnimplementedMirrorServer must be embedded to have forward compatible implementations.
-type UnimplementedMirrorServer struct {
-}
+type UnimplementedMirrorServer struct{}
 
 func (UnimplementedMirrorServer) Register(
 	context.Context, *payload.Mirror_Targets,