From eb61d12c7d00969037b8ac4b3282766d74609bd2 Mon Sep 17 00:00:00 2001
From: nexustar <inexustar@gmail.com>
Date: Thu, 27 Oct 2022 17:22:03 +0800
Subject: [PATCH] add metricfilter for k8s (#417)

---
 api/doc.html                     | 2 +-
 api/generate-codes.sh            | 2 +-
 api/swagger.yaml                 | 4 ++++
 api/types/collect_job_request.go | 3 +++
 k8s/server/collectors.go         | 6 ++++--
 5 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/api/doc.html b/api/doc.html
index 3127f20c..b9a296e3 100644
--- a/api/doc.html
+++ b/api/doc.html
@@ -35,7 +35,7 @@
 <script>
 window.onload = function() {
 
-  var spec = {"swagger": "2.0", "info": {"description": "RESTful API definitions for Diag", "title": "Diag API", "version": "1.0.0"}, "basePath": "/api/v1", "schemes": ["http", "https"], "consumes": ["application/json"], "produces": ["application/json", "text/plain"], "paths": {"/collectors": {"get": {"operationId": "getJobList", "responses": {"200": {"description": "list all collect jobs", "schema": {"type": "array", "items": {"$ref": "#/definitions/CollectJob"}}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "post": {"operationId": "collectData", "parameters": [{"name": "body", "in": "body", "schema": {"$ref": "#/definitions/CollectJobRequest"}}], "responses": {"202": {"description": "collector started", "schema": {"$ref": "#/definitions/CollectJob"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/collectors/{id}": {"get": {"operationId": "getCollectJob", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "responses": {"200": {"description": "get a collect job", "schema": {"$ref": "#/definitions/CollectJob"}}, "404": {"description": "job not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "post": {"operationId": "operateCollectJob", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}, {"name": "body", "in": "body", "schema": {"$ref": "#/definitions/OperateJobRequest"}}], "responses": {"202": {"description": "collector restarted", "schema": {"$ref": "#/definitions/CollectJob"}}, "405": {"description": "unknown operation", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "406": {"description": "the status of this job cannot perform the operation", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "404": {"description": "job not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "delete": {"operationId": "cancelCollectJob", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "responses": {"202": {"description": "job cancelled", "schema": {"$ref": "#/definitions/CollectJob"}}, "404": {"description": "job not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "410": {"description": "the job has already been cancelled", "schema": {"$ref": "#/definitions/CollectJob"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/collectors/{id}/logs": {"get": {"operationId": "getCollectJobLogs", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "produces": ["application/json", "text/plain"], "responses": {"200": {"description": "get logs of a collect job", "schema": {"type": "string"}}, "404": {"description": "job not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/data": {"get": {"operationId": "getDataList", "parameters": [{"name": "status", "in": "query", "type": "string"}], "responses": {"200": {"description": "list all available data sets", "schema": {"type": "array", "items": {"$ref": "#/definitions/DataSet"}}}, "400": {"description": "unknown status", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/data/{id}": {"get": {"operationId": "getDataSet", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "responses": {"200": {"description": "get a data set", "schema": {"$ref": "#/definitions/DataSet"}}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "delete": {"operationId": "deleteDataSet", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "responses": {"204": {"description": "deleted"}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "503": {"description": "collect job not finished", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/data/{id}/upload": {"get": {"operationId": "getUploadTask", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "responses": {"200": {"description": "show last upload task of the data set", "schema": {"$ref": "#/definitions/UploadTask"}}, "204": {"description": "no upload task available"}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "post": {"operationId": "uploadDataSet", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}, {"name": "rebuild", "in": "query", "type": "boolean", "required": false}], "responses": {"202": {"description": "package and upload a data set to the clinic server", "schema": {"$ref": "#/definitions/UploadTask"}}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "409": {"description": "an upload task is already running", "schema": {"$ref": "#/definitions/UploadTask"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "delete": {"operationId": "cancelDataUpload", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "responses": {"202": {"description": "upload has finished, no need to cancal", "schema": {"$ref": "#/definitions/UploadTask"}}, "204": {"description": "upload cancelled"}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "410": {"description": "the upload has already been cancelled", "schema": {"$ref": "#/definitions/UploadTask"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/data/{id}/check": {"get": {"operationId": "getCheckResult", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "produces": ["application/json", "text/plain"], "responses": {"200": {"description": "show last check result of the data set", "schema": {"description": "check result of the data set", "type": "string"}}, "204": {"description": "no check result available"}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "post": {"operationId": "checkDataSet", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}, {"name": "body", "in": "body", "schema": {"$ref": "#/definitions/CheckDataRequest"}}], "produces": ["application/json", "text/plain"], "responses": {"200": {"description": "run checker on a data set", "schema": {"description": "check result of the data set", "type": "string"}}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "delete": {"operationId": "cancelCheck", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "produces": ["application/json", "text/plain"], "responses": {"202": {"description": "check has finished, no need to cancal", "schema": {"description": "check result of the data set", "type": "string"}}, "204": {"description": "check cancelled"}, "404": {"description": "data set not found or check result unavailable", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/version": {"get": {"operationId": "getVersion", "responses": {"200": {"description": "get server version", "schema": {"type": "object", "properties": {"version": {"type": "string"}, "go": {"type": "string"}}}}}}}, "/status": {"get": {"operationId": "getStatus", "responses": {"200": {"description": "get server status (empty for now)", "schema": {"type": "object"}}}}}}, "definitions": {"DataSet": {"type": "object", "properties": {"id": {"type": "string"}, "clusterName": {"type": "string"}, "size": {"type": "integer"}, "date": {"type": "string", "format": "dateTime"}}}, "CollectJobRequest": {"type": "object", "properties": {"clusterName": {"type": "string"}, "namespace": {"type": "string"}, "monitor_namespace": {"type": "string"}, "collectors": {"type": "array", "items": {"type": "string"}}, "from": {"type": "string", "format": "dateTime"}, "to": {"type": "string", "format": "dateTime"}}}, "CollectJob": {"type": "object", "properties": {"id": {"type": "string"}, "clusterName": {"type": "string"}, "collectors": {"type": "array", "items": {"type": "string"}}, "from": {"type": "string", "format": "dateTime"}, "to": {"type": "string", "format": "dateTime"}, "date": {"type": "string", "format": "dateTime"}, "status": {"type": "string"}, "dir": {"type": "string"}}}, "OperateJobRequest": {"type": "object", "properties": {"operation": {"type": "string"}}}, "CheckDataRequest": {"type": "object", "properties": {"types": {"type": "array", "items": {"type": "string"}}}}, "UploadTask": {"type": "object", "properties": {"id": {"type": "string"}, "date": {"type": "string", "format": "dateTime"}, "status": {"type": "string"}, "result": {"type": "string"}}}, "ResponseMsg": {"type": "object", "properties": {"message": {"type": "string"}}}}};
+  var spec = {"swagger": "2.0", "info": {"description": "RESTful API definitions for Diag", "title": "Diag API", "version": "1.0.0"}, "basePath": "/api/v1", "schemes": ["http", "https"], "consumes": ["application/json"], "produces": ["application/json", "text/plain"], "paths": {"/collectors": {"get": {"operationId": "getJobList", "responses": {"200": {"description": "list all collect jobs", "schema": {"type": "array", "items": {"$ref": "#/definitions/CollectJob"}}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "post": {"operationId": "collectData", "parameters": [{"name": "body", "in": "body", "schema": {"$ref": "#/definitions/CollectJobRequest"}}], "responses": {"202": {"description": "collector started", "schema": {"$ref": "#/definitions/CollectJob"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/collectors/{id}": {"get": {"operationId": "getCollectJob", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "responses": {"200": {"description": "get a collect job", "schema": {"$ref": "#/definitions/CollectJob"}}, "404": {"description": "job not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "post": {"operationId": "operateCollectJob", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}, {"name": "body", "in": "body", "schema": {"$ref": "#/definitions/OperateJobRequest"}}], "responses": {"202": {"description": "collector restarted", "schema": {"$ref": "#/definitions/CollectJob"}}, "405": {"description": "unknown operation", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "406": {"description": "the status of this job cannot perform the operation", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "404": {"description": "job not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "delete": {"operationId": "cancelCollectJob", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "responses": {"202": {"description": "job cancelled", "schema": {"$ref": "#/definitions/CollectJob"}}, "404": {"description": "job not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "410": {"description": "the job has already been cancelled", "schema": {"$ref": "#/definitions/CollectJob"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/collectors/{id}/logs": {"get": {"operationId": "getCollectJobLogs", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "produces": ["application/json", "text/plain"], "responses": {"200": {"description": "get logs of a collect job", "schema": {"type": "string"}}, "404": {"description": "job not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/data": {"get": {"operationId": "getDataList", "parameters": [{"name": "status", "in": "query", "type": "string"}], "responses": {"200": {"description": "list all available data sets", "schema": {"type": "array", "items": {"$ref": "#/definitions/DataSet"}}}, "400": {"description": "unknown status", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/data/{id}": {"get": {"operationId": "getDataSet", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "responses": {"200": {"description": "get a data set", "schema": {"$ref": "#/definitions/DataSet"}}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "delete": {"operationId": "deleteDataSet", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "responses": {"204": {"description": "deleted"}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "503": {"description": "collect job not finished", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/data/{id}/upload": {"get": {"operationId": "getUploadTask", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "responses": {"200": {"description": "show last upload task of the data set", "schema": {"$ref": "#/definitions/UploadTask"}}, "204": {"description": "no upload task available"}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "post": {"operationId": "uploadDataSet", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}, {"name": "rebuild", "in": "query", "type": "boolean", "required": false}], "responses": {"202": {"description": "package and upload a data set to the clinic server", "schema": {"$ref": "#/definitions/UploadTask"}}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "409": {"description": "an upload task is already running", "schema": {"$ref": "#/definitions/UploadTask"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "delete": {"operationId": "cancelDataUpload", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "responses": {"202": {"description": "upload has finished, no need to cancal", "schema": {"$ref": "#/definitions/UploadTask"}}, "204": {"description": "upload cancelled"}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "410": {"description": "the upload has already been cancelled", "schema": {"$ref": "#/definitions/UploadTask"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/data/{id}/check": {"get": {"operationId": "getCheckResult", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "produces": ["application/json", "text/plain"], "responses": {"200": {"description": "show last check result of the data set", "schema": {"description": "check result of the data set", "type": "string"}}, "204": {"description": "no check result available"}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "post": {"operationId": "checkDataSet", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}, {"name": "body", "in": "body", "schema": {"$ref": "#/definitions/CheckDataRequest"}}], "produces": ["application/json", "text/plain"], "responses": {"200": {"description": "run checker on a data set", "schema": {"description": "check result of the data set", "type": "string"}}, "404": {"description": "data set not found", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}, "delete": {"operationId": "cancelCheck", "parameters": [{"name": "id", "in": "path", "type": "string", "required": true}], "produces": ["application/json", "text/plain"], "responses": {"202": {"description": "check has finished, no need to cancal", "schema": {"description": "check result of the data set", "type": "string"}}, "204": {"description": "check cancelled"}, "404": {"description": "data set not found or check result unavailable", "schema": {"$ref": "#/definitions/ResponseMsg"}}, "500": {"description": "server side error", "schema": {"$ref": "#/definitions/ResponseMsg"}}}}}, "/version": {"get": {"operationId": "getVersion", "responses": {"200": {"description": "get server version", "schema": {"type": "object", "properties": {"version": {"type": "string"}, "go": {"type": "string"}}}}}}}, "/status": {"get": {"operationId": "getStatus", "responses": {"200": {"description": "get server status (empty for now)", "schema": {"type": "object"}}}}}}, "definitions": {"DataSet": {"type": "object", "properties": {"id": {"type": "string"}, "clusterName": {"type": "string"}, "size": {"type": "integer"}, "date": {"type": "string", "format": "dateTime"}}}, "CollectJobRequest": {"type": "object", "properties": {"clusterName": {"type": "string"}, "namespace": {"type": "string"}, "monitor_namespace": {"type": "string"}, "collectors": {"type": "array", "items": {"type": "string"}}, "explain_sqls": {"type": "array", "items": {"type": "string"}}, "from": {"type": "string", "format": "dateTime"}, "to": {"type": "string", "format": "dateTime"}, "metricfilter": {"type": "array", "items": {"type": "string"}}}}, "CollectJob": {"type": "object", "properties": {"id": {"type": "string"}, "clusterName": {"type": "string"}, "collectors": {"type": "array", "items": {"type": "string"}}, "from": {"type": "string", "format": "dateTime"}, "to": {"type": "string", "format": "dateTime"}, "date": {"type": "string", "format": "dateTime"}, "status": {"type": "string"}, "dir": {"type": "string"}}}, "OperateJobRequest": {"type": "object", "properties": {"operation": {"type": "string"}}}, "CheckDataRequest": {"type": "object", "properties": {"types": {"type": "array", "items": {"type": "string"}}}}, "UploadTask": {"type": "object", "properties": {"id": {"type": "string"}, "date": {"type": "string", "format": "dateTime"}, "status": {"type": "string"}, "result": {"type": "string"}}}, "ResponseMsg": {"type": "object", "properties": {"message": {"type": "string"}}}}};
 
   // Build a system
   const ui = SwaggerUIBundle({
diff --git a/api/generate-codes.sh b/api/generate-codes.sh
index 1be3b999..40162fd2 100755
--- a/api/generate-codes.sh
+++ b/api/generate-codes.sh
@@ -11,4 +11,4 @@ swagger generate model \
     -t "${scriptdir}" \
     -m types
 
-"${scriptdir}"/swagger-yaml-to-html.py < "${scriptdir}/swagger.yaml" > "${scriptdir}/doc.html"
+python3 "${scriptdir}"/swagger-yaml-to-html.py < "${scriptdir}/swagger.yaml" > "${scriptdir}/doc.html"
diff --git a/api/swagger.yaml b/api/swagger.yaml
index 511222fe..1a181651 100644
--- a/api/swagger.yaml
+++ b/api/swagger.yaml
@@ -424,6 +424,10 @@ definitions:
       to:
         type: string
         format: dateTime
+      metricfilter:
+        type: array
+        items:
+          type: string
   CollectJob:
     type: object
     properties:
diff --git a/api/types/collect_job_request.go b/api/types/collect_job_request.go
index d40627db..39d410ea 100644
--- a/api/types/collect_job_request.go
+++ b/api/types/collect_job_request.go
@@ -29,6 +29,9 @@ type CollectJobRequest struct {
 	// from
 	From string `json:"from,omitempty"`
 
+	// metricfilter
+	Metricfilter []string `json:"metricfilter"`
+
 	// monitor namespace
 	MonitorNamespace string `json:"monitor_namespace,omitempty"`
 
diff --git a/k8s/server/collectors.go b/k8s/server/collectors.go
index 66784728..4fcf77ad 100644
--- a/k8s/server/collectors.go
+++ b/k8s/server/collectors.go
@@ -111,7 +111,7 @@ func collectData(c *gin.Context) {
 	explainSQLs := req.ExplainSqls
 
 	// run collector
-	go runCollector(diagCtx, &opt, worker, req, explainSQLs)
+	go runCollector(diagCtx, &opt, worker, req, explainSQLs, req.Metricfilter)
 
 	c.JSON(http.StatusAccepted, worker.job)
 }
@@ -122,6 +122,7 @@ func runCollector(
 	worker *collectJobWorker,
 	req interface{},
 	explainSQLs []string,
+	metricFilters []string,
 ) {
 	gOpt := operator.Options{
 		Concurrency: 2,
@@ -140,6 +141,7 @@ func runCollector(
 		RawRequest:      req,
 		Dir:             filepath.Join(collectDir, "diag-"+worker.job.ID), // set default k8s package dir
 		ExplainSqls:     explainSQLs,
+		MetricsFilter:   metricFilters,
 		CompressMetrics: false,
 	}
 
@@ -416,7 +418,7 @@ func reCollectData(c *gin.Context, diagCtx *context, id string) {
 	os.RemoveAll(requestDir)
 
 	// run collector
-	go runCollector(diagCtx, &opt, worker, cluster.RawRequest, []string{})
+	go runCollector(diagCtx, &opt, worker, cluster.RawRequest, []string{}, []string{})
 	diagCtx.setJobStatus(worker.job.ID, taskStatusRunning)
 
 	c.JSON(http.StatusAccepted, worker.job)