Skip to content

Commit

Permalink
Extract MetricsClient and NodeMetrics to support other metrics platform
Browse files Browse the repository at this point in the history
Signed-off-by: ZhangJian He <shoothzj@gmail.com>
  • Loading branch information
shoothzj committed Feb 11, 2023
1 parent 3859cd3 commit b9e7632
Show file tree
Hide file tree
Showing 4 changed files with 145 additions and 50 deletions.
58 changes: 8 additions & 50 deletions pkg/scheduler/cache/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,6 @@ import (
"sync"
"time"

"github.com/prometheus/client_golang/api"
prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
pmodel "github.com/prometheus/common/model"

v1 "k8s.io/api/core/v1"
schedulingv1 "k8s.io/api/scheduling/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -70,10 +66,6 @@ import (
)

const (
// record name of cpu average usage defined in prometheus rules
cpuUsageAvg = "cpu_usage_avg"
// record name of mem average usage defined in prometheus rules
memUsageAvg = "mem_usage_avg"
// default interval for sync data from metrics server, the value is 5s
defaultMetricsInternal = 5000000000
)
Expand Down Expand Up @@ -1266,19 +1258,11 @@ func (sc *SchedulerCache) SetMetricsConf(conf map[string]string) {
}

func (sc *SchedulerCache) GetMetricsData() {
address := sc.metricsConf["address"]
if len(address) == 0 {
return
}
klog.V(4).Infof("Get metrics from Prometheus: %s", address)
client, err := api.NewClient(api.Config{
Address: address,
})
client, err := newMetricsClient(sc.metricsConf)
if err != nil {
klog.Errorf("Error creating client: %v\n", err)
return
}
v1api := prometheusv1.NewAPI(client)
ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
defer cancel()
nodeUsageMap := make(map[string]*schedulingapi.NodeUsage)
Expand All @@ -1292,42 +1276,16 @@ func (sc *SchedulerCache) GetMetricsData() {
sc.Mutex.Unlock()

supportedPeriods := []string{"5m"}
supportedMetrics := []string{cpuUsageAvg, memUsageAvg}
for node := range nodeUsageMap {
for _, period := range supportedPeriods {
for _, metric := range supportedMetrics {
queryStr := fmt.Sprintf("%s_%s{instance=\"%s\"}", metric, period, node)
klog.V(4).Infof("Query prometheus by %s", queryStr)
res, warnings, err := v1api.Query(ctx, queryStr, time.Now())
if err != nil {
klog.Errorf("Error querying Prometheus: %v", err)
}
if len(warnings) > 0 {
klog.V(3).Infof("Warning querying Prometheus: %v", warnings)
}
if res == nil || res.String() == "" {
klog.Warningf("Warning querying Prometheus: no data found for %s", queryStr)
continue
}
// plugin.usage only need type pmodel.ValVector in Prometheus.rulues
if res.Type() != pmodel.ValVector {
continue
}
// only method res.String() can get data, dataType []pmodel.ValVector, eg: "{k1:v1, ...} => #[value] @#[timespace]\n {k2:v2, ...} => ..."
firstRowValVector := strings.Split(res.String(), "\n")[0]
rowValues := strings.Split(strings.TrimSpace(firstRowValVector), "=>")
value := strings.Split(strings.TrimSpace(rowValues[1]), " ")
switch metric {
case cpuUsageAvg:
cpuUsage, _ := strconv.ParseFloat(value[0], 64)
nodeUsageMap[node].CPUUsageAvg[period] = cpuUsage
klog.V(4).Infof("node: %v, CpuUsageAvg: %v, period:%v", node, cpuUsage, period)
case memUsageAvg:
memUsage, _ := strconv.ParseFloat(value[0], 64)
nodeUsageMap[node].MEMUsageAvg[period] = memUsage
klog.V(4).Infof("node: %v, MemUsageAvg: %v, period:%v", node, memUsage, period)
}
nodeMetrics, err := client.NodeMetricsAvg(ctx, node, period)
if err != nil {
klog.Errorf("Error getting node metrics: %v\n", err)
continue
}
klog.V(4).Infof("node: %v, CpuUsageAvg: %v, MemUsageAvg: %v, period:%v", node, nodeMetrics.Cpu, nodeMetrics.Memory, period)
nodeUsageMap[node].CPUUsageAvg[period] = nodeMetrics.Cpu
nodeUsageMap[node].MEMUsageAvg[period] = nodeMetrics.Memory
}
}
sc.setMetricsData(nodeUsageMap)
Expand Down
34 changes: 34 additions & 0 deletions pkg/scheduler/cache/metrics_client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
Copyright 2023 The Volcano Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
"context"
"errors"
)

type MetricsClient interface {
NodeMetricsAvg(ctx context.Context, nodeName string, period string) (*NodeMetrics, error)
}

func newMetricsClient(metricsConf map[string]string) (MetricsClient, error) {
address := metricsConf["address"]
if len(address) == 0 {
return nil, errors.New("metrics address is empty")
}
return &PrometheusMetricsClient{address: address}, nil
}
81 changes: 81 additions & 0 deletions pkg/scheduler/cache/metrics_client_prometheus.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/*
Copyright 2023 The Volcano Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
"context"
"fmt"
"github.com/prometheus/client_golang/api"
prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
"k8s.io/klog"
"strconv"
"strings"
"time"
)

const (
// promCpuUsageAvg record name of cpu average usage defined in prometheus rules
promCpuUsageAvg = "cpu_usage_avg"
// promMemUsageAvg record name of mem average usage defined in prometheus rules
promMemUsageAvg = "mem_usage_avg"
)

type PrometheusMetricsClient struct {
address string
}

func NewPrometheusMetricsClient(address string) (*PrometheusMetricsClient, error) {
return &PrometheusMetricsClient{address: address}, nil
}

func (p *PrometheusMetricsClient) NodeMetricsAvg(ctx context.Context, nodeName string, period string) (*NodeMetrics, error) {
klog.V(4).Infof("Get node metrics from Prometheus: %s", p.address)
client, err := api.NewClient(api.Config{
Address: p.address,
})
if err != nil {
return nil, err
}
v1api := prometheusv1.NewAPI(client)
nodeMetrics := &NodeMetrics{}
supportedMetrics := []string{"cpu_usage_avg", "mem_usage_avg"}
for _, metric := range supportedMetrics {
queryStr := fmt.Sprintf("%s_%s{instance=\"%s\"}", metric, period, nodeName)
res, warnings, err := v1api.Query(ctx, queryStr, time.Now())
if err != nil {
return nil, err
}
if res == nil || res.String() == "" {
continue
}
if len(warnings) > 0 {
return nil, err
}
firstRowValVector := strings.Split(res.String(), "\n")[0]
rowValues := strings.Split(strings.TrimSpace(firstRowValVector), "=>")
value := strings.Split(strings.TrimSpace(rowValues[1]), " ")
switch metric {
case promCpuUsageAvg:
cpuUsage, _ := strconv.ParseFloat(value[0], 64)
nodeMetrics.Cpu = cpuUsage
case promMemUsageAvg:
memUsage, _ := strconv.ParseFloat(value[0], 64)
nodeMetrics.Memory = memUsage
}
}
return nodeMetrics, nil
}
22 changes: 22 additions & 0 deletions pkg/scheduler/cache/module.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
Copyright 2023 The Volcano Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

type NodeMetrics struct {
Cpu float64
Memory float64
}

0 comments on commit b9e7632

Please sign in to comment.