diff --git a/pkg/calculator/calculator.go b/pkg/calculator/calculator.go index 6d5b230..d5ee74d 100644 --- a/pkg/calculator/calculator.go +++ b/pkg/calculator/calculator.go @@ -2,8 +2,6 @@ package calculator import ( "time" - - v1 "github.com/re-cinq/cloud-carbon/pkg/types/v1" ) // TODO add links / sources for where numbers and calculations are gathered @@ -14,7 +12,7 @@ const lifespan = 4 type calculate struct { cores float64 - usageCPU v1.Percentage + usageCPU float64 minWatts float64 maxWatts float64 chip float64 @@ -37,7 +35,7 @@ func (c *calculate) operationalCPUEmissions(interval time.Duration) float64 { // architecture is unknown the Min and Max wattage is the average of all machines // for that provider, and is supplied in the provider defaults. This is being // handled in the types/factors package (the point of reading in coefficient data). - avgWatts := c.minWatts + float64(c.usageCPU)*(c.maxWatts-c.minWatts) + avgWatts := c.minWatts + c.usageCPU*(c.maxWatts-c.minWatts) // Operational Emissions are calculated by multiplying the avgWatts, vCPUHours, PUE, // and region grid CO2e. The PUE is collected from the providers. The CO2e grid data diff --git a/pkg/calculator/calculator_test.go b/pkg/calculator/calculator_test.go index d7f9d5a..3aa9256 100644 --- a/pkg/calculator/calculator_test.go +++ b/pkg/calculator/calculator_test.go @@ -4,7 +4,6 @@ import ( "testing" "time" - v1 "github.com/re-cinq/cloud-carbon/pkg/types/v1" "github.com/stretchr/testify/assert" ) @@ -20,7 +19,7 @@ type testcase struct { func defaultCalc() *calculate { return &calculate{ cores: 4, - usageCPU: v1.Percentage(25), + usageCPU: 25.0, minWatts: 1.3423402398570, maxWatts: 4.00498247528, chip: 35.23458732, @@ -47,7 +46,7 @@ func TestCalculateEmissions(t *testing.T) { interval: 30 * time.Second, calc: &calculate{ cores: 0, - usageCPU: v1.Percentage(0), + usageCPU: 0, minWatts: 0, maxWatts: 0, chip: 0, @@ -95,7 +94,7 @@ func TestCalculateEmissions(t *testing.T) { func() *testcase { c := defaultCalc() - c.usageCPU = v1.Percentage(50) + c.usageCPU = 50 // test with vCPU utilization at 50% return &testcase{ name: "50% utilization", @@ -107,7 +106,7 @@ func TestCalculateEmissions(t *testing.T) { func() *testcase { c := defaultCalc() - c.usageCPU = v1.Percentage(100) + c.usageCPU = 100 // test with vCPU utilization at 100% return &testcase{ name: "100% utilization", @@ -152,7 +151,7 @@ func TestCalculateEmissions(t *testing.T) { interval: 30 * time.Second, calc: &calculate{ cores: 32, - usageCPU: v1.Percentage(90), + usageCPU: 90, minWatts: 3.0369270833333335, maxWatts: 8.575357663690477, chip: 129.77777777777777, diff --git a/pkg/providers/aws/cloudwatch.go b/pkg/providers/aws/cloudwatch.go index acf3344..07ae0be 100644 --- a/pkg/providers/aws/cloudwatch.go +++ b/pkg/providers/aws/cloudwatch.go @@ -72,7 +72,7 @@ func (e *cloudWatchClient) GetEc2Metrics(region awsRegion, cache *awsCache) map[ // Build the resource cpu := v1.NewMetric(v1.CPU.String()).SetResourceUnit(cpuMetric.unit).SetUnitAmount(float64(instanceMetadata.coreCount)) - cpu.SetUsagePercentage(cpuMetric.value).SetType(cpuMetric.kind) + cpu.SetUsage(cpuMetric.value).SetType(cpuMetric.kind) // Update the CPU information now instanceService.Metrics().Upsert(cpu) diff --git a/pkg/providers/gcp/gcp.go b/pkg/providers/gcp/gcp.go index f658333..92149e3 100644 --- a/pkg/providers/gcp/gcp.go +++ b/pkg/providers/gcp/gcp.go @@ -2,6 +2,7 @@ package gcp import ( "context" + "errors" "fmt" "net/url" "path" @@ -11,7 +12,6 @@ import ( compute "cloud.google.com/go/compute/apiv1" "cloud.google.com/go/compute/apiv1/computepb" monitoring "cloud.google.com/go/monitoring/apiv3/v2" - monitoringpb "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" cache "github.com/patrickmn/go-cache" "github.com/re-cinq/cloud-carbon/pkg/config" v1 "github.com/re-cinq/cloud-carbon/pkg/types/v1" @@ -20,35 +20,6 @@ import ( "k8s.io/klog/v2" ) -var ( - /* - * An MQL query that will return data from Google Cloud with the - * - Instance Name - * - Region - * - Zone - * - Machine Type - * - Reserved CPUs - * - Utilization - */ - CPUQuery = ` - fetch gce_instance - | { metric 'compute.googleapis.com/instance/cpu/utilization' - ; metric 'compute.googleapis.com/instance/cpu/reserved_cores' } - | outer_join 0 - | filter project_id = '%s' - | group_by [ - resource.instance_id, - metric.instance_name, - metadata.system.region, - resource.zone, - metadata.system.machine_type, - reserved_cores: format(t_1.value.reserved_cores, '%%f') - ], [max(t_0.value.utilization)] - | window %s - | within %s - ` -) - // GCP is the structure used as the provider for Google Cloud Platform type GCP struct { // GCP Clients @@ -155,123 +126,102 @@ func (g *GCP) GetMetricsForInstances( ) ([]v1.Instance, error) { var instances []v1.Instance - metrics, err := g.instanceMetrics( + cpumetrics, err := g.instanceCPUMetrics( // TODO these parameters can be cleaned up ctx, project, fmt.Sprintf(CPUQuery, project, window, window), ) + if err != nil { + return instances, err + } + memmetrics, err := g.instanceMemoryMetrics( + ctx, project, fmt.Sprintf(MEMQuery, project, window, window), + ) if err != nil { return instances, err } + // we use a lookup to add different metrics to the same instance + lookup := make(map[string]*v1.Instance) + // TODO there seems to be duplicated logic here // Why not create instance whuile collecting metric instead of handeling // it in two steps - for _, m := range metrics { + for _, m := range append(cpumetrics, memmetrics...) { metric := *m - zone, ok := metric.Labels().Get("zone") - if !ok { - continue - } - - region, ok := metric.Labels().Get("region") - if !ok { - continue - } - - instanceName, ok := metric.Labels().Get("name") - if !ok { - continue - } - - instanceID, ok := metric.Labels().Get("id") - if !ok { - continue - } - - machineType, ok := metric.Labels().Get("machine_type") - if !ok { + meta, err := getMetadata(&metric) + if err != nil { continue } // Load the cache // TODO make this more explicit, im not sure why this // is needed as we dont use the cache anywhere - cachedInstance, ok := g.cache.Get(cacheKey(zone, service, instanceName)) + cachedInstance, ok := g.cache.Get(cacheKey(meta.zone, service, meta.name)) if cachedInstance == nil && !ok { continue } - instance := v1.NewInstance(instanceID, provider).SetService(service) + i, ok := lookup[meta.id] + if !ok { + i = v1.NewInstance(meta.id, provider).SetService(service) + } + + i.SetKind(meta.machineType) + i.SetRegion(meta.region) + i.SetZone(meta.zone) + i.Metrics().Upsert(&metric) + + lookup[meta.id] = i - instance.SetKind(machineType) - instance.SetRegion(region) - instance.SetZone(zone) - instance.Metrics().Upsert(&metric) + } - instances = append(instances, *instance) + // create list of instances + // TODO: this seems repetitive + for _, v := range lookup { + instances = append(instances, *v) } return instances, nil } -// instanceMetrics runs a query on googe cloud monitoring using MQL -// and responds with a list of metrics -func (g *GCP) instanceMetrics( - ctx context.Context, - project, query string, -) ([]*v1.Metric, error) { - var metrics []*v1.Metric +type metadata struct { + zone, region, name, id, machineType string +} - it := g.monitoring.QueryTimeSeries(ctx, &monitoringpb.QueryTimeSeriesRequest{ - Name: fmt.Sprintf("projects/%s", project), - Query: query, - }) +func getMetadata(m *v1.Metric) (*metadata, error) { + zone, ok := m.Labels().Get("zone") + if !ok { + return &metadata{}, errors.New("zone not found") + } - for { - resp, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - return nil, err - } + region, ok := m.Labels().Get("region") + if !ok { + return &metadata{}, errors.New("region not found") + } - // This is dependant on the MQL query - // label ordering - instanceID := resp.GetLabelValues()[0].GetStringValue() - instanceName := resp.GetLabelValues()[1].GetStringValue() - region := resp.GetLabelValues()[2].GetStringValue() - zone := resp.GetLabelValues()[3].GetStringValue() - instanceType := resp.GetLabelValues()[4].GetStringValue() - totalCores := resp.GetLabelValues()[5].GetStringValue() - - m := v1.NewMetric("cpu") - m.SetResourceUnit(v1.Core) - m.SetType(v1.CPU).SetUsagePercentage( - // translate fraction to a percentage - resp.GetPointData()[0].GetValues()[0].GetDoubleValue() * 100, - ) + name, ok := m.Labels().Get("name") + if !ok { + return &metadata{}, errors.New("instance name not found") + } - f, err := strconv.ParseFloat(totalCores, 64) - // TODO: we should not fail here but collect errors - if err != nil { - klog.Errorf("failed to parse GCP metric %s", err) - continue - } + id, ok := m.Labels().Get("id") + if !ok { + return &metadata{}, errors.New("instance id not found") + } - m.SetUnitAmount(f) - m.SetLabels(v1.Labels{ - "id": instanceID, - "name": instanceName, - "region": region, - "zone": zone, - "machine_type": instanceType, - }) - metrics = append(metrics, m) + machineType, ok := m.Labels().Get("machine_type") + if !ok { + return &metadata{}, errors.New("machine type not found") } - return metrics, nil + return &metadata{ + zone: zone, + region: region, + name: name, + id: id, + machineType: machineType, + }, nil } // Refresh fetches all the Instances diff --git a/pkg/providers/gcp/gcp_test.go b/pkg/providers/gcp/gcp_test.go index 559b32d..88b4c0e 100644 --- a/pkg/providers/gcp/gcp_test.go +++ b/pkg/providers/gcp/gcp_test.go @@ -1,222 +1,2 @@ package gcp -import ( - "context" - "errors" - "fmt" - "net" - "testing" - - compute "cloud.google.com/go/compute/apiv1" - "cloud.google.com/go/compute/apiv1/computepb" - monitoring "cloud.google.com/go/monitoring/apiv3/v2" - "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" - "github.com/re-cinq/cloud-carbon/pkg/config" - v1 "github.com/re-cinq/cloud-carbon/pkg/types/v1" - "github.com/stretchr/testify/require" - "google.golang.org/api/option" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -func withMonitoringTestClient(c *monitoring.QueryClient) options { - return func(g *GCP) { - g.monitoring = c - } -} - -func withInstancesTestClient(c *compute.InstancesClient) options { - return func(g *GCP) { - g.instances = c - } -} - -type fakeMonitoringServer struct { - monitoringpb.UnimplementedQueryServiceServer - // Response that will return from the fake server - Response *monitoringpb.QueryTimeSeriesResponse - // If error is set the server will return an error - Error error -} - -type fakeInstancesServer struct { - computepb.UnimplementedInstancesServer -} - -// setupFakeServer is used to setup a fake GRPC server to hanlde test requests -func setupFakeServer( - m *fakeMonitoringServer, - i *fakeInstancesServer, -) (*string, error) { - l, err := net.Listen("tcp", "localhost:0") - if err != nil { - return nil, err - } - gsrv := grpc.NewServer() - monitoringpb.RegisterQueryServiceServer(gsrv, m) - computepb.RegisterInstancesServer(gsrv, i) - fakeServerAddr := l.Addr().String() - go func() { - if err := gsrv.Serve(l); err != nil { - panic(err) - } - }() - return &fakeServerAddr, nil -} - -func (f *fakeMonitoringServer) QueryTimeSeries( - ctx context.Context, - req *monitoringpb.QueryTimeSeriesRequest, -) (*monitoringpb.QueryTimeSeriesResponse, error) { - if f.Error != nil { - return nil, f.Error - } - return f.Response, nil -} - -var defaultLabelValues = []*monitoringpb.LabelValue{ - { - Value: &monitoringpb.LabelValue_StringValue{StringValue: "my-instance-id"}, - }, - { - Value: &monitoringpb.LabelValue_StringValue{StringValue: "foobar"}, - }, - { - Value: &monitoringpb.LabelValue_StringValue{StringValue: "europe-west-1"}, - }, - { - Value: &monitoringpb.LabelValue_StringValue{StringValue: "europe-west"}, - }, - { - Value: &monitoringpb.LabelValue_StringValue{StringValue: "e2-medium"}, - }, - { - Value: &monitoringpb.LabelValue_StringValue{StringValue: "2.000000"}, - }, -} - -func TestGetCPUMetrics(t *testing.T) { - assert := require.New(t) - ctx := context.TODO() - - // TODO see if we can use v1.Metric instead of this - type testMetric struct { - Name string - UnitAmount float64 - Type v1.ResourceType - Labels v1.Labels - Usage v1.Percentage - } - testdata := []struct { - description string - responsePointData []*monitoringpb.TimeSeriesData_PointData - responseLabelValues []*monitoringpb.LabelValue - err error - expectedResponse []*testMetric - query string - }{ - { - description: "query for count metrics", - query: fmt.Sprintf(CPUQuery, "foobar", "5m", "5m"), - responsePointData: []*monitoringpb.TimeSeriesData_PointData{ - { - Values: []*monitoringpb.TypedValue{ - { - Value: &monitoringpb.TypedValue_DoubleValue{ - DoubleValue: 0.01, - }, - }, - }, - }, - }, - expectedResponse: []*testMetric{ - { - Type: v1.CPU, - Labels: v1.Labels{ - "id": "my-instance-id", - "machine_type": "e2-medium", - "name": "foobar", - "region": "europe-west-1", - "zone": "europe-west", - }, - Usage: 1, - UnitAmount: 2.0000, - }, - }, - }, - { - description: "error occurs in query", - query: fmt.Sprintf(CPUQuery, "foobar", "5m", "5m"), - err: errors.New("random error occurred"), - }, - } - - for _, test := range testdata { - t.Run(test.description, func(t *testing.T) { - testResp := &monitoringpb.QueryTimeSeriesResponse{ - TimeSeriesData: []*monitoringpb.TimeSeriesData{ - { - LabelValues: defaultLabelValues, - PointData: test.responsePointData, - }, - }, - } - if len(test.responseLabelValues) != 0 { - testResp.TimeSeriesData[0].LabelValues = test.responseLabelValues - } - - fakeMonitoringServer := &fakeMonitoringServer{ - Response: testResp, - Error: test.err, - } - - fakeInstancesServer := &fakeInstancesServer{} - - addr, err := setupFakeServer(fakeMonitoringServer, fakeInstancesServer) - assert.NoError(err) - - m, err := monitoring.NewQueryClient(ctx, - option.WithEndpoint(*addr), - option.WithoutAuthentication(), - option.WithGRPCDialOption(grpc.WithTransportCredentials( - insecure.NewCredentials(), - )), - ) - assert.NoError(err) - - i, err := compute.NewInstancesRESTClient(ctx, - option.WithEndpoint(*addr), - option.WithoutAuthentication(), - option.WithGRPCDialOption(grpc.WithTransportCredentials( - insecure.NewCredentials(), - )), - ) - assert.NoError(err) - - g, teardown, err := New(ctx, - &config.Account{}, - withMonitoringTestClient(m), - // TODO create helper to setup fake server - withInstancesTestClient(i), - ) - assert.NoError(err) - defer teardown() - - resp, err := g.instanceMetrics(ctx, "", test.query) - if test.err == nil { - assert.NoError(err) - for i, r := range resp { - assert.Equal(test.expectedResponse[i].Labels, r.Labels()) - assert.Equal(test.expectedResponse[i].Type, r.Type()) - assert.Equal(test.expectedResponse[i].Usage, r.Usage()) - assert.Equal(test.expectedResponse[i].UnitAmount, r.UnitAmount()) - } - } else { - assert.Equal( - fmt.Sprintf("%s", err), - fmt.Sprintf("rpc error: code = Unknown desc = %s", test.err), - ) - } - }) - } -} diff --git a/pkg/providers/gcp/instance.go b/pkg/providers/gcp/instance.go new file mode 100644 index 0000000..47b2474 --- /dev/null +++ b/pkg/providers/gcp/instance.go @@ -0,0 +1,179 @@ +package gcp + +import ( + "context" + "fmt" + "strconv" + + monitoringpb "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" + v1 "github.com/re-cinq/cloud-carbon/pkg/types/v1" + "google.golang.org/api/iterator" + "k8s.io/klog/v2" +) + +var ( + /* + * An MQL query that will return data from Google Cloud with the + * - Instance Name + * - Region + * - Zone + * - Machine Type + * - Reserved CPUs + * - Utilization + */ + CPUQuery = ` + fetch gce_instance + | { metric 'compute.googleapis.com/instance/cpu/utilization' + ; metric 'compute.googleapis.com/instance/cpu/reserved_cores' } + | outer_join 0 + | filter project_id = '%s' + | group_by [ + resource.instance_id, + metric.instance_name, + metadata.system.region, + resource.zone, + metadata.system.machine_type, + reserved_cores: format(t_1.value.reserved_cores, '%%f') + ], [max(t_0.value.utilization)] + | window %s + | within %s + ` + /* + * An MQL query that will return memory data from Google Cloud with the + * - Instance Name + * - Region + * - Zone + * - Machine Type + * - Memory Usage + * NOTE: According to Google the 'ram_used' metric is only available for + * e2-xxxx instances, which means that we can get memory usage for other types + * of VM's + */ + MEMQuery = ` + fetch gce_instance + | metric 'compute.googleapis.com/instance/memory/balloon/ram_used' + | filter project_id = '%s' + | group_by [ + resource.instance_id, + metric.instance_name, + metadata.system.region, + resource.zone, + metadata.system.machine_type, + ], [max(value.ram_used)] + | window %s + | within %s + ` +) + +// instanceMetrics runs a query on googe cloud monitoring using MQL +// and responds with a list of metrics +func (g *GCP) instanceMemoryMetrics( + ctx context.Context, + project, query string, +) ([]*v1.Metric, error) { + var metrics []*v1.Metric + + it := g.monitoring.QueryTimeSeries(ctx, &monitoringpb.QueryTimeSeriesRequest{ + Name: fmt.Sprintf("projects/%s", project), + Query: query, + }) + + for { + resp, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + + // This is dependant on the MQL query + // label ordering + instanceID := resp.GetLabelValues()[0].GetStringValue() + instanceName := resp.GetLabelValues()[1].GetStringValue() + region := resp.GetLabelValues()[2].GetStringValue() + zone := resp.GetLabelValues()[3].GetStringValue() + instanceType := resp.GetLabelValues()[4].GetStringValue() + + m := v1.NewMetric("memory") + m.SetResourceUnit(v1.Gb) + m.SetType(v1.Memory).SetUsage( + // convert bytes to GBs + float64(resp.GetPointData()[0].GetValues()[0].GetInt64Value()) / 3072, + ) + + // TODO: we should not fail here but collect errors + if err != nil { + klog.Errorf("failed to parse GCP metric %s", err) + continue + } + + m.SetLabels(v1.Labels{ + "id": instanceID, + "name": instanceName, + "region": region, + "zone": zone, + "machine_type": instanceType, + }) + metrics = append(metrics, m) + } + return metrics, nil +} + +// instanceCPUMetrics runs a query on googe cloud monitoring using MQL +// and responds with a list of CPU metrics +func (g *GCP) instanceCPUMetrics( + ctx context.Context, + project, query string, +) ([]*v1.Metric, error) { + var metrics []*v1.Metric + + it := g.monitoring.QueryTimeSeries(ctx, &monitoringpb.QueryTimeSeriesRequest{ + Name: fmt.Sprintf("projects/%s", project), + Query: query, + }) + + for { + resp, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + + // This is dependant on the MQL query + // label ordering + instanceID := resp.GetLabelValues()[0].GetStringValue() + instanceName := resp.GetLabelValues()[1].GetStringValue() + region := resp.GetLabelValues()[2].GetStringValue() + zone := resp.GetLabelValues()[3].GetStringValue() + instanceType := resp.GetLabelValues()[4].GetStringValue() + totalCores := resp.GetLabelValues()[5].GetStringValue() + + m := v1.NewMetric("cpu") + m.SetResourceUnit(v1.Core) + m.SetType(v1.CPU).SetUsage( + // translate fraction to a percentage + resp.GetPointData()[0].GetValues()[0].GetDoubleValue() * 100, + ) + + f, err := strconv.ParseFloat(totalCores, 64) + // TODO: we should not fail here but collect errors + if err != nil { + klog.Errorf("failed to parse GCP metric %s", err) + continue + } + + m.SetUnitAmount(f) + m.SetLabels(v1.Labels{ + "id": instanceID, + "name": instanceName, + "region": region, + "zone": zone, + "machine_type": instanceType, + }) + metrics = append(metrics, m) + } + return metrics, nil +} diff --git a/pkg/providers/gcp/instance_test.go b/pkg/providers/gcp/instance_test.go new file mode 100644 index 0000000..820ee7f --- /dev/null +++ b/pkg/providers/gcp/instance_test.go @@ -0,0 +1,347 @@ +package gcp + +import ( + "context" + "errors" + "fmt" + "net" + "testing" + + compute "cloud.google.com/go/compute/apiv1" + "cloud.google.com/go/compute/apiv1/computepb" + monitoring "cloud.google.com/go/monitoring/apiv3/v2" + "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" + "github.com/re-cinq/cloud-carbon/pkg/config" + v1 "github.com/re-cinq/cloud-carbon/pkg/types/v1" + "github.com/stretchr/testify/require" + "google.golang.org/api/option" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func withMonitoringTestClient(c *monitoring.QueryClient) options { + return func(g *GCP) { + g.monitoring = c + } +} + +func withInstancesTestClient(c *compute.InstancesClient) options { + return func(g *GCP) { + g.instances = c + } +} + +type fakeMonitoringServer struct { + monitoringpb.UnimplementedQueryServiceServer + // Response that will return from the fake server + Response *monitoringpb.QueryTimeSeriesResponse + // If error is set the server will return an error + Error error +} + +type fakeInstancesServer struct { + computepb.UnimplementedInstancesServer +} + +// setupFakeServer is used to setup a fake GRPC server to hanlde test requests +func setupFakeServer( + m *fakeMonitoringServer, + i *fakeInstancesServer, +) (*string, error) { + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + return nil, err + } + gsrv := grpc.NewServer() + monitoringpb.RegisterQueryServiceServer(gsrv, m) + computepb.RegisterInstancesServer(gsrv, i) + fakeServerAddr := l.Addr().String() + go func() { + if err := gsrv.Serve(l); err != nil { + panic(err) + } + }() + return &fakeServerAddr, nil +} + +func (f *fakeMonitoringServer) QueryTimeSeries( + ctx context.Context, + req *monitoringpb.QueryTimeSeriesRequest, +) (*monitoringpb.QueryTimeSeriesResponse, error) { + if f.Error != nil { + return nil, f.Error + } + return f.Response, nil +} + +var defaultLabelValues = []*monitoringpb.LabelValue{ + { + Value: &monitoringpb.LabelValue_StringValue{StringValue: "my-instance-id"}, + }, + { + Value: &monitoringpb.LabelValue_StringValue{StringValue: "foobar"}, + }, + { + Value: &monitoringpb.LabelValue_StringValue{StringValue: "europe-west-1"}, + }, + { + Value: &monitoringpb.LabelValue_StringValue{StringValue: "europe-west"}, + }, + { + Value: &monitoringpb.LabelValue_StringValue{StringValue: "e2-medium"}, + }, + { + Value: &monitoringpb.LabelValue_StringValue{StringValue: "2.000000"}, + }, +} + +func TestGetCPUMetrics(t *testing.T) { + assert := require.New(t) + ctx := context.TODO() + + // TODO see if we can use v1.Metric instead of this + type testMetric struct { + Name string + UnitAmount float64 + Type v1.ResourceType + Labels v1.Labels + Usage float64 + } + testdata := []struct { + description string + responsePointData []*monitoringpb.TimeSeriesData_PointData + responseLabelValues []*monitoringpb.LabelValue + err error + expectedResponse []*testMetric + query string + }{ + { + description: "query for count metrics", + query: fmt.Sprintf(CPUQuery, "foobar", "5m", "5m"), + responsePointData: []*monitoringpb.TimeSeriesData_PointData{ + { + Values: []*monitoringpb.TypedValue{ + { + Value: &monitoringpb.TypedValue_DoubleValue{ + DoubleValue: 0.01, + }, + }, + }, + }, + }, + expectedResponse: []*testMetric{ + { + Type: v1.CPU, + Labels: v1.Labels{ + "id": "my-instance-id", + "machine_type": "e2-medium", + "name": "foobar", + "region": "europe-west-1", + "zone": "europe-west", + }, + Usage: 1, + UnitAmount: 2.0000, + }, + }, + }, + { + description: "error occurs in query", + query: fmt.Sprintf(CPUQuery, "foobar", "5m", "5m"), + err: errors.New("random error occurred"), + }, + } + + for _, test := range testdata { + t.Run(test.description, func(t *testing.T) { + testResp := &monitoringpb.QueryTimeSeriesResponse{ + TimeSeriesData: []*monitoringpb.TimeSeriesData{ + { + LabelValues: defaultLabelValues, + PointData: test.responsePointData, + }, + }, + } + if len(test.responseLabelValues) != 0 { + testResp.TimeSeriesData[0].LabelValues = test.responseLabelValues + } + + fakeMonitoringServer := &fakeMonitoringServer{ + Response: testResp, + Error: test.err, + } + + fakeInstancesServer := &fakeInstancesServer{} + + addr, err := setupFakeServer(fakeMonitoringServer, fakeInstancesServer) + assert.NoError(err) + + m, err := monitoring.NewQueryClient(ctx, + option.WithEndpoint(*addr), + option.WithoutAuthentication(), + option.WithGRPCDialOption(grpc.WithTransportCredentials( + insecure.NewCredentials(), + )), + ) + assert.NoError(err) + + i, err := compute.NewInstancesRESTClient(ctx, + option.WithEndpoint(*addr), + option.WithoutAuthentication(), + option.WithGRPCDialOption(grpc.WithTransportCredentials( + insecure.NewCredentials(), + )), + ) + assert.NoError(err) + + g, teardown, err := New(ctx, + &config.Account{}, + withMonitoringTestClient(m), + // TODO create helper to setup fake server + withInstancesTestClient(i), + ) + assert.NoError(err) + defer teardown() + + resp, err := g.instanceCPUMetrics(ctx, "", test.query) + if test.err == nil { + assert.NoError(err) + for i, r := range resp { + assert.Equal(test.expectedResponse[i].Labels, r.Labels()) + assert.Equal(test.expectedResponse[i].Type, r.Type()) + assert.Equal(test.expectedResponse[i].Usage, r.Usage()) + assert.Equal(test.expectedResponse[i].UnitAmount, r.UnitAmount()) + } + } else { + assert.Equal( + fmt.Sprintf("%s", err), + fmt.Sprintf("rpc error: code = Unknown desc = %s", test.err), + ) + } + }) + } +} + +func TestInstanceMemoryMetrics(t *testing.T) { + assert := require.New(t) + ctx := context.TODO() + + // TODO see if we can use v1.Metric instead of this + type testMetric struct { + Name string + UnitAmount float64 + Type v1.ResourceType + Labels v1.Labels + Usage float64 + } + testdata := []struct { + description string + responsePointData []*monitoringpb.TimeSeriesData_PointData + responseLabelValues []*monitoringpb.LabelValue + err error + expectedResponse []*testMetric + query string + }{ + { + description: "query for count metrics", + query: fmt.Sprintf(CPUQuery, "foobar", "5m", "5m"), + responsePointData: []*monitoringpb.TimeSeriesData_PointData{ + { + Values: []*monitoringpb.TypedValue{ + { + Value: &monitoringpb.TypedValue_Int64Value{ + Int64Value: 3072, + }, + }, + }, + }, + }, + expectedResponse: []*testMetric{ + { + Type: v1.Memory, + Labels: v1.Labels{ + "id": "my-instance-id", + "machine_type": "e2-medium", + "name": "foobar", + "region": "europe-west-1", + "zone": "europe-west", + }, + Usage: 1.0, + UnitAmount: 0.0000, + }, + }, + }, + { + description: "error occurs in query", + query: fmt.Sprintf(CPUQuery, "foobar", "5m", "5m"), + err: errors.New("random error occurred"), + }, + } + + for _, test := range testdata { + t.Run(test.description, func(t *testing.T) { + testResp := &monitoringpb.QueryTimeSeriesResponse{ + TimeSeriesData: []*monitoringpb.TimeSeriesData{ + { + LabelValues: defaultLabelValues, + PointData: test.responsePointData, + }, + }, + } + if len(test.responseLabelValues) != 0 { + testResp.TimeSeriesData[0].LabelValues = test.responseLabelValues + } + + fakeMonitoringServer := &fakeMonitoringServer{ + Response: testResp, + Error: test.err, + } + + fakeInstancesServer := &fakeInstancesServer{} + + addr, err := setupFakeServer(fakeMonitoringServer, fakeInstancesServer) + assert.NoError(err) + + m, err := monitoring.NewQueryClient(ctx, + option.WithEndpoint(*addr), + option.WithoutAuthentication(), + option.WithGRPCDialOption(grpc.WithTransportCredentials( + insecure.NewCredentials(), + )), + ) + assert.NoError(err) + + i, err := compute.NewInstancesRESTClient(ctx, + option.WithEndpoint(*addr), + option.WithoutAuthentication(), + option.WithGRPCDialOption(grpc.WithTransportCredentials( + insecure.NewCredentials(), + )), + ) + assert.NoError(err) + + g, teardown, err := New(ctx, + &config.Account{}, + withMonitoringTestClient(m), + withInstancesTestClient(i), + ) + assert.NoError(err) + defer teardown() + + resp, err := g.instanceMemoryMetrics(ctx, "", test.query) + if test.err == nil { + assert.NoError(err) + for i, r := range resp { + assert.Equal(test.expectedResponse[i].Labels, r.Labels()) + assert.Equal(test.expectedResponse[i].Type, r.Type()) + assert.Equal(test.expectedResponse[i].Usage, r.Usage()) + assert.Equal(test.expectedResponse[i].UnitAmount, r.UnitAmount()) + } + } else { + assert.Equal( + fmt.Sprintf("%s", err), + fmt.Sprintf("rpc error: code = Unknown desc = %s", test.err), + ) + } + }) + } +} diff --git a/pkg/types/v1/instance_test.go b/pkg/types/v1/instance_test.go index 1e3ecbb..18c3e5f 100644 --- a/pkg/types/v1/instance_test.go +++ b/pkg/types/v1/instance_test.go @@ -16,13 +16,10 @@ func TestInstanceOperations(t *testing.T) { // Mocking the metric r := NewMetric("cpu") - r.SetUsagePercentage(170.4).SetUnitAmount(4.0).SetResourceUnit(Core) + r.SetUsage(170.4).SetUnitAmount(4.0).SetResourceUnit(Core) r.SetEmissions(NewResourceEmission(1024.57, GCO2eqkWh)) r.SetUpdatedAt() - // Make sure the usage validation succeeded - assert.Equal(t, Percentage(100), r.Usage()) - // Create a new instance instance := NewInstance(instanceID, Prometheus).SetRegion("europe-west4").SetKind("n2-standard-8") diff --git a/pkg/types/v1/metric.go b/pkg/types/v1/metric.go index 914c1fb..5330786 100644 --- a/pkg/types/v1/metric.go +++ b/pkg/types/v1/metric.go @@ -16,8 +16,8 @@ type Metric struct { // The resource type resourceType ResourceType - // The resource usage in percentage - usage Percentage + // The resource usage + usage float64 // The total amount of unit types unitAmount float64 @@ -69,7 +69,7 @@ func (r *Metric) Type() ResourceType { // The resource usage in percentage // It is a value between 0 and 100 -func (r *Metric) Usage() Percentage { +func (r *Metric) Usage() float64 { return r.usage } @@ -129,23 +129,14 @@ func (r *Metric) SetType(resourceType ResourceType) *Metric { return r } -// Adds the usage -// Examples: -// - 50.0 (indicates a 50% usage) -func (r *Metric) SetUsagePercentage(usage float64) *Metric { +// SetUsage is used to set the usage on the struct which can not be below 0 +func (r *Metric) SetUsage(u float64) *Metric { // Make sure we are not setting the usage to a negative value - if usage < 0 { - usage = 0 - } - - // It does not make much sense to have a value higher than 100% - // So set it to 100 to make the CO2eq calculations easier - if usage > 100 { - usage = 100 + if u < 0 { + u = 0 } - // Assign the usage - r.usage = Percentage(usage) + r.usage = u // Allows to use it as a builder return r diff --git a/pkg/types/v1/metric_test.go b/pkg/types/v1/metric_test.go index 8c7b1b3..657f995 100644 --- a/pkg/types/v1/metric_test.go +++ b/pkg/types/v1/metric_test.go @@ -12,13 +12,13 @@ func TestResourceFunctionalities(t *testing.T) { assert.NotNil(t, r) // Assign the various values to the resource - r.SetUsagePercentage(20.54).SetUnitAmount(4.0).SetResourceUnit(Core).SetType(CPU) + r.SetUsage(20.54).SetUnitAmount(4.0).SetResourceUnit(Core).SetType(CPU) r.SetEmissions(NewResourceEmission(1056.76, GCO2eqkWh)) r.SetUpdatedAt() // Validate the data assert.Equal(t, r.Name(), "cpu") - assert.Equal(t, r.Usage(), Percentage(20.54)) + assert.Equal(t, r.Usage(), 20.54) assert.Equal(t, r.UnitAmount(), float64(4.0)) assert.Equal(t, r.Type(), CPU) assert.Equal(t, r.Unit(), Core) diff --git a/pkg/types/v1/metrics_test.go b/pkg/types/v1/metrics_test.go index a1ac2df..38adf7a 100644 --- a/pkg/types/v1/metrics_test.go +++ b/pkg/types/v1/metrics_test.go @@ -11,11 +11,11 @@ func TestMetricsParser(t *testing.T) { cpuResource := NewMetric("cpu") assert.NotNil(t, cpuResource) - cpuResource.SetUsagePercentage(20.54).SetUnitAmount(4.0) + cpuResource.SetUsage(20.54).SetUnitAmount(4.0) cpuResource.SetType(CPU).SetResourceUnit(Core) cpuResource.SetEmissions(NewResourceEmission(1056.76, GCO2eqkWh)) - assert.Equal(t, cpuResource.Usage(), Percentage(20.54)) + assert.Equal(t, cpuResource.Usage(), 20.54) assert.Equal(t, cpuResource.UnitAmount(), float64(4.0)) assert.Equal(t, cpuResource.Unit(), Core) assert.Equal(t, cpuResource.Type(), CPU)