Skip to content

Commit

Permalink
[GCP] Instance Group Manager (#28)
Browse files Browse the repository at this point in the history
  • Loading branch information
@obierlaire
obierlaire authored Feb 18, 2023
1 parent bb01d9c commit f9f906d
Show file tree
Hide file tree
Showing 14 changed files with 358 additions and 89 deletions.
14 changes: 8 additions & 6 deletions internal/estimate/estimate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ import (
"github.com/spf13/viper"
)

func EstimateResources(resourceList []resources.Resource) EstimationReport {
func EstimateResources(resourceList map[string]resources.Resource) EstimationReport {

var estimationResources []EstimationResource
var unsupportedResources []resources.Resource
estimationTotal := EstimationTotal{
Power: decimal.Zero,
CarbonEmissions: decimal.Zero,
ResourcesCount: 0,
ResourcesCount: decimal.Zero,
}
for _, resource := range resourceList {
estimationResource, uerr := EstimateResource(resource)
Expand All @@ -33,9 +33,9 @@ func EstimateResources(resourceList []resources.Resource) EstimationReport {
unsupportedResources = append(unsupportedResources, resource)
}

estimationTotal.Power = estimationTotal.Power.Add(estimationResource.Power)
estimationTotal.CarbonEmissions = estimationTotal.CarbonEmissions.Add(estimationResource.CarbonEmissions)
estimationTotal.ResourcesCount += 1
estimationTotal.Power = estimationTotal.Power.Add(estimationResource.Power.Mul(estimationResource.Count))
estimationTotal.CarbonEmissions = estimationTotal.CarbonEmissions.Add(estimationResource.CarbonEmissions.Mul(estimationResource.Count))
estimationTotal.ResourcesCount = estimationTotal.ResourcesCount.Add(estimationResource.Count)
}

return EstimationReport{
Expand Down Expand Up @@ -103,7 +103,7 @@ func estimateGCP(resource resources.Resource) *EstimationResource {
carbonEmissionPerTime := avgWatt.Mul(regionEmissions.GridCarbonIntensity)

log.Debugf(
"estimating resource %v.%v (%v): %v %v%v * %v %vCO2/%v%v = %v %vCO2/%v%v",
"estimating resource %v.%v (%v): %v %v%v * %v %vCO2/%v%v = %v %vCO2/%v%v * %v",
computeResource.Identification.ResourceType,
computeResource.Identification.Name,
regionEmissions.Region,
Expand All @@ -118,13 +118,15 @@ func estimateGCP(resource resources.Resource) *EstimationResource {
viper.Get("unit.carbon").(string),
viper.Get("unit.power").(string),
viper.Get("unit.time").(string),
resource.GetIdentification().Count,
)

return &EstimationResource{
Resource: &computeResource,
Power: avgWatt.RoundFloor(10),
CarbonEmissions: carbonEmissionPerTime.RoundFloor(10),
AverageCPUUsage: decimal.NewFromFloat(viper.GetFloat64("provider.gcp.avg_cpu_use")).RoundFloor(10),
Count: decimal.NewFromInt(int64(computeResource.Identification.Count)),
}
}

Expand Down
58 changes: 51 additions & 7 deletions internal/estimate/estimate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ var resourceGCPComputeBasic = resources.ComputeResource{
ResourceType: "type-1",
Provider: providers.GCP,
Region: "europe-west9",
Count: 1,
},
Specs: &resources.ComputeResourceSpecs{
VCPUs: 2,
Expand All @@ -31,6 +32,7 @@ var resourceGCPComputeCPUType = resources.ComputeResource{
ResourceType: "type-1",
Provider: providers.GCP,
Region: "europe-west9",
Count: 1,
},
Specs: &resources.ComputeResourceSpecs{
VCPUs: 2,
Expand All @@ -47,6 +49,21 @@ var resourceAWSComputeBasic = resources.ComputeResource{
ResourceType: "type-1",
Provider: providers.AWS,
Region: "europe-west9",
Count: 1,
},
Specs: &resources.ComputeResourceSpecs{
VCPUs: 2,
MemoryMb: 4096,
},
}

var resourceGCPInstanceGroup = resources.ComputeResource{
Identification: &resources.ResourceIdentification{
Name: "machine-group-1",
ResourceType: "type-1",
Provider: providers.GCP,
Region: "europe-west9",
Count: 3,
},
Specs: &resources.ComputeResourceSpecs{
VCPUs: 2,
Expand All @@ -72,6 +89,7 @@ func TestEstimateResource(t *testing.T) {
Power: decimal.NewFromFloat(7.600784000).RoundFloor(10),
CarbonEmissions: decimal.NewFromFloat(0.448446256).RoundFloor(10),
AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
Count: decimal.NewFromInt(1),
},
},
{
Expand All @@ -82,6 +100,18 @@ func TestEstimateResource(t *testing.T) {
Power: decimal.NewFromFloat(9.5565660741),
CarbonEmissions: decimal.NewFromFloat(0.5638373983),
AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
Count: decimal.NewFromInt(1),
},
},
{
name: "gcp_group",
args: args{resourceGCPInstanceGroup},
want: &EstimationResource{
Resource: &resourceGCPInstanceGroup,
Power: decimal.NewFromFloat(7.600784000).RoundFloor(10),
CarbonEmissions: decimal.NewFromFloat(0.448446256).RoundFloor(10),
AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
Count: decimal.NewFromInt(3),
},
},
}
Expand Down Expand Up @@ -113,6 +143,7 @@ func TestEstimateResourceKilo(t *testing.T) {
Power: decimal.NewFromFloat(5472.56448).RoundFloor(10),
CarbonEmissions: decimal.NewFromFloat(232.4745391104).RoundFloor(10),
AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
Count: decimal.NewFromInt(1),
},
},
{
Expand All @@ -123,6 +154,7 @@ func TestEstimateResourceKilo(t *testing.T) {
Power: decimal.NewFromFloat(6880.7275733647).RoundFloor(10),
CarbonEmissions: decimal.NewFromFloat(292.2933073165).RoundFloor(10),
AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
Count: decimal.NewFromInt(1),
},
},
}
Expand Down Expand Up @@ -165,21 +197,23 @@ func EqualsEstimationResource(t *testing.T, expected *EstimationResource, actual
assert.Equal(t, expected.Power.String(), actual.Power.String())
assert.Equal(t, expected.CarbonEmissions.String(), actual.CarbonEmissions.String())
assert.Equal(t, expected.AverageCPUUsage.String(), actual.AverageCPUUsage.String())
assert.Equal(t, expected.Count.String(), actual.Count.String())

}

func EqualsTotal(t *testing.T, expected *EstimationTotal, actual *EstimationTotal) {
assert.Equal(t, expected.ResourcesCount, actual.ResourcesCount)
assert.Equal(t, expected.Power.String(), actual.Power.String())
assert.Equal(t, expected.CarbonEmissions.String(), actual.CarbonEmissions.String())
assert.Equal(t, expected.ResourcesCount.String(), actual.ResourcesCount.String())
}

func TestEstimateResources(t *testing.T) {
avg_cpu_use := viper.GetFloat64("provider.gcp.avg_cpu_use")
viper.Set("unit.carbon", "g")
viper.Set("unit.time", "h")
type args struct {
resources []resources.Resource
resources map[string]resources.Resource
}
tests := []struct {
name string
Expand All @@ -189,9 +223,10 @@ func TestEstimateResources(t *testing.T) {
{
name: "gcp_array",
args: args{
[]resources.Resource{
resourceGCPComputeBasic,
resourceGCPComputeCPUType,
map[string]resources.Resource{
"type-1.machine-name-1": resourceGCPComputeBasic,
"type-1.machine-name-2": resourceGCPComputeCPUType,
"type-group.machine-group-1": resourceGCPInstanceGroup,
},
},
want: EstimationReport{
Expand All @@ -206,18 +241,27 @@ func TestEstimateResources(t *testing.T) {
Power: decimal.NewFromFloat(7.600784).Round(10),
CarbonEmissions: decimal.NewFromFloat(0.448446256).Round(10),
AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
Count: decimal.NewFromInt(1),
},
{
Resource: &resourceGCPComputeCPUType,
Power: decimal.NewFromFloat(9.5565660741),
CarbonEmissions: decimal.NewFromFloat(0.5638373983),
AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
Count: decimal.NewFromInt(1),
},
{
Resource: &resourceGCPInstanceGroup,
Power: decimal.NewFromFloat(7.600784).Round(10),
CarbonEmissions: decimal.NewFromFloat(0.448446256).Round(10),
AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
Count: decimal.NewFromInt(3),
},
},
Total: EstimationTotal{
Power: decimal.NewFromFloat(17.1573500741),
CarbonEmissions: decimal.NewFromFloat(1.0122836543),
ResourcesCount: 2,
Power: decimal.NewFromFloat(39.9597020741),
CarbonEmissions: decimal.NewFromFloat(2.3576224223),
ResourcesCount: decimal.NewFromInt(5),
},
},
},
Expand Down
7 changes: 4 additions & 3 deletions internal/estimate/estimation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,16 @@ type EstimationReport struct {

type EstimationResource struct {
Resource resources.Resource
Power decimal.Decimal
CarbonEmissions decimal.Decimal
Power decimal.Decimal `json:"PowerPerInstance"`
CarbonEmissions decimal.Decimal `json:"CarbonEmissionsPerInstance"`
AverageCPUUsage decimal.Decimal
Count decimal.Decimal
}

type EstimationTotal struct {
Power decimal.Decimal
CarbonEmissions decimal.Decimal
ResourcesCount int
ResourcesCount decimal.Decimal
}

type EstimationInfo struct {
Expand Down
6 changes: 4 additions & 2 deletions internal/estimate/gpu_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ import (

var noGPUResource = resources.ComputeResource{
Identification: &resources.ResourceIdentification{
Name: "no-gpu",
Name: "no-gpu",
Count: 1,
},
Specs: &resources.ComputeResourceSpecs{
GpuTypes: nil,
Expand All @@ -19,7 +20,8 @@ var noGPUResource = resources.ComputeResource{

var twoGPUResources = resources.ComputeResource{
Identification: &resources.ResourceIdentification{
Name: "two-gpu",
Name: "two-gpu",
Count: 1,
},
Specs: &resources.ComputeResourceSpecs{
GpuTypes: []string{
Expand Down
2 changes: 1 addition & 1 deletion internal/output/json_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func TestGenerateReportJson_Empty(t *testing.T) {
Total: estimate.EstimationTotal{
Power: decimal.Decimal{},
CarbonEmissions: decimal.Decimal{},
ResourcesCount: 0,
ResourcesCount: decimal.Zero,
},
}

Expand Down
6 changes: 4 additions & 2 deletions internal/output/text.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,13 @@ func GenerateReportText(report estimate.EstimationReport) string {
tableString.WriteString("\n Average estimation of CO2 emissions per instance: \n\n")

table := tablewriter.NewWriter(tableString)
table.SetHeader([]string{"resource type", "name", "emissions"})
table.SetHeader([]string{"resource type", "name", "count", "emissions per instance"})

for _, resource := range report.Resources {
table.Append([]string{
resource.Resource.GetIdentification().ResourceType,
resource.Resource.GetIdentification().Name,
fmt.Sprintf("%v", resource.Count),
fmt.Sprintf(" %v %v", resource.CarbonEmissions.StringFixed(4), report.Info.UnitCarbonEmissionsTime),
})
}
Expand All @@ -29,11 +30,12 @@ func GenerateReportText(report estimate.EstimationReport) string {
table.Append([]string{
resource.GetIdentification().ResourceType,
resource.GetIdentification().Name,
"",
"unsupported",
})
}

table.SetFooter([]string{"", "Total", fmt.Sprintf(" %v %v", report.Total.CarbonEmissions.StringFixed(4), report.Info.UnitCarbonEmissionsTime)})
table.SetFooter([]string{"", "Total", report.Total.ResourcesCount.String(), fmt.Sprintf(" %v %v", report.Total.CarbonEmissions.StringFixed(4), report.Info.UnitCarbonEmissionsTime)})

// Format
table.SetAutoFormatHeaders(false)
Expand Down
1 change: 1 addition & 0 deletions internal/resources/compute.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ type ResourceIdentification struct {
Provider providers.Provider
Region string
SelfLink string
Count int64
}

type ComputeResource struct {
Expand Down
Loading

0 comments on commit f9f906d

Please sign in to comment.