diff --git a/internal/estimate/estimate.go b/internal/estimate/estimate.go index 204c2a0..6da4d9f 100644 --- a/internal/estimate/estimate.go +++ b/internal/estimate/estimate.go @@ -12,14 +12,14 @@ import ( "github.com/spf13/viper" ) -func EstimateResources(resourceList []resources.Resource) EstimationReport { +func EstimateResources(resourceList map[string]resources.Resource) EstimationReport { var estimationResources []EstimationResource var unsupportedResources []resources.Resource estimationTotal := EstimationTotal{ Power: decimal.Zero, CarbonEmissions: decimal.Zero, - ResourcesCount: 0, + ResourcesCount: decimal.Zero, } for _, resource := range resourceList { estimationResource, uerr := EstimateResource(resource) @@ -33,9 +33,9 @@ func EstimateResources(resourceList []resources.Resource) EstimationReport { unsupportedResources = append(unsupportedResources, resource) } - estimationTotal.Power = estimationTotal.Power.Add(estimationResource.Power) - estimationTotal.CarbonEmissions = estimationTotal.CarbonEmissions.Add(estimationResource.CarbonEmissions) - estimationTotal.ResourcesCount += 1 + estimationTotal.Power = estimationTotal.Power.Add(estimationResource.Power.Mul(estimationResource.Count)) + estimationTotal.CarbonEmissions = estimationTotal.CarbonEmissions.Add(estimationResource.CarbonEmissions.Mul(estimationResource.Count)) + estimationTotal.ResourcesCount = estimationTotal.ResourcesCount.Add(estimationResource.Count) } return EstimationReport{ @@ -103,7 +103,7 @@ func estimateGCP(resource resources.Resource) *EstimationResource { carbonEmissionPerTime := avgWatt.Mul(regionEmissions.GridCarbonIntensity) log.Debugf( - "estimating resource %v.%v (%v): %v %v%v * %v %vCO2/%v%v = %v %vCO2/%v%v", + "estimating resource %v.%v (%v): %v %v%v * %v %vCO2/%v%v = %v %vCO2/%v%v * %v", computeResource.Identification.ResourceType, computeResource.Identification.Name, regionEmissions.Region, @@ -118,6 +118,7 @@ func estimateGCP(resource resources.Resource) *EstimationResource { viper.Get("unit.carbon").(string), viper.Get("unit.power").(string), viper.Get("unit.time").(string), + resource.GetIdentification().Count, ) return &EstimationResource{ @@ -125,6 +126,7 @@ func estimateGCP(resource resources.Resource) *EstimationResource { Power: avgWatt.RoundFloor(10), CarbonEmissions: carbonEmissionPerTime.RoundFloor(10), AverageCPUUsage: decimal.NewFromFloat(viper.GetFloat64("provider.gcp.avg_cpu_use")).RoundFloor(10), + Count: decimal.NewFromInt(int64(computeResource.Identification.Count)), } } diff --git a/internal/estimate/estimate_test.go b/internal/estimate/estimate_test.go index 5ab58f3..1e365d6 100644 --- a/internal/estimate/estimate_test.go +++ b/internal/estimate/estimate_test.go @@ -18,6 +18,7 @@ var resourceGCPComputeBasic = resources.ComputeResource{ ResourceType: "type-1", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ VCPUs: 2, @@ -31,6 +32,7 @@ var resourceGCPComputeCPUType = resources.ComputeResource{ ResourceType: "type-1", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ VCPUs: 2, @@ -47,6 +49,21 @@ var resourceAWSComputeBasic = resources.ComputeResource{ ResourceType: "type-1", Provider: providers.AWS, Region: "europe-west9", + Count: 1, + }, + Specs: &resources.ComputeResourceSpecs{ + VCPUs: 2, + MemoryMb: 4096, + }, +} + +var resourceGCPInstanceGroup = resources.ComputeResource{ + Identification: &resources.ResourceIdentification{ + Name: "machine-group-1", + ResourceType: "type-1", + Provider: providers.GCP, + Region: "europe-west9", + Count: 3, }, Specs: &resources.ComputeResourceSpecs{ VCPUs: 2, @@ -72,6 +89,7 @@ func TestEstimateResource(t *testing.T) { Power: decimal.NewFromFloat(7.600784000).RoundFloor(10), CarbonEmissions: decimal.NewFromFloat(0.448446256).RoundFloor(10), AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use), + Count: decimal.NewFromInt(1), }, }, { @@ -82,6 +100,18 @@ func TestEstimateResource(t *testing.T) { Power: decimal.NewFromFloat(9.5565660741), CarbonEmissions: decimal.NewFromFloat(0.5638373983), AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use), + Count: decimal.NewFromInt(1), + }, + }, + { + name: "gcp_group", + args: args{resourceGCPInstanceGroup}, + want: &EstimationResource{ + Resource: &resourceGCPInstanceGroup, + Power: decimal.NewFromFloat(7.600784000).RoundFloor(10), + CarbonEmissions: decimal.NewFromFloat(0.448446256).RoundFloor(10), + AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use), + Count: decimal.NewFromInt(3), }, }, } @@ -113,6 +143,7 @@ func TestEstimateResourceKilo(t *testing.T) { Power: decimal.NewFromFloat(5472.56448).RoundFloor(10), CarbonEmissions: decimal.NewFromFloat(232.4745391104).RoundFloor(10), AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use), + Count: decimal.NewFromInt(1), }, }, { @@ -123,6 +154,7 @@ func TestEstimateResourceKilo(t *testing.T) { Power: decimal.NewFromFloat(6880.7275733647).RoundFloor(10), CarbonEmissions: decimal.NewFromFloat(292.2933073165).RoundFloor(10), AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use), + Count: decimal.NewFromInt(1), }, }, } @@ -165,6 +197,7 @@ func EqualsEstimationResource(t *testing.T, expected *EstimationResource, actual assert.Equal(t, expected.Power.String(), actual.Power.String()) assert.Equal(t, expected.CarbonEmissions.String(), actual.CarbonEmissions.String()) assert.Equal(t, expected.AverageCPUUsage.String(), actual.AverageCPUUsage.String()) + assert.Equal(t, expected.Count.String(), actual.Count.String()) } @@ -172,6 +205,7 @@ func EqualsTotal(t *testing.T, expected *EstimationTotal, actual *EstimationTota assert.Equal(t, expected.ResourcesCount, actual.ResourcesCount) assert.Equal(t, expected.Power.String(), actual.Power.String()) assert.Equal(t, expected.CarbonEmissions.String(), actual.CarbonEmissions.String()) + assert.Equal(t, expected.ResourcesCount.String(), actual.ResourcesCount.String()) } func TestEstimateResources(t *testing.T) { @@ -179,7 +213,7 @@ func TestEstimateResources(t *testing.T) { viper.Set("unit.carbon", "g") viper.Set("unit.time", "h") type args struct { - resources []resources.Resource + resources map[string]resources.Resource } tests := []struct { name string @@ -189,9 +223,10 @@ func TestEstimateResources(t *testing.T) { { name: "gcp_array", args: args{ - []resources.Resource{ - resourceGCPComputeBasic, - resourceGCPComputeCPUType, + map[string]resources.Resource{ + "type-1.machine-name-1": resourceGCPComputeBasic, + "type-1.machine-name-2": resourceGCPComputeCPUType, + "type-group.machine-group-1": resourceGCPInstanceGroup, }, }, want: EstimationReport{ @@ -206,18 +241,27 @@ func TestEstimateResources(t *testing.T) { Power: decimal.NewFromFloat(7.600784).Round(10), CarbonEmissions: decimal.NewFromFloat(0.448446256).Round(10), AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use), + Count: decimal.NewFromInt(1), }, { Resource: &resourceGCPComputeCPUType, Power: decimal.NewFromFloat(9.5565660741), CarbonEmissions: decimal.NewFromFloat(0.5638373983), AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use), + Count: decimal.NewFromInt(1), + }, + { + Resource: &resourceGCPInstanceGroup, + Power: decimal.NewFromFloat(7.600784).Round(10), + CarbonEmissions: decimal.NewFromFloat(0.448446256).Round(10), + AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use), + Count: decimal.NewFromInt(3), }, }, Total: EstimationTotal{ - Power: decimal.NewFromFloat(17.1573500741), - CarbonEmissions: decimal.NewFromFloat(1.0122836543), - ResourcesCount: 2, + Power: decimal.NewFromFloat(39.9597020741), + CarbonEmissions: decimal.NewFromFloat(2.3576224223), + ResourcesCount: decimal.NewFromInt(5), }, }, }, diff --git a/internal/estimate/estimation.go b/internal/estimate/estimation.go index cab07cf..9606f91 100644 --- a/internal/estimate/estimation.go +++ b/internal/estimate/estimation.go @@ -16,15 +16,16 @@ type EstimationReport struct { type EstimationResource struct { Resource resources.Resource - Power decimal.Decimal - CarbonEmissions decimal.Decimal + Power decimal.Decimal `json:"PowerPerInstance"` + CarbonEmissions decimal.Decimal `json:"CarbonEmissionsPerInstance"` AverageCPUUsage decimal.Decimal + Count decimal.Decimal } type EstimationTotal struct { Power decimal.Decimal CarbonEmissions decimal.Decimal - ResourcesCount int + ResourcesCount decimal.Decimal } type EstimationInfo struct { diff --git a/internal/estimate/gpu_test.go b/internal/estimate/gpu_test.go index 2b65057..51f8cf8 100644 --- a/internal/estimate/gpu_test.go +++ b/internal/estimate/gpu_test.go @@ -10,7 +10,8 @@ import ( var noGPUResource = resources.ComputeResource{ Identification: &resources.ResourceIdentification{ - Name: "no-gpu", + Name: "no-gpu", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ GpuTypes: nil, @@ -19,7 +20,8 @@ var noGPUResource = resources.ComputeResource{ var twoGPUResources = resources.ComputeResource{ Identification: &resources.ResourceIdentification{ - Name: "two-gpu", + Name: "two-gpu", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ GpuTypes: []string{ diff --git a/internal/output/json_test.go b/internal/output/json_test.go index 6840f09..be1c59d 100644 --- a/internal/output/json_test.go +++ b/internal/output/json_test.go @@ -34,7 +34,7 @@ func TestGenerateReportJson_Empty(t *testing.T) { Total: estimate.EstimationTotal{ Power: decimal.Decimal{}, CarbonEmissions: decimal.Decimal{}, - ResourcesCount: 0, + ResourcesCount: decimal.Zero, }, } diff --git a/internal/output/text.go b/internal/output/text.go index 3f03999..02a16d1 100644 --- a/internal/output/text.go +++ b/internal/output/text.go @@ -15,12 +15,13 @@ func GenerateReportText(report estimate.EstimationReport) string { tableString.WriteString("\n Average estimation of CO2 emissions per instance: \n\n") table := tablewriter.NewWriter(tableString) - table.SetHeader([]string{"resource type", "name", "emissions"}) + table.SetHeader([]string{"resource type", "name", "count", "emissions per instance"}) for _, resource := range report.Resources { table.Append([]string{ resource.Resource.GetIdentification().ResourceType, resource.Resource.GetIdentification().Name, + fmt.Sprintf("%v", resource.Count), fmt.Sprintf(" %v %v", resource.CarbonEmissions.StringFixed(4), report.Info.UnitCarbonEmissionsTime), }) } @@ -29,11 +30,12 @@ func GenerateReportText(report estimate.EstimationReport) string { table.Append([]string{ resource.GetIdentification().ResourceType, resource.GetIdentification().Name, + "", "unsupported", }) } - table.SetFooter([]string{"", "Total", fmt.Sprintf(" %v %v", report.Total.CarbonEmissions.StringFixed(4), report.Info.UnitCarbonEmissionsTime)}) + table.SetFooter([]string{"", "Total", report.Total.ResourcesCount.String(), fmt.Sprintf(" %v %v", report.Total.CarbonEmissions.StringFixed(4), report.Info.UnitCarbonEmissionsTime)}) // Format table.SetAutoFormatHeaders(false) diff --git a/internal/resources/compute.go b/internal/resources/compute.go index 7437323..7834d90 100644 --- a/internal/resources/compute.go +++ b/internal/resources/compute.go @@ -24,6 +24,7 @@ type ResourceIdentification struct { Provider providers.Provider Region string SelfLink string + Count int64 } type ComputeResource struct { diff --git a/internal/terraform/GCP.go b/internal/terraform/GCP.go index 8df322a..e2a0ea0 100644 --- a/internal/terraform/GCP.go +++ b/internal/terraform/GCP.go @@ -12,10 +12,10 @@ import ( "github.com/spf13/viper" ) -func GetResource(tfResource tfjson.ConfigResource, dataResources *map[string]resources.DataResource) resources.Resource { +func GetResource(tfResource tfjson.ConfigResource, dataResources *map[string]resources.DataResource, resourceTemplates *map[string]*tfjson.ConfigResource) resources.Resource { resourceId := getResourceIdentification(tfResource) if resourceId.ResourceType == "google_compute_instance" { - specs := getComputeResourceSpecs(tfResource, dataResources) + specs := getComputeResourceSpecs(tfResource, dataResources, nil) return resources.ComputeResource{ Identification: resourceId, Specs: specs, @@ -36,16 +36,38 @@ func GetResource(tfResource tfjson.ConfigResource, dataResources *map[string]res Specs: specs, } } + if resourceId.ResourceType == "google_compute_instance_group_manager" { + specs, count := getComputeInstanceGroupManagerSpecs(tfResource, dataResources, resourceTemplates) + if specs != nil { + resourceId.Count = count + return resources.ComputeResource{ + Identification: resourceId, + Specs: specs, + } + } + } return resources.UnsupportedResource{ Identification: resourceId, } } +func GetResourceTemplate(tfResource tfjson.ConfigResource, dataResources *map[string]resources.DataResource, zone string) resources.Resource { + resourceId := getResourceIdentification(tfResource) + if resourceId.ResourceType == "google_compute_instance_template" { + specs := getComputeResourceSpecs(tfResource, dataResources, zone) + return resources.ComputeResource{ + Identification: resourceId, + Specs: specs, + } + } + return nil +} + func getResourceIdentification(resource tfjson.ConfigResource) *resources.ResourceIdentification { - region := GetValueExpression(&resource, "region") + region := GetConstFromConfig(&resource, "region") if region == nil { - zone := GetValueExpression(&resource, "zone") - replica_zones := GetValueExpression(&resource, "replica_zones") + zone := GetConstFromConfig(&resource, "zone") + replica_zones := GetConstFromConfig(&resource, "replica_zones") if zone != nil { region = strings.Join(strings.Split(zone.(string), "-")[:2], "-") } else if replica_zones != nil { @@ -54,10 +76,10 @@ func getResourceIdentification(resource tfjson.ConfigResource) *resources.Resour region = "" } } - selfLinkExpr := GetValueExpression(&resource, "self_link") + selfLinkExpr := GetConstFromConfig(&resource, "self_link") var selfLink string if selfLinkExpr != nil { - selfLink = GetValueExpression(&resource, "self_link").(string) + selfLink = GetConstFromConfig(&resource, "self_link").(string) } return &resources.ResourceIdentification{ @@ -66,17 +88,24 @@ func getResourceIdentification(resource tfjson.ConfigResource) *resources.Resour Provider: providers.GCP, Region: fmt.Sprint(region), SelfLink: selfLink, + Count: 1, } } func getComputeResourceSpecs( resource tfjson.ConfigResource, - dataResources *map[string]resources.DataResource) *resources.ComputeResourceSpecs { + dataResources *map[string]resources.DataResource, groupZone interface{}) *resources.ComputeResourceSpecs { + + machine_type := GetConstFromConfig(&resource, "machine_type").(string) + var zone string + if groupZone != nil { + zone = groupZone.(string) + } else { + zone = GetConstFromConfig(&resource, "zone").(string) + } - machine_type := GetValueExpression(&resource, "machine_type").(string) - zone := GetValueExpression(&resource, "zone").(string) machineType := providers.GetGCPMachineType(machine_type, zone) - CPUType, ok := GetValueExpression(&resource, "cpu_platform").(string) + CPUType, ok := GetConstFromConfig(&resource, "cpu_platform").(string) if !ok { CPUType = "" } @@ -91,6 +120,16 @@ func getComputeResourceSpecs( } } + diskExpr, ok_bd := resource.Expressions["disk"] + if ok_bd { + disksBlocks := diskExpr.NestedBlocks + for _, diskBlock := range disksBlocks { + + bootDisk := getDisk(resource.Address, diskBlock, false, dataResources) + disks = append(disks, bootDisk) + } + } + sdExpr, ok_sd := resource.Expressions["scratch_disk"] if ok_sd { scratchDisks := sdExpr.NestedBlocks @@ -112,7 +151,7 @@ func getComputeResourceSpecs( } gpus := machineType.GPUTypes - gasI := GetValueExpression(&resource, "guest_accelerator") + gasI := GetConstFromConfig(&resource, "guest_accelerator") if gasI != nil { guestAccelerators := gasI.([]interface{}) for _, gaI := range guestAccelerators { @@ -126,12 +165,13 @@ func getComputeResourceSpecs( } return &resources.ComputeResourceSpecs{ - GpuTypes: gpus, - VCPUs: machineType.Vcpus, - MemoryMb: machineType.MemoryMb, - CPUType: CPUType, - SsdStorage: ssdSize, - HddStorage: hddSize, + GpuTypes: gpus, + VCPUs: machineType.Vcpus, + MemoryMb: machineType.MemoryMb, + CPUType: CPUType, + SsdStorage: ssdSize, + HddStorage: hddSize, + ReplicationFactor: 1, } } @@ -170,12 +210,21 @@ func getBootDisk(resourceAddress string, bootDiskBlock map[string]*tfjson.Expres return disk } -func getDisk(resourceAddress string, diskBlock map[string]*tfjson.Expression, isBootDisk bool, dataResources *map[string]resources.DataResource) disk { +func getDisk(resourceAddress string, diskBlock map[string]*tfjson.Expression, isBootDiskParam bool, dataResources *map[string]resources.DataResource) disk { disk := disk{ sizeGb: viper.GetFloat64("provider.gcp.boot_disk.size"), isSSD: true, replicationFactor: 1, } + + // Is Boot disk + isBootDisk := isBootDiskParam + isBootDiskI := GetConstFromExpression(diskBlock["boot"]) + if isBootDiskI != nil { + isBootDisk = isBootDiskI.(bool) + } + + // Get disk type var diskType string diskTypeExpr := diskBlock["type"] if diskTypeExpr == nil { @@ -192,10 +241,12 @@ func getDisk(resourceAddress string, diskBlock map[string]*tfjson.Expression, is disk.isSSD = false } - sizeParamExpr := diskBlock["size"] - if sizeParamExpr != nil { - disk.sizeGb = sizeParamExpr.ConstantValue.(float64) - } else { + // Get Disk size + declaredSize := GetConstFromExpression(diskBlock["size"]) + if declaredSize == nil { + declaredSize = GetConstFromExpression(diskBlock["disk_size_gb"]) + } + if declaredSize == nil { if isBootDisk { disk.sizeGb = viper.GetFloat64("provider.gcp.boot_disk.size") } else { @@ -215,13 +266,16 @@ func getDisk(resourceAddress string, diskBlock map[string]*tfjson.Expression, is log.Warningf("%v : Boot disk size not declared. Please set it! (otherwise we assume 10gb) ", resourceAddress) } - + } else { + disk.sizeGb = declaredSize.(float64) } replicaZonesExpr := diskBlock["replica_zones"] if replicaZonesExpr != nil { rz := replicaZonesExpr.ConstantValue.([]interface{}) disk.replicationFactor = int32(len(rz)) + } else { + disk.replicationFactor = 1 } return disk @@ -281,8 +335,46 @@ func getSQLResourceSpecs( } } -func GetValueExpression(resource *tfjson.ConfigResource, key string) interface{} { +func getComputeInstanceGroupManagerSpecs(tfResource tfjson.ConfigResource, dataResources *map[string]resources.DataResource, resourceTemplates *map[string]*tfjson.ConfigResource) (*resources.ComputeResourceSpecs, int64) { + targetSize := int64(0) + targetSizeExpr := GetConstFromConfig(&tfResource, "target_size") + if targetSizeExpr != nil { + targetSize = decimal.NewFromFloat(targetSizeExpr.(float64)).BigInt().Int64() + } + versionExpr := tfResource.Expressions["version"] + var template *tfjson.ConfigResource + if versionExpr != nil { + for _, version := range versionExpr.NestedBlocks { + instanceTemplate := version["instance_template"] + if instanceTemplate != nil { + references := instanceTemplate.References + for _, reference := range references { + if !strings.HasSuffix(reference, ".id") { + template = (*resourceTemplates)[reference] + } + } + } + } + } + if template != nil { + zone := GetConstFromConfig(&tfResource, "zone").(string) + templateResource := GetResourceTemplate(*template, dataResources, zone) + computeTemplate, ok := templateResource.(resources.ComputeResource) + if ok { + return computeTemplate.Specs, targetSize + } else { + log.Fatalf("Type mismatch, not a esources.ComputeResource template %v", computeTemplate.GetAddress()) + } + } + return nil, 0 +} + +func GetConstFromConfig(resource *tfjson.ConfigResource, key string) interface{} { expr := resource.Expressions[key] + return GetConstFromExpression(expr) +} + +func GetConstFromExpression(expr *tfjson.Expression) interface{} { if expr != nil { if expr.ConstantValue != nil { return expr.ConstantValue diff --git a/internal/terraform/GCP_test.go b/internal/terraform/GCP_test.go index 349803b..ebf9152 100644 --- a/internal/terraform/GCP_test.go +++ b/internal/terraform/GCP_test.go @@ -97,6 +97,7 @@ func TestGetResource(t *testing.T) { ResourceType: "google_compute_disk", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ HddStorage: decimal.NewFromInt(1024), @@ -116,6 +117,7 @@ func TestGetResource(t *testing.T) { ResourceType: "google_compute_disk", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ HddStorage: decimal.New(50, 1), @@ -135,6 +137,7 @@ func TestGetResource(t *testing.T) { ResourceType: "google_compute_region_disk", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ HddStorage: decimal.Zero, @@ -154,14 +157,16 @@ func TestGetResource(t *testing.T) { ResourceType: "google_compute_instance", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ GpuTypes: []string{ "nvidia-tesla-k80", "nvidia-tesla-k80", }, - HddStorage: decimal.Zero, - SsdStorage: decimal.Zero, + HddStorage: decimal.Zero, + SsdStorage: decimal.Zero, + ReplicationFactor: 1, }, }, }, @@ -176,22 +181,24 @@ func TestGetResource(t *testing.T) { ResourceType: "google_compute_instance", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ GpuTypes: []string{ "nvidia-tesla-a100", }, - VCPUs: int32(12), - MemoryMb: int32(87040), - HddStorage: decimal.Zero, - SsdStorage: decimal.Zero, + VCPUs: int32(12), + MemoryMb: int32(87040), + HddStorage: decimal.Zero, + SsdStorage: decimal.Zero, + ReplicationFactor: 1, }, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := GetResource(tt.args.tfResource, nil) + got := GetResource(tt.args.tfResource, nil, nil) assert.Equal(t, tt.want, got) }) } diff --git a/internal/terraform/terraform.go b/internal/terraform/terraform.go index 2c51437..29909e6 100644 --- a/internal/terraform/terraform.go +++ b/internal/terraform/terraform.go @@ -151,7 +151,7 @@ func TerraformPlan() (*tfjson.Plan, error) { return tfplan, nil } -func GetResources() ([]resources.Resource, error) { +func GetResources() (map[string]resources.Resource, error) { log.Debug("Reading planned resources from Terraform plan") tfPlan, err := TerraformPlan() if err != nil { @@ -162,7 +162,8 @@ func GetResources() ([]resources.Resource, error) { } } log.Debugf("Reading resources from Terraform plan: %d resources", len(tfPlan.PlannedValues.RootModule.Resources)) - var resourcesList []resources.Resource + resourcesMap := make(map[string]resources.Resource) + resourceTemplates := make(map[string]*tfjson.ConfigResource) dataResources := make(map[string]resources.DataResource) if tfPlan.PriorState != nil { for _, priorRes := range tfPlan.PriorState.Values.RootModule.Resources { @@ -176,25 +177,32 @@ func GetResources() ([]resources.Resource, error) { } } + // Find template first for _, res := range tfPlan.Config.RootModule.Resources { log.Debugf("Reading resource %v", res.Address) - if strings.HasPrefix(res.Type, "google") { - var resource resources.Resource + if strings.HasPrefix(res.Type, "google") && strings.HasSuffix(res.Type, "_template") { if res.Mode == "managed" { - resource := GetResource(*res, &dataResources) - resourcesList = append(resourcesList, resource) + resourceTemplates[res.Address] = res } + } + } - if log.IsLevelEnabled(log.DebugLevel) { - computeJsonStr := "" - if resource.IsSupported() { - computeJson, _ := json.Marshal(resource) - computeJsonStr = string(computeJson) + for _, res := range tfPlan.Config.RootModule.Resources { + log.Debugf("Reading resource %v", res.Address) + if strings.HasPrefix(res.Type, "google") && !strings.HasSuffix(res.Type, "_template") { + if res.Mode == "managed" { + resource := GetResource(*res, &dataResources, &resourceTemplates) + resourcesMap[resource.GetAddress()] = resource + if log.IsLevelEnabled(log.DebugLevel) { + computeJsonStr := "" + if resource.IsSupported() { + computeJson, _ := json.Marshal(resource) + computeJsonStr = string(computeJson) + } + log.Debugf(" Compute resource : %v", string(computeJsonStr)) } - log.Debugf(" Compute resource : %v", string(computeJsonStr)) } - } } - return resourcesList, nil + return resourcesMap, nil } diff --git a/internal/terraform/terraform_test.go b/internal/terraform/terraform_test.go index 1c4eeaf..0c11fb0 100644 --- a/internal/terraform/terraform_test.go +++ b/internal/terraform/terraform_test.go @@ -105,13 +105,14 @@ func TestGetResources(t *testing.T) { wd := path.Join(testutils.RootDir, "test/terraform/gcp_1") viper.Set("workdir", wd) - wantResources := []resources.Resource{ - resources.ComputeResource{ + wantResources := map[string]resources.Resource{ + "google_compute_disk.first": resources.ComputeResource{ Identification: &resources.ResourceIdentification{ Name: "first", ResourceType: "google_compute_disk", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ GpuTypes: nil, @@ -123,12 +124,13 @@ func TestGetResources(t *testing.T) { ReplicationFactor: 1, }, }, - resources.ComputeResource{ + "google_compute_instance.first": resources.ComputeResource{ Identification: &resources.ResourceIdentification{ Name: "first", ResourceType: "google_compute_instance", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ HddStorage: decimal.Zero, @@ -140,38 +142,43 @@ func TestGetResources(t *testing.T) { "nvidia-tesla-k80", // Added by user in main.tf "nvidia-tesla-k80", // Added by user in main.tf }, + ReplicationFactor: 1, }, }, - resources.ComputeResource{ + "google_compute_instance.second": resources.ComputeResource{ Identification: &resources.ResourceIdentification{ Name: "second", ResourceType: "google_compute_instance", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ - GpuTypes: nil, - HddStorage: decimal.NewFromFloat(10), - SsdStorage: decimal.Zero, - MemoryMb: 4098, - VCPUs: 2, - CPUType: "", + GpuTypes: nil, + HddStorage: decimal.NewFromFloat(10), + SsdStorage: decimal.Zero, + MemoryMb: 4098, + VCPUs: 2, + CPUType: "", + ReplicationFactor: 1, }, }, - resources.UnsupportedResource{ + "google_compute_network.vpc_network": resources.UnsupportedResource{ Identification: &resources.ResourceIdentification{ Name: "vpc_network", ResourceType: "google_compute_network", Provider: providers.GCP, Region: "", + Count: 1, }, }, - resources.ComputeResource{ + "google_compute_region_disk.regional-first": resources.ComputeResource{ Identification: &resources.ResourceIdentification{ Name: "regional-first", ResourceType: "google_compute_region_disk", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ GpuTypes: nil, @@ -183,20 +190,22 @@ func TestGetResources(t *testing.T) { ReplicationFactor: 2, }, }, - resources.UnsupportedResource{ + "google_compute_subnetwork.first": resources.UnsupportedResource{ Identification: &resources.ResourceIdentification{ Name: "first", ResourceType: "google_compute_subnetwork", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, }, - resources.ComputeResource{ + "google_sql_database_instance.instance": resources.ComputeResource{ Identification: &resources.ResourceIdentification{ Name: "instance", ResourceType: "google_sql_database_instance", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ GpuTypes: nil, @@ -239,13 +248,14 @@ func TestGetResources_DiskImage(t *testing.T) { wd := path.Join(testutils.RootDir, "test/terraform/gcp_images") viper.Set("workdir", wd) - wantResources := []resources.Resource{ - resources.ComputeResource{ + wantResources := map[string]resources.Resource{ + "google_compute_disk.diskImage": resources.ComputeResource{ Identification: &resources.ResourceIdentification{ Name: "diskImage", ResourceType: "google_compute_disk", Provider: providers.GCP, Region: "europe-west9", + Count: 1, }, Specs: &resources.ComputeResourceSpecs{ GpuTypes: nil, @@ -269,3 +279,63 @@ func TestGetResources_DiskImage(t *testing.T) { } } + +func TestGetResources_GroupInstance(t *testing.T) { + testutils.SkipWithCreds(t) + // reset + terraformExec = nil + + t.Setenv("GOOGLE_OAUTH_ACCESS_TOKEN", "") + + wd := path.Join(testutils.RootDir, "test/terraform/gcp_group") + viper.Set("workdir", wd) + + wantResources := map[string]resources.Resource{ + "google_compute_network.vpc_network": resources.UnsupportedResource{ + Identification: &resources.ResourceIdentification{ + Name: "vpc_network", + ResourceType: "google_compute_network", + Provider: providers.GCP, + Region: "", + Count: 1, + }, + }, + "google_compute_subnetwork.first": resources.UnsupportedResource{ + Identification: &resources.ResourceIdentification{ + Name: "first", + ResourceType: "google_compute_subnetwork", + Provider: providers.GCP, + Region: "europe-west9", + Count: 1, + }, + }, + "google_compute_instance_group_manager.my-group-manager": resources.ComputeResource{ + Identification: &resources.ResourceIdentification{ + Name: "my-group-manager", + ResourceType: "google_compute_instance_group_manager", + Provider: providers.GCP, + Region: "europe-west9", + Count: 3, + }, + Specs: &resources.ComputeResourceSpecs{ + GpuTypes: nil, + HddStorage: decimal.NewFromFloat(20), + SsdStorage: decimal.Zero, + MemoryMb: 8192, + VCPUs: 2, + CPUType: "", + ReplicationFactor: 1, + }, + }, + } + + resources, err := GetResources() + if assert.NoError(t, err) { + assert.Equal(t, len(wantResources), len(resources)) + for i, resource := range resources { + wantResource := wantResources[i] + assert.EqualValues(t, wantResource, resource) + } + } + +} diff --git a/test/outputs/nothing.txt b/test/outputs/nothing.txt index 36b377f..022f056 100644 --- a/test/outputs/nothing.txt +++ b/test/outputs/nothing.txt @@ -1,9 +1,9 @@ Average estimation of CO2 emissions per instance: - --------------- ------- ------------------ - resource type name emissions - --------------- ------- ------------------ - --------------- ------- ------------------ - Total 0.0000 gCO2eq/h - --------------- ------- ------------------ + --------------- ------- ------- ------------------------ + resource type name count emissions per instance + --------------- ------- ------- ------------------------ + --------------- ------- ------- ------------------------ + Total 0 0.0000 gCO2eq/h + --------------- ------- ------- ------------------------ \ No newline at end of file diff --git a/test/terraform/gcp_group/main.tf b/test/terraform/gcp_group/main.tf new file mode 100644 index 0000000..6c8158f --- /dev/null +++ b/test/terraform/gcp_group/main.tf @@ -0,0 +1,36 @@ +resource "google_compute_network" "vpc_network" { + name = "cbf-network" + auto_create_subnetworks = false + mtu = 1460 +} + +resource "google_compute_subnetwork" "first" { + name = "cbf-subnet" + ip_cidr_range = "10.0.1.0/24" + region = "europe-west9" + network = google_compute_network.vpc_network.id +} + +resource "google_compute_instance_template" "my-instance-template" { + name = "my-instance-template" + machine_type = "e2-standard-2" + + disk { + boot = true + disk_size_gb = 20 + } + +} + +resource "google_compute_instance_group_manager" "my-group-manager" { + name = "my-group-manager" + + base_instance_name = "managed" + zone = "europe-west9-a" + + version { + instance_template = google_compute_instance_template.my-instance-template.id + } + + target_size = 3 +} \ No newline at end of file diff --git a/test/terraform/gcp_group/provider.tf b/test/terraform/gcp_group/provider.tf new file mode 100644 index 0000000..7fb6a3d --- /dev/null +++ b/test/terraform/gcp_group/provider.tf @@ -0,0 +1,4 @@ +provider "google" { + region = "europe-west9" +} +