[GCP] Compute Instance from template

README.md

@@ -19,7 +19,7 @@ It can estimate Carbon Emissions of:
 
 - **Google Cloud Platform**
   - [x] **Compute Engine**
-    - [x] Compute Instances (generic and custom machine types)
+    - [x] Compute Instances (generic and custom machine types, and from template)
     - [x] Disks (boot, persistent and region-persistent, HDD or SSD)
     - [X] Machines with GPUs
     - [x] Cloud SQL

doc/scope.md

@@ -24,6 +24,7 @@ Not all resource types need to be supported if their energy use is negligible or
 | `google_compute_instance`  | | Custom machine, nested boot disk type and GPU supported |
 | `google_compute_instance_group_manager`  | | Count will be the target size. Uses machine specifications from `google_compute_instance_template` |
 | `google_compute_region_instance_group_manager`  | | Count will be the target size. Uses machine specifications from `google_compute_instance_template` |
+| `google_compute_instance_group_manager`  | | Uses machine specs from `google_compute_instance_template` |
 | `google_compute_autoscaler`  | Takes an average size  | Will set target size of `google_compute_instance_group_manager` |
 | `google_compute_disk`| `size` needs to be set, otherwise get it from image| |
 | `google_compute_region_disk` | `size` needs to be set, otherwise get it from image| |

go.mod

@@ -3,10 +3,11 @@ module github.com/carboniferio/carbonifer
 go 1.17
 
 require (
+	github.com/forestgiant/sliceutil v0.0.0-20160425183142-94783f95db6c
 	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/hc-install v0.4.0
 	github.com/hashicorp/terraform-exec v0.17.2
-	github.com/hashicorp/terraform-json v0.14.0
+	github.com/hashicorp/terraform-json v0.15.0
 	github.com/heirko/go-contrib v0.0.0-20200825160048-11fc5e2235fa
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/shopspring/decimal v1.3.1

internal/estimate/gcp/GCP.go

@@ -23,6 +23,7 @@ func EstimateGCP(resource resources.Resource) *estimation.EstimationResource {
 	if viper.Get("unit.time").(string) == "y" {
 		avgWatt = avgWatt.Mul(decimal.NewFromInt(24 * 365))
 	}
+	avgWattStr := avgWatt.String()
 
 	// Regional grid emission per unit of time
 	regionEmissions, err := GCPRegionEmission(resource.GetIdentification().Region) // gCO2eq /kWh
@@ -44,20 +45,21 @@ func EstimateGCP(resource resources.Resource) *estimation.EstimationResource {
 
 	// Carbon Emissions
 	carbonEmissionPerTime := avgWatt.Mul(regionEmissions.GridCarbonIntensity)
+	carbonEmissionPerTimeStr := carbonEmissionPerTime.String()
 
 	log.Debugf(
 		"estimating resource %v.%v (%v): %v %v%v * %v %vCO2/%v%v = %v %vCO2/%v%v * %v",
 		computeResource.Identification.ResourceType,
 		computeResource.Identification.Name,
 		regionEmissions.Region,
-		avgWatt,
+		avgWattStr,
 		viper.Get("unit.power").(string),
 		viper.Get("unit.time").(string),
 		regionEmissions.GridCarbonIntensity,
 		viper.Get("unit.carbon").(string),
 		viper.Get("unit.power").(string),
 		viper.Get("unit.time").(string),
-		carbonEmissionPerTime,
+		carbonEmissionPerTimeStr,
 		viper.Get("unit.carbon").(string),
 		viper.Get("unit.power").(string),
 		viper.Get("unit.time").(string),

internal/estimate/gcp/storage.go

@@ -3,7 +3,6 @@ package gcp
 import (
 	"github.com/carboniferio/carbonifer/internal/estimate/coefficients"
 	"github.com/carboniferio/carbonifer/internal/resources"
-	_ "github.com/carboniferio/carbonifer/internal/testutils"
 	"github.com/shopspring/decimal"
 )
 

internal/terraform/gcp/ComputeResource.go

@@ -35,7 +35,6 @@ func getComputeResourceSpecs(
 		}
 	}
 
-	// TODO Disks
 	diskListI, ok_disks := resource.AttributeValues["disk"]
 	if ok_disks {
 		diskList := diskListI.([]interface{})

internal/terraform/gcp/ComputeResourceFromTemplate.go

@@ -0,0 +1,75 @@
+package gcp
+
+import (
+	"strings"
+
+	"github.com/carboniferio/carbonifer/internal/resources"
+	tfjson "github.com/hashicorp/terraform-json"
+	log "github.com/sirupsen/logrus"
+)
+
+func getComputeResourceFromTemplateSpecs(
+	tfResource tfjson.StateResource,
+	dataResources *map[string]resources.DataResource,
+	resourceReferences *map[string]*tfjson.StateResource,
+	resourceConfigs *map[string]*tfjson.ConfigResource) *resources.ComputeResourceSpecs {
+
+	// Get template of instance
+	specs := getTemplateSpecs(tfResource, dataResources, resourceReferences, resourceConfigs)
+	if specs != nil {
+		return specs
+	}
+	return nil
+
+}
+
+func getTemplateSpecs(
+	tfResource tfjson.StateResource,
+	dataResources *map[string]resources.DataResource,
+	resourceReferences *map[string]*tfjson.StateResource,
+	resourceConfigs *map[string]*tfjson.ConfigResource) *resources.ComputeResourceSpecs {
+
+	// Find google_compute_instance_from_template resourceConfig
+	iftConfig := (*resourceConfigs)[tfResource.Address]
+
+	var template *tfjson.StateResource
+	sourceTemplateExpr := iftConfig.Expressions["source_instance_template"]
+	if sourceTemplateExpr != nil {
+		references := sourceTemplateExpr.References
+		for _, reference := range references {
+			if !strings.HasSuffix(reference, ".id") {
+				template = (*resourceReferences)[reference]
+				break
+			}
+		}
+	}
+
+	if template != nil {
+		var zones []string
+		zoneAttr := tfResource.AttributeValues["zone"]
+		if zoneAttr != nil {
+			zones = append(zones, zoneAttr.(string))
+		}
+		distributionPolicyZonesI := tfResource.AttributeValues["distribution_policy_zones"]
+		if distributionPolicyZonesI != nil {
+			distributionPolicyZones := distributionPolicyZonesI.([]interface{})
+			for _, z := range distributionPolicyZones {
+				zones = append(zones, z.(string))
+			}
+		}
+
+		if len(zones) == 0 {
+			log.Fatalf("No zone or distribution policy declared for %v", tfResource.Address)
+		}
+		templateResource := GetResourceTemplate(*template, dataResources, zones[0])
+		computeTemplate, ok := templateResource.(resources.ComputeResource)
+		if ok {
+			return computeTemplate.Specs
+		} else {
+			log.Fatalf("Type mismatch, not a esources.ComputeResource template %v", computeTemplate.GetAddress())
+		}
+	} else {
+		log.Fatalf("Cannot find template of %v", tfResource.Address)
+	}
+	return nil
+}

internal/terraform/gcp/ManagedInstanceGroupResource.go

@@ -17,7 +17,7 @@ func getComputeInstanceGroupManagerSpecs(
 	resourceConfigs *map[string]*tfjson.ConfigResource) (*resources.ComputeResourceSpecs, int64) {
 
 	// Get template of instance
-	specs, targetSize := getTemplateSpecs(tfResource, dataResources, resourceReferences, resourceConfigs)
+	specs, targetSize := getGroupInstanceTemplateSpecs(tfResource, dataResources, resourceReferences, resourceConfigs)
 	if specs == nil {
 		return specs, targetSize
 	}
@@ -74,7 +74,7 @@ func computeTargetSize(minSize decimal.Decimal, maxSize decimal.Decimal) int64 {
 	return avgAutoscalerSizePercent.Mul(maxSize.Sub(minSize)).Ceil().IntPart()
 }
 
-func getTemplateSpecs(
+func getGroupInstanceTemplateSpecs(
 	tfResource tfjson.StateResource,
 	dataResources *map[string]resources.DataResource,
 	resourceReferences *map[string]*tfjson.StateResource,

internal/terraform/gcp/resources.go

@@ -2,6 +2,7 @@ package gcp
 
 import (
 	"github.com/carboniferio/carbonifer/internal/resources"
+	"github.com/forestgiant/sliceutil"
 	tfjson "github.com/hashicorp/terraform-json"
 )
 
@@ -19,6 +20,15 @@ func GetResource(
 			Specs:          specs,
 		}
 	}
+	if resourceId.ResourceType == "google_compute_instance_from_template" {
+		specs := getComputeResourceFromTemplateSpecs(tfResource, dataResources, resourceTemplates, resourceConfigs)
+		if specs != nil {
+			return resources.ComputeResource{
+				Identification: resourceId,
+				Specs:          specs,
+			}
+		}
+	}
 	if resourceId.ResourceType == "google_compute_disk" ||
 		resourceId.ResourceType == "google_compute_region_disk" {
 		specs := getComputeDiskResourceSpecs(tfResource, dataResources)
@@ -45,9 +55,14 @@ func GetResource(
 			}
 		}
 	}
-	if resourceId.ResourceType == "google_compute_autoscaler" {
+	ignoredResourceType := []string{
+		"google_compute_autoscaler",
+		"google_compute_instance_template",
+	}
+	if sliceutil.Contains(ignoredResourceType, resourceId.ResourceType) {
 		return nil
 	}
+
 	return resources.UnsupportedResource{
 		Identification: resourceId,
 	}

internal/terraform/terraform.go

@@ -203,19 +203,19 @@ func GetResources() (map[string]resources.Resource, error) {
 	// Get All resources
 	for _, res := range tfPlan.PlannedValues.RootModule.Resources {
 		log.Debugf("Reading resource %v", res.Address)
-		if strings.HasPrefix(res.Type, "google") && !strings.HasSuffix(res.Type, "_template") {
+		if strings.HasPrefix(res.Type, "google") {
 			if res.Mode == "managed" {
 				resource := gcp.GetResource(*res, &dataResources, &resourceReferences, &resourceConfigs)
 				if resource != nil {
 					resourcesMap[resource.GetAddress()] = resource
-				}
-				if log.IsLevelEnabled(log.DebugLevel) {
-					computeJsonStr := "<RESOURCE TYPE CURRENTLY NOT SUPPORTED>"
-					if resource.IsSupported() {
-						computeJson, _ := json.Marshal(resource)
-						computeJsonStr = string(computeJson)
+					if log.IsLevelEnabled(log.DebugLevel) {
+						computeJsonStr := "<RESOURCE TYPE CURRENTLY NOT SUPPORTED>"
+						if resource.IsSupported() {
+							computeJson, _ := json.Marshal(resource)
+							computeJsonStr = string(computeJson)
+						}
+						log.Debugf("  Compute resource : %v", string(computeJsonStr))
 					}
-					log.Debugf("  Compute resource : %v", string(computeJsonStr))
 				}
 			}
 		}

internal/terraform/terraform_test.go

@@ -340,3 +340,62 @@ func TestGetResources_GroupInstance(t *testing.T) {
 	}
 
 }
+
+func TestGetResources_InstanceFromTemplate(t *testing.T) {
+	testutils.SkipWithCreds(t)
+	// reset
+	terraformExec = nil
+
+	t.Setenv("GOOGLE_OAUTH_ACCESS_TOKEN", "")
+
+	wd := path.Join(testutils.RootDir, "test/terraform/gcp_cit")
+	viper.Set("workdir", wd)
+
+	wantResources := map[string]resources.Resource{
+		"google_compute_network.vpc_network": resources.UnsupportedResource{
+			Identification: &resources.ResourceIdentification{
+				Name:         "vpc_network",
+				ResourceType: "google_compute_network",
+				Provider:     providers.GCP,
+				Region:       "",
+				Count:        1,
+			},
+		},
+		"google_compute_subnetwork.first": resources.UnsupportedResource{
+			Identification: &resources.ResourceIdentification{
+				Name:         "first",
+				ResourceType: "google_compute_subnetwork",
+				Provider:     providers.GCP,
+				Region:       "europe-west9",
+				Count:        1,
+			},
+		},
+		"google_compute_instance_from_template.ifromtpl": resources.ComputeResource{
+			Identification: &resources.ResourceIdentification{
+				Name:         "ifromtpl",
+				ResourceType: "google_compute_instance_from_template",
+				Provider:     providers.GCP,
+				Region:       "europe-west9",
+				Count:        1,
+			},
+			Specs: &resources.ComputeResourceSpecs{
+				GpuTypes:          nil,
+				HddStorage:        decimal.NewFromFloat(20),
+				SsdStorage:        decimal.Zero,
+				MemoryMb:          8192,
+				VCPUs:             2,
+				CPUType:           "",
+				ReplicationFactor: 1,
+			},
+		},
+	}
+
+	resources, err := GetResources()
+	if assert.NoError(t, err) {
+		for i, resource := range resources {
+			wantResource := wantResources[i]
+			assert.EqualValues(t, wantResource, resource)
+		}
+	}
+
+}

test/terraform/gcp_cit/main.tf

@@ -0,0 +1,36 @@
+resource "google_compute_network" "vpc_network" {
+  name                    = "cbf-network"
+  auto_create_subnetworks = false
+  mtu                     = 1460
+}
+
+resource "google_compute_subnetwork" "first" {
+  name          = "cbf-subnet"
+  ip_cidr_range = "10.0.1.0/24"
+  region        = "europe-west9"
+  network       = google_compute_network.vpc_network.id
+}
+
+resource "google_compute_instance_template" "my-instance-template" {
+  name             = "my-instance-template"
+  machine_type     = "e2-standard-2"
+
+ disk {
+    boot              = true
+    disk_size_gb = 20
+  }
+
+}
+
+resource "google_compute_instance_from_template" "ifromtpl" {
+  name = "instance-from-template"
+  zone = "europe-west9-a"
+
+  source_instance_template = google_compute_instance_template.my-instance-template.id
+
+  // Override fields from instance template
+  can_ip_forward = false
+  labels = {
+    my_key = "my_value"
+  }
+}

test/terraform/gcp_cit/provider.tf

@@ -0,0 +1,4 @@
+provider "google" {
+  region = "europe-west9"
+}
+