Adding first support for Google Kubernetes Engine (#87)

README.md

@@ -10,6 +10,8 @@ Reading Terraform files, `carbonifer plan` will estimate future Carbon Emissions
 
 ## Scope
 
+This tool currently estimates usage emissions, not embodied emissions (manufacturing, transport, recycling...). It is not a full LCA (Life Cycle Assessment) tool.
+
 This tool can analyze Infrastructure as Code definitions such as:
 
 - [Terraform](https://www.terraform.io/) files
@@ -24,6 +26,7 @@ It can estimate Carbon Emissions of:
     - [X] Machines with GPUs
     - [x] Cloud SQL
     - [x] Instance Group (including regional and Autoscaler)
+    - [x] Google Kubernetes Engine (GKE) cluster
 - Amazon Web Services
   - [x] EC2 (including inline root, elastic, and ephemeral block storages)
   - [x] EBS Volumes
@@ -324,10 +327,10 @@ Those calculations and estimations are detailed in the [Methodology document](do
 
 We are currently supporting only
 
-- resources with a significative power usage (basically anything which has CPU, GPU, memory or disk)
+- resources with a significative power usage (basically anything that has CPU, GPU, memory or disk)
 - resources that can be estimated beforehand (we discard for now data transfer)
 
-Because this is just an estimation, the actual power usage and carbon emission should probably differ depending on the actual usage of the resource (CPU %), and actual grid energy mix (could be weather dependent), ... But that should be enough to take decisions about the choice of provider/region, instance type...
+Because this is just an estimation, the actual power usage and carbon emission should probably differ depending on the actual usage of the resource (CPU %), and actual grid energy mix (could be weather dependent), ... But that should be enough to make decisions about the choice of provider/region, instance type...
 
 See the [Scope](doc/scope.md) document for more details.
 

doc/methodology.md

@@ -5,6 +5,8 @@ Those are just "estimations" and will probably differ from the actual energy use
 
 In summary, for each resource, Carbonifer calculate an [Energy Estimate](#energy-estimate) (Watt per Hour) used by it, and multiply it by the [Carbon Intensity](#carbon-intensity) of the underlying data center.
 
+This tool currently estimates usage emissions, not embodied emissions (manufacturing, transport, recycling...). It is not a full LCA (Life Cycle Assessment) tool.
+
 ```text
 Estimated Carbon Emissions (gCO2eq/h) = Energy Estimate (Wh) x Carbon Intensity (gCO2eq/Wh)
 ```

doc/scope.md

@@ -30,6 +30,7 @@ Not all resource types need to be supported if their energy use is negligible or
 | `google_compute_disk`| `size` needs to be set, otherwise get it from image| |
 | `google_compute_region_disk` | `size` needs to be set, otherwise get it from image| |
 | `google_sql_database_instance`  | | Custom machine also supported |
+| `google_container_cluster`  | | With default or referenced pool (`google_container_node_pool`) |
 
 Data resources:
 

internal/output/text.go

@@ -35,7 +35,7 @@ func GenerateReportText(report estimation.EstimationReport) string {
 	for _, resource := range report.UnsupportedResources {
 		table.Append([]string{
 			resource.GetIdentification().ResourceType,
-			resource.GetIdentification().Name,
+			resource.GetIdentification().Address,
 			"",
 			"unsupported",
 		})

internal/plan/json.go

@@ -6,12 +6,10 @@ import (
 	"github.com/carboniferio/carbonifer/internal/utils"
 )
 
-const allResourcesQuery = ".planned_values | .. | objects | select(has(\"resources\")) | .resources[]"
-
 func getJSON(query string, json interface{}) ([]interface{}, error) {
 
-	if strings.HasPrefix(query, "select(") {
-		results, err := utils.GetJSON(allResourcesQuery+" | "+query, *TfPlan)
+	if strings.Contains(query, "all_select(") {
+		results, err := utils.GetJSON(query, *TfPlan)
 		if len(results) > 0 && err == nil {
 			return results, nil
 		}

internal/plan/json_getters.go

@@ -253,6 +253,11 @@ func getValue(key string, context *tfContext) (*valueWithUnit, error) {
 			}
 		}
 
+		err = applyValidator(valueFound, &propertyMapping, context)
+		if err != nil {
+			return nil, err
+		}
+
 		if valueFound != nil {
 			return &valueWithUnit{
 				Value: valueFound,
@@ -375,6 +380,10 @@ func getDefaultValue(key string, context *tfContext) (*valueWithUnit, error) {
 					return nil, err
 				}
 			}
+			err = applyValidator(valueFound, &propertyMapping, context)
+			if err != nil {
+				return nil, err
+			}
 
 			if valueFound != nil {
 				return &valueWithUnit{

internal/plan/mappingStructs.go

@@ -33,6 +33,7 @@ type PropertyDefinition struct {
 	Reference *Reference         `yaml:"reference,omitempty"`
 	Regex     *Regex             `yaml:"regex,omitempty"`
 	Item      *[]ResourceMapping `yaml:"item,omitempty"`
+	Validator *string            `yaml:"validator,omitempty"`
 }
 
 type Reference struct {

internal/plan/mappings/aws/ec2_ebs.yaml

@@ -1,7 +1,7 @@
 compute_resource:
   aws_ebs_volume:
     paths:
-      - select(.type == "aws_ebs_volume")
+      - cbf::all_select("type";  "aws_ebs_volume")
     type: resource
     properties:
       name:

internal/plan/mappings/aws/ec2_instance.yaml

@@ -1,13 +1,13 @@
 compute_resource:
   aws_instance: 
     paths: 
-      - select(.type == "aws_instance")
+      - cbf::all_select("type";  "aws_instance")
     type: resource
     variables:
       properties:
         ami:
           - paths:
-            - select(.values.image_id == "${this.values.ami}")
+            - cbf::all_select("values.image_id";  "${this.values.ami}")
             reference:
               return_path: true
         provider_region:

internal/plan/mappings/aws/rds_instance.yaml

@@ -1,17 +1,17 @@
 compute_resource:
   aws_db_instance: 
     paths: 
-      - select(.type == "aws_db_instance")
+      - cbf::all_select("type";  "aws_db_instance")
     type: resource
     variables:
       properties:
         replicate_source_db:
           - paths:
-            - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.replicate_source_db?.references[]? | select(endswith("id")) | gsub("\\.id$"; "")'
+            - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.replicate_source_db?.references[]? | select(endswith(".id") or endswith(".name")) | gsub("\\.(id|name)$"; "")'
             reference:
               paths:
-                - select(.address == "${key}")
-                - select(.address == ("${key}" | split(".")[0:2] | join("."))) | .resources[] | select(.name == ("${key}" | split(".")[2]))
+                - cbf::all_select("address";  "${key}")
+                - cbf::all_select("address"; ("${key}" | split(".")[0:2] | join("."))) | .resources[] | select(.name",  ("${key}" | split(".")[2]))
                 - .prior_state.values.root_module.resources[] | select(.address == "${key}")
               return_path: true
     properties:

internal/plan/mappings/gcp/compute.yaml

@@ -1,6 +1,6 @@
 compute_resource:
   google_compute_instance:
-    paths: 'select(.type == "google_compute_instance")'
+    paths: 'cbf::all_select("type";  "google_compute_instance")'
     type: resource
     properties:
       name:
@@ -91,16 +91,16 @@ compute_resource:
                   - default: ssd
   google_compute_instance_from_template:
     paths:
-      - select(.type == "google_compute_instance_from_template")
+      - cbf::all_select("type";  "google_compute_instance_from_template")
     type: resource
     variables:
       properties:
         template_config:
           - paths:
-            - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.source_instance_template.references[] | select(endswith("id")) | gsub("\\.id$"; "")'
+            - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.source_instance_template.references[] | select(endswith(".id") or endswith(".name")) | gsub("\\.(id|name)$"; "")'
             reference:
               paths:
-                - select(.address == "${key}")
+                - cbf::all_select("address";  "${key}")
                 - .planned_values.root_module.child_modules[] | select(.address == ("${key}" | split(".")[0:2] | join("."))) | .resources[] | select(.name == ("${key}" | split(".")[2]))
                 - .prior_state.values.root_module.resources[] | select(.address == "${key}")
               return_path: true

internal/plan/mappings/gcp/compute_group.yaml

@@ -1,25 +1,25 @@
 compute_resource:
   google_compute_instance_group_manager:
     paths:
-      - select(.type == "google_compute_instance_group_manager")
-      - select(.type == "google_compute_region_instance_group_manager")
+      - cbf::all_select("type";  "google_compute_instance_group_manager")
+      - cbf::all_select("type";  "google_compute_region_instance_group_manager")
     type: resource
     variables:
       properties:
         template_config:
           - paths:
-              - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.version[0].instance_template.references[] | select(endswith("id")) | gsub("\\.id$"; "")'
+              - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.version[0].instance_template.references[] | select(endswith(".id") or endswith(".name")) | gsub("\\.(id|name)$"; "")'
             reference:
               paths:
-                - select(.address == "${key}")
+                - cbf::all_select("address";  "${key}")
                 - .prior_state.values.root_module.resources[] | select(.address == "${key}")
               return_path: true
         autoscaler:
           - paths:
             - '(.configuration.root_module.resources[] | select(.expressions.target?.references[]? == "${this.address}") | .address)'
             reference:
               paths:
-                - select(.address == "${key}")
+                - cbf::all_select("address";  "${key}")
                 -  map(select(.address == "${key}"))
                 - .prior_state.values.root_module.resources[] | select(.address == "${key}")
               return_path: true
@@ -35,12 +35,23 @@ compute_resource:
           reference:
             json_file: gcp_machines_types
             property: "vcpus"
+        - paths: "${template_config}.values.machine_type"
+          regex:
+            pattern: ".*custom-([0-9]+)-.*"
+            group: 1
+            value_type: integer
       memory:
         - paths: "${template_config}.values.machine_type"
           unit: mb
           reference:
             json_file: gcp_machines_types
             property: "memoryMb"
+        - paths: "${template_config}.values.machine_type"
+          unit: mb
+          regex:
+            pattern: ".*custom-[0-9]+-([0-9]+).*"
+            group: 1
+            value_type: integer
       zone:
         - paths: ".values.zone"
       region:

internal/plan/mappings/gcp/general.yaml

@@ -9,4 +9,5 @@ general:
       gcp_sql_tiers: "gcp_sql_tiers.json"
     ignored_resources:
       - ".*_template"
-      - "google_compute_autoscaler"
+      - "google_compute_autoscaler"
+      - "google_container_node_pool"

internal/plan/mappings/gcp/gke.yaml

@@ -0,0 +1,154 @@
+compute_resource:
+  google_container_cluster:
+    paths:
+      - cbf::all_select("type";  "google_container_cluster")
+    type: resource
+    variables:
+      properties:
+        node_pool:
+          - paths:
+              - '.configuration.root_module.resources[] | select(any(.expressions.cluster.references[]?; . == "${this.address}")) | .address'
+            reference:
+              paths:
+                - cbf::all_select("address";  "${key}") | .values
+                - .prior_state.values.root_module.resources[] | select(.address == "${key}") | .values
+              return_path: true
+          - default: '.values.node_pool[0]'
+        nb_zones :
+          - paths: 
+            - ".values.node_locations | length"
+            - "${node_pool}.node_locations | length"
+    properties:
+      name:
+        - paths: ".name"
+      address:
+        - paths: ".address"
+      type:
+        - paths: ".type"
+      vCPUs:
+        - paths: 
+          - ".values.node_config[].machine_type"
+          - "${node_pool}.node_config[].machine_type"
+          reference:
+            json_file: gcp_machines_types
+            property: "vcpus"
+        - paths: 
+          - ".values.node_config[].machine_type"
+          - "${node_pool}.node_config[].machine_type"
+          regex:
+            pattern: ".*custom-([0-9]+)-.*"
+            group: 1
+            value_type: integer
+        - paths: 
+          - ".values.cluster_autoscaling[0] | select(.enabled != false) | .resource_limits[] | select(.resource_type == \"cpu\" and .maximum != null) | ((.minimum // 1) + (${config.provider.gcp.avg_autoscaler_size_percent} * (.maximum - (.minimum // 1))))"
+          - ".values.cluster_autoscaling[0] | select(.enabled == false) | .resource_limits[] | select(.resource_type == \"cpu\") | (.minimum // 1)"
+          validator : "if . == null then error(\"The number of vCPUs of nodes must set. Does it have a minimum and a maxium value? \") else . end"
+      memory:
+        - paths: 
+          - ".values.node_config[].machine_type"
+          - "${node_pool}.node_config[].machine_type"
+          unit: mb
+          reference:
+            json_file: gcp_machines_types
+            property: "memoryMb"
+        - paths: 
+          - ".values.node_config[].machine_type"
+          - "${node_pool}.node_config[].machine_type"
+          unit: mb
+          regex:
+            pattern: ".*custom-[0-9]+-([0-9]+).*"
+            group: 1
+            value_type: integer
+        - paths: 
+          - ".values.cluster_autoscaling[0] | select(.enabled != false) | .resource_limits[] | select(.resource_type == \"memory\" and .maximum != null) | ((.minimum // 1) + (${config.provider.gcp.avg_autoscaler_size_percent} * (.maximum - (.minimum // 1))))"
+          - ".values.cluster_autoscaling[0] | select(.enabled == false) | .resource_limits[] | select(.resource_type == \"memory\") | (.minimum // 1)"
+          validator : "if . == null then error(\"The memory size of nodes must be set. Does it have a minimum and a maxium value? \") else . end"
+          unit: gb
+      zone:
+        - paths: 
+          - ".values.node_locations"
+          - "${node_pool}.node_locations"
+      region:
+        - paths: 
+          - ".values.location"
+          - ".values.node_locations[0]"
+          - "${node_pool}.location"
+          - "${node_pool}.node_locations[0]"
+          regex:
+            pattern: "^([^-]+-[^-]+)(-.*)?$"
+            group: 1
+      count:
+        - paths: 
+          - "${node_pool}.autoscaling[0] | select(.total_max_node_count != null) |(.total_min_node_count // 1) + (${config.provider.gcp.avg_autoscaler_size_percent} * (.total_max_node_count - (.total_min_node_count // 1)))"
+          - "(if ${nb_zones} == 0 then 1 else ${nb_zones} end) * (${node_pool}.autoscaling[0] | select(.max_node_count != null) | (.min_node_count // 1) + (${config.provider.gcp.avg_autoscaler_size_percent} * (.max_node_count - (.min_node_count // 1))))"
+          - "(( (${node_pool}.node_locations ) // .values.node_locations // [\"dummy_region\"]) | length) * (${node_pool}.autoscaling[0] | select(.max_node_count != null) | (.min_node_count // 1) + (${config.provider.gcp.avg_autoscaler_size_percent} * (.max_node_count - (.min_node_count // 1))))"
+          - "${node_pool}.node_count"
+          - ".values.initial_node_count"
+      guest_accelerator:
+        - type: list
+          item:
+            - paths: 
+              - ".values.node_config[]?.guest_accelerator"
+              - "${node_pool}.node_config[].guest_accelerator"
+              properties:
+                count:
+                  - paths: ".count"
+                    type: integer
+                type:
+                  - paths: ".type"
+                    type: string
+            - paths: ".values.cluster_autoscaling[0] | select(.enabled != false) | .resource_limits[] | select(.resource_type != \"memory\" and .resource_type != \"cpu\")"
+              properties:
+                count:
+                  - paths: "(.minimum // 1) + (${config.provider.gcp.avg_autoscaler_size_percent} * (.maximum - (.minimum // 1)))"
+                    validator : "if . <= 0 then error(\"The number of GPU of nodes must be bigger than zero. Does it have a minimum and a maxium value? \") else . end"
+                    type: integer
+                type:
+                  - paths: ".resource_type"
+                    type: string
+      storage:
+        - type: list
+          item:
+            - paths: 
+              - .values.node_config[]
+              - ${node_pool}.node_config[]
+              - .values.cluster_autoscaling[0].auto_provisioning_defaults
+              properties:
+                size:
+                  - paths: 
+                    - ".disk_size_gb"
+                    - ".disk_size"
+                    unit: gb
+                  - default: 10
+                type:
+                  - paths: ".disk_type"
+                    default: pd-standard
+                    reference:
+                      general: disk_types
+            - paths: 
+              - .values.node_config[]
+              - ${node_pool}.node_config[]
+              properties:
+                size:
+                  - paths: "(.local_ssd_count? // 0 )* 375"
+                    unit: gb
+                type: 
+                  - default : ssd
+            - paths: 
+              - .values.node_config[].ephemeral_storage_local_ssd_config
+              - ${node_pool}.node_config[].ephemeral_storage_local_ssd_config
+              properties:
+                size:
+                  - paths: "(.local_ssd_count? // 0 )* 375"
+                    unit: gb
+                type: 
+                  - default : ssd
+            - paths: 
+              - .values.node_config[].local_nvme_ssd_block_config
+              - ${node_pool}.node_config[].local_nvme_ssd_block_config
+              properties:
+                size:
+                  - paths: "(.local_ssd_count? // 0 )* 375"
+                    unit: gb
+                type: 
+                  - default : ssd