Add missing AWS/EC2 ephemeral storages (#91)

carboniferio · Sep 12, 2023 · ae97944 · ae97944
1 parent c448c54
commit ae97944
Show file tree

Hide file tree

Showing 12 changed files with 83 additions and 33 deletions.
diff --git a/internal/plan/json_getters.go b/internal/plan/json_getters.go
@@ -89,6 +89,7 @@ func getSliceItems(context tfContext) ([]interface{}, error) {
 			if err != nil {
 				return nil, err
 			}
+			fmt.Println(path)
 		}
 		jsonResults, err := getJSON(path, context.Resource)
 		if err != nil {
@@ -136,7 +137,7 @@ func getItem(context tfContext, itemMappingProperties *ResourceMapping, jsonResu
 			Resource:        jsonResultI,
 			Mapping:         itemMappingProperties,
 			ResourceAddress: context.ResourceAddress,
-			ParentContext:   &context,
+			ParentContext:   context.ParentContext,
 			Provider:        context.Provider,
 		}
 		property, err := getValue(key, &itemContext)
@@ -398,8 +399,15 @@ func getDefaultValue(key string, context *tfContext) (*valueWithUnit, error) {
 
 }
 
-func getVariable(name string, context *tfContext) (interface{}, error) {
+func getVariable(name string, contextParam *tfContext) (interface{}, error) {
+	context := contextParam
 	variablesMappings := context.Mapping.Variables
+	if variablesMappings == nil {
+		context = contextParam.ParentContext
+		if context != nil {
+			variablesMappings = context.Mapping.Variables
+		}
+	}
 	if variablesMappings == nil {
 		return nil, nil
 	}

diff --git a/internal/plan/mappings/aws/ec2_ebs.yaml b/internal/plan/mappings/aws/ec2_ebs.yaml
@@ -30,7 +30,7 @@ compute_resource:
                   - paths: ".snapshot_id"
                     reference:
                       paths: .prior_state.values.root_module.resources[] | select(.values.id == "${key}") | .values
-                      property: "volume_size"
+                      property: ".volume_size"
                   - default: 8
                     unit: gb
                 type:

diff --git a/internal/plan/mappings/aws/ec2_instance.yaml b/internal/plan/mappings/aws/ec2_instance.yaml
@@ -14,6 +14,8 @@ compute_resource:
           - paths:
             - '.configuration'
             property: "region"
+        instance_type:
+          - paths: '.values.instance_type'
     properties:
       name:
         - paths: ".name"
@@ -25,13 +27,13 @@ compute_resource:
         - paths: ".values.instance_type"
           reference:
             json_file: aws_instances
-            property: "VCPU"
+            property: ".VCPU"
       memory:
         - paths: ".values.instance_type"
           unit: mb
           reference:
             json_file: aws_instances
-            property: "MemoryMb"
+            property: ".MemoryMb"
             zone:
       zone:
         - paths: ".values.availability_zone"
@@ -65,11 +67,25 @@ compute_resource:
                   - paths: ".snapshot_id"
                     reference:
                       paths: .prior_state.values.root_module.resources[] | select(.values.id == "${key}") | .values
-                      property: "volume_size"
+                      property: ".volume_size"
                   - default: 8
                     unit: gb
                 type:
                   - paths: ".volume_type"
                     default: standard
                     reference:
-                      general: disk_types
+                      general: disk_types
+            - paths: '.values.ephemeral_block_device[] | select(length > 0)'
+              properties:
+                size:
+                  - paths: '"${instance_type}"'
+                    unit: gb
+                    reference:
+                      json_file: aws_instances
+                      property: ".InstanceStorage.SizePerDiskGB"
+                type:
+                  - paths: '"${instance_type}"'
+                    default: standard
+                    reference:
+                      json_file: aws_instances
+                      property: ".InstanceStorage.Type"
diff --git a/internal/plan/mappings/aws/rds_instance.yaml b/internal/plan/mappings/aws/rds_instance.yaml
@@ -38,7 +38,7 @@ compute_resource:
             group: 1
           reference:
             json_file: aws_instances
-            property: "VCPU"
+            property: ".VCPU"
       memory:
         - paths: ".values.instance_class"
           unit: mb
@@ -47,7 +47,7 @@ compute_resource:
             group: 1
           reference:
             json_file: aws_instances
-            property: "MemoryMb"
+            property: ".MemoryMb"
       storage:
         - type: list
           item:

diff --git a/internal/plan/mappings/gcp/compute.yaml b/internal/plan/mappings/gcp/compute.yaml
@@ -13,7 +13,7 @@ compute_resource:
         - paths: ".values.machine_type"
           reference:
             json_file: gcp_machines_types
-            property: "vcpus"
+            property: ".vcpus"
         - paths: ".values.machine_type"
           regex:
             pattern: ".*custom-([0-9]+)-.*"
@@ -24,7 +24,7 @@ compute_resource:
           unit: mb
           reference:
             json_file: gcp_machines_types
-            property: "memoryMb"
+            property: ".memoryMb"
         - paths: ".values.machine_type"
           unit: mb
           regex:
@@ -73,7 +73,7 @@ compute_resource:
                     unit: gb
                     reference:
                       paths: '.prior_state.values.root_module.resources[] | select(.values.self_link == "${key}") | .values'
-                      property: "disk_size_gb"
+                      property: ".disk_size_gb"
                 type:
                   - paths: 
                     - ".type"
@@ -115,13 +115,13 @@ compute_resource:
         - paths: "${template_config}.values.machine_type"
           reference:
             json_file: gcp_machines_types
-            property: "vcpus"
+            property: ".vcpus"
       memory:
         - paths: "${template_config}.values.machine_type"
           unit: mb
           reference:
             json_file: gcp_machines_types
-            property: "memoryMb"
+            property: ".memoryMb"
       zone:
         - paths: ".values.zone"
       region:
@@ -156,7 +156,7 @@ compute_resource:
                     unit: gb
                     reference:
                       paths: .prior_state.values.root_module.resources[] | select(.values.self_link == "${key}") | .values
-                      property: "disk_size_gb"
+                      property: ".disk_size_gb"
                   - default: 10
                 type:
                   - paths: ".disk_type"

diff --git a/internal/plan/mappings/gcp/compute_group.yaml b/internal/plan/mappings/gcp/compute_group.yaml
@@ -34,7 +34,7 @@ compute_resource:
         - paths: "${template_config}.values.machine_type"
           reference:
             json_file: gcp_machines_types
-            property: "vcpus"
+            property: ".vcpus"
         - paths: "${template_config}.values.machine_type"
           regex:
             pattern: ".*custom-([0-9]+)-.*"
@@ -45,7 +45,7 @@ compute_resource:
           unit: mb
           reference:
             json_file: gcp_machines_types
-            property: "memoryMb"
+            property: ".memoryMb"
         - paths: "${template_config}.values.machine_type"
           unit: mb
           regex:
@@ -93,7 +93,7 @@ compute_resource:
                     unit: gb
                     reference:
                       paths: .prior_state.values.root_module.resources[] | select(.values.self_link == "${key}") | .values
-                      property: "disk_size_gb"
+                      property: ".disk_size_gb"
                   - default: 10
                 type:
                   - paths: ".disk_type"

diff --git a/internal/plan/mappings/gcp/disk.yaml b/internal/plan/mappings/gcp/disk.yaml
@@ -38,7 +38,7 @@ compute_resource:
                     unit: gb
                     reference:
                       paths: .prior_state.values.root_module.resources[] | select(.values.self_link == "${key}") | .values
-                      property: "disk_size_gb"
+                      property: ".disk_size_gb"
                   - default: 10
                     unit: gb
                 type:

diff --git a/internal/plan/mappings/gcp/gke.yaml b/internal/plan/mappings/gcp/gke.yaml
@@ -33,7 +33,7 @@ compute_resource:
           - "${node_pool}.node_config[].machine_type"
           reference:
             json_file: gcp_machines_types
-            property: "vcpus"
+            property: ".vcpus"
         - paths: 
           - ".values.node_config[].machine_type"
           - "${node_pool}.node_config[].machine_type"
@@ -52,7 +52,7 @@ compute_resource:
           unit: mb
           reference:
             json_file: gcp_machines_types
-            property: "memoryMb"
+            property: ".memoryMb"
         - paths: 
           - ".values.node_config[].machine_type"
           - "${node_pool}.node_config[].machine_type"

diff --git a/internal/plan/mappings/gcp/sql_database.yaml b/internal/plan/mappings/gcp/sql_database.yaml
@@ -13,13 +13,13 @@ compute_resource:
         - paths: ".values.settings[0].tier"
           reference:
             json_file: gcp_sql_tiers
-            property: "vcpus"
+            property: ".vcpus"
       memory:
         - paths: ".values.settings[0].tier"
           unit: mb
           reference:
             json_file: gcp_sql_tiers
-            property: "memoryMb"
+            property: ".memoryMb"
       zone:
         - paths: ".values.zone"
       region:

diff --git a/internal/plan/resolver.go b/internal/plan/resolver.go
@@ -6,6 +6,7 @@ import (
 	"regexp"
 
 	"github.com/carboniferio/carbonifer/internal/data"
+	"github.com/carboniferio/carbonifer/internal/utils"
 	"github.com/pkg/errors"
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
@@ -44,12 +45,21 @@ func resolveReference(key string, reference *Reference, context *tfContext) (int
 			log.Debugf("Cannot find key %v in file %v", key, reference.JSONFile)
 			return nil, nil
 		}
+
 		var value interface{}
 		property := reference.Property
 		if property != "" {
-			value, ok = item.(map[string]interface{})[reference.Property]
-			if !ok {
-				log.Fatalf("Cannot find property %v in file %v", reference.Property, reference.JSONFile)
+			valueArray, err := utils.GetJSON(reference.Property, item)
+			if err != nil {
+				return nil, errors.Wrapf(err, "Cannot find property %v in file %v", reference.Property, reference.JSONFile)
+			}
+			if len(valueArray) != 0 {
+				if valueArray[0] == nil {
+					return nil, fmt.Errorf("Cannot find property %v in file %v", reference.Property, reference.JSONFile)
+				}
+				value = valueArray[0]
+			} else {
+				return nil, fmt.Errorf("Cannot find property %v in file %v", reference.Property, reference.JSONFile)
 			}
 		}
 		return value, nil
@@ -82,10 +92,16 @@ func resolveReference(key string, reference *Reference, context *tfContext) (int
 			}
 			for _, referencedItem := range referencedItems {
 				if reference.Property != "" {
-					value := referencedItem.(map[string]interface{})[reference.Property]
-					if value != nil {
-						return value, nil
+					value, err := utils.GetJSON(reference.Property, referencedItem)
+					if err != nil {
+						return nil, errors.Wrapf(err, "Cannot find property %v in path %v", reference.Property, path)
+					}
+					if len(value) != 0 {
+						if value[0] != nil {
+							return value[0], nil
+						}
 					}
+
 				} else if reference.ReturnPath {
 					return path, nil
 				} else {

diff --git a/internal/plan/test/resources_aws_test.go b/internal/plan/test/resources_aws_test.go
@@ -36,11 +36,11 @@ func TestGetResource_DiskFromAMI(t *testing.T) {
 				ReplicationFactor: 1,
 			},
 			Specs: &resources.ComputeResourceSpecs{
-				VCPUs:    int32(2),
-				MemoryMb: int32(8192),
+				VCPUs:    int32(4),
+				MemoryMb: int32(16384),
 
 				HddStorage: decimal.NewFromInt(80),
-				SsdStorage: decimal.NewFromInt(30),
+				SsdStorage: decimal.NewFromInt(330),
 			},
 		},
 		"aws_ebs_volume.ebs_volume": resources.ComputeResource{

diff --git a/test/terraform/aws_ec2/main.tf b/test/terraform/aws_ec2/main.tf
@@ -36,7 +36,7 @@ resource "aws_ebs_volume" "ebs_volume" {
 
 resource "aws_instance" "foo" {
   ami           = data.aws_ami.ubuntu.id
-  instance_type = "m4.large"
+  instance_type = "m5d.xlarge"
 
   network_interface {
     network_interface_id = aws_network_interface.foo.id
@@ -53,6 +53,16 @@ resource "aws_instance" "foo" {
     device_name = "/dev/sdj"
     snapshot_id = data.aws_ebs_snapshot.ebs_snapshot.id
   }
+
+  ephemeral_block_device {
+    device_name  = "/dev/sdk"
+    virtual_name = "ephemeral0"
+  }
+
+  ephemeral_block_device {
+    device_name  = "/dev/sdl"
+    virtual_name = "ephemeral1"
+  }
 }
 
 resource "aws_volume_attachment" "ebs_att" {