[AWS] Supports EBS volumes and snapshot (#73)

README.md

@@ -26,11 +26,11 @@ It can estimate Carbon Emissions of:
     - [x] Instance Group (including regional and Autoscaler)
 - Amazon Web Services
   - [x] EC2 (including inline root, elastic, and ephemeral block storages)
+  - [x] EBS Volumes
 
 The following will also be supported soon:
 
 - Amazon Web Services
-  - [ ] EBS
   - [ ] RDS
   - [ ] AutoScaling Group
 - Azure

doc/scope.md

@@ -41,13 +41,15 @@ Data resources:
 
 | Resource | Limitations  | Comment |
 |---|---|---|
-| `aws_instance`| Only inline EBS (not attached), no GPU | |
+| `aws_instance`| No GPU | |
+| `aws_ebs_volume`| if size set, or if snapshot declared as data resource | |
 
 Data resources:
 
 | Resource | Limitations  | Comment |
 |---|---|---|
-| `aws_ami`| `ebs.volume_size` can be set, otherwise get it from image only if GCP credentials are provided| |
+| `aws_ami`| `ebs.volume_size` can be set, otherwise get it from image only if AWS credentials are provided| |
+| `aws_ebs_snapshot`| `volume_size` can be set, otherwise get it from image only if AWS credentials are provided| |
 
 
 _more to be implemented_

internal/resources/aws_ami.go → internal/resources/aws_ebs.go

@@ -4,20 +4,20 @@ import (
 	"fmt"
 )
 
-type AmiDataResource struct {
+type EbsDataResource struct {
 	Identification *ResourceIdentification
 	DataImageSpecs []*DataImageSpecs
-	AmiId          string
+	AwsId          string
 }
 
-func (r AmiDataResource) GetIdentification() *ResourceIdentification {
+func (r EbsDataResource) GetIdentification() *ResourceIdentification {
 	return r.Identification
 }
 
-func (r AmiDataResource) GetAddress() string {
+func (r EbsDataResource) GetAddress() string {
 	return fmt.Sprintf("data.%v.%v", r.GetIdentification().ResourceType, r.GetIdentification().Name)
 }
 
-func (r AmiDataResource) GetKey() string {
-	return r.AmiId
+func (r EbsDataResource) GetKey() string {
+	return r.AwsId
 }

internal/terraform/aws/Data.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/carboniferio/carbonifer/internal/providers"
 	"github.com/carboniferio/carbonifer/internal/resources"
+	"github.com/carboniferio/carbonifer/internal/terraform/tfrefs"
 	tfjson "github.com/hashicorp/terraform-json"
 )
 
@@ -31,13 +32,27 @@ func GetDataResource(tfResource tfjson.StateResource) resources.DataResource {
 					specs[i] = &diskSpecs
 				}
 			}
-			return resources.AmiDataResource{
+			return resources.EbsDataResource{
 				Identification: resourceId,
 				DataImageSpecs: specs,
-				AmiId:          tfResource.AttributeValues["id"].(string),
+				AwsId:          tfResource.AttributeValues["id"].(string),
 			}
 		}
 	}
+	if resourceId.ResourceType == "aws_ebs_snapshot" {
+		diskSize := tfResource.AttributeValues["volume_size"]
+		diskSizeGb := diskSize.(float64)
+		return resources.EbsDataResource{
+			Identification: resourceId,
+			DataImageSpecs: []*resources.DataImageSpecs{
+				{
+					DiskSizeGb: diskSizeGb,
+				},
+			},
+			AwsId: tfResource.AttributeValues["id"].(string),
+		}
+	}
+
 	return resources.DataImageResource{
 		Identification: resourceId,
 	}
@@ -51,3 +66,14 @@ func getDataResourceIdentification(resource tfjson.StateResource) *resources.Res
 		Provider:     providers.AWS,
 	}
 }
+
+func getAwsImage(tfRefs *tfrefs.References, awsImageId string) *resources.EbsDataResource {
+	imageI := tfRefs.DataResources[awsImageId]
+
+	var image *resources.EbsDataResource
+	if imageI != nil {
+		i := imageI.(resources.EbsDataResource)
+		image = &i
+	}
+	return image
+}

internal/terraform/aws/Data_test.go

@@ -40,7 +40,7 @@ func TestGetDataResource(t *testing.T) {
 					},
 				},
 			},
-			want: resources.AmiDataResource{
+			want: resources.EbsDataResource{
 				Identification: &resources.ResourceIdentification{
 					Name:         "foo",
 					ResourceType: "aws_ami",
@@ -53,7 +53,34 @@ func TestGetDataResource(t *testing.T) {
 						VolumeType: "gp2",
 					},
 				},
-				AmiId: "ami-1234567890",
+				AwsId: "ami-1234567890",
+			},
+		},
+		{
+			name: "Snapshot of size 60 Gb",
+			args: args{
+				tfResource: tfjson.StateResource{
+					Address: "data.aws_ebs_snapshot.test_snapshot",
+					Type:    "aws_ebs_snapshot",
+					Name:    "test_snapshot",
+					AttributeValues: map[string]interface{}{
+						"id":          "snap-1234567890",
+						"volume_size": float64(60),
+					},
+				},
+			},
+			want: resources.EbsDataResource{
+				Identification: &resources.ResourceIdentification{
+					Name:         "test_snapshot",
+					ResourceType: "aws_ebs_snapshot",
+					Provider:     providers.AWS,
+				},
+				DataImageSpecs: []*resources.DataImageSpecs{
+					{
+						DiskSizeGb: 60,
+					},
+				},
+				AwsId: "snap-1234567890",
 			},
 		},
 	}

internal/terraform/aws/EBS.go

@@ -4,6 +4,9 @@ import (
 	"strings"
 
 	"github.com/carboniferio/carbonifer/internal/resources"
+	"github.com/carboniferio/carbonifer/internal/terraform/tfrefs"
+	tfjson "github.com/hashicorp/terraform-json"
+	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 )
@@ -14,7 +17,7 @@ type disk struct {
 	replicationFactor int32
 }
 
-func getDisk(resourceAddress string, diskBlock map[string]interface{}, isBootDisk bool, image *resources.AmiDataResource) disk {
+func getDisk(resourceAddress string, diskBlock map[string]interface{}, isBootDisk bool, image *resources.EbsDataResource, trRefs *tfrefs.References) disk {
 	disk := disk{
 		sizeGb:            viper.GetInt64("provider.aws.boot_disk.size"),
 		isSSD:             true,
@@ -25,13 +28,23 @@ func getDisk(resourceAddress string, diskBlock map[string]interface{}, isBootDis
 
 	diskType := viper.GetString("provider.aws.disk.type")
 	diskTypeI := diskBlock["volume_type"]
+	if diskTypeI == nil {
+		diskTypeI = diskBlock["type"]
+	}
 	if diskTypeI != nil {
 		diskType = diskTypeI.(string)
 	} else {
 		if image != nil && diskBlock["device_name"] != nil {
-			for _, bd := range image.DataImageSpecs {
-				if strings.HasPrefix(bd.DeviceName, diskBlock["device_name"].(string)) {
-					diskType = bd.VolumeType
+			if strings.HasPrefix(image.Identification.ResourceType, "aws_ebs_snapshot") {
+				disk.sizeGb = int64(image.DataImageSpecs[0].DiskSizeGb)
+			}
+			if strings.HasPrefix(image.Identification.ResourceType, "aws_ami") {
+				for _, bd := range image.DataImageSpecs {
+					if bd != nil {
+						if strings.HasPrefix(bd.DeviceName, diskBlock["device_name"].(string)) {
+							diskType = bd.VolumeType
+						}
+					}
 				}
 			}
 		}
@@ -41,16 +54,37 @@ func getDisk(resourceAddress string, diskBlock map[string]interface{}, isBootDis
 
 	// Get Disk size
 	declaredSize := diskBlock["volume_size"]
+	if declaredSize == nil {
+		declaredSize = diskBlock["size"]
+	}
+	if declaredSize == nil && diskBlock["snapshot_id"] != nil {
+		snapshotId := diskBlock["snapshot_id"].(string)
+		snapshot := getAwsImage(trRefs, snapshotId)
+		declaredSize = snapshot.DataImageSpecs[0].DiskSizeGb
+	}
 	if declaredSize == nil {
 		if image != nil {
-			for _, bd := range image.DataImageSpecs {
-				if strings.HasPrefix(bd.DeviceName, "/dev/sda") {
-					disk.sizeGb = int64(bd.DiskSizeGb)
+			// Case of snapshot, no device name
+			if strings.HasPrefix(image.Identification.ResourceType, "aws_ebs_snapshot") {
+				disk.sizeGb = int64(image.DataImageSpecs[0].DiskSizeGb)
+			}
+			// Case of ami, we use device name, except for boot disk
+			if strings.HasPrefix(image.Identification.ResourceType, "aws_ami") {
+				searchedDeviceName := "/dev/sda1"
+				if !isBootDisk {
+					searchedDeviceName = diskBlock["device_name"].(string)
+				}
+				for _, bd := range image.DataImageSpecs {
+					if bd != nil {
+						if strings.HasPrefix(bd.DeviceName, searchedDeviceName) {
+							disk.sizeGb = int64(bd.DiskSizeGb)
+						}
+					}
 				}
 			}
 		} else {
 			disk.sizeGb = viper.GetInt64("provider.aws.disk.size")
-			log.Warningf("%v : Boot disk size not declared. Please set it! (otherwise we assume %vsgb) ", resourceAddress, disk.sizeGb)
+			log.Warningf("%v : Disk size not declared. Please set it! (otherwise we assume %vsgb) ", resourceAddress, disk.sizeGb)
 
 		}
 	} else {
@@ -66,3 +100,29 @@ func IsSSD(diskType string) bool {
 	}
 	return isSSD
 }
+
+func getEbsVolume(tfResource tfjson.StateResource, tfRefs *tfrefs.References) *resources.ComputeResourceSpecs {
+
+	// Get image if it comes from a snapshot
+	var image *resources.EbsDataResource
+	if tfResource.AttributeValues["snapshot_id"] != nil {
+		awsImageId := tfResource.AttributeValues["snapshot_id"].(string)
+		image = getAwsImage(tfRefs, awsImageId)
+	}
+
+	// Get disk specifications
+	disk := getDisk(tfResource.Address, tfResource.AttributeValues, false, image, tfRefs)
+	hddSize := decimal.Zero
+	ssdSize := decimal.Zero
+	if disk.isSSD {
+		ssdSize = decimal.NewFromInt(disk.sizeGb)
+	} else {
+		hddSize = decimal.NewFromInt(disk.sizeGb)
+	}
+	computeResourceSpecs := resources.ComputeResourceSpecs{
+		SsdStorage: ssdSize,
+		HddStorage: hddSize,
+	}
+
+	return &computeResourceSpecs
+}

internal/terraform/aws/EC2Instance.go

@@ -14,33 +14,30 @@ import (
 
 func getEC2Instance(
 	resource tfjson.StateResource,
-	tfRefs *tfrefs.References, groupZone interface{}) *resources.ComputeResourceSpecs {
+	tfRefs *tfrefs.References) *resources.ComputeResourceSpecs {
 
 	instanceType := resource.AttributeValues["instance_type"].(string)
 
 	awsInstanceType := aws.GetAWSInstanceType(instanceType)
 
 	var disks []disk
 
-	amiId := ""
+	awsImageId := ""
 	if resource.AttributeValues["ami"] != nil {
-		amiId = resource.AttributeValues["ami"].(string)
+		awsImageId = resource.AttributeValues["ami"].(string)
 	}
-
-	imageI := tfRefs.DataResources[amiId]
-
-	var image *resources.AmiDataResource
-	if imageI != nil {
-		i := imageI.(resources.AmiDataResource)
-		image = &i
+	if resource.AttributeValues["snapshot_id"] != nil {
+		awsImageId = resource.AttributeValues["snapshot_id"].(string)
 	}
 
+	image := getAwsImage(tfRefs, awsImageId)
+
 	// Root block device
 	bd, ok_rd := resource.AttributeValues["root_block_device"]
 	if ok_rd {
 		rootDevices := bd.([]interface{})
 		for _, rootDevice := range rootDevices {
-			rootDisk := getDisk(resource.Address, rootDevice.(map[string]interface{}), true, image)
+			rootDisk := getDisk(resource.Address, rootDevice.(map[string]interface{}), true, image, tfRefs)
 			disks = append(disks, rootDisk)
 		}
 	} else {
@@ -66,7 +63,7 @@ func getEC2Instance(
 	if ok_ebd {
 		ebds := bd.([]interface{})
 		for _, blockDevice := range ebds {
-			blockDisk := getDisk(resource.Address, blockDevice.(map[string]interface{}), false, image)
+			blockDisk := getDisk(resource.Address, blockDevice.(map[string]interface{}), false, image, tfRefs)
 			disks = append(disks, blockDisk)
 		}
 	}

internal/terraform/aws/resources.go

@@ -12,7 +12,14 @@ func GetResource(
 
 	resourceId := getResourceIdentification(tfResource, tfRefs)
 	if resourceId.ResourceType == "aws_instance" {
-		specs := getEC2Instance(tfResource, tfRefs, nil)
+		specs := getEC2Instance(tfResource, tfRefs)
+		return resources.ComputeResource{
+			Identification: resourceId,
+			Specs:          specs,
+		}
+	}
+	if resourceId.ResourceType == "aws_ebs_volume" {
+		specs := getEbsVolume(tfResource, tfRefs)
 		return resources.ComputeResource{
 			Identification: resourceId,
 			Specs:          specs,

internal/terraform/aws/resources_test.go

@@ -96,6 +96,24 @@ var machineWithEBSSizeAndEphemeral tfjson.StateResource = tfjson.StateResource{
 	},
 }
 
+var machineWithEbsFromSnapshotSizeSpecified tfjson.StateResource = tfjson.StateResource{
+	Address: "aws_instance.foo",
+	Type:    "aws_instance",
+	Name:    "machineWithEbsFromSnapshotSizeNotSpecified",
+	AttributeValues: map[string]interface{}{
+		"name":          "machineWithEbsFromSnapshotSizeNotSpecified",
+		"instance_type": "t2.micro",
+		"ebs_block_device": []interface{}{
+			map[string]interface{}{
+				"delete_on_termination": true,
+				"snapshot_id":           "snap-1234567890",
+				"volume_type":           "st1",
+				"volume_size":           float64(50),
+			},
+		},
+	},
+}
+
 var tfRefs *tfrefs.References = &tfrefs.References{
 	ProviderConfigs: map[string]string{
 		"region": "eu-west-3",
@@ -227,6 +245,29 @@ func TestGetResource(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "aws_instance with ebs from snapshot size specified",
+			args: args{
+				tfResource: machineWithEbsFromSnapshotSizeSpecified,
+				tfRefs:     tfRefs,
+			},
+			want: resources.ComputeResource{
+				Identification: &resources.ResourceIdentification{
+					Name:         "machineWithEbsFromSnapshotSizeNotSpecified",
+					ResourceType: "aws_instance",
+					Provider:     providers.AWS,
+					Region:       "eu-west-3",
+					Count:        1,
+				},
+				Specs: &resources.ComputeResourceSpecs{
+					VCPUs:             int32(1),
+					MemoryMb:          int32(1024),
+					ReplicationFactor: 1,
+					HddStorage:        decimal.NewFromInt(50),
+					SsdStorage:        decimal.NewFromInt(8),
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {