diff --git a/README.md b/README.md index 9c76f80..4749867 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,11 @@ It can estimate Carbon Emissions of: - [x] Instance Group (including regional and Autoscaler) - Amazon Web Services - [x] EC2 (including inline root, elastic, and ephemeral block storages) + - [x] EBS Volumes The following will also be supported soon: - Amazon Web Services - - [ ] EBS - [ ] RDS - [ ] AutoScaling Group - Azure diff --git a/doc/scope.md b/doc/scope.md index 1971b91..a3a77d9 100644 --- a/doc/scope.md +++ b/doc/scope.md @@ -41,13 +41,15 @@ Data resources: | Resource | Limitations | Comment | |---|---|---| -| `aws_instance`| Only inline EBS (not attached), no GPU | | +| `aws_instance`| No GPU | | +| `aws_ebs_volume`| if size set, or if snapshot declared as data resource | | Data resources: | Resource | Limitations | Comment | |---|---|---| -| `aws_ami`| `ebs.volume_size` can be set, otherwise get it from image only if GCP credentials are provided| | +| `aws_ami`| `ebs.volume_size` can be set, otherwise get it from image only if AWS credentials are provided| | +| `aws_ebs_snapshot`| `volume_size` can be set, otherwise get it from image only if AWS credentials are provided| | _more to be implemented_ diff --git a/internal/resources/aws_ami.go b/internal/resources/aws_ebs.go similarity index 53% rename from internal/resources/aws_ami.go rename to internal/resources/aws_ebs.go index cc0cfd1..df3308f 100644 --- a/internal/resources/aws_ami.go +++ b/internal/resources/aws_ebs.go @@ -4,20 +4,20 @@ import ( "fmt" ) -type AmiDataResource struct { +type EbsDataResource struct { Identification *ResourceIdentification DataImageSpecs []*DataImageSpecs - AmiId string + AwsId string } -func (r AmiDataResource) GetIdentification() *ResourceIdentification { +func (r EbsDataResource) GetIdentification() *ResourceIdentification { return r.Identification } -func (r AmiDataResource) GetAddress() string { +func (r EbsDataResource) GetAddress() string { return fmt.Sprintf("data.%v.%v", r.GetIdentification().ResourceType, r.GetIdentification().Name) } -func (r AmiDataResource) GetKey() string { - return r.AmiId +func (r EbsDataResource) GetKey() string { + return r.AwsId } diff --git a/internal/terraform/aws/Data.go b/internal/terraform/aws/Data.go index 23d49d9..8a7d4a7 100644 --- a/internal/terraform/aws/Data.go +++ b/internal/terraform/aws/Data.go @@ -5,6 +5,7 @@ import ( "github.com/carboniferio/carbonifer/internal/providers" "github.com/carboniferio/carbonifer/internal/resources" + "github.com/carboniferio/carbonifer/internal/terraform/tfrefs" tfjson "github.com/hashicorp/terraform-json" ) @@ -31,13 +32,27 @@ func GetDataResource(tfResource tfjson.StateResource) resources.DataResource { specs[i] = &diskSpecs } } - return resources.AmiDataResource{ + return resources.EbsDataResource{ Identification: resourceId, DataImageSpecs: specs, - AmiId: tfResource.AttributeValues["id"].(string), + AwsId: tfResource.AttributeValues["id"].(string), } } } + if resourceId.ResourceType == "aws_ebs_snapshot" { + diskSize := tfResource.AttributeValues["volume_size"] + diskSizeGb := diskSize.(float64) + return resources.EbsDataResource{ + Identification: resourceId, + DataImageSpecs: []*resources.DataImageSpecs{ + { + DiskSizeGb: diskSizeGb, + }, + }, + AwsId: tfResource.AttributeValues["id"].(string), + } + } + return resources.DataImageResource{ Identification: resourceId, } @@ -51,3 +66,14 @@ func getDataResourceIdentification(resource tfjson.StateResource) *resources.Res Provider: providers.AWS, } } + +func getAwsImage(tfRefs *tfrefs.References, awsImageId string) *resources.EbsDataResource { + imageI := tfRefs.DataResources[awsImageId] + + var image *resources.EbsDataResource + if imageI != nil { + i := imageI.(resources.EbsDataResource) + image = &i + } + return image +} diff --git a/internal/terraform/aws/Data_test.go b/internal/terraform/aws/Data_test.go index eafd757..5fa1ca1 100644 --- a/internal/terraform/aws/Data_test.go +++ b/internal/terraform/aws/Data_test.go @@ -40,7 +40,7 @@ func TestGetDataResource(t *testing.T) { }, }, }, - want: resources.AmiDataResource{ + want: resources.EbsDataResource{ Identification: &resources.ResourceIdentification{ Name: "foo", ResourceType: "aws_ami", @@ -53,7 +53,34 @@ func TestGetDataResource(t *testing.T) { VolumeType: "gp2", }, }, - AmiId: "ami-1234567890", + AwsId: "ami-1234567890", + }, + }, + { + name: "Snapshot of size 60 Gb", + args: args{ + tfResource: tfjson.StateResource{ + Address: "data.aws_ebs_snapshot.test_snapshot", + Type: "aws_ebs_snapshot", + Name: "test_snapshot", + AttributeValues: map[string]interface{}{ + "id": "snap-1234567890", + "volume_size": float64(60), + }, + }, + }, + want: resources.EbsDataResource{ + Identification: &resources.ResourceIdentification{ + Name: "test_snapshot", + ResourceType: "aws_ebs_snapshot", + Provider: providers.AWS, + }, + DataImageSpecs: []*resources.DataImageSpecs{ + { + DiskSizeGb: 60, + }, + }, + AwsId: "snap-1234567890", }, }, } diff --git a/internal/terraform/aws/EBS.go b/internal/terraform/aws/EBS.go index 9250714..7763ed1 100644 --- a/internal/terraform/aws/EBS.go +++ b/internal/terraform/aws/EBS.go @@ -4,6 +4,9 @@ import ( "strings" "github.com/carboniferio/carbonifer/internal/resources" + "github.com/carboniferio/carbonifer/internal/terraform/tfrefs" + tfjson "github.com/hashicorp/terraform-json" + "github.com/shopspring/decimal" log "github.com/sirupsen/logrus" "github.com/spf13/viper" ) @@ -14,7 +17,7 @@ type disk struct { replicationFactor int32 } -func getDisk(resourceAddress string, diskBlock map[string]interface{}, isBootDisk bool, image *resources.AmiDataResource) disk { +func getDisk(resourceAddress string, diskBlock map[string]interface{}, isBootDisk bool, image *resources.EbsDataResource, trRefs *tfrefs.References) disk { disk := disk{ sizeGb: viper.GetInt64("provider.aws.boot_disk.size"), isSSD: true, @@ -25,13 +28,23 @@ func getDisk(resourceAddress string, diskBlock map[string]interface{}, isBootDis diskType := viper.GetString("provider.aws.disk.type") diskTypeI := diskBlock["volume_type"] + if diskTypeI == nil { + diskTypeI = diskBlock["type"] + } if diskTypeI != nil { diskType = diskTypeI.(string) } else { if image != nil && diskBlock["device_name"] != nil { - for _, bd := range image.DataImageSpecs { - if strings.HasPrefix(bd.DeviceName, diskBlock["device_name"].(string)) { - diskType = bd.VolumeType + if strings.HasPrefix(image.Identification.ResourceType, "aws_ebs_snapshot") { + disk.sizeGb = int64(image.DataImageSpecs[0].DiskSizeGb) + } + if strings.HasPrefix(image.Identification.ResourceType, "aws_ami") { + for _, bd := range image.DataImageSpecs { + if bd != nil { + if strings.HasPrefix(bd.DeviceName, diskBlock["device_name"].(string)) { + diskType = bd.VolumeType + } + } } } } @@ -41,16 +54,37 @@ func getDisk(resourceAddress string, diskBlock map[string]interface{}, isBootDis // Get Disk size declaredSize := diskBlock["volume_size"] + if declaredSize == nil { + declaredSize = diskBlock["size"] + } + if declaredSize == nil && diskBlock["snapshot_id"] != nil { + snapshotId := diskBlock["snapshot_id"].(string) + snapshot := getAwsImage(trRefs, snapshotId) + declaredSize = snapshot.DataImageSpecs[0].DiskSizeGb + } if declaredSize == nil { if image != nil { - for _, bd := range image.DataImageSpecs { - if strings.HasPrefix(bd.DeviceName, "/dev/sda") { - disk.sizeGb = int64(bd.DiskSizeGb) + // Case of snapshot, no device name + if strings.HasPrefix(image.Identification.ResourceType, "aws_ebs_snapshot") { + disk.sizeGb = int64(image.DataImageSpecs[0].DiskSizeGb) + } + // Case of ami, we use device name, except for boot disk + if strings.HasPrefix(image.Identification.ResourceType, "aws_ami") { + searchedDeviceName := "/dev/sda1" + if !isBootDisk { + searchedDeviceName = diskBlock["device_name"].(string) + } + for _, bd := range image.DataImageSpecs { + if bd != nil { + if strings.HasPrefix(bd.DeviceName, searchedDeviceName) { + disk.sizeGb = int64(bd.DiskSizeGb) + } + } } } } else { disk.sizeGb = viper.GetInt64("provider.aws.disk.size") - log.Warningf("%v : Boot disk size not declared. Please set it! (otherwise we assume %vsgb) ", resourceAddress, disk.sizeGb) + log.Warningf("%v : Disk size not declared. Please set it! (otherwise we assume %vsgb) ", resourceAddress, disk.sizeGb) } } else { @@ -66,3 +100,29 @@ func IsSSD(diskType string) bool { } return isSSD } + +func getEbsVolume(tfResource tfjson.StateResource, tfRefs *tfrefs.References) *resources.ComputeResourceSpecs { + + // Get image if it comes from a snapshot + var image *resources.EbsDataResource + if tfResource.AttributeValues["snapshot_id"] != nil { + awsImageId := tfResource.AttributeValues["snapshot_id"].(string) + image = getAwsImage(tfRefs, awsImageId) + } + + // Get disk specifications + disk := getDisk(tfResource.Address, tfResource.AttributeValues, false, image, tfRefs) + hddSize := decimal.Zero + ssdSize := decimal.Zero + if disk.isSSD { + ssdSize = decimal.NewFromInt(disk.sizeGb) + } else { + hddSize = decimal.NewFromInt(disk.sizeGb) + } + computeResourceSpecs := resources.ComputeResourceSpecs{ + SsdStorage: ssdSize, + HddStorage: hddSize, + } + + return &computeResourceSpecs +} diff --git a/internal/terraform/aws/EC2Instance.go b/internal/terraform/aws/EC2Instance.go index a41bec0..80cbeeb 100644 --- a/internal/terraform/aws/EC2Instance.go +++ b/internal/terraform/aws/EC2Instance.go @@ -14,7 +14,7 @@ import ( func getEC2Instance( resource tfjson.StateResource, - tfRefs *tfrefs.References, groupZone interface{}) *resources.ComputeResourceSpecs { + tfRefs *tfrefs.References) *resources.ComputeResourceSpecs { instanceType := resource.AttributeValues["instance_type"].(string) @@ -22,25 +22,22 @@ func getEC2Instance( var disks []disk - amiId := "" + awsImageId := "" if resource.AttributeValues["ami"] != nil { - amiId = resource.AttributeValues["ami"].(string) + awsImageId = resource.AttributeValues["ami"].(string) } - - imageI := tfRefs.DataResources[amiId] - - var image *resources.AmiDataResource - if imageI != nil { - i := imageI.(resources.AmiDataResource) - image = &i + if resource.AttributeValues["snapshot_id"] != nil { + awsImageId = resource.AttributeValues["snapshot_id"].(string) } + image := getAwsImage(tfRefs, awsImageId) + // Root block device bd, ok_rd := resource.AttributeValues["root_block_device"] if ok_rd { rootDevices := bd.([]interface{}) for _, rootDevice := range rootDevices { - rootDisk := getDisk(resource.Address, rootDevice.(map[string]interface{}), true, image) + rootDisk := getDisk(resource.Address, rootDevice.(map[string]interface{}), true, image, tfRefs) disks = append(disks, rootDisk) } } else { @@ -66,7 +63,7 @@ func getEC2Instance( if ok_ebd { ebds := bd.([]interface{}) for _, blockDevice := range ebds { - blockDisk := getDisk(resource.Address, blockDevice.(map[string]interface{}), false, image) + blockDisk := getDisk(resource.Address, blockDevice.(map[string]interface{}), false, image, tfRefs) disks = append(disks, blockDisk) } } diff --git a/internal/terraform/aws/resources.go b/internal/terraform/aws/resources.go index c5dc3b1..b240311 100644 --- a/internal/terraform/aws/resources.go +++ b/internal/terraform/aws/resources.go @@ -12,7 +12,14 @@ func GetResource( resourceId := getResourceIdentification(tfResource, tfRefs) if resourceId.ResourceType == "aws_instance" { - specs := getEC2Instance(tfResource, tfRefs, nil) + specs := getEC2Instance(tfResource, tfRefs) + return resources.ComputeResource{ + Identification: resourceId, + Specs: specs, + } + } + if resourceId.ResourceType == "aws_ebs_volume" { + specs := getEbsVolume(tfResource, tfRefs) return resources.ComputeResource{ Identification: resourceId, Specs: specs, diff --git a/internal/terraform/aws/resources_test.go b/internal/terraform/aws/resources_test.go index 7479d7b..5420070 100644 --- a/internal/terraform/aws/resources_test.go +++ b/internal/terraform/aws/resources_test.go @@ -96,6 +96,24 @@ var machineWithEBSSizeAndEphemeral tfjson.StateResource = tfjson.StateResource{ }, } +var machineWithEbsFromSnapshotSizeSpecified tfjson.StateResource = tfjson.StateResource{ + Address: "aws_instance.foo", + Type: "aws_instance", + Name: "machineWithEbsFromSnapshotSizeNotSpecified", + AttributeValues: map[string]interface{}{ + "name": "machineWithEbsFromSnapshotSizeNotSpecified", + "instance_type": "t2.micro", + "ebs_block_device": []interface{}{ + map[string]interface{}{ + "delete_on_termination": true, + "snapshot_id": "snap-1234567890", + "volume_type": "st1", + "volume_size": float64(50), + }, + }, + }, +} + var tfRefs *tfrefs.References = &tfrefs.References{ ProviderConfigs: map[string]string{ "region": "eu-west-3", @@ -227,6 +245,29 @@ func TestGetResource(t *testing.T) { }, }, }, + { + name: "aws_instance with ebs from snapshot size specified", + args: args{ + tfResource: machineWithEbsFromSnapshotSizeSpecified, + tfRefs: tfRefs, + }, + want: resources.ComputeResource{ + Identification: &resources.ResourceIdentification{ + Name: "machineWithEbsFromSnapshotSizeNotSpecified", + ResourceType: "aws_instance", + Provider: providers.AWS, + Region: "eu-west-3", + Count: 1, + }, + Specs: &resources.ComputeResourceSpecs{ + VCPUs: int32(1), + MemoryMb: int32(1024), + ReplicationFactor: 1, + HddStorage: decimal.NewFromInt(50), + SsdStorage: decimal.NewFromInt(8), + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/terraform/resources_aws_test.go b/internal/terraform/resources_aws_test.go index f23d4d8..e193dda 100644 --- a/internal/terraform/resources_aws_test.go +++ b/internal/terraform/resources_aws_test.go @@ -37,7 +37,20 @@ func TestGetResource_DiskFromAMI(t *testing.T) { MemoryMb: int32(8192), ReplicationFactor: 1, HddStorage: decimal.NewFromInt(20), - SsdStorage: decimal.NewFromInt(30), + SsdStorage: decimal.NewFromInt(90), + }, + }, + "aws_ebs_volume.ebs_volume": resources.ComputeResource{ + Identification: &resources.ResourceIdentification{ + Name: "ebs_volume", + ResourceType: "aws_ebs_volume", + Provider: providers.AWS, + Region: "eu-west-3", + Count: 1, + }, + Specs: &resources.ComputeResourceSpecs{ + HddStorage: decimal.Zero, + SsdStorage: decimal.NewFromInt(100), }, }, } @@ -51,5 +64,8 @@ func TestGetResource_DiskFromAMI(t *testing.T) { if res.GetIdentification().ResourceType == "aws_instance" { assert.Equal(t, wantResources["aws_instance.foo"], res) } + if res.GetIdentification().ResourceType == "aws_ebs_volume" { + assert.Equal(t, wantResources["aws_ebs_volume.ebs_volume"], res) + } } } diff --git a/test/terraform/aws_ec2/main.tf b/test/terraform/aws_ec2/main.tf index 1883945..0aba414 100644 --- a/test/terraform/aws_ec2/main.tf +++ b/test/terraform/aws_ec2/main.tf @@ -7,6 +7,33 @@ data "aws_ami" "ubuntu" { } } +data "aws_ebs_snapshot" "ebs_snapshot" { + most_recent = true + + filter { + name = "volume-size" + values = ["60"] + } +} + +data "aws_ebs_snapshot" "ebs_snapshot_bigger" { + most_recent = true + + filter { + name = "volume-size" + values = ["100"] + } +} + +resource "aws_ebs_volume" "ebs_volume" { + availability_zone = "eu-west-3a" + snapshot_id = data.aws_ebs_snapshot.ebs_snapshot_bigger.id + type = "gp2" + tags = { + Name = "ebs_volume" + } +} + resource "aws_instance" "foo" { ami = data.aws_ami.ubuntu.id instance_type = "m4.large" @@ -21,4 +48,15 @@ resource "aws_instance" "foo" { volume_size = 20 volume_type = "standard" } -} \ No newline at end of file + + ebs_block_device { + device_name = "/dev/sdj" + snapshot_id = data.aws_ebs_snapshot.ebs_snapshot.id + } +} + +resource "aws_volume_attachment" "ebs_att" { + device_name = "/dev/sdi" + volume_id = aws_ebs_volume.ebs_volume.id + instance_id = aws_instance.foo.id +}