feat: Added models for EC2 dashboard (#646)

* feat: Added models for ec2 dashboard

* feat: Added models for ec2 dashboard - CUR2

transformations/aws/cost/dbt_project.yml

@@ -246,3 +246,9 @@ models:
       +enabled: true
     aws_usage__data_usage_capacity:
       +enabled: true
+    aws_cost__ec2_cost_breakdown_cur_2:
+      +enabled: false
+    aws_cost__ec2_cost_by_region_cur_2:
+      +enabled: false
+    aws_cost__ec2_instance_utilization_cost_cur_2:
+      +enabled: false

transformations/aws/cost/models/ec2/aws_cost__ec2_cost_breakdown.sql

@@ -0,0 +1,19 @@
+with ec2_cost as (
+
+select 
+'arn:aws:ec2:' || product_region::text || ':' || line_item_usage_account_id::text || ':instance/' || line_item_resource_id::text AS arn,
+product_region as region,
+SUM(line_item_unblended_cost) AS cost
+from {{ var('cost_usage_table') }}
+where line_item_product_code = 'AmazonEC2'
+and line_item_resource_id like 'i-%'
+group by arn, region
+
+)
+select 
+ec.arn,
+instance_type,
+ec.region,
+ec.cost
+from ec2_cost as ec
+join aws_ec2_instances as ei on ei.arn = ec.arn

transformations/aws/cost/models/ec2/aws_cost__ec2_cost_breakdown_cur_2.sql

@@ -0,0 +1,19 @@
+with ec2_cost as (
+
+select 
+'arn:aws:ec2:' || product_region_code::text || ':' || line_item_usage_account_id::text || ':instance/' || line_item_resource_id::text AS arn,
+product_region_code as region,
+SUM(line_item_unblended_cost) AS cost
+from {{ var('cost_usage_table') }}
+where line_item_product_code = 'AmazonEC2'
+and line_item_resource_id like 'i-%'
+group by arn, region
+
+)
+select 
+ec.arn,
+instance_type,
+ec.region,
+ec.cost
+from ec2_cost as ec
+join aws_ec2_instances as ei on ei.arn = ec.arn

transformations/aws/cost/models/ec2/aws_cost__ec2_cost_by_instance_type.sql

@@ -0,0 +1,18 @@
+SELECT
+  ec2.instance_type,
+  SUM(cu.line_item_unblended_cost) AS cost
+FROM
+  {{ var('cost_usage_table') }} AS cu
+JOIN
+  aws_ec2_instances AS ec2
+ON
+  cu.line_item_resource_id = ec2.instance_id
+WHERE
+  cu.line_item_line_item_type = 'Usage'
+  AND line_item_product_code = 'AmazonEC2'
+GROUP BY
+  ec2.instance_type
+HAVING
+  SUM(cu.line_item_unblended_cost) > 0
+ORDER BY
+  SUM(cu.line_item_unblended_cost) DESC

transformations/aws/cost/models/ec2/aws_cost__ec2_cost_by_region.sql

@@ -0,0 +1,8 @@
+SELECT product_region, SUM(line_item_unblended_cost) as cost
+FROM {{ var('cost_usage_table') }}
+where line_item_line_item_type = 'Usage'
+AND line_item_product_code = 'AmazonEC2'
+AND line_item_resource_id like 'i-%'
+GROUP BY product_region
+HAVING SUM(line_item_unblended_cost) > 0
+ORDER BY SUM(line_item_unblended_cost) DESC

transformations/aws/cost/models/ec2/aws_cost__ec2_cost_by_region_cur_2.sql

@@ -0,0 +1,8 @@
+SELECT product_region_code, SUM(line_item_unblended_cost) as cost
+FROM {{ var('cost_usage_table') }}
+where line_item_line_item_type = 'Usage'
+AND line_item_product_code = 'AmazonEC2'
+AND line_item_resource_id like 'i-%'
+GROUP BY product_region_code
+HAVING SUM(line_item_unblended_cost) > 0
+ORDER BY SUM(line_item_unblended_cost) DESC

transformations/aws/cost/models/ec2/aws_cost__ec2_cost_over_time.sql

@@ -0,0 +1,11 @@
+SELECT
+  DATE(line_item_usage_end_date) AS usage_date,
+  SUM(line_item_unblended_cost) AS total_cost
+FROM {{ var('cost_usage_table') }}
+WHERE
+  line_item_product_code = 'AmazonEC2'
+  AND
+  line_item_resource_id like 'i-%'
+GROUP BY DATE(line_item_usage_end_date)
+HAVING SUM(line_item_unblended_cost) > 0
+ORDER BY usage_date ASC

transformations/aws/cost/models/ec2/aws_cost__ec2_instance_count.sql

@@ -0,0 +1,2 @@
+SELECT COUNT(*) AS ec2_instance_count
+FROM aws_ec2_instances

transformations/aws/cost/models/ec2/aws_cost__ec2_instance_status.sql

@@ -0,0 +1,7 @@
+SELECT 
+    instance_status->>'Status' AS status,
+    COUNT(*) AS count
+FROM 
+    aws_ec2_instance_statuses
+GROUP BY 
+    instance_status->>'Status'

transformations/aws/cost/models/ec2/aws_cost__ec2_instance_utilization_cost.sql

@@ -0,0 +1,56 @@
+WITH ec2_instance_resource_utilization AS (
+    SELECT
+        'arn:aws:ec2:' || cws.region::text || ':' || cws.account_id::text || ':instance/' || (elem.value ->> 'Value')::text AS arn,
+        elem.value ->> 'Value' AS instance_id,
+        cws.region,
+        cws.label,
+        AVG(cws.average) AS mean_usage
+    FROM
+        aws_cloudwatch_metric_statistics cws,
+        jsonb_array_elements(cws.input_json -> 'Dimensions') AS elem
+    WHERE
+        cws.label = 'CPUUtilization'
+        AND cws.input_json ->> 'Namespace' = 'AWS/EC2'
+        AND elem ->> 'Name' = 'InstanceId'
+    GROUP BY
+        1, 2, 3, 4
+),
+cost_by_region_resource AS (
+    SELECT
+        product_region,
+        line_item_resource_id,
+        SUM(line_item_unblended_cost) AS cost
+    FROM
+        {{ var('cost_usage_table') }}
+    WHERE
+        line_item_resource_id != ''
+        AND line_item_product_code = 'AmazonEC2'
+    GROUP BY
+        1, 2
+    ORDER BY
+        cost DESC
+)
+SELECT
+    cw_usage.arn,
+    ec2.instance_type,
+    cw_usage.mean_usage as mean_usage,
+    cost.cost
+FROM
+    ec2_instance_resource_utilization cw_usage
+    LEFT JOIN aws_ec2_instances ec2 ON (
+        (
+            cw_usage.instance_id = ec2.instance_id
+            and cw_usage.region = ec2.region
+        )
+        or cw_usage.arn = ec2.arn
+    )
+    LEFT JOIN cost_by_region_resource cost ON (
+        (
+            cw_usage.instance_id = cost.line_item_resource_id
+            and cw_usage.region = cost.product_region
+        )
+        or cw_usage.arn = cost.line_item_resource_id
+    )
+WHERE
+    cw_usage.label = 'CPUUtilization'
+    and cost > 0 

transformations/aws/cost/models/ec2/aws_cost__ec2_instance_utilization_cost_cur_2.sql

@@ -0,0 +1,56 @@
+WITH ec2_instance_resource_utilization AS (
+    SELECT
+        'arn:aws:ec2:' || cws.region::text || ':' || cws.account_id::text || ':instance/' || (elem.value ->> 'Value')::text AS arn,
+        elem.value ->> 'Value' AS instance_id,
+        cws.region,
+        cws.label,
+        AVG(cws.average) AS mean_usage
+    FROM
+        aws_cloudwatch_metric_statistics cws,
+        jsonb_array_elements(cws.input_json -> 'Dimensions') AS elem
+    WHERE
+        cws.label = 'CPUUtilization'
+        AND cws.input_json ->> 'Namespace' = 'AWS/EC2'
+        AND elem ->> 'Name' = 'InstanceId'
+    GROUP BY
+        1, 2, 3, 4
+),
+cost_by_region_resource AS (
+    SELECT
+        product_region_code,
+        line_item_resource_id,
+        SUM(line_item_unblended_cost) AS cost
+    FROM
+        {{ var('cost_usage_table') }}
+    WHERE
+        line_item_resource_id != ''
+        AND line_item_product_code = 'AmazonEC2'
+    GROUP BY
+        1, 2
+    ORDER BY
+        cost DESC
+)
+SELECT
+    cw_usage.arn,
+    ec2.instance_type,
+    cw_usage.mean_usage as mean_usage,
+    cost.cost
+FROM
+    ec2_instance_resource_utilization cw_usage
+    LEFT JOIN aws_ec2_instances ec2 ON (
+        (
+            cw_usage.instance_id = ec2.instance_id
+            and cw_usage.region = ec2.region
+        )
+        or cw_usage.arn = ec2.arn
+    )
+    LEFT JOIN cost_by_region_resource cost ON (
+        (
+            cw_usage.instance_id = cost.line_item_resource_id
+            and cw_usage.region = cost.product_region_code
+        )
+        or cw_usage.arn = cost.line_item_resource_id
+    )
+WHERE
+    cw_usage.label = 'CPUUtilization'
+    and cost > 0 

transformations/aws/cost/models/ec2/aws_cost__ec2_overall_cost.sql

@@ -0,0 +1,9 @@
+SELECT
+  SUM(line_item_unblended_cost) AS total_cost
+FROM {{ var('cost_usage_table') }}
+WHERE
+  line_item_product_code = 'AmazonEC2'
+  AND
+  line_item_resource_id like 'i-%'
+group by 
+line_item_resource_id