[7.x](backport #28028) Azure billing metricset - update consumption API's

go.mod

@@ -11,10 +11,10 @@ require (
 	code.cloudfoundry.org/go-loggregator v7.4.0+incompatible
 	code.cloudfoundry.org/rfc5424 v0.0.0-20180905210152-236a6d29298a // indirect
 	github.com/Azure/azure-event-hubs-go/v3 v3.1.2
-	github.com/Azure/azure-sdk-for-go v37.1.0+incompatible
+	github.com/Azure/azure-sdk-for-go v57.0.0+incompatible
 	github.com/Azure/azure-storage-blob-go v0.8.0
 	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
-	github.com/Azure/go-autorest/autorest v0.11.12
+	github.com/Azure/go-autorest/autorest v0.11.16
 	github.com/Azure/go-autorest/autorest/adal v0.9.15
 	github.com/Azure/go-autorest/autorest/azure/auth v0.4.2
 	github.com/Azure/go-autorest/autorest/date v0.3.0
@@ -144,7 +144,6 @@ require (
 	github.com/samuel/go-parser v0.0.0-20130731160455-ca8abbf65d0e // indirect
 	github.com/samuel/go-thrift v0.0.0-20140522043831-2187045faa54
 	github.com/sanathkr/yaml v1.0.1-0.20170819201035-0056894fa522 // indirect
-	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/shirou/gopsutil v3.20.12+incompatible
 	github.com/shopspring/decimal v1.2.0
 	github.com/spf13/cobra v0.0.5

go.sum

@@ -58,8 +58,9 @@ github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9a
 github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
 github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo=
 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
-github.com/Azure/azure-sdk-for-go v37.1.0+incompatible h1:aFlw3lP7ZHQi4m1kWCpcwYtczhDkGhDoRaMTaxcOf68=
 github.com/Azure/azure-sdk-for-go v37.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-sdk-for-go v57.0.0+incompatible h1:isVki3PbIFrwKvKdVP1byxo73/pt+Nn174YxW1k4PNw=
+github.com/Azure/azure-sdk-for-go v57.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
 github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o=
 github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0=
@@ -71,8 +72,9 @@ github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0=
-github.com/Azure/go-autorest/autorest v0.11.12 h1:gI8ytXbxMfI+IVbI9mP2JGCTXIuhHLgRlvQ9X4PsnHE=
 github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
+github.com/Azure/go-autorest/autorest v0.11.16 h1:3jkFG3SL0fFXmvmPF9Kc8LscIbeXUhmt3yuzUSqv3pI=
+github.com/Azure/go-autorest/autorest v0.11.16/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
 github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
 github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
@@ -749,8 +751,6 @@ github.com/sanathkr/yaml v1.0.1-0.20170819201035-0056894fa522 h1:39BJIaZIhIBmXAT
 github.com/sanathkr/yaml v1.0.1-0.20170819201035-0056894fa522/go.mod h1:tQTYKOQgxoH3v6dEmdHiz4JG+nbxWwM5fgPQUpSZqVQ=
 github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis=
 github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4=
-github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
-github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=

x-pack/metricbeat/module/azure/billing/_meta/docs.asciidoc

@@ -4,5 +4,14 @@ This metricset allows users to retrieve usage details and forecast information o
 
 include::../../_meta/shared-azure.asciidoc[]
 
+[float]
+==== Config options to identify resources
+
+`billing_scope_department`:: (_string_) Retrieve usage details based on the department scope.
+
+`billing_scope_account_id`:: (_string_) Retrieve usage details based on the billing account ID scope.
+
+If none of the 2 options are entered then the subscription ID will be used as scope.
+
 
 

x-pack/metricbeat/module/azure/billing/billing.go

@@ -5,6 +5,8 @@
 package billing
 
 import (
+	"time"
+
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/x-pack/metricbeat/module/azure"
@@ -58,11 +60,13 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 // It publishes the event which is then forwarded to the output. In case
 // of an error set the Error field of mb.Event or simply call report.Error().
 func (m *MetricSet) Fetch(report mb.ReporterV2) error {
-	results, err := m.client.GetMetrics()
+	startTime := time.Now().UTC().Truncate(24 * time.Hour).Add((-48) * time.Hour)
+	endTime := startTime.Add(time.Hour * 24).Add(time.Second * (-1))
+	results, err := m.client.GetMetrics(startTime, endTime)
 	if err != nil {
 		return errors.Wrap(err, "error retrieving usage information")
 	}
-	events := EventsMapping(results)
+	events := EventsMapping(results, startTime, endTime, m.client.Config.SubscriptionId)
 	for _, event := range events {
 		isOpen := report.Event(event)
 		if !isOpen {

x-pack/metricbeat/module/azure/billing/client.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption"
+	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-10-01/consumption"
 
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
@@ -25,7 +25,7 @@ type Client struct {
 }
 
 type Usage struct {
-	UsageDetails  []consumption.UsageDetail
+	UsageDetails  []consumption.BasicUsageDetail
 	ActualCosts   []consumption.Forecast
 	ForecastCosts []consumption.Forecast
 }
@@ -45,13 +45,17 @@ func NewClient(config azure.Config) (*Client, error) {
 }
 
 // GetMetrics returns the usage detail and forecast values.
-func (client *Client) GetMetrics() (Usage, error) {
+func (client *Client) GetMetrics(startTime time.Time, endTime time.Time) (Usage, error) {
 	var usage Usage
-	startTime := time.Now().UTC().Truncate(24 * time.Hour).Add((-24) * time.Hour)
-	endTime := startTime.Add(time.Hour * 24).Add(time.Second * (-1))
-	usageDetails, err := client.BillingService.GetUsageDetails(fmt.Sprintf("subscriptions/%s", client.Config.SubscriptionId), "properties/meterDetails",
-		fmt.Sprintf("properties/usageStart eq '%s' and properties/usageEnd eq '%s'", startTime.Format(time.RFC3339Nano), endTime.Format(time.RFC3339Nano)),
-		"", nil, "properties/instanceLocation")
+	scope := fmt.Sprintf("subscriptions/%s", client.Config.SubscriptionId)
+	if client.Config.BillingScopeDepartment != "" {
+		scope = fmt.Sprintf("/providers/Microsoft.Billing/departments/%s", client.Config.BillingScopeDepartment)
+	} else if client.Config.BillingScopeAccountId != "" {
+		scope = fmt.Sprintf("/providers/Microsoft.Billing/billingAccounts/%s", client.Config.BillingScopeAccountId)
+	}
+
+	filter := fmt.Sprintf("properties/usageStart eq '%s' and properties/usageEnd eq '%s'", startTime.Format(time.RFC3339Nano), endTime.Format(time.RFC3339Nano))
+	usageDetails, err := client.BillingService.GetUsageDetails(scope, "properties/meterDetails", filter, "", nil, consumption.MetrictypeActualCostMetricType)
 	if err != nil {
 		return usage, errors.Wrap(err, "Retrieving usage details failed in client")
 	}

x-pack/metricbeat/module/azure/billing/client_test.go

@@ -7,8 +7,9 @@ package billing
 import (
 	"errors"
 	"testing"
+	"time"
 
-	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption"
+	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-10-01/consumption"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 
@@ -27,7 +28,9 @@ func TestClient(t *testing.T) {
 		m.On("GetForcast", mock.Anything).Return(consumption.ForecastsListResult{}, errors.New("invalid query"))
 		m.On("GetUsageDetails", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(consumption.UsageDetailsListResultPage{}, nil)
 		client.BillingService = m
-		results, err := client.GetMetrics()
+		startTime := time.Now().UTC().Truncate(24 * time.Hour).Add((-48) * time.Hour)
+		endTime := startTime.Add(time.Hour * 24).Add(time.Second * (-1))
+		results, err := client.GetMetrics(startTime, endTime)
 		assert.Error(t, err)
 		assert.Equal(t, len(results.ActualCosts), 0)
 		m.AssertExpectations(t)
@@ -40,7 +43,9 @@ func TestClient(t *testing.T) {
 		m.On("GetForcast", mock.Anything).Return(consumption.ForecastsListResult{Value: &forecasts}, nil)
 		m.On("GetUsageDetails", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(consumption.UsageDetailsListResultPage{}, nil)
 		client.BillingService = m
-		results, err := client.GetMetrics()
+		startTime := time.Now().UTC().Truncate(24 * time.Hour).Add((-48) * time.Hour)
+		endTime := startTime.Add(time.Hour * 24).Add(time.Second * (-1))
+		results, err := client.GetMetrics(startTime, endTime)
 		assert.NoError(t, err)
 		assert.Equal(t, len(results.ActualCosts), 2)
 		assert.Equal(t, len(results.ForecastCosts), 2)

x-pack/metricbeat/module/azure/billing/data.go

@@ -5,48 +5,81 @@
 package billing
 
 import (
-	"fmt"
-	"strings"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption"
+	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-10-01/consumption"
 
 	"github.com/shopspring/decimal"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
-func EventsMapping(results Usage) []mb.Event {
+func EventsMapping(results Usage, startTime time.Time, endTime time.Time, subscriptionId string) []mb.Event {
 	var events []mb.Event
+	// usage details come in different forms, most common for this api call is LegacyUsageDetail
 	if len(results.UsageDetails) > 0 {
-		for _, usageDetail := range results.UsageDetails {
-			event := mb.Event{
-				ModuleFields: common.MapStr{
+		for _, ud := range results.UsageDetails {
+			event := mb.Event{Timestamp: time.Now().UTC()}
+			if legacyUsageDetail, err := ud.AsLegacyUsageDetail(); err == true {
+				event.ModuleFields = common.MapStr{
 					"resource": common.MapStr{
-						"type":  usageDetail.ConsumedService,
-						"group": getResourceGroupFromId(*usageDetail.InstanceID),
-						"name":  usageDetail.InstanceName,
+						"type":  legacyUsageDetail.ConsumedService,
+						"group": legacyUsageDetail.ResourceGroup,
+						"name":  legacyUsageDetail.ResourceName,
 					},
-					"subscription_id": usageDetail.SubscriptionGUID,
-				},
-				MetricSetFields: common.MapStr{
-					"pretax_cost":       usageDetail.PretaxCost,
-					"department_name":   usageDetail.DepartmentName,
-					"product":           usageDetail.Product,
-					"usage_start":       usageDetail.UsageStart.ToTime(),
-					"usage_end":         usageDetail.UsageEnd.ToTime(),
-					"currency":          usageDetail.Currency,
-					"billing_period_id": usageDetail.BillingPeriodID,
-					"account_name":      usageDetail.AccountName,
-				},
-				Timestamp: time.Now().UTC(),
+					"subscription_id": legacyUsageDetail.SubscriptionID,
+				}
+				event.MetricSetFields = common.MapStr{
+					"pretax_cost":          legacyUsageDetail.Cost,
+					"department_name":      legacyUsageDetail.InvoiceSection,
+					"product":              legacyUsageDetail.Product,
+					"usage_start":          startTime,
+					"usage_end":            endTime,
+					"billing_period_start": legacyUsageDetail.BillingPeriodStartDate.ToTime(),
+					"billing_period_end":   legacyUsageDetail.BillingPeriodEndDate.ToTime(),
+					"currency":             legacyUsageDetail.BillingCurrency,
+					"effective_price":      legacyUsageDetail.EffectivePrice,
+					"account_name":         legacyUsageDetail.BillingAccountName,
+					"account_id":           legacyUsageDetail.BillingAccountID,
+					"subscription_name":    legacyUsageDetail.SubscriptionName,
+					"unit_price":           legacyUsageDetail.UnitPrice,
+					"quantity":             legacyUsageDetail.Quantity,
+				}
+				event.RootFields = common.MapStr{}
+				event.RootFields.Put("cloud.provider", "azure")
+				event.RootFields.Put("cloud.region", legacyUsageDetail.ResourceLocation)
+				event.RootFields.Put("cloud.instance.name", legacyUsageDetail.ResourceName)
+				event.RootFields.Put("cloud.instance.id", legacyUsageDetail.ResourceID)
+			}
+			if modernUsageDetail, err := ud.AsModernUsageDetail(); err == true {
+				event.ModuleFields = common.MapStr{
+					"resource": common.MapStr{
+						"type":  modernUsageDetail.ConsumedService,
+						"group": modernUsageDetail.ResourceGroup,
+						"name":  modernUsageDetail.InstanceName,
+					},
+					"subscription_id": modernUsageDetail.SubscriptionGUID,
+				}
+				event.MetricSetFields = common.MapStr{
+					"product":              modernUsageDetail.Product,
+					"usage_start":          startTime,
+					"usage_end":            endTime,
+					"billing_period_start": modernUsageDetail.BillingPeriodStartDate.ToTime(),
+					"billing_period_end":   modernUsageDetail.BillingPeriodEndDate.ToTime(),
+					"currency":             modernUsageDetail.BillingCurrencyCode,
+					"account_id":           modernUsageDetail.BillingAccountID,
+					"billing_account_name": modernUsageDetail.BillingAccountName,
+					"subscription_name":    modernUsageDetail.SubscriptionName,
+					"unit_price":           modernUsageDetail.UnitPrice,
+				}
+				event.RootFields = common.MapStr{}
+				event.RootFields.Put("cloud.provider", "azure")
+				event.RootFields.Put("cloud.region", modernUsageDetail.ResourceLocation)
+			}
+			if _, err := ud.AsUsageDetail(); err == true {
+				continue
 			}
-			event.RootFields = common.MapStr{}
-			event.RootFields.Put("cloud.provider", "azure")
-			event.RootFields.Put("cloud.region", usageDetail.InstanceLocation)
-			event.RootFields.Put("cloud.instance.name", usageDetail.InstanceName)
-			event.RootFields.Put("cloud.instance.id", usageDetail.InstanceID)
 			events = append(events, event)
 		}
 	}
@@ -76,6 +109,9 @@ func EventsMapping(results Usage) []mb.Event {
 			RootFields: common.MapStr{
 				"cloud.provider": "azure",
 			},
+			ModuleFields: common.MapStr{
+				"subscription_id": subscriptionId,
+			},
 			MetricSetFields: common.MapStr{
 				"actual_cost":   actualCost,
 				"forecast_cost": forecastCost,
@@ -89,14 +125,3 @@ func EventsMapping(results Usage) []mb.Event {
 	}
 	return events
 }
-
-// getResourceGroupFromId maps resource group from resource ID
-func getResourceGroupFromId(path string) string {
-	params := strings.Split(path, "/")
-	for i, param := range params {
-		if param == "resourceGroups" {
-			return fmt.Sprintf("%s", params[i+1])
-		}
-	}
-	return ""
-}

x-pack/metricbeat/module/azure/billing/data_test.go

@@ -8,15 +8,17 @@ import (
 	"testing"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption"
 	"github.com/Azure/go-autorest/autorest/date"
+
+	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-10-01/consumption"
 	"github.com/shopspring/decimal"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestEventMapping(t *testing.T) {
 	usageDate := "2020-08-08"
 	name := "test"
+	billingAccountId := "123"
 	startDate := date.Time{}
 
 	var charge decimal.Decimal = decimal.NewFromFloat(8.123456)
@@ -36,44 +38,37 @@ func TestEventMapping(t *testing.T) {
 		ChargeType:       "Actual",
 		ConfidenceLevels: nil,
 	}
-	var prop1 = consumption.UsageDetailProperties{
-		InstanceName:     &name,
-		SubscriptionName: &name,
-		AccountName:      &name,
-		DepartmentName:   &name,
-		Product:          &name,
-		InstanceID:       &name,
-		UsageStart:       &startDate,
-		UsageEnd:         &startDate,
+	var pros = consumption.LegacyUsageDetailProperties{
+		BillingAccountID:       &billingAccountId,
+		BillingAccountName:     &name,
+		BillingPeriodStartDate: &startDate,
+		BillingPeriodEndDate:   &startDate,
+		Cost:                   &charge,
+		InvoiceSection:         &name,
+		Product:                &name,
 	}
-	usage := Usage{
-		UsageDetails: []consumption.UsageDetail{
-			{
-				UsageDetailProperties: &prop1,
-				ID:                    nil,
-				Name:                  nil,
-				Type:                  nil,
-				Tags:                  nil,
-			},
-		},
+	var legacy = consumption.LegacyUsageDetail{
+		LegacyUsageDetailProperties: &pros,
+	}
+	var usage = Usage{UsageDetails: []consumption.BasicUsageDetail{legacy},
 		ActualCosts: []consumption.Forecast{
 			{
 				ForecastProperties: &prop2,
 				ID:                 nil,
 				Name:               nil,
 				Type:               nil,
 				Tags:               nil,
-			}},
-		ForecastCosts: []consumption.Forecast{
+			}}, ForecastCosts: []consumption.Forecast{
 			{
 				ForecastProperties: &prop,
 				ID:                 nil,
 				Name:               nil,
 				Type:               nil,
 				Tags:               nil,
-			}},
-	}
-	events := EventsMapping(usage)
+			}}}
+	startTime := time.Now().UTC().Truncate(24 * time.Hour).Add((-48) * time.Hour)
+	endTime := startTime.Add(time.Hour * 24).Add(time.Second * (-1))
+	events := EventsMapping(usage, startTime, endTime, "sub")
 	assert.Equal(t, len(events), 2)
 	for _, event := range events {