[7.17](backport #31970) [Azure Billing] Upgrade Usage Details API to version 2019-10-01

CHANGELOG.next.asciidoc

@@ -91,6 +91,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Metricbeat*
 
+- Azure Billing: upgrade Usage Details API to version 2019-10-01 {pull}31970[31970]
 
 *Packetbeat*
 

NOTICE.txt

@@ -862,12 +862,12 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-event-hubs
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/Azure/azure-sdk-for-go
-Version: v59.0.0+incompatible
+Dependency : github.com/elastic/azure-sdk-for-go
+Version: v59.0.0-elastic-1+incompatible
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-sdk-for-go@v59.0.0+incompatible/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/azure-sdk-for-go@v59.0.0-elastic-1+incompatible/LICENSE.txt:
 
 The MIT License (MIT)
 

go.mod

@@ -294,6 +294,7 @@ require (
 )
 
 replace (
+	github.com/Azure/azure-sdk-for-go => github.com/elastic/azure-sdk-for-go v59.0.0-elastic-1+incompatible
 	github.com/Microsoft/go-winio => github.com/bi-zone/go-winio v0.4.15
 	github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20210823122811-11c3ef800752
 	github.com/apoydence/eachers => github.com/poy/eachers v0.0.0-20181020210610-23942921fe77 //indirect, see https://github.com/elastic/beats/pull/29780 for details.

go.sum

@@ -72,9 +72,6 @@ github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9a
 github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
 github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo=
 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
-github.com/Azure/azure-sdk-for-go v51.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/azure-sdk-for-go v59.0.0+incompatible h1:I1ULJqny1qQhUBFy11yDXHhW3pLvbhwV0PTn7mjp9V0=
-github.com/Azure/azure-sdk-for-go v59.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
 github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o=
 github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0=
@@ -322,6 +319,8 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
+github.com/elastic/azure-sdk-for-go v59.0.0-elastic-1+incompatible h1:jlUO91EFZuvAO+2Zg+WdV0iTWe/x1X8maTxdYIKCWu4=
+github.com/elastic/azure-sdk-for-go v59.0.0-elastic-1+incompatible/go.mod h1:4zuQekLQi489ShcqTmS1Zj1ta0qrcNBlSuGa+ziu2vM=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/ecs v1.12.0 h1:u6WZ2AWtxv5vHvTQ4EuVZdWZ51mKHQ2UIltRePcta5U=

metricbeat/docs/fields.asciidoc

@@ -5693,11 +5693,6 @@ azure module
 
 
 
-[float]
-=== azure
-
-
-
 
 *`azure.timegrain`*::
 +
@@ -5772,6 +5767,16 @@ type: keyword
 The subscription ID
 
 
+type: keyword
+
+--
+
+*`azure.subscription_name`*::
++
+--
+The subscription name
+
+
 type: keyword
 
 --
@@ -6060,7 +6065,7 @@ billing and usage details
 *`azure.billing.currency`*::
 +
 --
-The currency
+Billing Currency.
 
 
 type: keyword
@@ -6070,7 +6075,27 @@ type: keyword
 *`azure.billing.pretax_cost`*::
 +
 --
-Cost
+The amount of cost before tax.
+
+
+type: float
+
+--
+
+*`azure.billing.unit_price`*::
++
+--
+Unit Price is the price applicable to you. (your EA or other contract price).
+
+
+type: float
+
+--
+
+*`azure.billing.quantity`*::
++
+--
+Measure the quantity purchased or consumed. The amount of the meter used during the billing period.
 
 
 type: float
@@ -6090,7 +6115,7 @@ type: keyword
 *`azure.billing.product`*::
 +
 --
-The product type
+Product name for the consumed service or purchase.
 
 
 type: keyword
@@ -6120,7 +6145,7 @@ type: date
 *`azure.billing.billing_period_id`*::
 +
 --
-The billing period id
+The billing period id.
 
 
 type: keyword
@@ -6130,7 +6155,17 @@ type: keyword
 *`azure.billing.account_name`*::
 +
 --
-The billing account name
+Name of the Billing Account.
+
+
+type: keyword
+
+--
+
+*`azure.billing.account_id`*::
++
+--
+Billing Account identifier.
 
 
 type: keyword

x-pack/metricbeat/module/azure/_meta/fields.yml

@@ -6,7 +6,6 @@
   fields:
     - name: azure
       type: group
-      description: >
       fields:
         - name: timegrain
           type: keyword
@@ -43,6 +42,10 @@
           type: keyword
           description: >
             The subscription ID
+        - name: subscription_name
+          type: keyword
+          description: >
+            The subscription name
         - name: application_id
           type: keyword
           description: >

x-pack/metricbeat/module/azure/billing/_meta/fields.yml

@@ -7,19 +7,27 @@
     - name: currency
       type: keyword
       description: >
-        The currency
+        Billing Currency.
     - name: pretax_cost
       type: float
       description: >
-        Cost
+        The amount of cost before tax.
+    - name: unit_price
+      type: float
+      description: >
+        Unit Price is the price applicable to you. (your EA or other contract price).
+    - name: quantity
+      type: float
+      description: >
+        Measure the quantity purchased or consumed. The amount of the meter used during the billing period.
     - name: department_name
       type: keyword
       description: >
         The department name
     - name: product
       type: keyword
       description: >
-        The product type
+        Product name for the consumed service or purchase.
     - name: usage_start
       type: date
       description: >
@@ -31,11 +39,15 @@
     - name: billing_period_id
       type: keyword
       description: >
-        The billing period id
+        The billing period id.
     - name: account_name
       type: keyword
       description: >
-        The billing account name
+        Name of the Billing Account.
+    - name: account_id
+      type: keyword
+      description: >
+        Billing Account identifier.
     - name: actual_cost
       type: float
       description: >

x-pack/metricbeat/module/azure/billing/billing.go

@@ -5,13 +5,14 @@
 package billing
 
 import (
-	"github.com/pkg/errors"
-
-	"github.com/elastic/beats/v7/x-pack/metricbeat/module/azure"
+	"fmt"
+	"time"
 
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/metricbeat/mb/parse"
+
+	"github.com/elastic/beats/v7/x-pack/metricbeat/module/azure"
 )
 
 // init registers the MetricSet with the central registry as soon as the program
@@ -24,7 +25,7 @@ func init() {
 
 // MetricSet holds any configuration or state information. It must implement
 // the mb.MetricSet interface. And this is best achieved by embedding
-// mb.BaseMetricSet because it implements all of the required mb.MetricSet
+// mb.BaseMetricSet because it implements all the required mb.MetricSet
 // interface methods except for Fetch.
 type MetricSet struct {
 	mb.BaseMetricSet
@@ -38,31 +39,45 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	var config azure.Config
 	err := base.Module().UnpackConfig(&config)
 	if err != nil {
-		return nil, errors.Wrap(err, "error unpack raw module config using UnpackConfig")
+		return nil, fmt.Errorf("error unpack raw module config using UnpackConfig: %w", err)
 	}
 	if err != nil {
 		return nil, err
 	}
 	// instantiate monitor client
 	billingClient, err := NewClient(config)
 	if err != nil {
-		return nil, errors.Wrap(err, "error initializing the billing client: module azure - billing metricset")
+		return nil, fmt.Errorf("error initializing the billing client: module azure - billing metricset: %w", err)
 	}
 	return &MetricSet{
 		BaseMetricSet: base,
 		client:        billingClient,
+		log:           logp.NewLogger("azure billing"),
 	}, nil
 }
 
 // Fetch methods implements the data gathering and data conversion to the right metricset
 // It publishes the event which is then forwarded to the output. In case
 // of an error set the Error field of mb.Event or simply call report.Error().
 func (m *MetricSet) Fetch(report mb.ReporterV2) error {
-	results, err := m.client.GetMetrics()
+	// The time interval is yesterday (00:00:00->23:59:59) in UTC.
+	startTime, endTime := previousDayFrom(time.Now())
+
+	m.log.
+		With("billing.start_time", startTime).
+		With("billing.end_time", endTime).
+		Infow("Fetching billing data")
+
+	results, err := m.client.GetMetrics(startTime, endTime)
 	if err != nil {
-		return errors.Wrap(err, "error retrieving usage information")
+		return fmt.Errorf("error retrieving usage information: %w", err)
 	}
-	events := EventsMapping(m.client.Config.SubscriptionId, results)
+
+	events, err := EventsMapping(m.client.Config.SubscriptionId, results, startTime, endTime)
+	if err != nil {
+		return fmt.Errorf("error mapping events: %w", err)
+	}
+
 	for _, event := range events {
 		isOpen := report.Event(event)
 		if !isOpen {
@@ -72,3 +87,10 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 
 	return nil
 }
+
+// previousDayFrom returns the start/end times (00:00:00->23:59:59 UTC) of the day before, given the `reference` time.
+func previousDayFrom(reference time.Time) (time.Time, time.Time) {
+	startTime := reference.UTC().Truncate(24 * time.Hour).Add((-24) * time.Hour)
+	endTime := startTime.Add(time.Hour * 24).Add(time.Second * (-1))
+	return startTime, endTime
+}

x-pack/metricbeat/module/azure/billing/billing_test.go

@@ -0,0 +1,28 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package billing
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPreviousDayFrom(t *testing.T) {
+	t.Run("returns the previous day as time interval to collect metrics", func(t *testing.T) {
+		referenceTime, err := time.Parse("2006-01-02 15:04:05", "2007-01-09 09:41:00")
+		assert.NoError(t, err)
+		expectedStartTime, err := time.Parse("2006-01-02 15:04:05", "2007-01-08 00:00:00")
+		assert.NoError(t, err)
+		expectedEndTime, err := time.Parse("2006-01-02 15:04:05", "2007-01-08 23:59:59")
+		assert.NoError(t, err)
+
+		actualStartTime, actualEndTime := previousDayFrom(referenceTime)
+
+		assert.Equal(t, expectedStartTime, actualStartTime)
+		assert.Equal(t, expectedEndTime, actualEndTime)
+	})
+}