RUMM-3506 Collect the batch deleted telemetry

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/SdkFeature.kt

@@ -17,18 +17,23 @@ import com.datadog.android.api.net.RequestFactory
 import com.datadog.android.api.storage.EventBatchWriter
 import com.datadog.android.api.storage.FeatureStorageConfiguration
 import com.datadog.android.core.configuration.UploadFrequency
+import com.datadog.android.core.internal.configuration.DataUploadConfiguration
 import com.datadog.android.core.internal.data.upload.DataOkHttpUploader
 import com.datadog.android.core.internal.data.upload.NoOpUploadScheduler
 import com.datadog.android.core.internal.data.upload.UploadScheduler
 import com.datadog.android.core.internal.data.upload.v2.DataFlusher
 import com.datadog.android.core.internal.data.upload.v2.DataUploadScheduler
 import com.datadog.android.core.internal.data.upload.v2.DataUploader
 import com.datadog.android.core.internal.data.upload.v2.NoOpDataUploader
+import com.datadog.android.core.internal.metrics.BatchMetricsDispatcher
+import com.datadog.android.core.internal.metrics.MetricsDispatcher
+import com.datadog.android.core.internal.metrics.NoOpMetricsDispatcher
 import com.datadog.android.core.internal.persistence.ConsentAwareStorage
 import com.datadog.android.core.internal.persistence.NoOpStorage
 import com.datadog.android.core.internal.persistence.Storage
 import com.datadog.android.core.internal.persistence.file.FileMover
 import com.datadog.android.core.internal.persistence.file.FileOrchestrator
+import com.datadog.android.core.internal.persistence.file.FilePersistenceConfig
 import com.datadog.android.core.internal.persistence.file.FileReaderWriter
 import com.datadog.android.core.internal.persistence.file.NoOpFileOrchestrator
 import com.datadog.android.core.internal.persistence.file.advanced.FeatureFileOrchestrator
@@ -47,27 +52,48 @@ internal class SdkFeature(
 
     internal val initialized = AtomicBoolean(false)
     internal val eventReceiver = AtomicReference<FeatureEventReceiver>(null)
-
     internal var storage: Storage = NoOpStorage()
     internal var uploader: DataUploader = NoOpDataUploader()
     internal var uploadScheduler: UploadScheduler = NoOpUploadScheduler()
     internal var fileOrchestrator: FileOrchestrator = NoOpFileOrchestrator()
+    private var metricsDispatcher: MetricsDispatcher = NoOpMetricsDispatcher()
 
-    // region SDK Feature
+    // region SdkFeature
 
     fun initialize(context: Context) {
         if (initialized.get()) {
             return
         }
 
+        var dataUploadConfiguration: DataUploadConfiguration? = null
         if (wrappedFeature is StorageBackedFeature) {
-            storage = createStorage(wrappedFeature.name, wrappedFeature.storageConfiguration)
+            val uploadFrequency = resolveUploadFrequency()
+            dataUploadConfiguration = DataUploadConfiguration(uploadFrequency)
+
+            val storageConfiguration = wrappedFeature.storageConfiguration
+            val recentDelayMs = resolveBatchingDelay(coreFeature, storageConfiguration)
+            val filePersistenceConfig = coreFeature.buildFilePersistenceConfig().copy(
+                maxBatchSize = storageConfiguration.maxBatchSize,
+                maxItemSize = storageConfiguration.maxItemSize,
+                maxItemsPerBatch = storageConfiguration.maxItemsPerBatch,
+                oldFileThreshold = storageConfiguration.oldBatchThreshold,
+                recentDelayMs = recentDelayMs
+            )
+            metricsDispatcher = BatchMetricsDispatcher(
+                wrappedFeature.name,
+                dataUploadConfiguration,
+                filePersistenceConfig,
+                internalLogger,
+                coreFeature.timeProvider
+            )
+
+            storage = createStorage(wrappedFeature.name, filePersistenceConfig)
         }
 
         wrappedFeature.onInitialize(context)
 
-        if (wrappedFeature is StorageBackedFeature) {
-            setupUploader(wrappedFeature.requestFactory, wrappedFeature.storageConfiguration)
+        if (wrappedFeature is StorageBackedFeature && dataUploadConfiguration != null) {
+            setupUploader(wrappedFeature.requestFactory, dataUploadConfiguration)
         }
 
         if (wrappedFeature is TrackingConsentProviderCallback) {
@@ -151,16 +177,17 @@ internal class SdkFeature(
             ?: coreFeature.batchSize.windowDurationMs
     }
 
-    private fun resolveUploadFrequency(
-        coreFeature: CoreFeature,
-        featureStorageConfiguration: FeatureStorageConfiguration
-    ): UploadFrequency {
-        return featureStorageConfiguration.uploadFrequency ?: coreFeature.uploadFrequency
+    private fun resolveUploadFrequency(): UploadFrequency {
+        return if (wrappedFeature is StorageBackedFeature) {
+            wrappedFeature.storageConfiguration.uploadFrequency ?: coreFeature.uploadFrequency
+        } else {
+            coreFeature.uploadFrequency
+        }
     }
 
     private fun setupUploader(
         requestFactory: RequestFactory,
-        storageConfiguration: FeatureStorageConfiguration
+        uploadConfiguration: DataUploadConfiguration
     ) {
         uploadScheduler = if (coreFeature.isMainProcess) {
             uploader = createUploader(requestFactory)
@@ -170,7 +197,7 @@ internal class SdkFeature(
                 coreFeature.contextProvider,
                 coreFeature.networkInfoProvider,
                 coreFeature.systemInfoProvider,
-                resolveUploadFrequency(coreFeature, storageConfiguration),
+                uploadConfiguration,
                 coreFeature.uploadExecutorService,
                 internalLogger
             )
@@ -184,14 +211,15 @@ internal class SdkFeature(
 
     private fun createStorage(
         featureName: String,
-        storageConfiguration: FeatureStorageConfiguration
+        filePersistenceConfig: FilePersistenceConfig
     ): Storage {
         val fileOrchestrator = FeatureFileOrchestrator(
             consentProvider = coreFeature.trackingConsentProvider,
             storageDir = coreFeature.storageDir,
             featureName = featureName,
             executorService = coreFeature.persistenceExecutorService,
-            internalLogger = internalLogger
+            internalLogger = internalLogger,
+            metricsDispatcher = metricsDispatcher
         )
         this.fileOrchestrator = fileOrchestrator
 
@@ -209,16 +237,8 @@ internal class SdkFeature(
             ),
             fileMover = FileMover(internalLogger),
             internalLogger = internalLogger,
-            filePersistenceConfig = coreFeature.buildFilePersistenceConfig().copy(
-                maxBatchSize = storageConfiguration.maxBatchSize,
-                maxItemSize = storageConfiguration.maxItemSize,
-                maxItemsPerBatch = storageConfiguration.maxItemsPerBatch,
-                oldFileThreshold = storageConfiguration.oldBatchThreshold,
-                recentDelayMs = resolveBatchingDelay(
-                    coreFeature,
-                    storageConfiguration
-                )
-            )
+            filePersistenceConfig = filePersistenceConfig,
+            metricsDispatcher = metricsDispatcher
         )
     }
 

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/configuration/DataUploadConfiguration.kt

@@ -0,0 +1,20 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2016-Present Datadog, Inc.
+ */
+
+package com.datadog.android.core.internal.configuration
+
+import com.datadog.android.core.configuration.UploadFrequency
+
+internal data class DataUploadConfiguration(internal val frequency: UploadFrequency) {
+    internal val minDelayMs = MIN_DELAY_FACTOR * frequency.baseStepMs
+    internal val maxDelayMs = MAX_DELAY_FACTOR * frequency.baseStepMs
+    internal val defaultDelayMs = DEFAULT_DELAY_FACTOR * frequency.baseStepMs
+    companion object {
+        internal const val MIN_DELAY_FACTOR = 1
+        internal const val MAX_DELAY_FACTOR = 10
+        internal const val DEFAULT_DELAY_FACTOR = 5
+    }
+}

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/data/upload/DataOkHttpUploader.kt

@@ -47,7 +47,7 @@ internal class DataOkHttpUploader(
                 },
                 e
             )
-            return UploadStatus.REQUEST_CREATION_ERROR
+            return UploadStatus.RequestCreationError
         }
 
         val uploadStatus = try {
@@ -59,7 +59,7 @@ internal class DataOkHttpUploader(
                 { "Unable to execute the request; we will retry later." },
                 e
             )
-            UploadStatus.NETWORK_ERROR
+            UploadStatus.NetworkError
         }
 
         uploadStatus.logStatus(
@@ -96,7 +96,7 @@ internal class DataOkHttpUploader(
             }
             ?.value
         if (apiKey != null && (apiKey.isEmpty() || !isValidHeaderValue(apiKey))) {
-            return UploadStatus.INVALID_TOKEN_ERROR
+            return UploadStatus.InvalidTokenError(UploadStatus.UNKNOWN_RESPONSE_CODE)
         }
 
         val okHttpRequest = buildOkHttpRequest(request)
@@ -152,22 +152,22 @@ internal class DataOkHttpUploader(
         request: DatadogRequest
     ): UploadStatus {
         return when (code) {
-            HTTP_ACCEPTED -> UploadStatus.SUCCESS
-            HTTP_BAD_REQUEST -> UploadStatus.HTTP_CLIENT_ERROR
-            HTTP_UNAUTHORIZED -> UploadStatus.INVALID_TOKEN_ERROR
-            HTTP_FORBIDDEN -> UploadStatus.INVALID_TOKEN_ERROR
-            HTTP_CLIENT_TIMEOUT -> UploadStatus.HTTP_CLIENT_RATE_LIMITING
-            HTTP_ENTITY_TOO_LARGE -> UploadStatus.HTTP_CLIENT_ERROR
-            HTTP_TOO_MANY_REQUESTS -> UploadStatus.HTTP_CLIENT_RATE_LIMITING
-            HTTP_INTERNAL_ERROR -> UploadStatus.HTTP_SERVER_ERROR
-            HTTP_UNAVAILABLE -> UploadStatus.HTTP_SERVER_ERROR
+            HTTP_ACCEPTED -> UploadStatus.Success(code)
+            HTTP_BAD_REQUEST -> UploadStatus.HttpClientError(code)
+            HTTP_UNAUTHORIZED -> UploadStatus.InvalidTokenError(code)
+            HTTP_FORBIDDEN -> UploadStatus.InvalidTokenError(code)
+            HTTP_CLIENT_TIMEOUT -> UploadStatus.HttpClientRateLimiting(code)
+            HTTP_ENTITY_TOO_LARGE -> UploadStatus.HttpClientError(code)
+            HTTP_TOO_MANY_REQUESTS -> UploadStatus.HttpClientRateLimiting(code)
+            HTTP_INTERNAL_ERROR -> UploadStatus.HttpServerError(code)
+            HTTP_UNAVAILABLE -> UploadStatus.HttpServerError(code)
             else -> {
                 internalLogger.log(
                     InternalLogger.Level.WARN,
                     listOf(InternalLogger.Target.MAINTAINER, InternalLogger.Target.TELEMETRY),
                     { "Unexpected status code $code on upload request: ${request.description}" }
                 )
-                UploadStatus.UNKNOWN_ERROR
+                UploadStatus.UnknownError(code)
             }
         }
     }

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/data/upload/DataUploader.kt

@@ -6,9 +6,6 @@
 
 package com.datadog.android.core.internal.data.upload
 
-import com.datadog.tools.annotation.NoOpImplementation
-
-@NoOpImplementation
 internal interface DataUploader {
 
     fun upload(data: ByteArray): UploadStatus

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/data/upload/NoOpDataUploader.kt

@@ -0,0 +1,13 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2016-Present Datadog, Inc.
+ */
+
+package com.datadog.android.core.internal.data.upload
+
+internal interface NoOpDataUploader : DataUploader {
+    override fun upload(data: ByteArray): UploadStatus {
+        return UploadStatus.UnknownStatus
+    }
+}

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/data/upload/UploadStatus.kt

@@ -9,16 +9,19 @@ package com.datadog.android.core.internal.data.upload
 import com.datadog.android.api.InternalLogger
 
 @Suppress("StringLiteralDuplication")
-internal enum class UploadStatus(val shouldRetry: Boolean) {
-    SUCCESS(shouldRetry = false),
-    NETWORK_ERROR(shouldRetry = true),
-    REQUEST_CREATION_ERROR(shouldRetry = false),
-    INVALID_TOKEN_ERROR(shouldRetry = false),
-    HTTP_REDIRECTION(shouldRetry = false),
-    HTTP_CLIENT_ERROR(shouldRetry = false),
-    HTTP_SERVER_ERROR(shouldRetry = true),
-    HTTP_CLIENT_RATE_LIMITING(shouldRetry = true),
-    UNKNOWN_ERROR(shouldRetry = false);
+internal sealed class UploadStatus(val shouldRetry: Boolean = false, val code: Int = UNKNOWN_RESPONSE_CODE) {
+
+    internal class Success(responseCode: Int) : UploadStatus(shouldRetry = false, code = responseCode)
+    internal object NetworkError : UploadStatus(shouldRetry = true)
+    internal object RequestCreationError : UploadStatus(shouldRetry = false)
+    internal class InvalidTokenError(responseCode: Int) : UploadStatus(shouldRetry = false, code = responseCode)
+    internal class HttpRedirection(responseCode: Int) : UploadStatus(shouldRetry = false, code = responseCode)
+    internal class HttpClientError(responseCode: Int) : UploadStatus(shouldRetry = false, code = responseCode)
+    internal class HttpServerError(responseCode: Int) : UploadStatus(shouldRetry = true, code = responseCode)
+    internal class HttpClientRateLimiting(responseCode: Int) : UploadStatus(shouldRetry = true, code = responseCode)
+    internal class UnknownError(responseCode: Int) : UploadStatus(shouldRetry = false, code = responseCode)
+
+    internal object UnknownStatus : UploadStatus(shouldRetry = false, code = UNKNOWN_RESPONSE_CODE)
 
     @SuppressWarnings("LongMethod")
     fun logStatus(
@@ -33,13 +36,13 @@ internal enum class UploadStatus(val shouldRetry: Boolean) {
             "Batch $requestId [$byteSize bytes] ($context)"
         }
         when (this) {
-            NETWORK_ERROR -> logger.log(
+            is NetworkError -> logger.log(
                 InternalLogger.Level.WARN,
                 InternalLogger.Target.USER,
                 { "$batchInfo failed because of a network error; we will retry later." }
             )
 
-            INVALID_TOKEN_ERROR -> logger.log(
+            is InvalidTokenError -> logger.log(
                 InternalLogger.Level.ERROR,
                 InternalLogger.Target.USER,
                 {
@@ -49,13 +52,13 @@ internal enum class UploadStatus(val shouldRetry: Boolean) {
                 }
             )
 
-            HTTP_REDIRECTION -> logger.log(
+            is HttpRedirection -> logger.log(
                 InternalLogger.Level.WARN,
                 InternalLogger.Target.USER,
                 { "$batchInfo failed because of a network redirection; the batch was dropped." }
             )
 
-            HTTP_CLIENT_ERROR -> {
+            is HttpClientError -> {
                 logger.log(
                     InternalLogger.Level.ERROR,
                     listOf(InternalLogger.Target.USER, InternalLogger.Target.TELEMETRY),
@@ -66,27 +69,27 @@ internal enum class UploadStatus(val shouldRetry: Boolean) {
                 )
             }
 
-            HTTP_CLIENT_RATE_LIMITING -> {
+            is HttpClientRateLimiting -> {
                 logger.log(
                     InternalLogger.Level.WARN,
                     listOf(InternalLogger.Target.USER, InternalLogger.Target.TELEMETRY),
                     { "$batchInfo not uploaded due to rate limitation; we will retry later." }
                 )
             }
 
-            HTTP_SERVER_ERROR -> logger.log(
+            is HttpServerError -> logger.log(
                 InternalLogger.Level.ERROR,
                 InternalLogger.Target.USER,
                 { "$batchInfo failed because of a server processing error; we will retry later." }
             )
 
-            UNKNOWN_ERROR -> logger.log(
+            is UnknownError -> logger.log(
                 InternalLogger.Level.ERROR,
                 InternalLogger.Target.USER,
                 { "$batchInfo failed because of an unknown error; the batch was dropped." }
             )
 
-            REQUEST_CREATION_ERROR -> logger.log(
+            is RequestCreationError -> logger.log(
                 InternalLogger.Level.ERROR,
                 InternalLogger.Target.USER,
                 {
@@ -95,11 +98,18 @@ internal enum class UploadStatus(val shouldRetry: Boolean) {
                 }
             )
 
-            SUCCESS -> logger.log(
+            is Success -> logger.log(
                 InternalLogger.Level.INFO,
                 InternalLogger.Target.USER,
                 { "$batchInfo sent successfully." }
             )
+
+            else -> {
+                // no-op
+            }
         }
     }
+    companion object {
+        internal const val UNKNOWN_RESPONSE_CODE = 0
+    }
 }