RUM-4098: Add TLVFormat DataStore persistence

dd-sdk-android-core/api/apiSurface

@@ -102,6 +102,7 @@ interface com.datadog.android.api.feature.FeatureContextUpdateReceiver
 interface com.datadog.android.api.feature.FeatureEventReceiver
   fun onReceive(Any)
 interface com.datadog.android.api.feature.FeatureScope
+  val dataStore: com.datadog.android.api.storage.datastore.DataStoreHandler
   fun withWriteContext(Boolean = false, (com.datadog.android.api.context.DatadogContext, com.datadog.android.api.storage.EventBatchWriter) -> Unit)
   fun sendEvent(Any)
   fun <T: Feature> unwrap(): T
@@ -151,6 +152,15 @@ data class com.datadog.android.api.storage.RawBatchEvent
   constructor(ByteArray, ByteArray = EMPTY_BYTE_ARRAY)
   override fun equals(Any?): Boolean
   override fun hashCode(): Int
+interface com.datadog.android.api.storage.datastore.DataStoreCallback<T: Any>
+  fun onSuccess(com.datadog.android.core.persistence.datastore.DataStoreContent<T>?)
+  fun onFailure()
+interface com.datadog.android.api.storage.datastore.DataStoreHandler
+  fun <T: Any> setValue(String, T, Int = 0, com.datadog.android.core.persistence.Serializer<T>)
+  fun <T: Any> value(String, Int? = null, DataStoreCallback<T>, com.datadog.android.core.internal.persistence.Deserializer<String, T>)
+  fun removeValue(String)
+  companion object 
+    const val CURRENT_DATASTORE_VERSION: Int
 interface com.datadog.android.core.InternalSdkCore : com.datadog.android.api.feature.FeatureSdkCore
   val networkInfo: com.datadog.android.api.context.NetworkInfo
   val trackingConsent: com.datadog.android.privacy.TrackingConsent
@@ -290,6 +300,8 @@ interface com.datadog.android.core.persistence.Serializer<T: Any>
   fun serialize(T): String?
   companion object 
 fun <T: Any> Serializer<T>.serializeToByteArray(T, com.datadog.android.api.InternalLogger): ByteArray?
+data class com.datadog.android.core.persistence.datastore.DataStoreContent<T: Any>
+  constructor(Int, T?)
 class com.datadog.android.core.sampling.RateBasedSampler : Sampler
   constructor(() -> Float)
   constructor(Float)

dd-sdk-android-core/api/dd-sdk-android-core.api

@@ -325,6 +325,7 @@ public abstract interface class com/datadog/android/api/feature/FeatureEventRece
 }
 
 public abstract interface class com/datadog/android/api/feature/FeatureScope {
+	public abstract fun getDataStore ()Lcom/datadog/android/api/storage/datastore/DataStoreHandler;
 	public abstract fun sendEvent (Ljava/lang/Object;)V
 	public abstract fun unwrap ()Lcom/datadog/android/api/feature/Feature;
 	public abstract fun withWriteContext (ZLkotlin/jvm/functions/Function2;)V
@@ -462,6 +463,28 @@ public final class com/datadog/android/api/storage/RawBatchEvent {
 	public fun toString ()Ljava/lang/String;
 }
 
+public abstract interface class com/datadog/android/api/storage/datastore/DataStoreCallback {
+	public abstract fun onFailure ()V
+	public abstract fun onSuccess (Lcom/datadog/android/core/persistence/datastore/DataStoreContent;)V
+}
+
+public abstract interface class com/datadog/android/api/storage/datastore/DataStoreHandler {
+	public static final field CURRENT_DATASTORE_VERSION I
+	public static final field Companion Lcom/datadog/android/api/storage/datastore/DataStoreHandler$Companion;
+	public abstract fun removeValue (Ljava/lang/String;)V
+	public abstract fun setValue (Ljava/lang/String;Ljava/lang/Object;ILcom/datadog/android/core/persistence/Serializer;)V
+	public abstract fun value (Ljava/lang/String;Ljava/lang/Integer;Lcom/datadog/android/api/storage/datastore/DataStoreCallback;Lcom/datadog/android/core/internal/persistence/Deserializer;)V
+}
+
+public final class com/datadog/android/api/storage/datastore/DataStoreHandler$Companion {
+	public static final field CURRENT_DATASTORE_VERSION I
+}
+
+public final class com/datadog/android/api/storage/datastore/DataStoreHandler$DefaultImpls {
+	public static synthetic fun setValue$default (Lcom/datadog/android/api/storage/datastore/DataStoreHandler;Ljava/lang/String;Ljava/lang/Object;ILcom/datadog/android/core/persistence/Serializer;ILjava/lang/Object;)V
+	public static synthetic fun value$default (Lcom/datadog/android/api/storage/datastore/DataStoreHandler;Ljava/lang/String;Ljava/lang/Integer;Lcom/datadog/android/api/storage/datastore/DataStoreCallback;Lcom/datadog/android/core/internal/persistence/Deserializer;ILjava/lang/Object;)V
+}
+
 public abstract interface class com/datadog/android/core/InternalSdkCore : com/datadog/android/api/feature/FeatureSdkCore {
 	public abstract fun deleteLastViewEvent ()V
 	public abstract fun getAllFeatures ()Ljava/util/List;
@@ -789,6 +812,19 @@ public final class com/datadog/android/core/persistence/SerializerKt {
 	public static final fun serializeToByteArray (Lcom/datadog/android/core/persistence/Serializer;Ljava/lang/Object;Lcom/datadog/android/api/InternalLogger;)[B
 }
 
+public final class com/datadog/android/core/persistence/datastore/DataStoreContent {
+	public fun <init> (ILjava/lang/Object;)V
+	public final fun component1 ()I
+	public final fun component2 ()Ljava/lang/Object;
+	public final fun copy (ILjava/lang/Object;)Lcom/datadog/android/core/persistence/datastore/DataStoreContent;
+	public static synthetic fun copy$default (Lcom/datadog/android/core/persistence/datastore/DataStoreContent;ILjava/lang/Object;ILjava/lang/Object;)Lcom/datadog/android/core/persistence/datastore/DataStoreContent;
+	public fun equals (Ljava/lang/Object;)Z
+	public final fun getData ()Ljava/lang/Object;
+	public final fun getVersionCode ()I
+	public fun hashCode ()I
+	public fun toString ()Ljava/lang/String;
+}
+
 public final class com/datadog/android/core/sampling/RateBasedSampler : com/datadog/android/core/sampling/Sampler {
 	public static final field SAMPLE_ALL_RATE F
 	public fun <init> (D)V

dd-sdk-android-core/src/main/kotlin/com/datadog/android/api/feature/FeatureScope.kt

@@ -9,12 +9,18 @@ package com.datadog.android.api.feature
 import androidx.annotation.AnyThread
 import com.datadog.android.api.context.DatadogContext
 import com.datadog.android.api.storage.EventBatchWriter
+import com.datadog.android.api.storage.datastore.DataStoreHandler
 
 /**
  * Represents a Datadog feature.
  */
 interface FeatureScope {
 
+    /**
+     * Property to enable interaction with the data store.
+     */
+    val dataStore: DataStoreHandler
+
     /**
      * Utility to write an event, asynchronously.
      * @param forceNewBatch if `true` forces the [EventBatchWriter] to write in a new file and

dd-sdk-android-core/src/main/kotlin/com/datadog/android/api/storage/datastore/DataStoreCallback.kt

@@ -0,0 +1,28 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2016-Present Datadog, Inc.
+ */
+
+package com.datadog.android.api.storage.datastore
+
+import com.datadog.android.core.persistence.datastore.DataStoreContent
+
+/**
+ * Callback for asynchronous operations on the datastore.
+ * @param T the datatype being retrieved.
+ */
+interface DataStoreCallback<T : Any> {
+
+    /**
+     * Triggered on successfully fetching data from the datastore.
+     *
+     * @param dataStoreContent (nullable) contains the datastore content if there was data to fetch, else null.
+     */
+    fun onSuccess(dataStoreContent: DataStoreContent<T>?)
+
+    /**
+     * Triggered when we failed to get data from the datastore.
+     */
+    fun onFailure()
+}

dd-sdk-android-core/src/main/kotlin/com/datadog/android/api/storage/datastore/DataStoreHandler.kt

@@ -0,0 +1,69 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2016-Present Datadog, Inc.
+ */
+
+package com.datadog.android.api.storage.datastore
+
+import com.datadog.android.core.internal.persistence.Deserializer
+import com.datadog.android.core.persistence.Serializer
+
+/**
+ * Interface for the datastore.
+ */
+interface DataStoreHandler {
+
+    /**
+     * Write data to the datastore.
+     * This executes on a worker thread and not on the caller thread.
+     *
+     * @param T datatype of the data to write to the datastore.
+     * @param key name of the datastore entry.
+     * @param data to write.
+     * @param version optional version for the entry.
+     * If not specified will give the entry version 0 - even if that would be a downgrade from the previous version.
+     * @param serializer to use to serialize the data.
+     */
+    fun <T : Any> setValue(
+        key: String,
+        data: T,
+        version: Int = 0,
+        serializer: Serializer<T>
+    )
+
+    /**
+     * Read data from the datastore.
+     * This executes on a worker thread and not on the caller thread.
+     *
+     * @param T datatype of the data to read from the datastore.
+     * @param key name of the datastore entry.
+     * @param version optional version to use when reading from the datastore.
+     * If specified, will only return data if the persistent entry exactly matches this version number.
+     * @param callback to return result asynchronously.
+     * @param deserializer to use to deserialize the data.
+     */
+    fun <T : Any> value(
+        key: String,
+        version: Int? = null,
+        callback: DataStoreCallback<T>,
+        deserializer: Deserializer<String, T>
+    )
+
+    /**
+     * Remove an entry from the datastore.
+     * This executes on a worker thread and not on the caller thread.
+     *
+     * @param key name of the datastore entry
+     */
+    fun removeValue(
+        key: String
+    )
+
+    companion object {
+        /**
+         * The current version of the datastore.
+         */
+        const val CURRENT_DATASTORE_VERSION: Int = 0
+    }
+}

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/SdkFeature.kt

@@ -20,6 +20,7 @@ import com.datadog.android.api.feature.StorageBackedFeature
 import com.datadog.android.api.net.RequestFactory
 import com.datadog.android.api.storage.EventBatchWriter
 import com.datadog.android.api.storage.FeatureStorageConfiguration
+import com.datadog.android.api.storage.datastore.DataStoreHandler
 import com.datadog.android.core.internal.configuration.DataUploadConfiguration
 import com.datadog.android.core.internal.data.upload.DataFlusher
 import com.datadog.android.core.internal.data.upload.DataOkHttpUploader
@@ -36,15 +37,22 @@ import com.datadog.android.core.internal.persistence.AbstractStorage
 import com.datadog.android.core.internal.persistence.ConsentAwareStorage
 import com.datadog.android.core.internal.persistence.NoOpStorage
 import com.datadog.android.core.internal.persistence.Storage
+import com.datadog.android.core.internal.persistence.datastore.DataStoreFileHandler
+import com.datadog.android.core.internal.persistence.datastore.DataStoreFileHelper
+import com.datadog.android.core.internal.persistence.datastore.DatastoreFileReader
+import com.datadog.android.core.internal.persistence.datastore.DatastoreFileWriter
+import com.datadog.android.core.internal.persistence.datastore.NoOpDataStoreHandler
 import com.datadog.android.core.internal.persistence.file.FileMover
 import com.datadog.android.core.internal.persistence.file.FileOrchestrator
 import com.datadog.android.core.internal.persistence.file.FilePersistenceConfig
 import com.datadog.android.core.internal.persistence.file.FileReaderWriter
 import com.datadog.android.core.internal.persistence.file.NoOpFileOrchestrator
 import com.datadog.android.core.internal.persistence.file.advanced.FeatureFileOrchestrator
 import com.datadog.android.core.internal.persistence.file.batch.BatchFileReaderWriter
+import com.datadog.android.core.internal.persistence.tlvformat.TLVBlockFileReader
 import com.datadog.android.core.persistence.PersistenceStrategy
 import com.datadog.android.privacy.TrackingConsentProviderCallback
+import com.datadog.android.security.Encryption
 import java.util.Collections
 import java.util.Locale
 import java.util.concurrent.ConcurrentHashMap
@@ -58,6 +66,8 @@ internal class SdkFeature(
     internal val internalLogger: InternalLogger
 ) : FeatureScope {
 
+    override var dataStore: DataStoreHandler = NoOpDataStoreHandler()
+
     internal val initialized = AtomicBoolean(false)
 
     @Suppress("UnsafeThirdPartyFunctionCall") // the argument is always empty
@@ -105,6 +115,10 @@ internal class SdkFeature(
             coreFeature.trackingConsentProvider.registerCallback(wrappedFeature)
         }
 
+        prepareDataStoreHandler(
+            encryption = coreFeature.localDataEncryption
+        )
+
         initialized.set(true)
 
         uploadScheduler.startScheduling()
@@ -129,6 +143,7 @@ internal class SdkFeature(
             uploadScheduler.stopScheduling()
             uploadScheduler = NoOpUploadScheduler()
             storage = NoOpStorage()
+            dataStore = NoOpDataStoreHandler()
             uploader = NoOpDataUploader()
             fileOrchestrator = NoOpFileOrchestrator()
             metricsDispatcher = NoOpMetricsDispatcher()
@@ -343,6 +358,47 @@ internal class SdkFeature(
         )
     }
 
+    private fun prepareDataStoreHandler(
+        encryption: Encryption?
+    ) {
+        val fileReaderWriter = FileReaderWriter.create(
+            internalLogger,
+            encryption
+        )
+
+        val dataStoreFileHelper = DataStoreFileHelper(internalLogger)
+        val featureName = wrappedFeature.name
+        val storageDir = coreFeature.storageDir
+
+        val tlvBlockFileReader = TLVBlockFileReader(
+            internalLogger = internalLogger,
+            fileReaderWriter = fileReaderWriter
+        )
+
+        val dataStoreFileReader = DatastoreFileReader(
+            dataStoreFileHelper = dataStoreFileHelper,
+            featureName = featureName,
+            internalLogger = internalLogger,
+            storageDir = storageDir,
+            tlvBlockFileReader = tlvBlockFileReader
+        )
+
+        val dataStoreFileWriter = DatastoreFileWriter(
+            dataStoreFileHelper = dataStoreFileHelper,
+            featureName = featureName,
+            fileReaderWriter = fileReaderWriter,
+            internalLogger = internalLogger,
+            storageDir = storageDir
+        )
+
+        dataStore = DataStoreFileHandler(
+            executorService = coreFeature.persistenceExecutorService,
+            internalLogger = internalLogger,
+            dataStoreFileReader = dataStoreFileReader,
+            datastoreFileWriter = dataStoreFileWriter
+        )
+    }
+
     // endregion
 
     // Used for nightly tests only

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/persistence/datastore/DataStoreFileHandler.kt

@@ -0,0 +1,51 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2016-Present Datadog, Inc.
+ */
+
+package com.datadog.android.core.internal.persistence.datastore
+
+import com.datadog.android.api.InternalLogger
+import com.datadog.android.api.storage.datastore.DataStoreCallback
+import com.datadog.android.api.storage.datastore.DataStoreHandler
+import com.datadog.android.core.internal.persistence.Deserializer
+import com.datadog.android.core.internal.utils.executeSafe
+import com.datadog.android.core.persistence.Serializer
+import java.util.concurrent.ExecutorService
+
+internal class DataStoreFileHandler(
+    private val executorService: ExecutorService,
+    private val internalLogger: InternalLogger,
+    private val dataStoreFileReader: DatastoreFileReader,
+    private val datastoreFileWriter: DatastoreFileWriter
+) : DataStoreHandler {
+
+    override fun <T : Any> setValue(
+        key: String,
+        data: T,
+        version: Int,
+        serializer: Serializer<T>
+    ) {
+        executorService.executeSafe("dataStoreWrite", internalLogger) {
+            datastoreFileWriter.write(key, data, serializer, version)
+        }
+    }
+
+    override fun removeValue(key: String) {
+        executorService.executeSafe("dataStoreRemove", internalLogger) {
+            datastoreFileWriter.delete(key)
+        }
+    }
+
+    override fun <T : Any> value(
+        key: String,
+        version: Int?,
+        callback: DataStoreCallback<T>,
+        deserializer: Deserializer<String, T>
+    ) {
+        executorService.executeSafe("dataStoreRead", internalLogger) {
+            dataStoreFileReader.read(key, deserializer, version, callback)
+        }
+    }
+}