feat: add remnant events migration plugin (#40)

* feat: add remnant events migration plugin

* delete events after migration

* fix lint errors and add test file

* slight refactor and add unit tests

* attempt to fix unit test failure

* attempt to fix unit test failure

* use testImplementation instead of core, remove extension of Storage interface

* added getDeviceId()/getUserId() methods, added instance name support

* fix lint warnings

* fix plugin logic (database version, library format)

* extend plugin logic (device/user id, identifies, intercepted identifies)

* added support for db version 3

* added test for db version 3

* move migration logic to 'migration' package

* added test for missing legacy db

* move MigrationPlugin to Initializer

* restore session data, do not clean device/user id, support first/next runs since upgrade

* fix tests

* add short delay after each test to complete asynchronous handlers - to remove errors like 'java.lang.RuntimeException: Method e in android.util.Log not mocked.' during test run

* remove short delay after each test to complete asynchronous handlers - now in-memory IMIdentityStorageProvider is used

* add try/catch for each migrated event

* refactoring: added flag 'migrateLegacyData' instead of initializers

* add test for migrateLegacyData=false

* remove unnecessary permission READ_PHONE_STATE

* move legacy 'uuid' field to 'insert_id'

* make `build()` protected open for backward compatibility

* write converted legacy event instead of json representation

* fix some "simple" build warnings

* make migrateLegacyData writable to set in Java code

---------

Co-authored-by: Andrey Sokolov <andrey.sokolov@amplitude.com>

android/build.gradle

@@ -64,12 +64,11 @@ dependencies {
 
     coreLibraryDesugaring 'com.android.tools:desugar_jdk_libs:1.1.5'
 
-    testImplementation 'io.mockk:mockk:1.10.6'
+    testImplementation 'io.mockk:mockk:1.12.4'
     testImplementation project(':core')
     testImplementation project(path: ':common-android')
     testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.4'
     testImplementation 'com.squareup.okhttp3:mockwebserver:4.10.0'
-
     testImplementation 'org.junit.jupiter:junit-jupiter:5.7.2'
     testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.7.2'
     testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2'

android/src/main/java/com/amplitude/android/Amplitude.kt

@@ -1,6 +1,7 @@
 package com.amplitude.android
 
 import android.content.Context
+import com.amplitude.android.migration.RemnantDataMigration
 import com.amplitude.android.plugins.AnalyticsConnectorIdentityPlugin
 import com.amplitude.android.plugins.AnalyticsConnectorPlugin
 import com.amplitude.android.plugins.AndroidContextPlugin
@@ -9,15 +10,8 @@ import com.amplitude.core.Amplitude
 import com.amplitude.core.events.BaseEvent
 import com.amplitude.core.platform.plugins.AmplitudeDestination
 import com.amplitude.core.platform.plugins.GetAmpliExtrasPlugin
-import com.amplitude.core.utilities.AnalyticsIdentityListener
 import com.amplitude.core.utilities.FileStorage
-import com.amplitude.id.FileIdentityStorageProvider
 import com.amplitude.id.IdentityConfiguration
-import com.amplitude.id.IdentityContainer
-import com.amplitude.id.IdentityUpdateType
-import kotlinx.coroutines.CoroutineStart
-import kotlinx.coroutines.Deferred
-import kotlinx.coroutines.async
 import kotlinx.coroutines.launch
 
 open class Amplitude(
@@ -33,45 +27,41 @@ open class Amplitude(
         }
 
     init {
-        (timeline as Timeline).start()
         registerShutdownHook()
     }
 
     override fun createTimeline(): Timeline {
         return Timeline().also { it.amplitude = this }
     }
 
-    override fun build(): Deferred<Boolean> {
-        val client = this
-
-        val built = amplitudeScope.async(amplitudeDispatcher, CoroutineStart.LAZY) {
-            storage = configuration.storageProvider.getStorage(client)
-
-            val storageDirectory = (configuration as Configuration).context.getDir("${FileStorage.STORAGE_PREFIX}-${configuration.instanceName}", Context.MODE_PRIVATE)
-            idContainer = IdentityContainer.getInstance(
-                IdentityConfiguration(
-                    instanceName = configuration.instanceName,
-                    apiKey = configuration.apiKey,
-                    identityStorageProvider = FileIdentityStorageProvider(),
-                    storageDirectory = storageDirectory,
-                    logger = configuration.loggerProvider.getLogger(client)
-                )
-            )
-            val listener = AnalyticsIdentityListener(store)
-            idContainer.identityManager.addIdentityListener(listener)
-            if (idContainer.identityManager.isInitialized()) {
-                listener.onIdentityChanged(idContainer.identityManager.getIdentity(), IdentityUpdateType.Initialized)
-            }
-            androidContextPlugin = AndroidContextPlugin()
-            add(androidContextPlugin)
-            add(GetAmpliExtrasPlugin())
-            add(AndroidLifecyclePlugin())
-            add(AnalyticsConnectorIdentityPlugin())
-            add(AnalyticsConnectorPlugin())
-            add(AmplitudeDestination())
-            true
+    override fun createIdentityConfiguration(): IdentityConfiguration {
+        val configuration = configuration as Configuration
+        val storageDirectory = configuration.context.getDir("${FileStorage.STORAGE_PREFIX}-${configuration.instanceName}", Context.MODE_PRIVATE)
+
+        return IdentityConfiguration(
+            instanceName = configuration.instanceName,
+            apiKey = configuration.apiKey,
+            identityStorageProvider = configuration.identityStorageProvider,
+            storageDirectory = storageDirectory,
+            logger = configuration.loggerProvider.getLogger(this)
+        )
+    }
+
+    override suspend fun buildInternal(identityConfiguration: IdentityConfiguration) {
+        if ((this.configuration as Configuration).migrateLegacyData) {
+            RemnantDataMigration(this).execute()
         }
-        return built
+        this.createIdentityContainer(identityConfiguration)
+
+        androidContextPlugin = AndroidContextPlugin()
+        add(androidContextPlugin)
+        add(GetAmpliExtrasPlugin())
+        add(AndroidLifecyclePlugin())
+        add(AnalyticsConnectorIdentityPlugin())
+        add(AnalyticsConnectorPlugin())
+        add(AmplitudeDestination())
+
+        (timeline as Timeline).start()
     }
 
     /**

android/src/main/java/com/amplitude/android/Configuration.kt

@@ -10,6 +10,8 @@ import com.amplitude.core.ServerZone
 import com.amplitude.core.StorageProvider
 import com.amplitude.core.events.IngestionMetadata
 import com.amplitude.core.events.Plan
+import com.amplitude.id.FileIdentityStorageProvider
+import com.amplitude.id.IdentityStorageProvider
 
 open class Configuration @JvmOverloads constructor(
     apiKey: String,
@@ -39,8 +41,10 @@ open class Configuration @JvmOverloads constructor(
     var minTimeBetweenSessionsMillis: Long = MIN_TIME_BETWEEN_SESSIONS_MILLIS,
     var trackingSessionEvents: Boolean = true,
     override var identifyBatchIntervalMillis: Long = IDENTIFY_BATCH_INTERVAL_MILLIS,
-    override var identifyInterceptStorageProvider: StorageProvider = AndroidStorageProvider()
-) : Configuration(apiKey, flushQueueSize, flushIntervalMillis, instanceName, optOut, storageProvider, loggerProvider, minIdLength, partnerId, callback, flushMaxRetries, useBatch, serverZone, serverUrl, plan, ingestionMetadata, identifyBatchIntervalMillis, identifyInterceptStorageProvider) {
+    override var identifyInterceptStorageProvider: StorageProvider = AndroidStorageProvider(),
+    override var identityStorageProvider: IdentityStorageProvider = FileIdentityStorageProvider(),
+    var migrateLegacyData: Boolean = false,
+) : Configuration(apiKey, flushQueueSize, flushIntervalMillis, instanceName, optOut, storageProvider, loggerProvider, minIdLength, partnerId, callback, flushMaxRetries, useBatch, serverZone, serverUrl, plan, ingestionMetadata, identifyBatchIntervalMillis, identifyInterceptStorageProvider, identityStorageProvider) {
     companion object {
         const val MIN_TIME_BETWEEN_SESSIONS_MILLIS: Long = 300000
     }

android/src/main/java/com/amplitude/android/Timeline.kt

@@ -21,8 +21,6 @@ class Timeline : Timeline() {
 
     internal fun start() {
         amplitude.amplitudeScope.launch(amplitude.storageIODispatcher) {
-            amplitude.isBuilt.await()
-
             _sessionId.set(amplitude.storage.read(Storage.Constants.PREVIOUS_SESSION_ID)?.toLongOrNull() ?: -1)
             lastEventId = amplitude.storage.read(Storage.Constants.LAST_EVENT_ID)?.toLongOrNull() ?: 0
             lastEventTime = amplitude.storage.read(Storage.Constants.LAST_EVENT_TIME)?.toLongOrNull() ?: -1