feat: add autocapture for element clicks (#199)

* Implement Autocapture for the Kotlin SDK

* Support Android API levels down to 16 for autocapture

* Revert linter changes

* Upgrade minSDK to 19 with targetSDK set to 34

* Add unit tests for Autocapture

* Revert linter changes and fix typo

* Add tracking start and stop for user interactions

* Find ViewTarget even if Resource.NotFoundException is thrown

* Change event name to [Amplitude] Element Tapped

* Add checks for Compose

* Fix failing tests for autocapture

* Fix typo

* Change element clicked event name

* Move compose to the android module

* Remove compose gradle plugin

* Add experimental opt-in annotation for user interactions tracking

* Fix lint issues

* Change build JDK version to 17

* Update gradle version

* Add Java 11 to the CI flow

* Cleanup the CI workflow

* Add name for workflow jobs

* Exclude tests for builds

* Opt-in for internalComposeUiApi usage

.github/workflows/pull-request-test.yml

@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   build:
     name: Build with JDK ${{ matrix.java-version }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         java-version: [11, 17]
@@ -20,7 +20,7 @@ jobs:
           cache: 'gradle'
 
       - name: Build
-        run: ./gradlew build
+        run: ./gradlew build -x test
 
       - name: Upload build results
         if: always()
@@ -35,7 +35,7 @@ jobs:
 
   test-and-lint:
     name: Test and Lint with JDK 17
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: build
     steps:
       - uses: actions/checkout@v4

android/build.gradle

@@ -16,13 +16,13 @@ apply from: "${rootDir}/gradle/publish-module.gradle"
 version = PUBLISH_VERSION
 
 android {
-    compileSdk 31
+    compileSdk 34
 
     defaultConfig {
         multiDexEnabled true
 
-        minSdk 16
-        targetSdk 31
+        minSdk 19
+        targetSdk 34
         versionName PUBLISH_VERSION
         buildConfigField "String", "AMPLITUDE_VERSION", "\"${version}\""
 
@@ -47,6 +47,7 @@ android {
     testOptions {
         unitTests {
             includeAndroidResources = true
+            returnDefaultValues = true
         }
     }
 }
@@ -61,6 +62,8 @@ dependencies {
     implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.2'
     implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.5.2'
     implementation 'com.amplitude:analytics-connector:1.0.0'
+    implementation 'androidx.core:core-ktx:1.7.0'
+    compileOnly 'androidx.compose.ui:ui:1.6.7'
 
     coreLibraryDesugaring 'com.android.tools:desugar_jdk_libs:1.1.5'
 
@@ -74,7 +77,7 @@ dependencies {
     testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2'
     testImplementation("junit:junit:4.13.2")
     testRuntimeOnly("org.junit.vintage:junit-vintage-engine:5.7.2")
-    testImplementation 'org.robolectric:robolectric:4.7.3'
+    testImplementation 'org.robolectric:robolectric:4.12.1'
     testImplementation 'androidx.test:core:1.4.0'
     testImplementation 'androidx.test.ext:junit:1.1.3'
     androidTestImplementation 'androidx.test.ext:junit:1.1.3'

android/src/main/AndroidManifest.xml

@@ -1,5 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
     package="com.amplitude.android">
-
 </manifest>

android/src/main/java/com/amplitude/android/DefaultTrackingOptions.kt

@@ -1,27 +1,46 @@
 package com.amplitude.android
 
-open class DefaultTrackingOptions @JvmOverloads constructor(
+open class DefaultTrackingOptions
+@JvmOverloads
+constructor(
     var sessions: Boolean = true,
     var appLifecycles: Boolean = false,
     var deepLinks: Boolean = false,
-    var screenViews: Boolean = false
+    var screenViews: Boolean = false,
 ) {
+    var userInteractions = false
+        @ExperimentalAmplitudeFeature
+        set
+
     // Prebuilt options for easier usage
     companion object {
         @JvmField
-        val ALL = DefaultTrackingOptions(
-            sessions = true,
-            appLifecycles = true,
-            deepLinks = true,
-            screenViews = true
-        )
+        val ALL =
+            DefaultTrackingOptions(
+                sessions = true,
+                appLifecycles = true,
+                deepLinks = true,
+                screenViews = true,
+            )
 
         @JvmField
-        val NONE = DefaultTrackingOptions(
-            sessions = false,
-            appLifecycles = false,
-            deepLinks = false,
-            screenViews = false
-        )
+        val NONE =
+            DefaultTrackingOptions(
+                sessions = false,
+                appLifecycles = false,
+                deepLinks = false,
+                screenViews = false,
+            )
+    }
+
+    @ExperimentalAmplitudeFeature
+    constructor(
+        sessions: Boolean = true,
+        appLifecycles: Boolean = false,
+        deepLinks: Boolean = false,
+        screenViews: Boolean = false,
+        userInteractions: Boolean = false,
+    ) : this(sessions, appLifecycles, deepLinks, screenViews) {
+        this.userInteractions = userInteractions
     }
 }

android/src/main/java/com/amplitude/android/ExperimentalAmplitudeFeature.kt

@@ -0,0 +1,9 @@
+package com.amplitude.android
+
+@RequiresOptIn(
+    message =
+    "This feature is experimental, and may change or break at any time. Use with caution.",
+)
+@Retention(AnnotationRetention.BINARY)
+@Target(AnnotationTarget.PROPERTY_SETTER, AnnotationTarget.CONSTRUCTOR)
+annotation class ExperimentalAmplitudeFeature

android/src/main/java/com/amplitude/android/internal/ViewHierarchyScanner.kt

@@ -0,0 +1,95 @@
+package com.amplitude.android.internal
+
+import android.os.Looper
+import android.view.View
+import android.view.ViewGroup
+import androidx.core.view.children
+import com.amplitude.android.internal.locators.ViewTargetLocator
+import com.amplitude.common.Logger
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
+
+/**
+ * Utility class for scanning the view hierarchy and finding a target at a given position.
+ */
+internal object ViewHierarchyScanner {
+    /**
+     * Finds a target in the view hierarchy at the given position and returns a [ViewTarget].
+     *
+     * If the found target is clickable and its [View] contains another clickable direct child in the
+     * target position, the child will be returned.
+     *
+     * @param position the position (x, y) to find the target at
+     * @param targetType the type of the target to find
+     * @param viewTargetLocators the locators to use to find the target
+     * @return the [ViewTarget] at the given position, or null if none was found
+     */
+    @JvmStatic
+    fun View.findTarget(
+        position: Pair<Float, Float>,
+        viewTargetLocators: List<ViewTargetLocator>,
+        targetType: ViewTarget.Type,
+        logger: Logger,
+    ): ViewTarget? =
+        runBlocking {
+            val viewLooper =
+                handler?.looper
+                    ?: Looper.getMainLooper()
+                    ?: logger.error("Unable to get main looper")
+                        .let { return@runBlocking null }
+
+            // The entire view tree is single threaded, and that's typically the main thread, but
+            // it doesn't have to be, and we don't know where the passed in view is coming from.
+            if (viewLooper.thread == Thread.currentThread()) {
+                findTargetWithLocators(position, targetType, viewTargetLocators, logger)
+            } else {
+                withContext(Dispatchers.Main) {
+                    findTargetWithLocators(position, targetType, viewTargetLocators, logger)
+                }
+            }
+        }
+
+    /** Applies the locators to the view hierarchy to find the target */
+    private fun View.findTargetWithLocators(
+        position: Pair<Float, Float>,
+        targetType: ViewTarget.Type,
+        viewTargetLocators: List<ViewTargetLocator>,
+        logger: Logger,
+    ): ViewTarget? {
+        val queue = ArrayDeque<View>().apply { add(this@findTargetWithLocators) }
+
+        var target: ViewTarget? = null
+        while (queue.isNotEmpty()) {
+            val view =
+                try {
+                    queue.removeFirst()
+                } catch (e: NoSuchElementException) {
+                    logger.error("Unable to get view from queue")
+                    continue
+                }
+
+            if (view is ViewGroup) {
+                queue.addAll(view.children)
+            }
+
+            // Applies the locators until a target is found. If the target type is clickable, check
+            // the children in case the target is a child which is also clickable.
+            viewTargetLocators.any { locator ->
+                with(locator) {
+                    view.locate(position, targetType)?.let { newTarget ->
+                        if (targetType == ViewTarget.Type.Clickable) {
+                            target = newTarget
+                            return@any true
+                        } else {
+                            return newTarget
+                        }
+                    }
+                    false
+                }
+            }
+        }
+
+        return target
+    }
+}

android/src/main/java/com/amplitude/android/internal/ViewTarget.kt

@@ -0,0 +1,25 @@
+package com.amplitude.android.internal
+
+import java.lang.ref.WeakReference
+
+/**
+ * Represents a UI element in the view hierarchy from [ViewHierarchyScanner].
+ *
+ * @property className the class name of the view.
+ * @property recourseName the resource name of the view.
+ * @property tag the tag of the view.
+ */
+data class ViewTarget(
+    private val _view: Any?,
+    val className: String?,
+    val recourseName: String?,
+    val tag: String?,
+    val source: String,
+) {
+    private val viewRef: WeakReference<Any> = WeakReference(_view)
+
+    val view: Any?
+        get() = viewRef.get()
+
+    enum class Type { Clickable }
+}

android/src/main/java/com/amplitude/android/internal/gestures/AutocaptureGestureListener.kt

@@ -0,0 +1,78 @@
+package com.amplitude.android.internal.gestures
+
+import android.app.Activity
+import android.view.GestureDetector
+import android.view.MotionEvent
+import androidx.annotation.VisibleForTesting
+import com.amplitude.android.internal.ViewHierarchyScanner.findTarget
+import com.amplitude.android.internal.ViewTarget
+import com.amplitude.android.internal.locators.ViewTargetLocator
+import com.amplitude.android.utilities.DefaultEventUtils.EventProperties.ELEMENT_CLASS
+import com.amplitude.android.utilities.DefaultEventUtils.EventProperties.ELEMENT_RESOURCE
+import com.amplitude.android.utilities.DefaultEventUtils.EventProperties.ELEMENT_SOURCE
+import com.amplitude.android.utilities.DefaultEventUtils.EventProperties.ELEMENT_TAG
+import com.amplitude.android.utilities.DefaultEventUtils.EventTypes.ELEMENT_CLICKED
+import com.amplitude.common.Logger
+import java.lang.ref.WeakReference
+
+@VisibleForTesting(otherwise = VisibleForTesting.PACKAGE_PRIVATE)
+class AutocaptureGestureListener(
+    activity: Activity,
+    private val track: (String, Map<String, Any?>) -> Unit,
+    private val logger: Logger,
+    private val viewTargetLocators: List<ViewTargetLocator>,
+) : GestureDetector.OnGestureListener {
+    private val activityRef: WeakReference<Activity> = WeakReference(activity)
+
+    override fun onDown(e: MotionEvent): Boolean {
+        return false
+    }
+
+    override fun onShowPress(e: MotionEvent) {}
+
+    override fun onSingleTapUp(e: MotionEvent): Boolean {
+        val decorView =
+            activityRef.get()?.window?.decorView
+                ?: logger.error("DecorView is null in onSingleTapUp()").let { return false }
+        val target: ViewTarget =
+            decorView.findTarget(
+                Pair(e.x, e.y),
+                viewTargetLocators,
+                ViewTarget.Type.Clickable,
+                logger,
+            ) ?: logger.error("Unable to find click target").let { return false }
+
+        mapOf(
+            ELEMENT_CLASS to target.className,
+            ELEMENT_RESOURCE to target.recourseName,
+            ELEMENT_TAG to target.tag,
+            ELEMENT_SOURCE to
+                target.source
+                    .replace("_", " ")
+                    .split(" ")
+                    .joinToString(" ") { it.replaceFirstChar { c -> c.uppercase() } },
+        ).let { track(ELEMENT_CLICKED, it) }
+
+        return false
+    }
+
+    override fun onScroll(
+        e1: MotionEvent?,
+        e2: MotionEvent,
+        distanceX: Float,
+        distanceY: Float,
+    ): Boolean {
+        return false
+    }
+
+    override fun onLongPress(e: MotionEvent) {}
+
+    override fun onFling(
+        e1: MotionEvent?,
+        e2: MotionEvent,
+        velocityX: Float,
+        velocityY: Float,
+    ): Boolean {
+        return false
+    }
+}

android/src/main/java/com/amplitude/android/internal/gestures/AutocaptureWindowCallback.kt

@@ -0,0 +1,41 @@
+package com.amplitude.android.internal.gestures
+
+import android.app.Activity
+import android.view.GestureDetector
+import android.view.MotionEvent
+import android.view.Window
+import com.amplitude.android.internal.locators.ViewTargetLocator
+import com.amplitude.common.Logger
+
+internal class AutocaptureWindowCallback(
+    delegate: Window.Callback,
+    activity: Activity,
+    track: (String, Map<String, Any?>) -> Unit,
+    viewTargetLocators: List<ViewTargetLocator>,
+    private val logger: Logger,
+    private val motionEventObtainer: MotionEventObtainer = object : MotionEventObtainer {},
+    private val gestureListener: AutocaptureGestureListener =
+        AutocaptureGestureListener(activity, track, logger, viewTargetLocators),
+    private val gestureDetector: GestureDetector = GestureDetector(activity, gestureListener),
+) : WindowCallbackAdapter(delegate) {
+    override fun dispatchTouchEvent(event: MotionEvent?): Boolean {
+        event?.let {
+            motionEventObtainer.obtain(event).let {
+                try {
+                    gestureDetector.onTouchEvent(it)
+                } catch (e: Exception) {
+                    logger.error("Error handling touch event: $e")
+                } finally {
+                    it.recycle()
+                }
+            }
+        }
+        return super.dispatchTouchEvent(event)
+    }
+
+    interface MotionEventObtainer {
+        fun obtain(origin: MotionEvent): MotionEvent {
+            return MotionEvent.obtain(origin)
+        }
+    }
+}