Implement bitmap downscaling

detekt_custom.yml

@@ -727,6 +727,7 @@ datadog:
       - "kotlin.collections.MutableList.add(com.datadog.android.plugin.DatadogPlugin)"
       - "kotlin.collections.MutableList.add(com.datadog.android.sessionreplay.internal.processor.MutationResolver.Entry)"
       - "kotlin.collections.MutableList.add(com.datadog.android.sessionreplay.model.MobileSegment.Add)"
+      - "kotlin.collections.MutableList.add(com.datadog.android.sessionreplay.model.MobileSegment.Wireframe)"
       - "kotlin.collections.MutableList.add(com.datadog.android.sessionreplay.model.MobileSegment.WireframeUpdateMutation)"
       - "kotlin.collections.MutableList.add(com.datadog.android.rum.internal.domain.scope.RumScope)"
       - "kotlin.collections.MutableList.add(com.datadog.android.rum.model.ActionEvent.Type)"
@@ -838,6 +839,7 @@ datadog:
       - "kotlin.Float.toFloat()"
       - "kotlin.Int.inv()"
       - "kotlin.Int.toChar()"
+      - "kotlin.Int.toDouble()"
       - "kotlin.Int.toFloat()"
       - "kotlin.Int.toLong()"
       - "kotlin.Int.and(kotlin.Int)"
@@ -857,6 +859,7 @@ datadog:
       - "kotlin.Pair.constructor(com.datadog.android.sessionreplay.internal.utils.SessionReplayRumContext, com.google.gson.JsonArray)"
       - "kotlin.Pair.constructor(com.datadog.android.sessionreplay.model.MobileSegment, com.google.gson.JsonObject)"
       - "kotlin.Pair.constructor(com.google.gson.JsonObject, kotlin.Long)"
+      - "kotlin.Pair.constructor(kotlin.Int, kotlin.Int)"
       - "kotlin.Triple.constructor(kotlin.String, kotlin.String, kotlin.String)"
       - "kotlin.Triple.constructor(kotlin.Nothing?, kotlin.Nothing?, kotlin.Nothing?)"
       # endregion

features/dd-sdk-android-session-replay/src/main/kotlin/com/datadog/android/sessionreplay/internal/recorder/base64/Base64Serializer.kt

@@ -55,7 +55,7 @@ class Base64Serializer private constructor(
             return
         }
 
-        val bitmap = drawableUtils.createBitmapFromDrawable(drawable, displayMetrics)
+        val bitmap = drawableUtils.createBitmapOfApproxSizeFromDrawable(drawable, displayMetrics)
 
         if (bitmap == null) {
             asyncImageProcessingCallback?.finishProcessingImage()
@@ -74,10 +74,6 @@ class Base64Serializer private constructor(
         this.asyncImageProcessingCallback = asyncImageProcessingCallback
     }
 
-    @VisibleForTesting
-    internal fun isOverSizeLimit(bitmapSize: Int): Boolean =
-        bitmapSize > BITMAP_SIZE_LIMIT_BYTES
-
     // endregion
 
     // region testing
@@ -120,25 +116,22 @@ class Base64Serializer private constructor(
 
     @WorkerThread
     private fun convertBmpToBase64(drawable: Drawable, bitmap: Bitmap): String {
-        val byteArrayOutputStream = webPImageCompression.compressBitmapToStream(bitmap)
+        val base64Result: String
 
-        if (isOverSizeLimit(byteArrayOutputStream.size())) {
-            return ""
-        }
+        val byteArrayOutputStream = webPImageCompression.compressBitmapToStream(bitmap)
 
-        val base64String: String
         try {
-            base64String = base64Utils.serializeToBase64String(byteArrayOutputStream)
+            base64Result = base64Utils.serializeToBase64String(byteArrayOutputStream)
 
-            if (base64String.isNotEmpty()) {
+            if (base64Result.isNotEmpty()) {
                 // if we got a base64 string then cache it
-                base64LruCache.put(drawable, base64String)
+                base64LruCache.put(drawable, base64Result)
             }
         } finally {
             bitmap.recycle()
         }
 
-        return base64String
+        return base64Result
     }
 
     private fun finalizeRecordedDataItem(
@@ -208,9 +201,6 @@ class Base64Serializer private constructor(
     // endregion
 
     internal companion object {
-        @VisibleForTesting
-        internal const val BITMAP_SIZE_LIMIT_BYTES = 15000 // 15 kbs
-
         internal const val DOES_NOT_IMPLEMENT_COMPONENTCALLBACKS =
             "Cache instance does not implement ComponentCallbacks2"
 

features/dd-sdk-android-session-replay/src/main/kotlin/com/datadog/android/sessionreplay/internal/recorder/mapper/ImageButtonMapper.kt

@@ -6,7 +6,9 @@
 
 package com.datadog.android.sessionreplay.internal.recorder.mapper
 
+import android.graphics.drawable.Drawable
 import android.widget.ImageButton
+import com.datadog.android.sessionreplay.internal.recorder.GlobalBounds
 import com.datadog.android.sessionreplay.internal.recorder.MappingContext
 import com.datadog.android.sessionreplay.internal.recorder.base64.Base64Serializer
 import com.datadog.android.sessionreplay.internal.recorder.base64.ImageCompression
@@ -18,15 +20,15 @@ internal class ImageButtonMapper(
     webPImageCompression: ImageCompression = WebPImageCompression(),
     base64Serializer: Base64Serializer = Base64Serializer.Builder().build(),
     uniqueIdentifierGenerator: UniqueIdentifierGenerator = UniqueIdentifierGenerator
-) : BaseWireframeMapper<ImageButton, MobileSegment.Wireframe.ImageWireframe>(
+) : BaseWireframeMapper<ImageButton, MobileSegment.Wireframe>(
     webPImageCompression = webPImageCompression,
     base64Serializer = base64Serializer,
     uniqueIdentifierGenerator = uniqueIdentifierGenerator
 ) {
     override fun map(
         view: ImageButton,
         mappingContext: MappingContext
-    ): List<MobileSegment.Wireframe.ImageWireframe> {
+    ): List<MobileSegment.Wireframe> {
         val resources = view.resources
         val drawable = view.drawable?.constantState?.newDrawable(resources)
         val id = resolveChildDrawableUniqueIdentifier(view)
@@ -39,22 +41,51 @@ internal class ImageButtonMapper(
         val (shapeStyle, border) = view.background?.resolveShapeStyleAndBorder(view.alpha)
             ?: (null to null)
 
+        val wireframes = mutableListOf<MobileSegment.Wireframe>()
+
+        // if the drawable has no width/height then there's no point trying to get a bitmap
+        if (drawable.intrinsicWidth > 0 && drawable.intrinsicHeight > 0) {
+            val imageWireframe = resolveImageWireframe(
+                view,
+                id,
+                bounds,
+                shapeStyle,
+                border,
+                drawable
+            )
+            wireframes.add(imageWireframe)
+        }
+
+        return wireframes
+    }
+
+    // region internal
+
+    private fun resolveImageWireframe(
+        view: ImageButton,
+        id: Long,
+        bounds: GlobalBounds,
+        shapeStyle: MobileSegment.ShapeStyle?,
+        border: MobileSegment.ShapeBorder?,
+        drawable: Drawable
+    ): MobileSegment.Wireframe.ImageWireframe {
         val mimeType = getWebPMimeType()
         val displayMetrics = view.resources.displayMetrics
         val applicationContext = view.context.applicationContext
 
-        val imageWireframe = MobileSegment.Wireframe.ImageWireframe(
-            id = id,
-            x = bounds.x,
-            y = bounds.y,
-            width = bounds.width,
-            height = bounds.height,
-            shapeStyle = shapeStyle,
-            border = border,
-            base64 = "",
-            mimeType = mimeType,
-            isEmpty = true
-        )
+        val imageWireframe =
+            MobileSegment.Wireframe.ImageWireframe(
+                id = id,
+                x = bounds.x,
+                y = bounds.y,
+                width = bounds.width,
+                height = bounds.height,
+                shapeStyle = shapeStyle,
+                border = border,
+                base64 = "",
+                mimeType = mimeType,
+                isEmpty = true
+            )
 
         @Suppress("ThreadSafety") // TODO REPLAY-1861 caller thread of .map is unknown?
         handleBitmap(
@@ -64,6 +95,8 @@ internal class ImageButtonMapper(
             imageWireframe = imageWireframe
         )
 
-        return listOf(imageWireframe)
+        return imageWireframe
     }
+
+    // endregion
 }

features/dd-sdk-android-session-replay/src/main/kotlin/com/datadog/android/sessionreplay/internal/recorder/wrappers/BitmapWrapper.kt

@@ -17,7 +17,7 @@ internal class BitmapWrapper {
         bitmapHeight: Int,
         config: Config
     ): Bitmap? {
-        @Suppress("SwallowedException", "TooGenericExceptionCaught")
+        @Suppress("SwallowedException")
         return try {
             Bitmap.createBitmap(displayMetrics, bitmapWidth, bitmapHeight, config)
         } catch (e: IllegalArgumentException) {

features/dd-sdk-android-session-replay/src/main/kotlin/com/datadog/android/sessionreplay/internal/utils/DrawableUtils.kt

@@ -13,22 +13,67 @@ import android.util.DisplayMetrics
 import androidx.annotation.MainThread
 import com.datadog.android.sessionreplay.internal.recorder.wrappers.BitmapWrapper
 import com.datadog.android.sessionreplay.internal.recorder.wrappers.CanvasWrapper
+import kotlin.math.sqrt
 
 internal class DrawableUtils(
     private val bitmapWrapper: BitmapWrapper = BitmapWrapper(),
     private val canvasWrapper: CanvasWrapper = CanvasWrapper()
 ) {
+    /**
+     * This method attempts to create a bitmap from a drawable, such that the bitmap file size will
+     * be equal or less than a given size. It does so by modifying the dimensions of the
+     * bitmap, since the file size of a bitmap can be known by the formula width*height*color depth
+     */
     @MainThread
     @Suppress("ReturnCount")
-    internal fun createBitmapFromDrawable(drawable: Drawable, displayMetrics: DisplayMetrics): Bitmap? {
-        val bitmapWidth = if (drawable.intrinsicWidth <= 0) 1 else drawable.intrinsicWidth
-        val bitmapHeight = if (drawable.intrinsicHeight <= 0) 1 else drawable.intrinsicHeight
-        val bitmap = bitmapWrapper.createBitmap(displayMetrics, bitmapWidth, bitmapHeight, Bitmap.Config.ARGB_8888)
+    internal fun createBitmapOfApproxSizeFromDrawable(
+        drawable: Drawable,
+        displayMetrics: DisplayMetrics,
+        requestedSizeInBytes: Int = MAX_BITMAP_SIZE_IN_BYTES
+    ): Bitmap? {
+        val (width, height) = getScaledWidthAndHeight(drawable, requestedSizeInBytes)
+
+        val bitmap = bitmapWrapper.createBitmap(
+            displayMetrics,
+            width,
+            height,
+            Bitmap.Config.ARGB_8888
+        )
             ?: return null
 
         val canvas = canvasWrapper.createCanvas(bitmap) ?: return null
         drawable.setBounds(0, 0, canvas.width, canvas.height)
         drawable.draw(canvas)
         return bitmap
     }
+
+    private fun getScaledWidthAndHeight(
+        drawable: Drawable,
+        requestedSizeInBytes: Int
+    ): Pair<Int, Int> {
+        var width = drawable.intrinsicWidth
+        var height = drawable.intrinsicHeight
+        val sizeAfterCreation = width * height * ARGB_8888_PIXEL_SIZE_BYTES
+
+        if (sizeAfterCreation > requestedSizeInBytes) {
+            val bitmapRatio = width.toDouble() / height.toDouble()
+            val totalMaxPixels = (requestedSizeInBytes / ARGB_8888_PIXEL_SIZE_BYTES).toDouble()
+            val maxSize = sqrt(totalMaxPixels).toInt()
+            width = maxSize
+            height = maxSize
+
+            if (bitmapRatio > 1) { // width gt height
+                height = (maxSize / bitmapRatio).toInt()
+            } else {
+                width = (maxSize * bitmapRatio).toInt()
+            }
+        }
+
+        return Pair(width, height)
+    }
+
+    internal companion object {
+        internal const val MAX_BITMAP_SIZE_IN_BYTES = 10240 // 10kb
+        private const val ARGB_8888_PIXEL_SIZE_BYTES = 4
+    }
 }

features/dd-sdk-android-session-replay/src/test/kotlin/com/datadog/android/sessionreplay/internal/recorder/base64/Base64SerializerTest.kt

@@ -13,7 +13,6 @@ import android.graphics.drawable.StateListDrawable
 import android.util.DisplayMetrics
 import com.datadog.android.sessionreplay.forge.ForgeConfigurator
 import com.datadog.android.sessionreplay.internal.AsyncImageProcessingCallback
-import com.datadog.android.sessionreplay.internal.recorder.base64.Base64Serializer.Companion.BITMAP_SIZE_LIMIT_BYTES
 import com.datadog.android.sessionreplay.internal.utils.Base64Utils
 import com.datadog.android.sessionreplay.internal.utils.DrawableUtils
 import com.datadog.android.sessionreplay.model.MobileSegment
@@ -30,6 +29,7 @@ import org.mockito.Mock
 import org.mockito.junit.jupiter.MockitoExtension
 import org.mockito.junit.jupiter.MockitoSettings
 import org.mockito.kotlin.any
+import org.mockito.kotlin.anyOrNull
 import org.mockito.kotlin.mock
 import org.mockito.kotlin.times
 import org.mockito.kotlin.verify
@@ -95,10 +95,17 @@ internal class Base64SerializerTest {
         fakeImageWireframe.isEmpty = true
 
         val fakeByteOutputStream: ByteArrayOutputStream = mock()
-        whenever(mockWebPImageCompression.compressBitmapToStream(any())).thenReturn(fakeByteOutputStream)
+        whenever(mockWebPImageCompression.compressBitmapToStream(any()))
+            .thenReturn(fakeByteOutputStream)
         whenever(mockBase64Utils.serializeToBase64String(any())).thenReturn(fakeBase64String)
 
-        whenever(mockDrawableUtils.createBitmapFromDrawable(any(), any())).thenReturn(mockBitmap)
+        whenever(
+            mockDrawableUtils.createBitmapOfApproxSizeFromDrawable(
+                any(),
+                any(),
+                anyOrNull()
+            )
+        ).thenReturn(mockBitmap)
 
         whenever(mockExecutorService.submit(any())).then {
             (it.arguments[0] as Runnable).run()
@@ -127,7 +134,8 @@ internal class Base64SerializerTest {
     @Test
     fun `M callback with finishProcessingImage W handleBitmap() { failed to create bmp }`() {
         // Given
-        whenever(mockDrawableUtils.createBitmapFromDrawable(any(), any())).thenReturn(null)
+        whenever(mockDrawableUtils.createBitmapOfApproxSizeFromDrawable(any(), any(), anyOrNull()))
+            .thenReturn(null)
 
         // When
         testedBase64Serializer.handleBitmap(
@@ -143,6 +151,10 @@ internal class Base64SerializerTest {
 
     @Test
     fun `M callback with finishProcessingImage W handleBitmap() { created bmp async }`() {
+        // Given
+        whenever(mockDrawableUtils.createBitmapOfApproxSizeFromDrawable(any(), any(), anyOrNull()))
+            .thenReturn(mockBitmap)
+
         // When
         testedBase64Serializer.handleBitmap(
             applicationContext = mockApplicationContext,
@@ -163,6 +175,12 @@ internal class Base64SerializerTest {
         val fakeBase64String = forge.anAsciiString()
         whenever(mockBase64LruCache.get(mockDrawable)).thenReturn(fakeBase64String)
 
+        whenever(mockDrawableUtils.createBitmapOfApproxSizeFromDrawable(any(), any(), anyOrNull()))
+            .thenReturn(mockBitmap)
+        val mockByteArrayOutputStream: ByteArrayOutputStream = mock()
+        whenever(mockWebPImageCompression.compressBitmapToStream(any()))
+            .thenReturn(mockByteArrayOutputStream)
+
         // When
         testedBase64Serializer.handleBitmap(
             applicationContext = mockApplicationContext,
@@ -189,7 +207,7 @@ internal class Base64SerializerTest {
         )
 
         // Then
-        verify(mockDrawableUtils).createBitmapFromDrawable(any(), any())
+        verify(mockDrawableUtils).createBitmapOfApproxSizeFromDrawable(any(), any(), anyOrNull())
     }
 
     @Test
@@ -218,28 +236,6 @@ internal class Base64SerializerTest {
         verify(mockApplicationContext, times(1)).registerComponentCallbacks(mockBase64LruCache)
     }
 
-    @Test
-    fun `M return empty base64 string W handleBitmap() { image over size limit }`() {
-        // Given
-        val mockByteArrayOutputStream: ByteArrayOutputStream = mock()
-        whenever(mockByteArrayOutputStream.size()).thenReturn(BITMAP_SIZE_LIMIT_BYTES + 1)
-        whenever(mockWebPImageCompression.compressBitmapToStream(any())).thenReturn(mockByteArrayOutputStream)
-        whenever(mockDrawableUtils.createBitmapFromDrawable(any(), any())).thenReturn(mockBitmap)
-
-        // When
-        testedBase64Serializer.handleBitmap(
-            applicationContext = mockApplicationContext,
-            displayMetrics = mockDisplayMetrics,
-            drawable = mockDrawable,
-            imageWireframe = fakeImageWireframe
-        )
-
-        // Then
-        assertThat(fakeImageWireframe.base64).isEmpty()
-        assertThat(fakeImageWireframe.isEmpty).isTrue()
-        verify(mockCallback).finishProcessingImage()
-    }
-
     @Test
     fun `M use the same ThreadPoolExecutor W build()`() {
         // When