Use in-memory thumbnail APIs when possible (#3817)

* Use in-memory thumbnail APIs when possible * Make an exception for animated image types. Also add `TimelineItemImageContent.thumbnailMediaRequestData` lazy property. * Try simplifying the logic a bit more.
element-hq · Nov 8, 2024 · 2873a68 · 2873a68
1 parent bee708c
commit 2873a68
Show file tree

Hide file tree

Showing 12 changed files with 62 additions and 19 deletions.
diff --git a/...element/android/features/messages/impl/timeline/components/event/TimelineItemImageView.kt b/...element/android/features/messages/impl/timeline/components/event/TimelineItemImageView.kt
@@ -51,7 +51,6 @@ import io.element.android.libraries.designsystem.components.blurhash.blurHashBac
 import io.element.android.libraries.designsystem.preview.ElementPreview
 import io.element.android.libraries.designsystem.preview.PreviewsDayNight
 import io.element.android.libraries.matrix.api.timeline.item.event.MessageFormat
-import io.element.android.libraries.matrix.ui.media.MediaRequestData
 import io.element.android.libraries.textcomposer.ElementRichTextEditorStyle
 import io.element.android.libraries.ui.strings.CommonStrings
 import io.element.android.wysiwyg.compose.EditorStyledText
@@ -86,13 +85,7 @@ fun TimelineItemImageView(
                     modifier = Modifier
                         .fillMaxWidth()
                         .then(if (isLoaded) Modifier.background(Color.White) else Modifier),
-                    model = MediaRequestData(
-                        source = content.preferredMediaSource,
-                        kind = MediaRequestData.Kind.File(
-                            fileName = content.filename,
-                            mimeType = content.mimeType,
-                        ),
-                    ),
+                    model = content.thumbnailMediaRequestData,
                     contentScale = ContentScale.Fit,
                     alignment = Alignment.Center,
                     contentDescription = description,

diff --git a/...element/android/features/messages/impl/timeline/components/event/TimelineItemVideoView.kt b/...element/android/features/messages/impl/timeline/components/event/TimelineItemVideoView.kt
@@ -57,6 +57,8 @@ import io.element.android.libraries.designsystem.modifiers.roundedBackground
 import io.element.android.libraries.designsystem.preview.ElementPreview
 import io.element.android.libraries.designsystem.preview.PreviewsDayNight
 import io.element.android.libraries.matrix.api.timeline.item.event.MessageFormat
+import io.element.android.libraries.matrix.ui.media.MAX_THUMBNAIL_HEIGHT
+import io.element.android.libraries.matrix.ui.media.MAX_THUMBNAIL_WIDTH
 import io.element.android.libraries.matrix.ui.media.MediaRequestData
 import io.element.android.libraries.textcomposer.ElementRichTextEditorStyle
 import io.element.android.libraries.ui.strings.CommonStrings
@@ -97,9 +99,9 @@ fun TimelineItemVideoView(
                         .then(if (isLoaded) Modifier.background(Color.White) else Modifier),
                     model = MediaRequestData(
                         source = content.thumbnailSource,
-                        kind = MediaRequestData.Kind.File(
-                            fileName = content.filename,
-                            mimeType = content.mimeType
+                        kind = MediaRequestData.Kind.Thumbnail(
+                            width = content.thumbnailWidth?.toLong() ?: MAX_THUMBNAIL_WIDTH,
+                            height = content.thumbnailHeight?.toLong() ?: MAX_THUMBNAIL_HEIGHT,
                         )
                     ),
                     contentScale = ContentScale.Fit,

diff --git a/...roid/features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactory.kt b/...roid/features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactory.kt
@@ -93,6 +93,8 @@ class TimelineItemContentMessageFactory @Inject constructor(
                     blurhash = messageType.info?.blurhash,
                     width = messageType.info?.width?.toInt(),
                     height = messageType.info?.height?.toInt(),
+                    thumbnailWidth = messageType.info?.thumbnailInfo?.width?.toInt(),
+                    thumbnailHeight = messageType.info?.thumbnailInfo?.height?.toInt(),
                     aspectRatio = aspectRatio,
                     formattedFileSize = fileSizeFormatter.format(messageType.info?.size ?: 0),
                     fileExtension = fileExtensionExtractor.extractFromName(messageType.filename)
@@ -146,6 +148,8 @@ class TimelineItemContentMessageFactory @Inject constructor(
                     mimeType = messageType.info?.mimetype ?: MimeTypes.OctetStream,
                     width = messageType.info?.width?.toInt(),
                     height = messageType.info?.height?.toInt(),
+                    thumbnailWidth = messageType.info?.thumbnailInfo?.width?.toInt(),
+                    thumbnailHeight = messageType.info?.thumbnailInfo?.height?.toInt(),
                     duration = messageType.info?.duration ?: Duration.ZERO,
                     blurHash = messageType.info?.blurhash,
                     aspectRatio = aspectRatio,

diff --git a/...o/element/android/features/messages/impl/timeline/model/event/TimelineItemImageContent.kt b/...o/element/android/features/messages/impl/timeline/model/event/TimelineItemImageContent.kt
@@ -7,9 +7,12 @@
 
 package io.element.android.features.messages.impl.timeline.model.event
 
-import io.element.android.libraries.core.mimetype.MimeTypes
+import io.element.android.libraries.core.mimetype.MimeTypes.isMimeTypeAnimatedImage
 import io.element.android.libraries.matrix.api.media.MediaSource
 import io.element.android.libraries.matrix.api.timeline.item.event.FormattedBody
+import io.element.android.libraries.matrix.ui.media.MAX_THUMBNAIL_HEIGHT
+import io.element.android.libraries.matrix.ui.media.MAX_THUMBNAIL_WIDTH
+import io.element.android.libraries.matrix.ui.media.MediaRequestData
 
 data class TimelineItemImageContent(
     override val filename: String,
@@ -23,15 +26,31 @@ data class TimelineItemImageContent(
     val blurhash: String?,
     val width: Int?,
     val height: Int?,
+    val thumbnailWidth: Int?,
+    val thumbnailHeight: Int?,
     val aspectRatio: Float?
 ) : TimelineItemEventContentWithAttachment {
     override val type: String = "TimelineItemImageContent"
 
     val showCaption = caption != null
 
-    val preferredMediaSource = if (mimeType == MimeTypes.Gif) {
-        mediaSource
-    } else {
-        thumbnailSource ?: mediaSource
+    val thumbnailMediaRequestData: MediaRequestData by lazy {
+        if (mimeType.isMimeTypeAnimatedImage()) {
+            MediaRequestData(
+                source = mediaSource,
+                kind = MediaRequestData.Kind.File(
+                    fileName = filename,
+                    mimeType = mimeType
+                )
+            )
+        } else {
+            MediaRequestData(
+                source = thumbnailSource ?: mediaSource,
+                kind = MediaRequestData.Kind.Thumbnail(
+                    width = thumbnailWidth?.toLong() ?: MAX_THUMBNAIL_WIDTH,
+                    height = thumbnailHeight?.toLong() ?: MAX_THUMBNAIL_HEIGHT
+                ),
+            )
+        }
     }
 }
diff --git a/...t/android/features/messages/impl/timeline/model/event/TimelineItemImageContentProvider.kt b/...t/android/features/messages/impl/timeline/model/event/TimelineItemImageContentProvider.kt
@@ -37,6 +37,8 @@ fun aTimelineItemImageContent(
     blurhash = blurhash,
     width = null,
     height = 300,
+    thumbnailWidth = null,
+    thumbnailHeight = 150,
     aspectRatio = aspectRatio,
     formattedFileSize = "4MB",
     fileExtension = "jpg"

diff --git a/...o/element/android/features/messages/impl/timeline/model/event/TimelineItemVideoContent.kt b/...o/element/android/features/messages/impl/timeline/model/event/TimelineItemVideoContent.kt
@@ -22,6 +22,8 @@ data class TimelineItemVideoContent(
     val blurHash: String?,
     val height: Int?,
     val width: Int?,
+    val thumbnailWidth: Int?,
+    val thumbnailHeight: Int?,
     val mimeType: String,
     val formattedFileSize: String,
     val fileExtension: String,

diff --git a/...t/android/features/messages/impl/timeline/model/event/TimelineItemVideoContentProvider.kt b/...t/android/features/messages/impl/timeline/model/event/TimelineItemVideoContentProvider.kt
@@ -35,8 +35,10 @@ fun aTimelineItemVideoContent(
     aspectRatio = aspectRatio,
     duration = 100.milliseconds,
     videoSource = MediaSource(""),
-    height = 300,
     width = 150,
+    height = 300,
+    thumbnailWidth = 150,
+    thumbnailHeight = 300,
     mimeType = MimeTypes.Mp4,
     formattedFileSize = "14MB",
     fileExtension = "mp4"

diff --git a/...s/impl/src/test/kotlin/io/element/android/features/messages/impl/MessagesPresenterTest.kt b/...s/impl/src/test/kotlin/io/element/android/features/messages/impl/MessagesPresenterTest.kt
@@ -324,6 +324,8 @@ class MessagesPresenterTest {
                     blurhash = null,
                     width = 20,
                     height = 20,
+                    thumbnailWidth = null,
+                    thumbnailHeight = null,
                     aspectRatio = 1.0f,
                     fileExtension = "jpg",
                     formattedFileSize = "4MB"
@@ -364,6 +366,8 @@ class MessagesPresenterTest {
                     blurHash = null,
                     width = 20,
                     height = 20,
+                    thumbnailWidth = 20,
+                    thumbnailHeight = 20,
                     aspectRatio = 1.0f,
                     fileExtension = "mp4",
                     formattedFileSize = "50MB"

diff --git a/.../features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactoryTest.kt b/.../features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactoryTest.kt
@@ -246,6 +246,8 @@ class TimelineItemContentMessageFactoryTest {
             width = null,
             mimeType = MimeTypes.OctetStream,
             formattedFileSize = "0 Bytes",
+            thumbnailWidth = null,
+            thumbnailHeight = null,
             fileExtension = "",
         )
         assertThat(result).isEqualTo(expected)
@@ -294,6 +296,8 @@ class TimelineItemContentMessageFactoryTest {
             width = 300,
             mimeType = MimeTypes.Mp4,
             formattedFileSize = "555 Bytes",
+            thumbnailWidth = 5,
+            thumbnailHeight = 10,
             fileExtension = "mp4",
         )
         assertThat(result).isEqualTo(expected)
@@ -458,6 +462,8 @@ class TimelineItemContentMessageFactoryTest {
             blurhash = null,
             width = null,
             height = null,
+            thumbnailWidth = null,
+            thumbnailHeight = null,
             aspectRatio = null
         )
         assertThat(result).isEqualTo(expected)
@@ -531,6 +537,8 @@ class TimelineItemContentMessageFactoryTest {
             blurhash = A_BLUR_HASH,
             width = 5,
             height = 10,
+            thumbnailWidth = 5,
+            thumbnailHeight = 10,
             aspectRatio = 0.5f,
         )
         assertThat(result).isEqualTo(expected)

diff --git a/libraries/core/src/main/kotlin/io/element/android/libraries/core/mimetype/MimeTypes.kt b/libraries/core/src/main/kotlin/io/element/android/libraries/core/mimetype/MimeTypes.kt
@@ -38,6 +38,7 @@ object MimeTypes {
     fun String?.normalizeMimeType() = if (this == BadJpg) Jpeg else this
 
     fun String?.isMimeTypeImage() = this?.startsWith("image/").orFalse()
+    fun String?.isMimeTypeAnimatedImage() = this == Gif || this == WebP
     fun String?.isMimeTypeVideo() = this?.startsWith("video/").orFalse()
     fun String?.isMimeTypeAudio() = this?.startsWith("audio/").orFalse()
     fun String?.isMimeTypeApplication() = this?.startsWith("application/").orFalse()

diff --git a/...ix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/RustMediaLoader.kt b/...ix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/RustMediaLoader.kt
@@ -37,7 +37,7 @@ class RustMediaLoader(
         withContext(mediaDispatcher) {
             runCatching {
                 source.toRustMediaSource().use { source ->
-                    innerClient.getMediaContent(source).toUByteArray().toByteArray()
+                    innerClient.getMediaContent(source)
                 }
             }
         }
@@ -55,7 +55,7 @@ class RustMediaLoader(
                         mediaSource = mediaSource,
                         width = width.toULong(),
                         height = height.toULong()
-                    ).toUByteArray().toByteArray()
+                    )
                 }
             }
         }

diff --git a/...matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/MediaRequestData.kt b/...matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/MediaRequestData.kt
@@ -37,3 +37,9 @@ data class MediaRequestData(
         }
     }
 }
+
+/** Max width a thumbnail can have according to [the spec](https://spec.matrix.org/v1.10/client-server-api/#thumbnails). */
+const val MAX_THUMBNAIL_WIDTH = 800L
+
+/** Max height a thumbnail can have according to [the spec](https://spec.matrix.org/v1.10/client-server-api/#thumbnails). */
+const val MAX_THUMBNAIL_HEIGHT = 600L