Fix handling of multiple HEVC tracks in JPEG motion photos

The MP4 data in JPEG motion photos can contain multiple `video/hevc` tracks, but only the first is at a playable frame rate while the others are low-fps, high-res tracks designed for specific use-cases (not direct video playback). ExoPlayer currently selects the unplayable track by default, because it has a higher resolution. This change introduces a flag to `Mp4Extractor` that results in the first video track being marked as `ROLE_FLAG_MAIN`, and all subsequent video tracks `ROLE_FLAG_ALTERNATE` - this then results in the playable lower-res track being selected by default. PiperOrigin-RevId: 589832072
androidx · Dec 11, 2023 · 5266c71 · 5266c71
1 parent 4c4c5f6
commit 5266c71
Show file tree

Hide file tree

Showing 14 changed files with 442 additions and 12 deletions.
diff --git a/RELEASENOTES.md b/RELEASENOTES.md
@@ -50,6 +50,9 @@
         `ColorInfo.colorSpace`, `ColorInfo.colorTransfer`, and
         `ColorInfo.colorRange` values
         ([#692](https://github.com/androidx/media/pull/692)).
+    *   Mark secondary (unplayable) HEVC tracks in JPEG motion photos as
+        `ROLE_FLAG_ALTERNATE` to prevent them being automatically selected for
+        playback because of their higher resolution.
 *   Audio:
 *   Video:
     *   Add workaround for a device issue on Galaxy Tab S7 FE, Chromecast with

diff --git a/...ries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegMotionPhotoExtractor.java b/...ries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegMotionPhotoExtractor.java
@@ -17,6 +17,7 @@
 
 import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.extractor.SingleSampleExtractor.IMAGE_TRACK_ID;
+import static androidx.media3.extractor.mp4.Mp4Extractor.FLAG_MARK_FIRST_VIDEO_TRACK_WITH_MAIN_ROLE;
 import static java.lang.annotation.ElementType.TYPE_USE;
 
 import androidx.annotation.IntDef;
@@ -244,7 +245,7 @@ private void sniffMotionPhotoVideo(ExtractorInput input) throws IOException {
     } else {
       input.resetPeekPosition();
       if (mp4Extractor == null) {
-        mp4Extractor = new Mp4Extractor();
+        mp4Extractor = new Mp4Extractor(FLAG_MARK_FIRST_VIDEO_TRACK_WITH_MAIN_ROLE);
       }
       mp4ExtractorStartOffsetExtractorInput =
           new StartOffsetExtractorInput(input, mp4StartPosition);

diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java
@@ -80,7 +80,8 @@ public final class Mp4Extractor implements Extractor, SeekMap {
       value = {
         FLAG_WORKAROUND_IGNORE_EDIT_LISTS,
         FLAG_READ_MOTION_PHOTO_METADATA,
-        FLAG_READ_SEF_DATA
+        FLAG_READ_SEF_DATA,
+        FLAG_MARK_FIRST_VIDEO_TRACK_WITH_MAIN_ROLE
       })
   public @interface Flags {}
 
@@ -102,6 +103,12 @@ public final class Mp4Extractor implements Extractor, SeekMap {
    */
   public static final int FLAG_READ_SEF_DATA = 1 << 2;
 
+  /**
+   * Flag to mark the first video track encountered as {@link C#ROLE_FLAG_MAIN} and all subsequent
+   * video tracks as {@link C#ROLE_FLAG_ALTERNATE}.
+   */
+  public static final int FLAG_MARK_FIRST_VIDEO_TRACK_WITH_MAIN_ROLE = 1 << 3;
+
   /** Parser states. */
   @Documented
   @Retention(RetentionPolicy.SOURCE)
@@ -555,11 +562,18 @@ private void processMoovAtom(ContainerAtom moov) throws ParserException {
 
       Format.Builder formatBuilder = track.format.buildUpon();
       formatBuilder.setMaxInputSize(maxInputSize);
-      if (track.type == C.TRACK_TYPE_VIDEO
-          && trackDurationUs > 0
-          && trackSampleTable.sampleCount > 1) {
-        float frameRate = trackSampleTable.sampleCount / (trackDurationUs / 1000000f);
-        formatBuilder.setFrameRate(frameRate);
+      if (track.type == C.TRACK_TYPE_VIDEO) {
+        if ((flags & FLAG_MARK_FIRST_VIDEO_TRACK_WITH_MAIN_ROLE) != 0) {
+          formatBuilder.setRoleFlags(
+              track.format.roleFlags
+                  | (firstVideoTrackIndex == C.INDEX_UNSET
+                      ? C.ROLE_FLAG_MAIN
+                      : C.ROLE_FLAG_ALTERNATE));
+        }
+        if (trackDurationUs > 0 && trackSampleTable.sampleCount > 1) {
+          float frameRate = trackSampleTable.sampleCount / (trackDurationUs / 1000000f);
+          formatBuilder.setFrameRate(frameRate);
+        }
       }
 
       MetadataUtil.setFormatGaplessInfo(track.type, gaplessInfoHolder, formatBuilder);

diff --git a/...test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-2-hevc-tracks.jpg.0.dump b/...test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-2-hevc-tracks.jpg.0.dump
@@ -24,6 +24,7 @@ track 0:
       colorTransfer = 3
       lumaBitdepth = 8
       chromaBitdepth = 8
+    roleFlags = 1
     metadata = entries=[Mp4Timestamp: creation time=3784612704, modification time=3784612704, timescale=10000]
     initializationData:
       data = length 82, hash C508E2F1
@@ -276,6 +277,7 @@ track 1:
       colorTransfer = 3
       lumaBitdepth = 8
       chromaBitdepth = 8
+    roleFlags = 2
     metadata = entries=[Mp4Timestamp: creation time=3784612704, modification time=3784612704, timescale=10000]
     initializationData:
       data = length 82, hash 1924973

diff --git a/...test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-2-hevc-tracks.jpg.1.dump b/...test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-2-hevc-tracks.jpg.1.dump
@@ -24,6 +24,7 @@ track 0:
       colorTransfer = 3
       lumaBitdepth = 8
       chromaBitdepth = 8
+    roleFlags = 1
     metadata = entries=[Mp4Timestamp: creation time=3784612704, modification time=3784612704, timescale=10000]
     initializationData:
       data = length 82, hash C508E2F1
@@ -220,6 +221,7 @@ track 1:
       colorTransfer = 3
       lumaBitdepth = 8
       chromaBitdepth = 8
+    roleFlags = 2
     metadata = entries=[Mp4Timestamp: creation time=3784612704, modification time=3784612704, timescale=10000]
     initializationData:
       data = length 82, hash 1924973

diff --git a/...test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-2-hevc-tracks.jpg.2.dump b/...test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-2-hevc-tracks.jpg.2.dump
@@ -24,6 +24,7 @@ track 0:
       colorTransfer = 3
       lumaBitdepth = 8
       chromaBitdepth = 8
+    roleFlags = 1
     metadata = entries=[Mp4Timestamp: creation time=3784612704, modification time=3784612704, timescale=10000]
     initializationData:
       data = length 82, hash C508E2F1
@@ -136,6 +137,7 @@ track 1:
       colorTransfer = 3
       lumaBitdepth = 8
       chromaBitdepth = 8
+    roleFlags = 2
     metadata = entries=[Mp4Timestamp: creation time=3784612704, modification time=3784612704, timescale=10000]
     initializationData:
       data = length 82, hash 1924973

diff --git a/...test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-2-hevc-tracks.jpg.3.dump b/...test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-2-hevc-tracks.jpg.3.dump
@@ -24,6 +24,7 @@ track 0:
       colorTransfer = 3
       lumaBitdepth = 8
       chromaBitdepth = 8
+    roleFlags = 1
     metadata = entries=[Mp4Timestamp: creation time=3784612704, modification time=3784612704, timescale=10000]
     initializationData:
       data = length 82, hash C508E2F1
@@ -52,6 +53,7 @@ track 1:
       colorTransfer = 3
       lumaBitdepth = 8
       chromaBitdepth = 8
+    roleFlags = 2
     metadata = entries=[Mp4Timestamp: creation time=3784612704, modification time=3784612704, timescale=10000]
     initializationData:
       data = length 82, hash 1924973

diff --git a/.../src/test/assets/extractordumps/jpeg/pixel-motion-photo-jfif-segment-shortened.jpg.0.dump b/.../src/test/assets/extractordumps/jpeg/pixel-motion-photo-jfif-segment-shortened.jpg.0.dump
@@ -20,6 +20,7 @@ track 0:
     colorInfo:
       lumaBitdepth = 8
       chromaBitdepth = 8
+    roleFlags = 1
     metadata = entries=[TSSE: description=null: values=[Lavf58.42.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
     initializationData:
       data = length 32, hash 1F3D6E87

diff --git a/.../src/test/assets/extractordumps/jpeg/pixel-motion-photo-jfif-segment-shortened.jpg.1.dump b/.../src/test/assets/extractordumps/jpeg/pixel-motion-photo-jfif-segment-shortened.jpg.1.dump
@@ -20,6 +20,7 @@ track 0:
     colorInfo:
       lumaBitdepth = 8
       chromaBitdepth = 8
+    roleFlags = 1
     metadata = entries=[TSSE: description=null: values=[Lavf58.42.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
     initializationData:
       data = length 32, hash 1F3D6E87

diff --git a/.../src/test/assets/extractordumps/jpeg/pixel-motion-photo-jfif-segment-shortened.jpg.2.dump b/.../src/test/assets/extractordumps/jpeg/pixel-motion-photo-jfif-segment-shortened.jpg.2.dump
@@ -20,6 +20,7 @@ track 0:
     colorInfo:
       lumaBitdepth = 8
       chromaBitdepth = 8
+    roleFlags = 1
     metadata = entries=[TSSE: description=null: values=[Lavf58.42.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
     initializationData:
       data = length 32, hash 1F3D6E87

diff --git a/.../src/test/assets/extractordumps/jpeg/pixel-motion-photo-jfif-segment-shortened.jpg.3.dump b/.../src/test/assets/extractordumps/jpeg/pixel-motion-photo-jfif-segment-shortened.jpg.3.dump
@@ -20,6 +20,7 @@ track 0:
     colorInfo:
       lumaBitdepth = 8
       chromaBitdepth = 8
+    roleFlags = 1
     metadata = entries=[TSSE: description=null: values=[Lavf58.42.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
     initializationData:
       data = length 32, hash 1F3D6E87