Calculate and show frame number (#8112)

### What
Show frame number in selection panel when selecting a video.


![image](https://github.com/user-attachments/assets/4cab373d-44f0-4bb0-ab9a-a92b10cd0277)

We may want to make it more prominent though? 🤔

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/8112?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/8112?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!
* [x] If have noted any breaking changes to the log API in
`CHANGELOG.md` and the migration guide

- [PR Build Summary](https://build.rerun.io/pr/8112)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.

crates/store/re_video/src/decode/av1.rs

@@ -255,6 +255,7 @@ fn create_frame(debug_name: &str, picture: &dav1d::Picture) -> Result<Frame> {
         info: FrameInfo {
             is_sync: None,    // TODO(emilk)
             sample_idx: None, // TODO(emilk),
+            frame_nr: None,   // TODO(emilk),
             presentation_timestamp: Time(picture.timestamp().unwrap_or(0)),
             duration: Time(picture.duration()),
             latest_decode_timestamp: None,

crates/store/re_video/src/decode/ffmpeg_h264/ffmpeg.rs

@@ -103,9 +103,22 @@ struct FFmpegFrameInfo {
     /// can be decoded from only this one sample (though I'm not 100% sure).
     is_sync: bool,
 
-    /// Which sample is this in the video?
+    /// Which sample in the video is this from?
+    ///
+    /// In MP4, one sample is one frame, but we may be reordering samples when decoding.
+    ///
+    /// This is the order of which the samples appear in the container,
+    /// which is usually ordered by [`Self::decode_timestamp`].
     sample_idx: usize,
 
+    /// Which frame is this?
+    ///
+    /// This is on the assumption that each sample produces a single frame,
+    /// which is true for MP4.
+    ///
+    /// This is the index of frames ordered by [`Self::presentation_timestamp`].
+    frame_nr: usize,
+
     presentation_timestamp: Time,
     duration: Time,
     decode_timestamp: Time,
@@ -321,6 +334,7 @@ impl FFmpegProcessAndListener {
         let frame_info = FFmpegFrameInfo {
             is_sync: chunk.is_sync,
             sample_idx: chunk.sample_idx,
+            frame_nr: chunk.frame_nr,
             presentation_timestamp: chunk.presentation_timestamp,
             decode_timestamp: chunk.decode_timestamp,
             duration: chunk.duration,
@@ -549,6 +563,7 @@ impl FrameBuffer {
             info: FrameInfo {
                 is_sync: Some(frame_info.is_sync),
                 sample_idx: Some(frame_info.sample_idx),
+                frame_nr: Some(frame_info.frame_nr),
                 presentation_timestamp: frame_info.presentation_timestamp,
                 duration: frame_info.duration,
                 latest_decode_timestamp: Some(frame_info.decode_timestamp),

crates/store/re_video/src/decode/mod.rs

@@ -161,6 +161,8 @@ pub fn new_decoder(
 ) -> Result<Box<dyn AsyncDecoder>> {
     #![allow(unused_variables, clippy::needless_return)] // With some feature flags
 
+    re_tracing::profile_function!();
+
     re_log::trace!(
         "Looking for decoder for {}",
         video.human_readable_codec_string()
@@ -226,8 +228,19 @@ pub struct Chunk {
     pub data: Vec<u8>,
 
     /// Which sample (frame) did this chunk come from?
+    ///
+    /// This is the order of which the samples appear in the container,
+    /// which is usually ordered by [`Self::decode_timestamp`].
     pub sample_idx: usize,
 
+    /// Which frame does this chunk belong to?
+    ///
+    /// This is on the assumption that each sample produces a single frame,
+    /// which is true for MP4.
+    ///
+    /// This is the index of samples ordered by [`Self::presentation_timestamp`].
+    pub frame_nr: usize,
+
     /// Decode timestamp of this sample.
     /// Chunks are expected to be submitted in the order of decode timestamp.
     ///
@@ -271,9 +284,22 @@ pub struct FrameInfo {
     ///
     /// In MP4, one sample is one frame, but we may be reordering samples when decoding.
     ///
+    /// This is the order of which the samples appear in the container,
+    /// which is usually ordered by [`Self::latest_decode_timestamp`].
+    ///
     /// None = unknown.
     pub sample_idx: Option<usize>,
 
+    /// Which frame is this?
+    ///
+    /// This is on the assumption that each sample produces a single frame,
+    /// which is true for MP4.
+    ///
+    /// This is the index of frames ordered by [`Self::presentation_timestamp`].
+    ///
+    /// None = unknown.
+    pub frame_nr: Option<usize>,
+
     /// The presentation timestamp of the frame.
     pub presentation_timestamp: Time,
 

crates/store/re_video/src/decode/webcodecs.rs

@@ -204,8 +204,9 @@ fn init_video_decoder(
             on_output(Ok(Frame {
                 content: WebVideoFrame(frame),
                 info: FrameInfo {
-                    sample_idx: None,
-                    is_sync: None,
+                    is_sync: None,    // TODO(emilk)
+                    sample_idx: None, // TODO(emilk)
+                    frame_nr: None,   // TODO(emilk)
                     presentation_timestamp,
                     duration,
                     latest_decode_timestamp: None,

crates/store/re_video/src/demux/mod.rs

@@ -466,8 +466,19 @@ pub struct Sample {
     pub is_sync: bool,
 
     /// Which sample is this in the video?
+    ///
+    /// This is the order of which the samples appear in the container,
+    /// which is usually ordered by [`Self::decode_timestamp`].
     pub sample_idx: usize,
 
+    /// Which frame does this sample belong to?
+    ///
+    /// This is on the assumption that each sample produces a single frame,
+    /// which is true for MP4.
+    ///
+    /// This is the index of samples ordered by [`Self::presentation_timestamp`].
+    pub frame_nr: usize,
+
     /// Time at which this sample appears in the decoded bitstream, in time units.
     ///
     /// Samples should be decoded in this order.
@@ -508,6 +519,7 @@ impl Sample {
         Some(Chunk {
             data,
             sample_idx: self.sample_idx,
+            frame_nr: self.frame_nr,
             decode_timestamp: self.decode_timestamp,
             presentation_timestamp: self.presentation_timestamp,
             duration: self.duration,
@@ -662,6 +674,7 @@ mod tests {
             .map(|(sample_idx, (pts, dts))| Sample {
                 is_sync: false,
                 sample_idx,
+                frame_nr: 0, // unused
                 decode_timestamp: Time(dts),
                 presentation_timestamp: Time(pts),
                 duration: Time(1),

crates/store/re_video/src/demux/mp4.rs

@@ -66,6 +66,7 @@ impl VideoData {
                 samples.push(Sample {
                     is_sync: sample.is_sync,
                     sample_idx,
+                    frame_nr: 0, // filled in after the loop
                     decode_timestamp,
                     presentation_timestamp,
                     duration,
@@ -75,6 +76,7 @@ impl VideoData {
             }
         }
 
+        // Append the last GOP if there are any samples left:
         if !samples.is_empty() {
             let start = samples[gop_sample_start_index].decode_timestamp;
             let sample_range = gop_sample_start_index as u32..samples.len() as u32;
@@ -84,6 +86,29 @@ impl VideoData {
             });
         }
 
+        {
+            re_tracing::profile_scope!("Sanity-check samples");
+            let mut samples_are_in_decode_order = true;
+            for window in samples.windows(2) {
+                samples_are_in_decode_order &=
+                    window[0].decode_timestamp <= window[1].decode_timestamp;
+            }
+            if !samples_are_in_decode_order {
+                re_log::warn!(
+                    "Video samples are NOT in decode order. This implies either invalid video data or a bug in parsing the mp4."
+                );
+            }
+        }
+
+        {
+            re_tracing::profile_scope!("Calculate frame numbers");
+            let mut samples_sorted_by_pts = samples.iter_mut().collect::<Vec<_>>();
+            samples_sorted_by_pts.sort_by_key(|s| s.presentation_timestamp);
+            for (frame_nr, sample) in samples_sorted_by_pts.into_iter().enumerate() {
+                sample.frame_nr = frame_nr;
+            }
+        }
+
         let samples_statistics = SamplesStatistics::new(&samples);
 
         Ok(Self {

crates/viewer/re_data_ui/src/image.rs

@@ -85,10 +85,13 @@ pub fn texture_preview_ui(
         )
         .inner
     } else {
+        // TODO(emilk): we should limit the HEIGHT primarily,
+        // since if the image uses up too much vertical space,
+        // it is really annoying in the selection panel.
         let size_range = if ui_layout == UiLayout::Tooltip {
             egui::Rangef::new(64.0, 128.0)
         } else {
-            egui::Rangef::new(240.0, 640.0)
+            egui::Rangef::new(240.0, 320.0)
         };
         let preview_size = Vec2::splat(
             size_range

crates/viewer/re_data_ui/src/video.rs

@@ -4,7 +4,10 @@ use re_renderer::{
     video::VideoFrameTexture,
 };
 use re_types::components::VideoTimestamp;
-use re_ui::{list_item::PropertyContent, DesignTokens, UiExt};
+use re_ui::{
+    list_item::{self, PropertyContent},
+    DesignTokens, UiExt,
+};
 use re_video::{decode::FrameInfo, demux::SamplesStatistics, VideoData};
 use re_viewer_context::UiLayout;
 
@@ -93,7 +96,7 @@ fn video_data_ui(ui: &mut egui::Ui, ui_layout: UiLayout, video_data: &VideoData)
     );
 
     if ui_layout != UiLayout::Tooltip {
-        ui.list_item_collapsible_noninteractive_label("MP4 tracks", true, |ui| {
+        ui.list_item_collapsible_noninteractive_label("MP4 tracks", false, |ui| {
             for (track_id, track_kind) in &video_data.mp4_tracks {
                 let track_kind_string = match track_kind {
                     Some(re_video::TrackKind::Audio) => "audio",
@@ -117,7 +120,13 @@ fn video_data_ui(ui: &mut egui::Ui, ui_layout: UiLayout, video_data: &VideoData)
         });
 
         ui.list_item_collapsible_noninteractive_label("Video samples", false, |ui| {
-            samples_table_ui(ui, video_data);
+            egui::Resize::default()
+                .with_stroke(true)
+                .resizable([false, true])
+                .max_height(611.0) // Odd value so the user can see half-hidden rows
+                .show(ui, |ui| {
+                    samples_table_ui(ui, video_data);
+                });
         });
     }
 }
@@ -128,14 +137,17 @@ fn samples_table_ui(ui: &mut egui::Ui, video_data: &VideoData) {
     egui_extras::TableBuilder::new(ui)
         .auto_shrink([false, true])
         .vscroll(true)
-        .max_scroll_height(800.0)
-        .columns(Column::auto(), 7)
+        .max_scroll_height(611.0) // Odd value so the user can see half-hidden rows
+        .columns(Column::auto(), 8)
         .cell_layout(egui::Layout::left_to_right(egui::Align::Center))
         .header(DesignTokens::table_header_height(), |mut header| {
             DesignTokens::setup_table_header(&mut header);
             header.col(|ui| {
                 ui.strong("Sample");
             });
+            header.col(|ui| {
+                ui.strong("Frame");
+            });
             header.col(|ui| {
                 ui.strong("GOP");
             });
@@ -166,16 +178,21 @@ fn samples_table_ui(ui: &mut egui::Ui, video_data: &VideoData) {
                     let sample = &video_data.samples[sample_idx];
                     let re_video::Sample {
                         is_sync,
-                        sample_idx: _,
+                        sample_idx: sample_idx_in_sample,
+                        frame_nr,
                         decode_timestamp,
                         presentation_timestamp,
                         duration,
                         byte_offset: _,
                         byte_length,
                     } = *sample;
+                    debug_assert_eq!(sample_idx, sample_idx_in_sample);
 
                     row.col(|ui| {
-                        ui.monospace(sample_idx.to_string());
+                        ui.monospace(re_format::format_uint(sample_idx));
+                    });
+                    row.col(|ui| {
+                        ui.monospace(re_format::format_uint(frame_nr));
                     });
                     row.col(|ui| {
                         if let Some(gop_index) = video_data
@@ -260,19 +277,31 @@ pub fn show_decoded_frame_info(
             frame_info,
             source_pixel_format,
         }) => {
-            re_ui::list_item::list_item_scope(ui, "decoded_frame_ui", |ui| {
-                let default_open = false;
-                if let Some(frame_info) = frame_info {
-                    ui.list_item_collapsible_noninteractive_label(
-                        "Current decoded frame",
-                        default_open,
-                        |ui| {
-                            frame_info_ui(ui, &frame_info, video.data());
-                            source_image_data_format_ui(ui, &source_pixel_format);
-                        },
-                    );
-                }
-            });
+            if let Some(frame_info) = frame_info {
+                re_ui::list_item::list_item_scope(ui, "decoded_frame_ui", |ui| {
+                    let id = ui.id().with("decoded_frame_collapsible");
+                    let default_open = false;
+                    let label = if let Some(frame_nr) = frame_info.frame_nr {
+                        format!("Decoded frame #{}", re_format::format_uint(frame_nr))
+                    } else {
+                        "Current decoded frame".to_owned()
+                    };
+                    ui.list_item()
+                        .interactive(false)
+                        .show_hierarchical_with_children(
+                            ui,
+                            id,
+                            default_open,
+                            list_item::LabelContent::new(label),
+                            |ui| {
+                                list_item::list_item_scope(ui, id, |ui| {
+                                    frame_info_ui(ui, &frame_info, video.data());
+                                    source_image_data_format_ui(ui, &source_pixel_format);
+                                });
+                            },
+                        )
+                });
+            }
 
             let response = crate::image::texture_preview_ui(
                 render_ctx,
@@ -336,6 +365,7 @@ fn frame_info_ui(ui: &mut egui::Ui, frame_info: &FrameInfo, video_data: &re_vide
     let FrameInfo {
         is_sync,
         sample_idx,
+        frame_nr,
         presentation_timestamp,
         duration,
         latest_decode_timestamp,
@@ -381,6 +411,13 @@ fn frame_info_ui(ui: &mut egui::Ui, frame_info: &FrameInfo, video_data: &re_vide
         );
     }
 
+    if let Some(frame_nr) = frame_nr {
+        ui.list_item_flat_noninteractive(PropertyContent::new("Frame").value_fn(move |ui, _| {
+            ui.monospace(re_format::format_uint(frame_nr));
+        }))
+        .on_hover_text("The frame number, as ordered by presentation time");
+    }
+
     if let Some(dts) = latest_decode_timestamp {
         ui.list_item_flat_noninteractive(
             PropertyContent::new("DTS").value_fn(value_fn_for_time(dts, video_data)),

crates/viewer/re_renderer/src/video/player.rs

@@ -266,8 +266,14 @@ impl VideoPlayer {
             // Therefore, it's important to compare presentation timestamps instead of sample indices.
             // (comparing decode timestamps should be equivalent to comparing sample indices)
             let current_pts = self.data.samples[self.current_sample_idx].presentation_timestamp;
-            let requested_pts = self.data.samples[requested_sample_idx].presentation_timestamp;
-            if requested_pts < current_pts {
+            let requested_sample = &self.data.samples[requested_sample_idx];
+
+            re_log::trace!(
+                "Seeking to sample {requested_sample_idx} (frame_nr {})",
+                requested_sample.frame_nr
+            );
+
+            if requested_sample.presentation_timestamp < current_pts {
                 self.reset()?;
                 self.enqueue_gop(requested_gop_idx, video_data)?;
                 self.enqueue_gop(requested_gop_idx + 1, video_data)?;
@@ -325,6 +331,8 @@ impl VideoPlayer {
 
         let samples = &self.data.samples[gop.sample_range_usize()];
 
+        re_log::trace!("Enqueueing GOP {gop_idx} ({} samples)", samples.len());
+
         for sample in samples {
             let chunk = sample.get(video_data).ok_or(VideoPlayerError::BadData)?;
             self.chunk_decoder.decode(chunk)?;