Allow showing the same videos at several timestamps at the same time (#…

…7473)

### What

* Fixes #7420


https://github.com/user-attachments/assets/5f0a1f76-fd56-4f80-87f9-c3e02cd6e8d2
Decoders are now lazily created. There can now be several per video,
each identified by a user provided identifier.
If decoders are not used for a frame, they get automatically destroyed.

Also fixes flickering issue when switching from an video error state to
pending - now whenever we enter the pending state after an error, we
clear out the target texture first.

Commit-by-commit review enabled.

![example
screenshot](https://static.rerun.io/video_manual_frames/9f41c00f84a98cc3f26875fba7c1d2fa2bad7151/1200w.png)

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7473?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7473?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!
* [x] If have noted any breaking changes to the log API in
`CHANGELOG.md` and the migration guide

- [PR Build Summary](https://build.rerun.io/pr/7473)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.

crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs

@@ -9,8 +9,7 @@ namespace rerun.archetypes;
 /// In order to display a video, you need to log a [archetypes.VideoFrameReference] for each frame.
 ///
 /// \example archetypes/video_auto_frames title="Video with automatically determined frames" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png"
-/// \example archetypes/video_manual_frames title="Demonstrates manual use of video frame references" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png"
-// TODO(#7420): update screenshot for manual frames example
+/// \example archetypes/video_manual_frames title="Demonstrates manual use of video frame references" image="https://static.rerun.io/video_manual_frames/9f41c00f84a98cc3f26875fba7c1d2fa2bad7151/1200w.png"
 table AssetVideo (
   "attr.docs.unreleased",
   "attr.rerun.experimental"

crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs

@@ -6,8 +6,7 @@ namespace rerun.archetypes;
 /// To show an entire video, a fideo frame reference for each frame of the video should be logged.
 ///
 /// \example archetypes/video_auto_frames title="Video with automatically determined frames" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png"
-/// \example archetypes/video_manual_frames title="Demonstrates manual use of video frame references" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png"
-// TODO(#7420): update screenshot for manual frames example
+/// \example archetypes/video_manual_frames title="Demonstrates manual use of video frame references" image="https://static.rerun.io/video_manual_frames/9f41c00f84a98cc3f26875fba7c1d2fa2bad7151/1200w.png"
 table VideoFrameReference (
     "attr.docs.unreleased",
     "attr.rerun.experimental"

crates/viewer/re_data_ui/src/blob.rs

@@ -1,3 +1,4 @@
+use re_renderer::external::re_video::VideoLoadError;
 use re_types::components::{Blob, MediaType};
 use re_ui::{list_item::PropertyContent, UiExt};
 use re_viewer_context::UiLayout;
@@ -104,15 +105,10 @@ pub fn blob_preview_and_save_ui(
         image_preview_ui(ctx, ui, ui_layout, query, entity_path, image);
     }
     // Try to treat it as a video if treating it as image didn't work:
-    else if let Some(render_ctx) = ctx.render_ctx {
+    else {
         let video_result = blob_row_id.map(|row_id| {
             ctx.cache.entry(|c: &mut re_viewer_context::VideoCache| {
-                c.entry(
-                    row_id,
-                    blob,
-                    media_type.as_ref().map(|mt| mt.as_str()),
-                    render_ctx,
-                )
+                c.entry(row_id, blob, media_type.as_ref().map(|mt| mt.as_str()))
             })
         });
         if let Some(video_result) = &video_result {
@@ -161,7 +157,7 @@ pub fn blob_preview_and_save_ui(
 fn show_video_blob_info(
     ui: &mut egui::Ui,
     ui_layout: UiLayout,
-    video_result: &Result<re_renderer::video::Video, re_renderer::video::VideoError>,
+    video_result: &Result<re_renderer::video::Video, VideoLoadError>,
 ) {
     match video_result {
         Ok(video) => {

crates/viewer/re_renderer/src/context.rs

@@ -475,6 +475,11 @@ This means, either a call to RenderContext::before_submit was omitted, or the pr
     pub(crate) fn read_lock_renderers(&self) -> RwLockReadGuard<'_, Renderers> {
         self.renderers.read()
     }
+
+    /// Returns the global frame index of the active frame.
+    pub fn active_frame_idx(&self) -> u64 {
+        self.active_frame.frame_index
+    }
 }
 
 pub struct FrameGlobalCommandEncoder(Option<wgpu::CommandEncoder>);
@@ -527,7 +532,7 @@ pub struct ActiveFrameContext {
     /// This counter is part of the `content timeline` and may be arbitrarily
     /// behind both of the `device timeline` and `queue timeline`.
     /// See <https://www.w3.org/TR/webgpu/#programming-model-timelines>
-    frame_index: u64,
+    pub frame_index: u64,
 
     /// Top level device error scope, created at startup and closed & reopened on every frame.
     ///

crates/viewer/re_renderer/src/lib.rs

@@ -88,6 +88,7 @@ pub use self::file_server::FileServer;
 pub use ecolor::{Color32, Hsva, Rgba};
 
 pub mod external {
+    pub use re_video;
     pub use wgpu;
 }
 

crates/viewer/re_renderer/src/video/decoder/native.rs

@@ -38,7 +38,11 @@ impl VideoDecoder {
     }
 
     #[allow(clippy::unused_self)]
-    pub fn frame_at(&mut self, timestamp_s: f64) -> FrameDecodingResult {
+    pub fn frame_at(
+        &mut self,
+        _render_ctx: &RenderContext,
+        _timestamp_s: f64,
+    ) -> FrameDecodingResult {
         FrameDecodingResult::Error(DecodingError::NoNativeSupport)
     }
 }

crates/viewer/re_renderer/src/video/decoder/web.rs

@@ -13,7 +13,7 @@ use super::latest_at_idx;
 use crate::{
     resource_managers::GpuTexture2D,
     video::{DecodingError, FrameDecodingResult},
-    RenderContext,
+    DebugLabel, RenderContext,
 };
 
 #[derive(Clone)]
@@ -55,6 +55,8 @@ pub struct VideoDecoder {
     last_used_frame_timestamp: Time,
     current_segment_idx: usize,
     current_sample_idx: usize,
+
+    error_on_last_frame_at: bool,
 }
 
 // SAFETY: There is no way to access the same JS object from different OS threads
@@ -79,6 +81,7 @@ unsafe impl Sync for VideoFrame {}
 
 impl Drop for VideoDecoder {
     fn drop(&mut self) {
+        re_log::debug!("Dropping VideoDecoder");
         if let Err(err) = self.decoder.close() {
             re_log::warn!(
                 "Error when closing video decoder: {}",
@@ -132,10 +135,40 @@ impl VideoDecoder {
             last_used_frame_timestamp: Time::new(u64::MAX),
             current_segment_idx: usize::MAX,
             current_sample_idx: usize::MAX,
+
+            error_on_last_frame_at: false,
         })
     }
 
-    pub fn frame_at(&mut self, timestamp_s: f64) -> FrameDecodingResult {
+    pub fn frame_at(
+        &mut self,
+        render_ctx: &RenderContext,
+        timestamp_s: f64,
+    ) -> FrameDecodingResult {
+        let result = self.frame_at_internal(timestamp_s);
+        match &result {
+            FrameDecodingResult::Ready(_) => {
+                self.error_on_last_frame_at = false;
+            }
+            FrameDecodingResult::Pending(_) => {
+                if self.error_on_last_frame_at {
+                    // If we switched from error to pending, clear the texture.
+                    // This is important to avoid flickering, in particular when switching from
+                    // benign errors like DecodingError::NegativeTimestamp.
+                    // If we don't do this, we see the last valid texture which can look really weird.
+                    self.clear_video_texture(render_ctx);
+                }
+
+                self.error_on_last_frame_at = false;
+            }
+            FrameDecodingResult::Error(_) => {
+                self.error_on_last_frame_at = true;
+            }
+        }
+        result
+    }
+
+    fn frame_at_internal(&mut self, timestamp_s: f64) -> FrameDecodingResult {
         if timestamp_s < 0.0 {
             return FrameDecodingResult::Error(DecodingError::NegativeTimestamp);
         }
@@ -234,6 +267,30 @@ impl VideoDecoder {
         FrameDecodingResult::Ready(self.texture.clone())
     }
 
+    /// Clears the texture that is shown on pending to black.
+    fn clear_video_texture(&self, render_ctx: &RenderContext) {
+        // Clear texture is a native only feature, so let's not do that.
+        // before_view_builder_encoder.clear_texture(texture, subresource_range);
+
+        // But our target is also a render target, so just create a dummy renderpass with clear.
+        let mut before_view_builder_encoder =
+            render_ctx.active_frame.before_view_builder_encoder.lock();
+        let _ = before_view_builder_encoder
+            .get()
+            .begin_render_pass(&wgpu::RenderPassDescriptor {
+                label: DebugLabel::from("clear_video_texture").get(),
+                color_attachments: &[Some(wgpu::RenderPassColorAttachment {
+                    view: &self.texture.default_view,
+                    resolve_target: None,
+                    ops: wgpu::Operations::<wgpu::Color> {
+                        load: wgpu::LoadOp::Clear(wgpu::Color::BLACK),
+                        store: wgpu::StoreOp::Store,
+                    },
+                })],
+                ..Default::default()
+            });
+    }
+
     /// Enqueue all samples in the given segment.
     ///
     /// Does nothing if the index is out of bounds.

crates/viewer/re_renderer/src/video/mod.rs

@@ -1,21 +1,13 @@
 mod decoder;
 
+use ahash::HashMap;
 use parking_lot::Mutex;
-use std::sync::Arc;
+use std::{collections::hash_map::Entry, sync::Arc};
 
 use re_video::VideoLoadError;
 
 use crate::{resource_managers::GpuTexture2D, RenderContext};
 
-#[derive(thiserror::Error, Debug)]
-pub enum VideoError {
-    #[error(transparent)]
-    Load(#[from] VideoLoadError),
-
-    #[error(transparent)]
-    Init(#[from] DecodingError),
-}
-
 /// Error that can occur during frame decoding.
 // TODO(jan, andreas): These errors are for the most part specific to the web decoder right now.
 #[derive(thiserror::Error, Debug)]
@@ -56,31 +48,37 @@ pub enum FrameDecodingResult {
     Error(DecodingError),
 }
 
+/// Identifier for an independent video decoding stream.
+///
+/// A single video may use several decoders at a time to simultaneously decode frames at different timestamps.
+/// The id does not need to be globally unique, just unique enough to distinguish streams of the same video.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+
+pub struct VideoDecodingStreamId(pub u64);
+
+struct DecoderEntry {
+    decoder: decoder::VideoDecoder,
+    frame_index: u64,
+}
+
 /// Video data + decoder(s).
 ///
 /// Supports asynchronously decoding video into GPU textures via [`Video::frame_at`].
 pub struct Video {
     data: Arc<re_video::VideoData>,
-
-    // TODO(#7420): Support several tracks of video decoders.
-    // TODO(andreas): Create lazily.
-    decoder: Mutex<decoder::VideoDecoder>,
+    decoders: Mutex<HashMap<VideoDecodingStreamId, DecoderEntry>>,
 }
 
 impl Video {
     /// Loads a video from the given data.
     ///
     /// Currently supports the following media types:
     /// - `video/mp4`
-    pub fn load(
-        render_context: &RenderContext,
-        data: &[u8],
-        media_type: Option<&str>,
-    ) -> Result<Self, VideoError> {
+    pub fn load(data: &[u8], media_type: Option<&str>) -> Result<Self, VideoLoadError> {
         let data = Arc::new(re_video::VideoData::load_from_bytes(data, media_type)?);
-        let decoder = Mutex::new(decoder::VideoDecoder::new(render_context, data.clone())?);
+        let decoders = Mutex::new(HashMap::default());
 
-        Ok(Self { data, decoder })
+        Ok(Self { data, decoders })
     }
 
     /// The video data
@@ -108,8 +106,52 @@ impl Video {
     /// This API is _asynchronous_, meaning that the decoder may not yet have decoded the frame
     /// at the given timestamp. If the frame is not yet available, the returned texture will be
     /// empty.
-    pub fn frame_at(&self, timestamp_s: f64) -> FrameDecodingResult {
+    pub fn frame_at(
+        &self,
+        render_context: &RenderContext,
+        decoder_stream_id: VideoDecodingStreamId,
+        timestamp_s: f64,
+    ) -> FrameDecodingResult {
         re_tracing::profile_function!();
-        self.decoder.lock().frame_at(timestamp_s)
+
+        let global_frame_idx = render_context.active_frame_idx();
+
+        // We could protect this hashmap by a RwLock and the individual decoders by a Mutex.
+        // However, dealing with the RwLock efficiently is complicated:
+        // Upgradable-reads exclude other upgradable-reads which means that if an element is not found,
+        // we have to drop the unlock and relock with a write lock, during which new elements may be inserted.
+        // This can be overcome by looping until successful, or instead we can just use a single Mutex lock and leave it there.
+        let mut decoders = self.decoders.lock();
+        let decoder_entry = match decoders.entry(decoder_stream_id) {
+            Entry::Occupied(occupied_entry) => occupied_entry.into_mut(),
+            Entry::Vacant(vacant_entry) => {
+                let new_decoder =
+                    match decoder::VideoDecoder::new(render_context, self.data.clone()) {
+                        Ok(decoder) => decoder,
+                        Err(err) => {
+                            return FrameDecodingResult::Error(err);
+                        }
+                    };
+                vacant_entry.insert(DecoderEntry {
+                    decoder: new_decoder,
+                    frame_index: global_frame_idx,
+                })
+            }
+        };
+
+        decoder_entry.frame_index = render_context.active_frame_idx();
+        decoder_entry.decoder.frame_at(render_context, timestamp_s)
+    }
+
+    /// Removes all decoders that have been unused in the last frame.
+    ///
+    /// Decoders are very memory intensive, so they should be cleaned up as soon they're no longer needed.
+    pub fn purge_unused_decoders(&self, active_frame_idx: u64) {
+        if active_frame_idx == 0 {
+            return;
+        }
+
+        let mut decoders = self.decoders.lock();
+        decoders.retain(|_, decoder| decoder.frame_index >= active_frame_idx - 1);
     }
 }

crates/viewer/re_space_view_spatial/src/mesh_cache.rs

@@ -71,8 +71,6 @@ impl MeshCache {
 }
 
 impl Cache for MeshCache {
-    fn begin_frame(&mut self) {}
-
     fn purge_memory(&mut self) {
         self.0.clear();
     }