feat: LEAP-1367: Add TimelineLabels to mark parts of the video

docs/source/tags/timelinelabels.md

@@ -0,0 +1,58 @@
+---
+title: TimelineLabels
+type: tags
+order: 429
+is_new: t
+meta_title: TimelineLabels tag
+meta_description: Classify video frames using TimelineLabels.
+---
+
+Use the TimelineLabels tag to classify video frames. This can be a single frame or a span of frames.
+
+First, select a label and then click once to annotate a single frame. Click and drag to annotate multiple frames.
+
+To move forward and backward in the timeline without labeling, ensure that no labels are selected before you click.
+
+![Screenshot of video with frame classification](../images/timelinelabels.png)
+
+Use with the `<Video>` control tag.
+
+!!! info Tip
+    You can increase the height of the timeline using the `timelineHeight` parameter on the `<Video>` tag.
+
+### Parameters
+
+| Param | Type | Description |
+| --- | --- | --- |
+| name | <code>string</code> | Name of the element |
+| toName | <code>string</code> | Name of the video element |
+
+### Sample Results JSON
+
+| Name | Type | Description |
+| --- | --- | --- |
+| value | <code>Object</code> |  |
+| value.ranges | <code>Array.&lt;object&gt;</code> | Array of ranges, each range is an object with `start` and `end` properties. One range per region. |
+| [value.timelinelabels] | <code>Array.&lt;string&gt;</code> | Regions are created by `TimelineLabels`, and the corresponding label is listed here. |
+
+### Example JSON
+```json
+{
+  "value": {
+    "ranges": [{"start": 3, "end": 5}],
+    "timelinelabels": ["Moving"]
+  }
+}
+```
+
+### Example
+```html
+<View>
+  <Header>Label timeline spans:</Header>
+  <Video name="video" value="$video" />
+  <TimelineLabels name="timelineLabels" toName="video">
+    <Label value="Nothing" background="#944BFF"/>
+    <Label value="Movement" background="#98C84E"/>
+  </TimelineLabels>
+</View>
+```

docs/source/tags/video.md

@@ -19,7 +19,8 @@ Use with the following data types: video
 | [frameRate] | <code>number</code> | <code>24</code> | video frame rate per second; default is 24; can use task data like `$fps` |
 | [sync] | <code>string</code> |  | object name to sync with |
 | [muted] | <code>boolean</code> | <code>false</code> | muted video |
-| [height] | <code>number</code> | <code>600</code> | height of the video |
+| [height] | <code>number</code> | <code>600</code> | height of the video player |
+| [timelineHeight] | <code>number</code> | <code>64</code> | height of the timeline with regions |
 
 ### Example
 

docs/source/tags/videorectangle.md

@@ -1,7 +1,7 @@
 ---
 title: VideoRectangle
 type: tags
-order: 429
+order: 430
 meta_title: Video Tag for Video Labeling
 meta_description: Customize Label Studio with the Video tag for basic video annotation tasks for machine learning and data science projects.
 ---

web/libs/editor/src/components/Timeline/Timeline.tsx

@@ -3,7 +3,7 @@ import { type FC, useEffect, useMemo, useRef, useState } from "react";
 import { useLocalStorageState } from "../../hooks/useLocalStorageState";
 import { useMemoizedHandlers } from "../../hooks/useMemoizedHandlers";
 import { Block, Elem } from "../../utils/bem";
-import { clamp, isDefined } from "../../utils/utilities";
+import { clamp, fixMobxObserve, isDefined } from "../../utils/utilities";
 import { TimelineContextProvider } from "./Context";
 import { Controls } from "./Controls";
 import { Seeker } from "./Seeker";
@@ -62,6 +62,8 @@ const TimelineComponent: FC<TimelineProps> = ({
     onAddRegion: props.onAddRegion,
     onDeleteRegion: props.onDeleteRegion,
     onSelectRegion: props.onSelectRegion,
+    onStartDrawing: props.onStartDrawing,
+    onFinishDrawing: props.onFinishDrawing,
     onAction: props.onAction,
     onFullscreenToggle: props.onFullscreenToggle,
     onSpeedChange: props.onSpeedChange,
@@ -172,6 +174,8 @@ const TimelineComponent: FC<TimelineProps> = ({
     </Elem>
   );
 
+  regions.map((reg) => fixMobxObserve(reg.sequence));
+
   const view = !viewCollapsed && !disableView && (
     <Elem name="view">
       <View.View
@@ -183,6 +187,7 @@ const TimelineComponent: FC<TimelineProps> = ({
         speed={speed}
         volume={props.volume}
         controls={props.controls}
+        height={props.height}
         position={currentPosition}
         offset={seekOffset}
         leftOffset={View.settings?.leftOffset}
@@ -197,6 +202,8 @@ const TimelineComponent: FC<TimelineProps> = ({
         onAddRegion={(reg) => handlers.onAddRegion?.(reg)}
         onDeleteRegion={(id) => handlers.onDeleteRegion?.(id)}
         onSelectRegion={(e, id, select) => handlers.onSelectRegion?.(e, id, select)}
+        onStartDrawing={(frame) => handlers.onStartDrawing?.(frame)}
+        onFinishDrawing={() => handlers.onFinishDrawing?.()}
         onSpeedChange={(speed) => handlers.onSpeedChange?.(speed)}
         onZoom={props.onZoom}
       />

web/libs/editor/src/components/Timeline/Types.ts

@@ -11,6 +11,7 @@ export interface TimelineProps<D extends ViewTypes = "frames"> {
   regions: any[];
   length: number;
   position: number;
+  height?: number;
   mode: D;
   framerate: number;
   playing: boolean;
@@ -39,6 +40,8 @@ export interface TimelineProps<D extends ViewTypes = "frames"> {
   onToggleVisibility?: (id: string, visibility: boolean) => void;
   onAddRegion?: (region: Record<string, any>) => any;
   onDeleteRegion?: (id: string) => void;
+  onStartDrawing?: (frame: number) => void;
+  onFinishDrawing?: () => void;
   onZoom?: (zoom: number) => void;
   onSelectRegion?: (event: MouseEvent<HTMLDivElement>, id: string, select?: boolean) => void;
   onAction?: (event: MouseEvent, action: string, data?: any) => void;
@@ -58,6 +61,7 @@ export interface TimelineViewProps {
   speed?: number;
   volume?: number;
   regions: TimelineRegion[];
+  height?: number;
   leftOffset?: number;
   controls?: TimelineControls;
   onScroll: (position: number) => void;
@@ -73,17 +77,22 @@ export interface TimelineViewProps {
   onAddRegion?: TimelineProps["onAddRegion"];
   onDeleteRegion?: TimelineProps["onDeleteRegion"];
   onSelectRegion?: TimelineProps["onSelectRegion"];
+  onStartDrawing?: TimelineProps["onStartDrawing"];
+  onFinishDrawing?: TimelineProps["onFinishDrawing"];
   onVolumeChange?: TimelineProps["onVolumeChange"];
   onSpeedChange?: TimelineProps["onSpeedChange"];
 }
 
 export interface TimelineRegion {
   id: string;
+  index?: number;
   label: string;
   color: string;
   visible: boolean;
   selected: boolean;
   sequence: TimelineRegionKeyframe[];
+  /** is this timeline region with spans */
+  timeline?: boolean;
 }
 
 export interface TimelineRegionKeyframe {
@@ -176,9 +185,9 @@ export interface TimelineControlsProps {
   onPause?: TimelineProps["onPause"];
   onFullScreenToggle: TimelineProps["onFullscreenToggle"];
   onVolumeChange: TimelineProps["onVolumeChange"];
-  onSpeedChange: TimelineProps["onSpeedChange"];
-  onZoom: TimelineProps["onZoom"];
-  onAmpChange: (amp: number) => void;
+  onSpeedChange?: TimelineProps["onSpeedChange"];
+  onZoom?: TimelineProps["onZoom"];
+  onAmpChange?: (amp: number) => void;
   toggleVisibility?: (layerName: string, isVisible: boolean) => void;
 }
 

web/libs/editor/src/components/Timeline/Views/Frames/Controls.tsx

@@ -16,9 +16,9 @@ type DataType = {
 
 export const Controls: FC<TimelineExtraControls<Actions, DataType>> = ({ onAction }) => {
   const { position, regions } = useContext(TimelineContext);
-  const hasSelectedRegion = regions.some(({ selected }) => selected);
+  const hasSelectedRegion = regions.some(({ selected, timeline }) => selected && !timeline);
   const closestKeypoint = useMemo(() => {
-    const region = regions.find((r) => r.selected);
+    const region = regions.find((r) => r.selected && !r.timeline);
 
     return region?.sequence.filter(({ frame }) => frame <= position).slice(-1)[0];
   }, [regions, position]);

web/libs/editor/src/components/Timeline/Views/Frames/Frames.tsx

@@ -3,9 +3,9 @@ import { type FC, type MouseEvent, useCallback, useEffect, useMemo, useRef, useS
 import { useMemoizedHandlers } from "../../../../hooks/useMemoizedHandlers";
 import { Block, Elem } from "../../../../utils/bem";
 import { isDefined } from "../../../../utils/utilities";
-import type { TimelineViewProps } from "../../Types";
-import "./Frames.scss";
+import type { TimelineRegion, TimelineViewProps } from "../../Types";
 import { Keypoints } from "./Keypoints";
+import "./Frames.scss";
 
 const toSteps = (num: number, step: number) => {
   return Math.floor(num / step);
@@ -71,7 +71,7 @@ export const Frames: FC<TimelineViewProps> = ({
   }, [step]);
 
   const setScroll = useCallback(
-    ({ left, top }) => {
+    ({ left, top }: { left?: number; top?: number }) => {
       if (!length) return;
 
       setHoverOffset(null);
@@ -196,6 +196,14 @@ export const Frames: FC<TimelineViewProps> = ({
       const dimensions = scrollable.current!.getBoundingClientRect();
       const offsetLeft = dimensions.left;
       const rightLimit = dimensions.width - timelineStartOffset;
+      const target = e.target as Element;
+      // every region has `data-id` attribute, so looking for them
+      const regionRow = target.closest("[data-id]") as HTMLElement | null;
+      // not clicking on labels area
+      const onKeyframes = e.pageX - offsetLeft > timelineStartOffset;
+      // don't draw on region lines, only on the empty space or special new line
+      const isDrawing = onKeyframes && (!regionRow || regionRow.dataset?.id === "new");
+      let region: any;
 
       const getMouseToFrame = (e: MouseEvent | globalThis.MouseEvent) => {
         const mouseOffset = e.pageX - offsetLeft - timelineStartOffset;
@@ -204,22 +212,35 @@ export const Frames: FC<TimelineViewProps> = ({
       };
 
       const offset = getMouseToFrame(e);
+      const baseFrame = toSteps(offset, step) + 1;
 
       setIndicatorOffset(offset);
 
+      if (isDrawing) {
+        // always a timeline region
+        region = props.onStartDrawing?.(baseFrame);
+      }
+
       const onMouseMove = (e: globalThis.MouseEvent) => {
         const offset = getMouseToFrame(e);
+        const frame = toSteps(offset, step) + 1;
 
         if (offset >= 0 && offset <= rightLimit) {
           setHoverEnabled(false);
           setRegionSelectionDisabled(true);
           setIndicatorOffset(offset);
         }
+
+        if (region) {
+          const [start, end] = frame > baseFrame ? [baseFrame, frame] : [frame, baseFrame];
+          region.setRanges([start, end]);
+        }
       };
 
       const onMouseUp = () => {
         setHoverEnabled(true);
         setRegionSelectionDisabled(false);
+        props.onFinishDrawing?.();
         document.removeEventListener("mousemove", onMouseMove);
         document.removeEventListener("mouseup", onMouseUp);
       };
@@ -282,8 +303,9 @@ export const Frames: FC<TimelineViewProps> = ({
 
     if (!isDefined(scroll) || framesInView < 1) return;
 
-    const firstFrame = toSteps(roundToStep(lastOffsetX.current, step), step);
-    const lastFrame = firstFrame + framesInView;
+    // offsets are zero based, but position is 1 based
+    const firstFrame = toSteps(roundToStep(lastOffsetX.current, step), step) + 1;
+    const lastFrame = firstFrame + framesInView - 1;
 
     const positionDelta = Math.abs(position - lastPosition.current);
 
@@ -293,19 +315,19 @@ export const Frames: FC<TimelineViewProps> = ({
     // this ensures the calculation of offset is kept correct.
     // This is needed because the position is not always a multiple of the step
     // and the offset used to calculate the position is always a multiple of the step.
-    if (positionDelta === 1 && position >= firstFrame && position <= lastFrame) {
-      // set to previous frame scroll
-      // if position is 0, then it will be set to 0
-      if (position <= firstFrame) {
+    if (positionDelta === 1 && (position < firstFrame || position > lastFrame)) {
+      // scroll to previous page if we are going outside of the current one
+      if (position < firstFrame) {
         const prevLeft = clamp((firstFrame - 1 - framesInView) * step, 0, scroll.scrollWidth - scroll.clientWidth);
 
         lastScrollPosition.current = roundToStep(prevLeft, step);
 
         setScroll({ left: prevLeft });
 
-        // set to next frame scroll
-        // if position is last frame, then it will be set to last frame scroll
+        // scroll to the next page if we are going outside of the current one
       } else if (position > lastFrame) {
+        // offsets are zero based, but position is 1 based,
+        // so technically that's +1 to go to the next page, but -1 to switch to offsets
         const nextLeft = clamp(lastFrame * step, 0, scroll.scrollWidth - scroll.clientWidth);
 
         lastScrollPosition.current = roundToStep(nextLeft, step);
@@ -319,15 +341,19 @@ export const Frames: FC<TimelineViewProps> = ({
     // Handle position change outside of the current scroll
     // This updates when the user clicks within the track to change the position
     // or when keyframe hops are used and the position is changed more than 1 frame
-    const scrollTo = roundToStep(position, framesInView);
+    const scrollTo = roundToStep(position - 1, framesInView);
+    // how far are we from the start of currently visible window
+    const diff = (position - 1) * step - lastScrollPosition.current;
 
-    if (lastScrollPosition.current !== scrollTo) {
+    if (diff > (framesInView - 1) * step || diff < 0) {
       setScroll({ left: scrollTo * step });
+      // frames
+      lastScrollPosition.current = scrollTo * step;
     }
-    lastScrollPosition.current = scrollTo;
   }, [position, framesInView, step]);
 
   const styles = {
+    "--view-height": props.height ? `${props.height}px` : null,
     "--frame-size": `${step}px`,
     "--view-size": `${viewWidth}px`,
     "--offset": `${timelineStartOffset}px`,
@@ -379,7 +405,7 @@ export const Frames: FC<TimelineViewProps> = ({
 };
 
 interface KeypointsVirtualProps {
-  regions: any[];
+  regions: TimelineRegion[];
   startOffset: number;
   scrollTop: number;
   disabled?: boolean;
@@ -399,10 +425,10 @@ const KeypointsVirtual: FC<KeypointsVirtualProps> = ({ regions, startOffset, scr
   return (
     <Elem name="keypoints" style={{ height: regions.length * height }}>
       {regions.map((region, i) => {
-        return region.sequence.length > 0 ? (
+        return region.sequence.length > 0 || region.timeline ? (
           <Keypoints
             key={region.id}
-            idx={i + 1}
+            idx={region.index}
             region={region}
             startOffset={startOffset}
             onSelectRegion={disabled ? undefined : onSelectRegion}