Skip to content

Commit

Permalink
Add the ability to manually move the smart framing crop area
Browse files Browse the repository at this point in the history 
Change-Id: Id1f8d52079f981bc463271f1ac0ef25540aef67d
  • Loading branch information
@mohabfekry
mohabfekry committed Aug 15, 2024
1 parent 418acbe commit 28fb3cf
Show file tree
Hide file tree
Showing 8 changed files with 196 additions and 46 deletions.
15 changes: 15 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ limitations under the License.
Update to the latest version by running `npm run update-app` after pulling the latest changes from the repository via `git pull --rebase --autostash`; you would need to redploy the *UI* for features marked as `frontend`, and *GCP components* for features marked as `backend`.

* [August 2024]
* `frontend`: You can now manually move the Smart Framing crop area to better capture the point of interest. Read more [here](#3-object-tracking-and-smart-framing).
* `frontend`: You can now share a page of the Web App containing your rendered videos and all associated image & text assets via a dedicated link. Read more [here](#6-output-videos).
* `frontend` + `backend`: Performance improvements for processing videos that are 10 minutes or longer.
* [July 2024]
Expand Down Expand Up @@ -182,6 +183,20 @@ The UI continuously queries GCS for updates while showing a preview of the uploa

<center><img src='./img/preview-format-settings.png' width="600px" alt="Vigenair UI: Video format preview settings" /></center>

* You can also manually move the crop area in case the smart framing weights were insufficient in capturing your desired point of interest. This is possible by doing the following:
* Select the desired format (square / vertical) from the toggle group and play the video.
* Pause the video at the point where you would like to manually move the crop area.
* Click on the "Move crop area" button that will appear above the video once paused.

<center><img src='./img/preview-format-move.png' width="700px" alt="Vigenair UI: Move crop area" /></center>

* Drag the crop area left or right as desired.
* Save the new position of the crop area by clicking on the "Save adjusted crop area" button.

<center><img src='./img/preview-format-save.png' width="700px" alt="Vigenair UI: Move crop area" /></center>

The crop area will be adjusted automatically for all preceding and subsequent video frames that had the same undesired position.

* Once the `data.json` is available, the extracted A/V Segments are displayed along with a set of user controls.
* Clicking on the link icon in the top-right corner of the "Video editing" panel will open the Cloud Storage browser UI and navigate to the associated video folder.

Expand Down
Binary file added img/preview-format-move.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added img/preview-format-save.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
25 changes: 10 additions & 15 deletions ui/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,10 @@ function renderVariants(gcsFolder: string, renderQueue: RenderQueue): string {
);

const encodedSquareCropCommands = Utilities.base64Encode(
PreviewHelper.generateCropCommands(
renderQueue.squareCropAnalysis,
renderQueue.sourceDimensions,
{ w: renderQueue.sourceDimensions.h, h: renderQueue.sourceDimensions.h }
),
PreviewHelper.generateCropCommands(renderQueue.squareCropAnalysis, {
w: renderQueue.sourceDimensions.h,
h: renderQueue.sourceDimensions.h,
}),
Utilities.Charset.UTF_8
);
StorageManager.uploadFile(
Expand All @@ -129,16 +128,12 @@ function renderVariants(gcsFolder: string, renderQueue: RenderQueue): string {
);

const encodedVerticalCropCommands = Utilities.base64Encode(
PreviewHelper.generateCropCommands(
renderQueue.verticalCropAnalysis,
renderQueue.sourceDimensions,
{
w:
renderQueue.sourceDimensions.h *
(renderQueue.sourceDimensions.h / renderQueue.sourceDimensions.w),
h: renderQueue.sourceDimensions.h,
}
),
PreviewHelper.generateCropCommands(renderQueue.verticalCropAnalysis, {
w:
renderQueue.sourceDimensions.h *
(renderQueue.sourceDimensions.h / renderQueue.sourceDimensions.w),
h: renderQueue.sourceDimensions.h,
}),
Utilities.Charset.UTF_8
);
StorageManager.uploadFile(
Expand Down
29 changes: 13 additions & 16 deletions ui/src/preview.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -459,24 +459,21 @@ export class PreviewHelper {
}

static generateCropCommands(
cropAnalysis: VideoIntelligence,
sourceDimensions: { w: number; h: number },
cropAnalysis: any,
targetDimensions: { w: number; h: number }
) {
const lines =
cropAnalysis.annotation_results[0].object_annotations![0].frames.map(
(frame, index) => {
const time =
frame.time_offset!.seconds! + frame.time_offset!.nanos! / 1e9;
const x = frame.normalized_bounding_box!.left! * sourceDimensions.w;
const y = 0;
const w = targetDimensions.w;
const h = targetDimensions.h;
return index === 0
? `${time} crop x ${x}, crop y ${y}, crop w ${w}, crop h ${h};`
: `${time} crop x ${x};`;
}
);
const lines = cropAnalysis[0].frames.map(
(frame: { time: number; x: number }, index: number) => {
const time = frame.time;
const x = frame.x;
const y = 0;
const w = targetDimensions.w;
const h = targetDimensions.h;
return index === 0
? `${time} crop x ${x}, crop y ${y}, crop w ${w}, crop h ${h};`
: `${time} crop x ${x};`;
}
);
return lines.join('\n');
}
}
5 changes: 5 additions & 0 deletions ui/src/ui/src/app/app.component.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ segments-list {
pointer-events: none;
}

.canvas-drag-element {
cursor: move;
overflow: hidden;
}

.hourglass_top {
background-color: #e0e0e0 !important;
}
Expand Down
26 changes: 26 additions & 0 deletions ui/src/ui/src/app/app.component.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,23 @@
<mat-button-toggle value="segments" matTooltip="Segments info">
<mat-icon>theaters</mat-icon>
</mat-button-toggle>
<mat-button-toggle
*ngIf="
(previewToggleGroup.value === 'square' ||
previewToggleGroup.value === 'vertical') &&
previewVideoElem.paused &&
previewVideoElem.currentTime > 0 &&
segmentModeToggle.value === 'preview'
"
value="movecrop"
[matTooltip]="
moveCropArea ? 'Save adjusted crop-area' : 'Move crop-area'
"
[disabled]="!verticalVideoObjects || !squareVideoObjects"
(click)="toggleMoveCropArea()"
>
<mat-icon>{{ moveCropArea ? 'save' : 'open_with' }}</mat-icon>
</mat-button-toggle>
</mat-button-toggle-group>
</div>
<div style="margin-left: auto">
Expand Down Expand Up @@ -241,6 +258,15 @@
/>
</video>
<canvas #magicCanvas></canvas>
<div
#canvasDragElement
class="canvas-drag-element"
cdkDrag
cdkDragLockAxis="x"
cdkDragBoundary=".video-container"
[cdkDragFreeDragPosition]="dragPosition"
style="visibility: hidden"
></div>
</div>
</div>
<div class="row">
Expand Down
142 changes: 127 additions & 15 deletions ui/src/ui/src/app/app.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
* limitations under the License.
*/

import { CdkDrag } from '@angular/cdk/drag-drop';
import { CommonModule } from '@angular/common';
import { Component, ElementRef, ViewChild } from '@angular/core';
import { FormsModule } from '@angular/forms';
Expand Down Expand Up @@ -104,6 +105,7 @@ export type FramingDialogData = {
MatCardModule,
MatDialogModule,
MatProgressSpinnerModule,
CdkDrag,
],
templateUrl: './app.component.html',
styleUrl: './app.component.css',
Expand All @@ -117,8 +119,6 @@ export class AppComponent {
selectedFile?: File;
videoPath?: string;
analysisJson?: any;
squarePreviewAnalysis?: any;
verticalPreviewAnalysis?: any;
activeVideoObjects?: any[];
videoObjects?: any[];
squareVideoObjects?: any[];
Expand Down Expand Up @@ -154,11 +154,14 @@ export class AppComponent {
renderQueueJsonArray: string[] = [];
negativePrompt = false;
displayObjectTracking = true;
moveCropArea = false;
weightsTextIndex = 3;
weightsPersonFaceIndex = 1;
weightSteps = [0, 10, 100, 1000];
subtitlesTrack = '';
webAppUrl = '';
dragPosition = { x: 0, y: 0 };
cropAreaRect?: DOMRect;

@ViewChild('VideoComboComponent') VideoComboComponent?: VideoComboComponent;
@ViewChild('previewVideoElem')
Expand All @@ -176,6 +179,8 @@ export class AppComponent {
renderQueueButtonSpan!: ElementRef<HTMLSpanElement>;
@ViewChild('reorderSegmentsToggle') reorderSegmentsToggle?: MatSlideToggle;
@ViewChild('previewToggleGroup') previewToggleGroup!: MatButtonToggleGroup;
@ViewChild('canvasDragElement')
canvasDragElement?: ElementRef<HTMLDivElement>;

constructor(
private apiCallsService: ApiCallsService,
Expand Down Expand Up @@ -248,6 +253,21 @@ export class AppComponent {
this.getPreviousRuns();
}

getCurrentCropAreaFrame(entities: any[]):
| {
currentFrame: { x: number; width: number; height: number };
idx: number;
}
| undefined {
const timestamp = this.previewVideoElem.nativeElement.currentTime;
for (let i = 0; i < entities[0].frames.length; i++) {
if (entities[0].frames[i].time >= timestamp) {
return { currentFrame: entities[0].frames[i], idx: i };
}
}
return;
}

drawFrame(entities?: any[]) {
const context = this.canvas;
if (!context || !entities) {
Expand Down Expand Up @@ -455,9 +475,7 @@ export class AppComponent {
this.activeVideoObjects = undefined;
this.videoObjects = undefined;
this.squareVideoObjects = undefined;
this.squarePreviewAnalysis = undefined;
this.verticalVideoObjects = undefined;
this.verticalPreviewAnalysis = undefined;
this.variants = undefined;
this.transcriptStatus = 'hourglass_top';
this.analysisStatus = 'hourglass_top';
Expand All @@ -468,8 +486,10 @@ export class AppComponent {
this.segmentModeToggle.value = 'preview';
this.previewToggleGroup.value = 'toggle';
this.displayObjectTracking = true;
this.moveCropArea = false;
this.previewTrackElem.nativeElement.src = '';
this.subtitlesTrack = '';
this.cropAreaRect = undefined;
this.previewVideoElem.nativeElement.pause();
this.VideoComboComponent?.videoElem.nativeElement.pause();
this.videoMagicPanel.close();
Expand Down Expand Up @@ -559,11 +579,7 @@ export class AppComponent {
generatePreviews(loading = false) {
this.loading = loading;
this.generatingPreviews = true;
this.squareVideoObjects =
this.squarePreviewAnalysis =
this.verticalVideoObjects =
this.verticalPreviewAnalysis =
undefined;
this.squareVideoObjects = this.verticalVideoObjects = undefined;
this.apiCallsService
.generatePreviews(this.analysisJson, this.avSegments, {
sourceDimensions: {
Expand All @@ -584,19 +600,115 @@ export class AppComponent {
this.loading = false;
}
const previewFilter = (e: any) => e.entity.description === 'crop-area';
this.squarePreviewAnalysis = JSON.parse(previews.square);
const squarePreviewAnalysis = JSON.parse(previews.square);
this.squareVideoObjects = this.parseAnalysis(
this.squarePreviewAnalysis,
squarePreviewAnalysis,
previewFilter
);
this.verticalPreviewAnalysis = JSON.parse(previews.vertical);
const verticalPreviewAnalysis = JSON.parse(previews.vertical);
this.verticalVideoObjects = this.parseAnalysis(
this.verticalPreviewAnalysis,
verticalPreviewAnalysis,
previewFilter
);
});
}

toggleMoveCropArea() {
this.segmentModeToggle.value = 'preview';
this.moveCropArea = !this.moveCropArea;
const { currentFrame, idx } = this.getCurrentCropAreaFrame(
this.activeVideoObjects!
)!;

if (this.moveCropArea) {
while (this.canvasDragElement?.nativeElement.firstChild) {
this.canvasDragElement?.nativeElement.removeChild(
this.canvasDragElement?.nativeElement.firstChild
);
}
const canvasViewWidth = this.magicCanvas?.nativeElement.scrollWidth;
const canvasViewHeight = this.magicCanvas?.nativeElement.scrollHeight;

const outputX =
(currentFrame.x * canvasViewWidth) /
this.previewVideoElem.nativeElement.videoWidth;
const outputWidth =
(currentFrame.width * canvasViewWidth) /
this.previewVideoElem.nativeElement.videoWidth;
const outputHeight =
(currentFrame.height * canvasViewHeight) /
this.previewVideoElem.nativeElement.videoHeight;

const img = document.createElement('img');
img.src = this.magicCanvas?.nativeElement.toDataURL('image/png');
img.setAttribute(
'style',
`object-position: -${outputX}px; clip-path: rect(0px ${outputWidth}px ${outputHeight}px 0px); width: ${canvasViewWidth}px; height: ${canvasViewHeight}px;`
);
this.canvasDragElement?.nativeElement.appendChild(img);
this.canvasDragElement?.nativeElement.setAttribute(
'style',
`position: absolute; display: block; left: ${outputX}px; width: ${outputWidth}px; height: ${outputHeight}px`
);
this.canvas!.clearRect(
0,
0,
this.previewVideoElem.nativeElement.videoWidth,
this.previewVideoElem.nativeElement.videoHeight
);
this.previewVideoElem.nativeElement.controls = false;
this.cropAreaRect = img.getBoundingClientRect();
} else {
const imgElement = this.canvasDragElement?.nativeElement
.firstChild as HTMLImageElement;
const newX =
currentFrame.x +
imgElement.getBoundingClientRect().x -
this.cropAreaRect!.x;
this.updateVideoObjects(currentFrame.x, newX, idx);

this.dragPosition = { x: 0, y: 0 };
this.canvasDragElement?.nativeElement.setAttribute(
'style',
'display: none'
);
this.previewVideoElem.nativeElement.controls = true;
}
}

updateVideoObjects(currentX: number, newX: number, idx: number) {
const cropArea = this.activeVideoObjects![0];
const [startIdx, endIdx] = this.getMatchingCropAreaIndexRange(
currentX,
idx
);
for (let i = startIdx; i < endIdx; i++) {
if (cropArea.frames[i].x === currentX) {
cropArea.frames[i].x = newX;
}
}
}

getMatchingCropAreaIndexRange(currentX: number, idx: number) {
const cropArea = this.activeVideoObjects![0];
let startIdx = 0,
endIdx = cropArea.frames.length;

for (let i = idx; i < cropArea.frames.length; i++) {
if (cropArea.frames[i].x !== currentX) {
endIdx = i;
break;
}
}
for (let i = idx; i >= 0; i--) {
if (cropArea.frames[i].x !== currentX) {
startIdx = i + 1;
break;
}
}
return [startIdx, endIdx];
}

loadPreview() {
this.activeVideoObjects = this.videoObjects;
this.previewTrackElem.nativeElement.src = this.subtitlesTrack;
Expand Down Expand Up @@ -871,8 +983,8 @@ export class AppComponent {
this.apiCallsService
.renderVariants(this.folder, {
queue: this.renderQueue,
squareCropAnalysis: this.squarePreviewAnalysis,
verticalCropAnalysis: this.verticalPreviewAnalysis,
squareCropAnalysis: this.squareVideoObjects,
verticalCropAnalysis: this.verticalVideoObjects,
sourceDimensions: {
w: this.previewVideoElem.nativeElement.videoWidth,
h: this.previewVideoElem.nativeElement.videoHeight,
Expand Down

0 comments on commit 28fb3cf

Please sign in to comment.