diff --git a/.circleci/config.yml b/.circleci/config.yml index b45e6029f..fc1bd23eb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -287,7 +287,7 @@ workflows: filters: branches: only: - - master + - rc/0.2.21 - cleanup-lightsail: requires: - rest-tests diff --git a/Makefile b/Makefile index 3b9352f0a..d114b2f67 100644 --- a/Makefile +++ b/Makefile @@ -148,8 +148,7 @@ dashboard-token: kubectl -n kube-system describe secret $$(kubectl -n kube-system get secret | grep tator-kubernetes-dashboard | awk '{print $$1}') .PHONY: tator-image -tator-image: - $(MAKE) webpack +tator-image: webpack docker build --network host -t $(DOCKERHUB_USER)/tator_online:$(GIT_VERSION) -f containers/tator/Dockerfile . || exit 255 docker push $(DOCKERHUB_USER)/tator_online:$(GIT_VERSION) @@ -182,9 +181,22 @@ experimental_docker: endif +USE_VPL=$(shell python3 -c 'import yaml; a = yaml.load(open("helm/tator/values.yaml", "r"),$(YAML_ARGS)); print(a.get("enableVpl","False"))') +ifeq ($(USE_VPL),True) +.PHONY: client-vpl +client-vpl: + docker build --platform linux/amd64 --network host -t $(SYSTEM_IMAGE_REGISTRY)/tator_client_vpl:$(GIT_VERSION) -f containers/tator_client/Dockerfile.vpl . || exit 255 + docker push $(SYSTEM_IMAGE_REGISTRY)/tator_client_vpl:$(GIT_VERSION) +else +.PHONY: client-vpl +client-vpl: + @echo "Skipping VPL Build" +endif + # Publish client image to dockerhub so it can be used cross-cluster .PHONY: client-image client-image: experimental_docker + make client-vpl docker build --platform linux/amd64 --network host -t $(SYSTEM_IMAGE_REGISTRY)/tator_client_amd64:$(GIT_VERSION) -f containers/tator_client/Dockerfile . || exit 255 docker build --platform linux/aarch64 --network host -t $(SYSTEM_IMAGE_REGISTRY)/tator_client_aarch64:$(GIT_VERSION) -f containers/tator_client/Dockerfile_arm . || exit 255 docker push $(SYSTEM_IMAGE_REGISTRY)/tator_client_amd64:$(GIT_VERSION) @@ -223,11 +235,11 @@ USE_MIN_JS=$(shell python3 -c 'import yaml; a = yaml.load(open("helm/tator/value ifeq ($(USE_MIN_JS),True) webpack: @echo "Building webpack bundles for production, because USE_MIN_JS is true" - cd ui && python3 make_index_files.py && npm run build + cd ui && npm install && python3 make_index_files.py && npm run build else webpack: @echo "Building webpack bundles for development, because USE_MIN_JS is false" - cd ui && python3 make_index_files.py && npm run buildDev + cd ui && npm install && python3 make_index_files.py && npm run buildDev endif .PHONY: migrate @@ -308,13 +320,13 @@ js-bindings: ./codegen.py tator-openapi-schema.yaml docker run -it --rm \ -v $(shell pwd)/scripts/packages/tator-js:/pwd \ - openapitools/openapi-generator-cli:v5.2.1 \ + openapitools/openapi-generator-cli:v6.1.0 \ generate -c /pwd/config.json \ -i /pwd/tator-openapi-schema.yaml \ -g javascript -o /pwd/pkg -t /pwd/templates docker run -it --rm \ -v $(shell pwd)/scripts/packages/tator-js:/pwd \ - openapitools/openapi-generator-cli:v5.2.1 \ + openapitools/openapi-generator-cli:v6.1.0 \ chmod -R 777 /pwd/pkg cp -r examples pkg/examples cp -r utils pkg/src/utils diff --git a/containers/tator/Dockerfile b/containers/tator/Dockerfile index 867f6dcd8..d03f66bc5 100644 --- a/containers/tator/Dockerfile +++ b/containers/tator/Dockerfile @@ -1,7 +1,7 @@ # Build librclone shared object FROM golang:latest as rclone_build WORKDIR /go -RUN git clone https://github.com/rclone/rclone.git +RUN git clone https://github.com/rclone/rclone.git --single-branch --depth 1 --branch v1.59.2 WORKDIR /go/rclone/librclone/ RUN go build --buildmode=c-shared -o librclone.so github.com/rclone/rclone/librclone diff --git a/containers/tator_client/Dockerfile b/containers/tator_client/Dockerfile index b80957697..c18d47e49 100644 --- a/containers/tator_client/Dockerfile +++ b/containers/tator_client/Dockerfile @@ -1,4 +1,4 @@ -FROM cvisionai/svt_encoder:v0.0.7 as builder +FROM cvisionai/svt_encoder:v0.0.8 as builder ENV LANG C.UTF-8 RUN apt-get update && apt-get install -y --no-install-recommends \ wget unzip \ diff --git a/containers/tator_client/Dockerfile.vpl b/containers/tator_client/Dockerfile.vpl new file mode 100644 index 000000000..8bc0e160d --- /dev/null +++ b/containers/tator_client/Dockerfile.vpl @@ -0,0 +1,44 @@ +FROM cvisionai/vpl_encoder:v0.0.1 as builder +ENV LANG C.UTF-8 +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget unzip \ + && rm -rf /var/lib/apt/lists +RUN mkdir -p /opt/cvision +RUN wget http://zebulon.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip +RUN unzip Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip +RUN cp Bento4-SDK-1-6-0-632.x86_64-unknown-linux/bin/mp4dump /opt/cvision/bin +RUN cp Bento4-SDK-1-6-0-632.x86_64-unknown-linux/bin/mp4info /opt/cvision/bin + +FROM ubuntu:20.04 AS cvtranscoder +MAINTAINER CVision AI +# Install apt packages +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-pip \ + python3-setuptools python3-dev gcc git vim curl unzip wget \ + && rm -rf /var/lib/apt/lists +COPY --from=builder /opt/cvision /opt/cvision +ENV PATH="/opt/cvision/bin/:$PATH" +RUN echo "/opt/cvision/lib" > /etc/ld.so.conf.d/cvision.conf +RUN ldconfig + +# Install pip packages +RUN pip3 --no-cache-dir --timeout=1000 install wheel +RUN pip3 --no-cache-dir --timeout=1000 install pillow==9.0.0 imageio==2.14.0 progressbar2==4.0.0 boto3==1.20.41 pandas==1.4.0 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends fastjar libsm6 libxext6 \ + libxrender-dev libx265-179 libx264-155 \ + libpng16-16 libfreetype6 python3-opencv \ + libx265-179 libx264-155 libx11-6 libxext6 libxfixes3 \ + libxcb1 libxcb-shm0 libxcb-shape0 libxcb-xfixes0 \ + libwayland-client0 libaom0 && rm -rf /var/lib/apt/lists + +# Copy over scripts +COPY scripts/transcoder /scripts +COPY scripts/packages/tator-py/dist/*.whl /tmp + +# Build tator-py +RUN pip3 install tmp/*.whl + +WORKDIR /scripts diff --git a/helm/tator/templates/tator.yaml b/helm/tator/templates/tator.yaml index 4d3803e1c..38b1a3bf0 100644 --- a/helm/tator/templates/tator.yaml +++ b/helm/tator/templates/tator.yaml @@ -24,6 +24,9 @@ {{- $filebeatSettings := dict "Values" .Values "name" "prune-filebeat-cron" "app" "prune-filebeat" "selector" "webServer: \"yes\"" "command" "[python3]" "args" "[\"manage.py\", \"prunefilebeat\"]" "schedule" "40 4 * * *" }} {{include "tatorCron.template" $filebeatSettings }} --- +{{- $fileSettings := dict "Values" .Values "name" "prune-files-cron" "app" "prune-files" "selector" "webServer: \"yes\"" "command" "[python3]" "args" "[\"manage.py\", \"prunefiles\"]" "schedule" "30 1 * * *" }} +{{include "tatorCron.template" $fileSettings }} +--- {{- $backupSettings := dict "Values" .Values "name" "backup-cron" "app" "backup" "selector" "webServer: \"yes\"" "command" "[/bin/sh]" "args" "[\"-c\", \"pg_dump -Fc -h $POSTGRES_HOST -U $POSTGRES_USERNAME -d tator_online -f /backup/tator_online_$(date +%Y_%m_%d__%H_%M_%S)_$(GIT_VERSION).sql;\"]" "schedule" "40 6 * * *" }} {{include "dbCron.template" $backupSettings }} --- diff --git a/main/backup.py b/main/backup.py index 93d4fc6ae..23c059e46 100644 --- a/main/backup.py +++ b/main/backup.py @@ -1,5 +1,6 @@ from collections import defaultdict from ctypes import CDLL, c_char_p, c_int, Structure +from enum import auto, Enum import logging import json import os @@ -13,7 +14,11 @@ logger = logging.getLogger(__name__) LIVE_STORAGE_CLASS = "STANDARD" -ALL_STORE_TYPES = ["backup", "live"] + + +class StoreType(Enum): + BACKUP = auto() + LIVE = auto() """ @@ -126,10 +131,9 @@ def _rpc(self, rclone_method: str, **kwargs): except: logger.error("Call to `RcloneRPC` failed") raise - else: - output = json.loads(resp.Output.value.decode("utf-8")) - finally: - TatorBackupManager.__rclone.RcloneFreeString(resp.Output) + + output = json.loads(resp.Output.value.decode("utf-8")) + TatorBackupManager.__rclone.RcloneFreeString(resp.Output) status = resp.Status if status != 200: @@ -150,11 +154,16 @@ def _get_bucket_info(self, project) -> dict: if project_id not in TatorBackupManager.__project_stores: TatorBackupManager.__project_stores[project_id] = {} + + # Determine if the default bucket is being used for all StoreTypes or none + use_default_bucket = project.get_bucket() is None + # Get the `TatorStore` object that connects to object storage for the given type - for store_type in ALL_STORE_TYPES: - is_backup = store_type == "backup" - project_bucket = project.get_bucket(backup=is_backup) - use_default_bucket = project_bucket is None + for store_type in StoreType: + is_backup = store_type == StoreType.BACKUP + project_bucket = ( + None if use_default_bucket else project.get_bucket(backup=is_backup) + ) try: store = get_tator_store(project_bucket, backup=is_backup and use_default_bucket) except: @@ -164,13 +173,13 @@ def _get_bucket_info(self, project) -> dict: exc_info=True, ) break - else: - if store: - TatorBackupManager.__project_stores[project_id][store_type] = { - "store": store, - "remote_name": f"{project_id}_{store_type}", - "bucket_name": store.bucket_name, - } + + if store: + TatorBackupManager.__project_stores[project_id][store_type] = { + "store": store, + "remote_name": f"{project_id}_{store_type}", + "bucket_name": store.bucket_name, + } if failed: TatorBackupManager.__project_stores.pop(project_id, None) @@ -188,10 +197,10 @@ def _create_rclone_remote(self, remote_name, bucket_name, remote_type, rclone_pa @staticmethod def get_backup_store(store_info): - if "backup" in store_info: - return True, store_info["backup"]["store"] - if "live" in store_info: - return True, store_info["live"]["store"] + if StoreType.BACKUP in store_info: + return True, store_info[StoreType.BACKUP]["store"] + if StoreType.LIVE in store_info: + return True, store_info[StoreType.LIVE]["store"] return False, None @@ -230,7 +239,7 @@ def get_store_info(self, project) -> bool: ) except: logger.error( - f"Failed to create remote config for bucket {bucket_name} in project " + f"Failed to create remote config for bucket {bucket_info['bucket_name']} in project " f"{project.id}", exc_info=True, ) @@ -257,36 +266,53 @@ def backup_resources(self, resource_qs) -> Generator[tuple, None, None]: """ successful_backups = set() for resource in resource_qs.iterator(): - project = self.project_from_resource(resource) + success = True path = resource.path - success, store_info = self.get_store_info(project) - success = success and "backup" in store_info + try: + project = self.project_from_resource(resource) + except: + logger.warning(f"Could not get project from resource with path '{path}', skipping", exc_info=True) + success = False + project = None if success: - if store_info["backup"]["store"].check_key(path): - logger.info(f"Resource {path} already backed up") - continue - - # Get presigned url from the live bucket, set to expire in 1h - download_url = store_info["live"]["store"].get_download_url(path, 3600) - - # Perform the actual copy operation directly from the presigned url - try: - self._rpc( - "operations/copyurl", - fs=f"{store_info['backup']['remote_name']}:", - remote=f"{store_info['backup']['bucket_name']}/{path}", - url=download_url, - ) - except: - success = False - logger.error( - f"Backing up resource '{path}' with presigned url {download_url} failed", - exc_info=True, - ) + success, store_info = self.get_store_info(project) + success = success and StoreType.BACKUP in store_info if success: - successful_backups.add(resource.id) + backup_info = store_info[StoreType.BACKUP] + backup_store = backup_info["store"] + live_info = store_info[StoreType.LIVE] + live_store = live_info["store"] + + backup_size = backup_store.get_size(path) + live_size = live_store.get_size(path) + if backup_size < 0 or live_size != backup_size: + # Get presigned url from the live bucket, set to expire in 1h + download_url = live_store.get_download_url(path, 3600) + + # Get the destination key + key = backup_store.path_to_key(path) + + # Perform the actual copy operation directly from the presigned url + try: + self._rpc( + "operations/copyurl", + fs=f"{backup_info['remote_name']}:", + remote=f"{backup_info['bucket_name']}/{key}", + url=download_url, + ) + except: + success = False + logger.error( + f"Backing up resource '{path}' with presigned url {download_url} failed", + exc_info=True, + ) + else: + logger.info(f"Resource {path} already backed up, updating its state.") + + if success: + successful_backups.add(resource.id) yield success, resource @@ -314,7 +340,9 @@ def request_restore_resource(self, path, project, min_exp_days) -> bool: success, store = self.get_backup_store(store_info) if success: - live_storage_class = store_info["live"]["store"].get_live_sc() or LIVE_STORAGE_CLASS + live_storage_class = ( + store_info[StoreType.LIVE]["store"].get_live_sc() or LIVE_STORAGE_CLASS + ) response = store.head_object(path) if not response: logger.warning(f"Object {path} not found, skipping") @@ -365,7 +393,7 @@ def finish_restore_resource(self, path, project) -> bool: if success: # If no backup store is defined, use the live bucket success, backup_store = self.get_backup_store(store_info) - live_store = store_info["live"]["store"] + live_store = store_info[StoreType.LIVE]["store"] if success: live_storage_class = live_store.get_live_sc() or LIVE_STORAGE_CLASS @@ -375,11 +403,11 @@ def finish_restore_resource(self, path, project) -> bool: return success request_state = response.get("Restore", "") - if 'true' in request_state: + if "true" in request_state: # There is an ongoing request and the object is not ready to be permanently restored logger.info(f"Object {path} not in standard access yet, skipping") success = False - elif 'false' in request_state: + elif "false" in request_state: # The request has completed and the object is ready for restoration logger.info(f"Object {path} restoration request is complete, restoring...") elif "StorageClass" not in response or response["StorageClass"] == live_storage_class: @@ -411,7 +439,7 @@ def finish_restore_resource(self, path, project) -> bool: # Perform the actual copy operation directly from the presigned url try: - live_info = store_info["live"] + live_info = store_info[StoreType.LIVE] self._rpc( "operations/copyurl", fs=f"{live_info['remote_name']}:", @@ -449,10 +477,10 @@ def get_size(self, resource): success, store_info = self.get_store_info(project) if success: - if resource.backed_up: - store_type = "backup" + if resource.backed_up and StoreType.BACKUP in store_info: + store_type = StoreType.BACKUP else: - store_type = "live" + store_type = StoreType.LIVE size = store_info[store_type]["store"].get_size(resource.path) return size diff --git a/main/management/commands/prunefiles.py b/main/management/commands/prunefiles.py new file mode 100644 index 000000000..4cbd28f2e --- /dev/null +++ b/main/management/commands/prunefiles.py @@ -0,0 +1,48 @@ +import logging +import datetime + +from django.core.management.base import BaseCommand +from main.models import File + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Deletes any files marked for deletion with null project or type." + + def add_arguments(self, parser): + parser.add_argument( + "--min-age-days", + type=int, + default=30, + help="Minimum age in days of file objects for deletion.", + ) + + def handle(self, **options): + BATCH_SIZE = 100 + num_deleted = 0 + min_delta = datetime.timedelta(days=options["min_age_days"]) + max_datetime = datetime.datetime.now(datetime.timezone.utc) - min_delta + while True: + # We cannot delete with a LIMIT query, so make a separate query + # using IDs. + deleted = File.objects.filter(deleted=True, modified_datetime__lte=max_datetime) + null_project = File.objects.filter( + project__isnull=True, modified_datetime__lte=max_datetime + ) + null_meta = File.objects.filter(meta__isnull=True, modified_datetime__lte=max_datetime) + file_ids = ( + (deleted | null_project | null_meta) + .distinct() + .values_list("pk", flat=True)[:BATCH_SIZE] + ) + files = File.objects.filter(pk__in=file_ids) + num_files = files.count() + if num_files == 0: + break + # Delete in a loop to avoid resource deletion errors. + for file_ in files: + file_.delete() + num_deleted += num_files + logger.info(f"Deleted a total of {num_deleted} files...") + logger.info(f"Deleted a total of {num_deleted} files!") diff --git a/main/models.py b/main/models.py index 44ff97455..ddabed4cf 100644 --- a/main/models.py +++ b/main/models.py @@ -1088,7 +1088,9 @@ def media_def_iterator(self, keys: List[str] = None) -> Generator[Tuple[str, dic if files is None: files = [] for obj in files: - yield (key, obj) + # Make sure `obj` is subscriptable + if hasattr(obj, "__getitem__"): + yield (key, obj) def path_iterator(self, keys: List[str] = None) -> Generator[str, None, None]: """ @@ -1104,10 +1106,17 @@ def path_iterator(self, keys: List[str] = None) -> Generator[str, None, None]: keys = [] for key, media_def in self.media_def_iterator(keys): - yield media_def["path"] + # Do not yield invalid media definitions; must have at least the `path` field and, if + # streaming, must also have the `segment_info` field + if "path" not in media_def: + continue + if key == "streaming" and "segment_info" not in media_def: + continue - if key == "streaming": - yield media_def["segment_info"] + yield media_def["path"] + + if key == "streaming": + yield media_def["segment_info"] class FileType(Model): @@ -1155,6 +1164,7 @@ class File(Model, ModelDiffMixin): """ Type associated with file """ attributes = JSONField(null=True, blank=True) """ Values of user defined attributes. """ + deleted = BooleanField(default=False) class Resource(Model): path = CharField(db_index=True, max_length=256) @@ -1310,7 +1320,7 @@ def media_delete(sender, instance, **kwargs): @receiver(post_delete, sender=Media) def media_post_delete(sender, instance, **kwargs): # Delete all the files referenced in media_files - project_id = instance.project.id + project_id = instance.project and instance.project.id for path in instance.path_iterator(): safe_delete(path, project_id) @@ -1376,6 +1386,9 @@ class Localization(Model, ModelDiffMixin): parent = ForeignKey("self", on_delete=SET_NULL, null=True, blank=True,db_column='parent') """ Pointer to localization in which this one was generated from """ deleted = BooleanField(default=False) + elemental_id = UUIDField(primary_key = False, db_index=True, editable = True, null=True, blank=True) + variant_deleted = BooleanField(default=False, null=True, blank=True) + """ Indicates this is a variant that is deleted """ @receiver(post_save, sender=Localization) def localization_save(sender, instance, created, **kwargs): @@ -1411,6 +1424,8 @@ class State(Model, ModelDiffMixin): modified_by = ForeignKey(User, on_delete=SET_NULL, null=True, blank=True, related_name='state_modified_by', db_column='modified_by') version = ForeignKey(Version, on_delete=SET_NULL, null=True, blank=True, db_column='version') + parent = ForeignKey("self", on_delete=SET_NULL, null=True, blank=True,db_column='parent') + """ Pointer to localization in which this one was generated from """ modified = BooleanField(default=True, null=True, blank=True) """ Indicates whether an annotation is original or modified. null: Original upload, no modifications. @@ -1429,6 +1444,9 @@ class State(Model, ModelDiffMixin): related_name='extracted', db_column='extracted') deleted = BooleanField(default=False) + elemental_id = UUIDField(primary_key = False, db_index=True, blank=True, null=True, editable = True) + variant_deleted = BooleanField(default=False, null=True, blank=True) + """ Indicates this is a variant that is deleted """ def selectOnMedia(media_id): return State.objects.filter(media__in=media_id) diff --git a/main/rest/_annotation_query.py b/main/rest/_annotation_query.py index e835029ad..2a4348eba 100644 --- a/main/rest/_annotation_query.py +++ b/main/rest/_annotation_query.py @@ -53,14 +53,10 @@ def get_annotation_es_query(project, params, annotation_type): query = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(dict)))) query['sort']['_postgres_id'] = 'asc' media_bools = [] + annotation_types = ["box", "line", "dot", "poly"] if annotation_type == 'localization': annotation_bools = [{'bool': { - 'should': [ - {'match': {'_dtype': 'box'}}, - {'match': {'_dtype': 'line'}}, - {'match': {'_dtype': 'dot'}}, - {'match': {'_dtype': 'poly'}}, - ], + "should": [{"match": {"_dtype": type_}} for type_ in annotation_types], 'minimum_should_match': 1, }}] elif annotation_type == 'state': @@ -69,31 +65,26 @@ def get_annotation_es_query(project, params, annotation_type): raise ValueError(f"Programming error: invalid annotation type {annotation_type}") media_ids = [] + media_types = ["image", "video", "multi"] if media_id_put is not None: - media_ids += [f'image_{id_}' for id_ in media_id_put]\ - + [f'video_{id_}' for id_ in media_id_put]\ - + [f'multi_{id_}' for id_ in media_id_put] + media_ids.extend(f"{type_}_{id_}" for type_ in media_types for id_ in media_id_put) if media_query is not None: media_query_ids = query_string_to_media_ids(project, media_query) - media_ids += [f'image_{id_}' for id_ in media_query_ids]\ - + [f'video_{id_}' for id_ in media_query_ids]\ - + [f'multi_{id_}' for id_ in media_query_ids] + media_ids.extend(f"{type_}_{id_}" for type_ in media_types for id_ in media_query_ids) if media_id is not None: - media_ids += [f'image_{id_}' for id_ in media_id]\ - + [f'video_{id_}' for id_ in media_id]\ - + [f'multi_{id_}' for id_ in media_id] + media_ids.extend(f"{type_}_{id_}" for type_ in media_types for id_ in media_id) if media_ids: media_bools.append({'ids': {'values': media_ids}}) annotation_ids = [] if localization_ids is not None: - annotation_ids += [f'box_{id_}' for id_ in localization_ids]\ - + [f'line_{id_}' for id_ in localization_ids]\ - + [f'dot_{id_}' for id_ in localization_ids] + annotation_ids.extend( + f"{type_}_{id_}" for type_ in annotation_types for id_ in localization_ids + ) if state_ids is not None: - annotation_ids += [f'state_{id_}' for id_ in state_ids] + annotation_ids.extend(f"state_{id_}" for id_ in state_ids) if annotation_ids: annotation_bools.append({'ids': {'values': annotation_ids}}) @@ -166,9 +157,11 @@ def _get_annotation_psql_queryset(project, filter_ops, params, annotation_type): if localization_id_put: localization_ids += localization_id_put if state_ids and (annotation_type == 'localization'): - localization_ids += list(State.localizations.through.objects\ - .filter(state__in=state_ids)\ - .values_list('localization_id', flat=True).distinct()) + localization_ids += list( + State.localizations.through.objects.filter(state__in=state_ids) + .values_list("localization_id", flat=True) + .distinct() + ) if localization_ids: if annotation_type == 'localization': qs = qs.filter(pk__in=localization_ids) @@ -196,7 +189,7 @@ def _get_annotation_psql_queryset(project, filter_ops, params, annotation_type): qs = get_attribute_psql_queryset(qs, params, filter_ops) if exclude_parents: - parent_set = Localization.objects.filter(pk__in=Subquery(qs.values('parent'))) + parent_set = ANNOTATION_LOOKUP[annotation_type].objects.filter(pk__in=Subquery(qs.values('parent'))) qs = qs.difference(parent_set) # Coalesce is a no-op that prevents PSQL from using the primary key index for small diff --git a/main/rest/_file_query.py b/main/rest/_file_query.py index d8afe2d76..e46e2d658 100644 --- a/main/rest/_file_query.py +++ b/main/rest/_file_query.py @@ -18,7 +18,6 @@ def get_file_es_query(params): # Get parameters. file_id = params.get('file_id', None) file_id_put = params.get('ids', None) # PUT request only - project = params['project'] filter_type = params.get('meta', None) start = params.get('start', None) stop = params.get('stop', None) @@ -139,7 +138,7 @@ def _get_file_psql_queryset(project, filter_ops, params): start = params.get('start') stop = params.get('stop') - qs = File.objects.filter(project=project) + qs = File.objects.filter(project=project, deleted=False) file_ids = [] if file_id is not None: diff --git a/main/rest/_leaf_query.py b/main/rest/_leaf_query.py index 764617678..c4c3e92f7 100644 --- a/main/rest/_leaf_query.py +++ b/main/rest/_leaf_query.py @@ -20,7 +20,6 @@ def get_leaf_es_query(params): # Get parameters. leaf_id = params.get('leaf_id', None) leaf_id_put = params.get('ids', None) # PUT request only - project = params['project'] filter_type = params.get('type', None) start = params.get('start', None) stop = params.get('stop', None) diff --git a/main/rest/_media_query.py b/main/rest/_media_query.py index 1d0466451..813de8ec7 100644 --- a/main/rest/_media_query.py +++ b/main/rest/_media_query.py @@ -56,36 +56,30 @@ def get_media_es_query(project, params): query = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(dict)))) query['sort'] = [{'_exact_name': 'asc'}, {'_postgres_id': 'asc'}] + media_types = ["image", "video", "multi"] bools = [{'bool': { - 'should': [ - {'match': {'_dtype': 'image'}}, - {'match': {'_dtype': 'video'}}, - {'match': {'_dtype': 'multi'}}, - ], + "should": [{"match": {"_dtype": type_}} for type_ in media_types], 'minimum_should_match': 1, }}] annotation_bools = [] media_ids = [] if media_id_put is not None: - media_ids += [f'image_{id_}' for id_ in media_id_put]\ - + [f'video_{id_}' for id_ in media_id_put]\ - + [f'multi_{id_}' for id_ in media_id_put] + media_ids.extend(f"{type_}_{id_}" for type_ in media_types for id_ in media_id_put) if media_id is not None: - media_ids += [f'image_{id_}' for id_ in media_id]\ - + [f'video_{id_}' for id_ in media_id]\ - + [f'multi_{id_}' for id_ in media_id] + media_ids.extend(f"{type_}_{id_}" for type_ in media_types for id_ in media_id) if media_ids: bools.append({'ids': {'values': media_ids}}) annotation_ids = [] + annotation_types = ["box", "line", "dot"] if localization_ids is not None: - annotation_ids += [f'box_{id_}' for id_ in localization_ids]\ - + [f'line_{id_}' for id_ in localization_ids]\ - + [f'dot_{id_}' for id_ in localization_ids] + annotation_ids.extend( + f"{type_}_{id_}" for type_ in annotation_types for id_ in localization_ids + ) if state_ids is not None: - annotation_ids += [f'state_{id_}' for id_ in state_ids] + annotation_ids.extend(f"state_{id_}" for id_ in state_ids) if annotation_ids: annotation_bools.append({'ids': {'values': annotation_ids}}) @@ -182,9 +176,11 @@ def _get_media_psql_queryset(project, section_uuid, filter_ops, params): if media_id is not None: media_ids += media_id if state_ids is not None: - media_ids += list(State.media.through.objects\ - .filter(state__in=state_ids)\ - .values_list('media_id', flat=True).distinct()) + media_ids += list( + State.media.through.objects.filter(state__in=state_ids) + .values_list("media_id", flat=True) + .distinct() + ) if media_ids: qs = qs.filter(pk__in=media_ids) diff --git a/main/rest/_types.py b/main/rest/_types.py new file mode 100644 index 000000000..0af626890 --- /dev/null +++ b/main/rest/_types.py @@ -0,0 +1,40 @@ +from ..search import TatorSearch + +from ._annotation_query import get_annotation_es_query +from ._leaf_query import get_leaf_es_query +from ._file_query import get_file_es_query +from ._media_query import get_media_es_query +from ._util import bulk_delete_and_log_changes + + +def delete_instances(inst_type, inst_model, user, es_query_type): + """ + Deletes the instances associated with their type. + """ + + project_id = inst_type.project.id + inst_qs = inst_model.objects.filter(meta=inst_type.id) + count = inst_qs.count() + + if count: + bulk_delete_and_log_changes(inst_qs, inst_type.project, user) + if es_query_type in ["state", "localization"]: + params = {"ids": list(inst_qs.values_list("id", flat=True))} + query = get_annotation_es_query(project_id, params, es_query_type) + elif es_query_type == "media": + params = {"media_id": list(inst_qs.values_list("id", flat=True))} + query = get_media_es_query(project_id, params) + elif es_query_type == "file": + params = {"file_id": list(inst_qs.values_list("id", flat=True))} + query = get_file_es_query(params) + elif es_query_type == "leaf": + params = {"leaf_id": list(inst_qs.values_list("id", flat=True))} + query = get_leaf_es_query(params) + else: + raise ValueError( + f"Got unsupported instance type '{es_query_type}', expected one of: " + f"'media', 'file', 'leaf', 'localization', or 'state'" + ) + TatorSearch().delete(project_id, query) + + return count diff --git a/main/rest/file.py b/main/rest/file.py index 79ee8f8e7..44834f252 100644 --- a/main/rest/file.py +++ b/main/rest/file.py @@ -121,7 +121,7 @@ class FileDetailAPI(BaseDetailView): def _delete(self, params: dict) -> dict: # Grab the file object and delete it from the database - obj = File.objects.get(pk=params[fields.id]) + obj = File.objects.get(pk=params[fields.id], deleted=False) # Delete the correlated file if obj.path and hasattr(obj.path, "path"): diff --git a/main/rest/file_type.py b/main/rest/file_type.py index 982c63539..3272843f7 100644 --- a/main/rest/file_type.py +++ b/main/rest/file_type.py @@ -11,6 +11,7 @@ from ._base_views import BaseDetailView from ._permissions import ProjectFullControlPermission from ._attribute_keywords import attribute_keywords +from ._types import delete_instances fields = ['id', 'project', 'name', 'description', 'attribute_types'] @@ -88,8 +89,12 @@ def _delete(self, params): name, description, and (like other entity types) may have any number of attribute types associated with it. """ - FileType.objects.get(pk=params['id']).delete() - return {'message': f'File type {params["id"]} deleted successfully!'} + file_type = FileType.objects.get(pk=params["id"]) + count = delete_instances(file_type, File, self.request.user, "file") + file_type.delete() + return { + "message": f"File type {params['id']} (and {count} instances) deleted successfully!" + } def get_queryset(self): - return FileType.objects.all() \ No newline at end of file + return FileType.objects.all() diff --git a/main/rest/leaf_type.py b/main/rest/leaf_type.py index 2f01e5ec8..39b854d34 100644 --- a/main/rest/leaf_type.py +++ b/main/rest/leaf_type.py @@ -10,6 +10,7 @@ from ._base_views import BaseDetailView from ._permissions import ProjectFullControlPermission from ._attribute_keywords import attribute_keywords +from ._types import delete_instances fields = ['id', 'project', 'name', 'description', 'dtype', 'attribute_types', 'visible'] @@ -67,9 +68,12 @@ def _patch(self, params): return {'message': f'Leaf type {obj.id} updated successfully!'} def _delete(self, params): - LeafType.objects.get(pk=params['id']).delete() - return {'message': f'Leaf type {params["id"]} deleted successfully!'} + leaf_type = LeafType.objects.get(pk=params["id"]) + count = delete_instances(leaf_type, Leaf, self.request.user, "leaf") + leaf_type.delete() + return { + "message": f"Leaf type {params['id']} (and {count} instances) deleted successfully!" + } def get_queryset(self): return LeafType.objects.all() - diff --git a/main/rest/localization_graphic.py b/main/rest/localization_graphic.py index 68e851b2b..c8cdb0615 100644 --- a/main/rest/localization_graphic.py +++ b/main/rest/localization_graphic.py @@ -6,7 +6,7 @@ from rest_framework.response import Response from rest_framework import status -from django.http import response +from django.http import Http404, response from ..models import Localization, Media from ..renderers import PngRenderer @@ -239,7 +239,10 @@ def _get(self, params: dict): """ # Get the localization associated with the given ID - obj = Localization.objects.get(pk=params['id']) + qs = Localization.objects.filter(pk=params['id'], deleted=False) + if not qs.exists(): + raise Http404 + obj = qs.first() # Extract the force image size argument and assert if there's a problem with the provided inputs force_image_size = params.get(self.schema.PARAMS_IMAGE_SIZE, None) diff --git a/main/rest/localization_type.py b/main/rest/localization_type.py index c4c6057a7..c0e4e777d 100644 --- a/main/rest/localization_type.py +++ b/main/rest/localization_type.py @@ -14,6 +14,7 @@ from ._base_views import BaseDetailView from ._permissions import ProjectFullControlPermission from ._attribute_keywords import attribute_keywords +from ._types import delete_instances fields = ['id', 'project', 'name', 'description', 'dtype', 'attribute_types', 'colorMap', 'line_width', 'visible', 'drawable', 'grouping_default'] @@ -151,9 +152,12 @@ def _delete(self, params): shape, name, description, and (like other entity types) may have any number of attribute types associated with it. """ - LocalizationType.objects.get(pk=params['id']).delete() - return {'message': f'Localization type {params["id"]} deleted successfully!'} + loc_type = LocalizationType.objects.get(pk=params["id"]) + count = delete_instances(loc_type, Localization, self.request.user, "localization") + loc_type.delete() + return { + "message": f"Localization type {params['id']} (and {count} instances) deleted successfully!" + } def get_queryset(self): return LocalizationType.objects.all() - diff --git a/main/rest/media_type.py b/main/rest/media_type.py index 9df6d7626..889d01219 100644 --- a/main/rest/media_type.py +++ b/main/rest/media_type.py @@ -10,6 +10,7 @@ from ._base_views import BaseDetailView from ._permissions import ProjectFullControlPermission from ._attribute_keywords import attribute_keywords +from ._types import delete_instances fields = ['id', 'project', 'name', 'description', 'dtype', 'attribute_types', 'file_format', 'default_volume', 'visible', 'archive_config', 'streaming_config', 'overlay_config', @@ -169,8 +170,12 @@ def _delete(self, params): name, description, and (like other entity types) may have any number of attribute types associated with it. """ - MediaType.objects.get(pk=params['id']).delete() - return {'message': f'Media type {params["id"]} deleted successfully!'} + media_type = MediaType.objects.get(pk=params["id"]) + count = delete_instances(media_type, Media, self.request.user, "media") + media_type.delete() + return { + "message": f"Media type {params['id']} (and {count} instances) deleted successfully!" + } def get_queryset(self): return MediaType.objects.all() diff --git a/main/rest/organization.py b/main/rest/organization.py index 111d037f1..7b68aaae8 100644 --- a/main/rest/organization.py +++ b/main/rest/organization.py @@ -3,6 +3,7 @@ from rest_framework.exceptions import PermissionDenied from django.db import transaction +from ..cache import TatorCache from ..models import Organization from ..models import Affiliation from ..models import database_qs @@ -16,12 +17,19 @@ from ._base_views import BaseDetailView def _serialize_organizations(organizations, user_id): + ttl = 28800 organization_data = database_qs(organizations) store = get_tator_store() + cache = TatorCache() for idx, organization in enumerate(organizations): organization_data[idx]['permission'] = str(organization.user_permission(user_id)) - if organization_data[idx]['thumb']: - organization_data[idx]['thumb'] = store.get_download_url(organization_data[idx]['thumb'], 28800) + thumb_path = organization_data[idx]['thumb'] + if thumb_path: + url = cache.get_presigned(user_id, thumb_path) + if url is None: + url = store.get_download_url(thumb_path, ttl) + cache.set_presigned(user_id, thumb_path, url, ttl) + organization_data[idx]['thumb'] = url return organization_data class OrganizationListAPI(BaseListView): diff --git a/main/rest/project.py b/main/rest/project.py index 59f9690e7..88751fb34 100644 --- a/main/rest/project.py +++ b/main/rest/project.py @@ -5,6 +5,7 @@ from django.db import transaction from django.shortcuts import get_object_or_404 +from ..cache import TatorCache from ..models import Project from ..models import Membership from ..models import Organization @@ -25,22 +26,52 @@ logger = logging.getLogger(__name__) def _serialize_projects(projects, user_id): + cache = TatorCache() + ttl = 28800 project_data = database_qs(projects) - default_store = get_tator_store(None, connect_timeout=1, read_timeout=1, max_attempts=1) + stores = {None: get_tator_store(None, connect_timeout=1, read_timeout=1, max_attempts=1)} for idx, project in enumerate(projects): if project.creator.pk == user_id: project_data[idx]['permission'] = 'Creator' else: project_data[idx]['permission'] = str(project.user_permission(user_id)) del project_data[idx]['attribute_type_uuids'] - if project_data[idx]['thumb']: - thumb_store = default_store - if project.bucket: - # If this project has a separate bucket, the thumbnail may be there or on the - # main bucket - if not default_store.check_key(project_data[idx]['thumb']): - thumb_store = get_tator_store(project.bucket, connect_timeout=1, read_timeout=1, max_attempts=1) - project_data[idx]['thumb'] = thumb_store.get_download_url(project_data[idx]['thumb'], 28800) + thumb = "" # TODO put default value here + thumb_path = project_data[idx]["thumb"] + if thumb_path: + url = cache.get_presigned(user_id, thumb_path) + if url is None: + try: + url = stores[None].get_download_url(thumb_path, ttl) + except: + bucket_id = project.bucket and project.bucket.id + if bucket_id: + logger.warning( + f"Could not find thumbnail for project {project.id} in the default bucket, " + f"looking in the project-specific one (legacy behavior)..." + ) + if bucket_id in stores: + project_store = stores[bucket_id] + else: + project_store = get_tator_store( + project.bucket, connect_timeout=1, read_timeout=1, max_attempts=1 + ) + stores[bucket_id] = project_store + try: + url = project_store.get_download_url(thumb_path, ttl) + except: + logger.warning( + f"Could not find thumbnail for project {project.id} the " + f"project-specific bucket either" + ) + else: + logger.warning(f"Could not find thumbnail for project {project.id}") + if url is not None: + cache.set_presigned(user_id, thumb_path, url, ttl) + + if url is not None: + thumb = url + project_data[idx]["thumb"] = thumb return project_data class ProjectListAPI(BaseListView): diff --git a/main/rest/state.py b/main/rest/state.py index 57da6edb5..cfb8fede4 100644 --- a/main/rest/state.py +++ b/main/rest/state.py @@ -241,6 +241,7 @@ def _post(self, params): modified_by=self.request.user, version=versions[state_spec.get("version", None)], frame=state_spec.get("frame", None), + parent=State.objects.get(pk=state_spec.get("parent")) if state_spec.get("parent") else None, ) for state_spec, attrs in zip(state_specs, attr_specs) ) diff --git a/main/rest/state_type.py b/main/rest/state_type.py index f8501ed8f..c625ad41d 100644 --- a/main/rest/state_type.py +++ b/main/rest/state_type.py @@ -13,6 +13,7 @@ from ._base_views import BaseDetailView from ._permissions import ProjectFullControlPermission from ._attribute_keywords import attribute_keywords +from ._types import delete_instances fields = ['id', 'project', 'name', 'description', 'dtype', 'attribute_types', 'interpolation', 'association', 'visible', 'grouping_default', @@ -164,8 +165,12 @@ def _delete(self, params): type, name, description, and (like other entity types) may have any number of attribute types associated with it. """ - StateType.objects.get(pk=params['id']).delete() - return {'message': f'State type {params["id"]} deleted successfully!'} + state_type = StateType.objects.get(pk=params["id"]) + count = delete_instances(state_type, State, self.request.user, "state") + state_type.delete() + return { + "message": f"State type {params['id']} (and {count} instances) deleted successfully!" + } def get_queryset(self): return StateType.objects.all() diff --git a/main/rest/transcode.py b/main/rest/transcode.py index e4e5c475c..bb6d63f6e 100644 --- a/main/rest/transcode.py +++ b/main/rest/transcode.py @@ -6,6 +6,7 @@ import requests from ..kube import TatorTranscode +from ..kube import get_jobs from ..store import get_tator_store from ..cache import TatorCache from ..models import Project @@ -127,6 +128,16 @@ def _post(self, params): 'gid': gid, 'media_id': media_id} + # Update Media object with workflow name + if media_id: + media = Media.objects.get(pk=media_id) + cache = TatorCache().get_jobs_by_uid(uid) + jobs = get_jobs(f'uid={uid}', cache) + workflow_names = media.attributes.get('_tator_import_workflow',[]) + workflow_names.append(jobs[0]['metadata']['name']) + media.attributes['_tator_import_workflow'] = workflow_names + media.save() + # Send notification that transcode started. logger.info(msg) return response_data diff --git a/main/schema/_annotation_query.py b/main/schema/_annotation_query.py index d460a404b..ace1eee9d 100644 --- a/main/schema/_annotation_query.py +++ b/main/schema/_annotation_query.py @@ -54,4 +54,17 @@ 'This search is applied to parent media of annotations only.', 'schema': {'type': 'string'}, }, + { + 'name': 'excludeParents', + 'in': 'query', + 'required': False, + 'description': 'If a clone is present, do not send parent. This parameter will cause an ' + 'exception if an Elasticsearch query is triggered and pagination parameters ' + '(start or stop) are included.', + 'schema': {'type': 'integer', + 'minimum': 0, + 'maximum': 1, + 'default': 0 + } + } ] diff --git a/main/schema/components/state.py b/main/schema/components/state.py index f54e5bdd4..6166053a0 100644 --- a/main/schema/components/state.py +++ b/main/schema/components/state.py @@ -3,6 +3,11 @@ 'description': 'Frame number this state applies to.', 'type': 'integer', }, + 'parent': { + 'description': 'If a clone, the pk of the parent.', + 'type': 'number', + 'nullable': True, + } } version_properties = { diff --git a/main/schema/download_info.py b/main/schema/download_info.py index 77f736aca..b1501e136 100644 --- a/main/schema/download_info.py +++ b/main/schema/download_info.py @@ -56,7 +56,7 @@ def get_request_body(self, path, method): def get_responses(self, path, method): responses = error_responses() if method == 'POST': - responses['200'] = { + responses['201'] = { 'description': 'Information required for download.', 'content': {'application/json': {'schema': { 'type': 'array', diff --git a/main/schema/localization.py b/main/schema/localization.py index 1cc40722e..559760346 100644 --- a/main/schema/localization.py +++ b/main/schema/localization.py @@ -9,19 +9,6 @@ from ._annotation_query import annotation_filter_parameter_schema localization_filter_schema = [ - { - 'name': 'excludeParents', - 'in': 'query', - 'required': False, - 'description': 'If a clone is present, do not send parent. This parameter will cause an ' - 'exception if an Elasticsearch query is triggered and pagination parameters ' - '(start or stop) are included.', - 'schema': {'type': 'integer', - 'minimum': 0, - 'maximum': 1, - 'default': 0 - } - }, { 'name': 'frame', 'in': 'query', diff --git a/main/static/demo/embedded_video.html b/main/static/demo/embedded_video.html new file mode 100644 index 000000000..9aeb5edfd --- /dev/null +++ b/main/static/demo/embedded_video.html @@ -0,0 +1,258 @@ + + + Embedded Tator Video Example + + + + + + + + + + +
+ + + + +
+ + diff --git a/main/store.py b/main/store.py index bef5243f4..32eac5013 100644 --- a/main/store.py +++ b/main/store.py @@ -67,6 +67,12 @@ def __init__(self, bucket, client, bucket_name, rclone_params, external_host=Non self._server = None self.remote_type = None + self._proto = "https" if os.getenv("REQUIRE_HTTPS") == "TRUE" else "http" + + @property + def proto(self): + return self._proto + def get_archive_sc(self) -> str: """ Gets the configured archive storage class for this object store. If a bucket is defined, get @@ -100,7 +106,7 @@ def get_tator_store(server, bucket, client, bucket_name, rclone_params, external raise ValueError(f"Server type '{server}' is not supported") - def _path_to_key(self, path: str) -> str: + def path_to_key(self, path: str) -> str: """Returns the storage key for the given path""" return path @@ -112,7 +118,7 @@ def server(self): def check_key(self, path: str) -> bool: """Checks that at least one key matches the given path""" - def head_object(self, path: str) -> dict: + def head_object(self, path: str, quiet: Optional[bool]=False) -> dict: """ Returns the object metadata for a given path using the concrete class implementation of `_head_object`. If the concrete implementation raises, this logs a warning and returns an @@ -121,7 +127,11 @@ def head_object(self, path: str) -> dict: try: return self._head_object(path) except: - logger.warning(f"Failed to call `head_object` on path '{path}'", exc_info=True) + msg = f"Failed to call `head_object` on path '{path}'" + if quiet: + logger.debug(msg, exc_info=True) + else: + logger.warning(msg, exc_info=True) return {} @@ -162,7 +172,7 @@ def get_upload_urls( self, path: str, expiration: int, num_parts: int, domain: str ) -> Tuple[List[str], str]: """Generates the pre-signed urls for uploading objects for a given path.""" - key = self._path_to_key(path) + key = self.path_to_key(path) if num_parts == 1: return self._get_single_upload_url(key, expiration, domain) @@ -173,7 +183,7 @@ def get_size(self, path: str) -> int: """ Returns the size of an object for the given path, if it exists, otherwise returns -1. """ - return self.head_object(path).get("ContentLength", -1) + return self.head_object(path, quiet=True).get("ContentLength", -1) def list_objects_v2(self, prefix: Optional[str] = None, **kwargs) -> list: """ @@ -286,11 +296,11 @@ def __init__(self, bucket, client, bucket_name, rclone_params, external_host=Non self.remote_type = "s3" def check_key(self, path): - return bool(self.list_objects_v2(self._path_to_key(path))) + return bool(self.list_objects_v2(self.path_to_key(path))) def object_tagged_for_archive(self, path): tag_set = self.client.get_object_tagging( - Bucket=self.bucket_name, Key=self._path_to_key(path) + Bucket=self.bucket_name, Key=self.path_to_key(path) ).get("TagSet", []) for tag in tag_set: @@ -302,25 +312,25 @@ def object_tagged_for_archive(self, path): def _put_archive_tag(self, path): self.client.put_object_tagging( Bucket=self.bucket_name, - Key=self._path_to_key(path), + Key=self.path_to_key(path), Tagging={"TagSet": [{"Key": ARCHIVE_KEY, "Value": "true"}]}, ) def put_media_id_tag(self, path, media_id): self.client.put_object_tagging( Bucket=self.bucket_name, - Key=self._path_to_key(path), + Key=self.path_to_key(path), Tagging={"TagSet": [{"Key": MEDIA_ID_KEY, "Value": str(media_id)}]}, ) def _head_object(self, path): - return self.client.head_object(Bucket=self.bucket_name, Key=self._path_to_key(path)) + return self.client.head_object(Bucket=self.bucket_name, Key=self.path_to_key(path)) def copy(self, source_path, dest_path, extra_args=None): self.client.copy( - CopySource={"Bucket": self.bucket_name, "Key": self._path_to_key(source_path)}, + CopySource={"Bucket": self.bucket_name, "Key": self.path_to_key(source_path)}, Bucket=self.bucket_name, - Key=self._path_to_key(dest_path), + Key=self.path_to_key(dest_path), ExtraArgs=extra_args, ) @@ -328,24 +338,20 @@ def restore_object(self, path, live_storage_class, min_exp_days): self._update_storage_class(path, live_storage_class) def delete_object(self, path): - self.client.delete_object(Bucket=self.bucket_name, Key=self._path_to_key(path)) + self.client.delete_object(Bucket=self.bucket_name, Key=self.path_to_key(path)) def get_download_url(self, path, expiration): """Gets the presigned url for accessing an object""" - if os.getenv("REQUIRE_HTTPS") == "TRUE": - PROTO = "https" - else: - PROTO = "http" # Generate presigned url. url = self.client.generate_presigned_url( ClientMethod="get_object", - Params={"Bucket": self.bucket_name, "Key": self._path_to_key(path)}, + Params={"Bucket": self.bucket_name, "Key": self.path_to_key(path)}, ExpiresIn=expiration, ) # Replace host if external host is given. if self.external_host: parsed = urlsplit(url) - parsed = parsed._replace(netloc=self.external_host, scheme=PROTO) + parsed = parsed._replace(netloc=self.external_host, scheme=self.proto) url = urlunsplit(parsed) return url @@ -385,13 +391,13 @@ def _list_objects_v2(self, prefix=None, **kwargs): def complete_multipart_upload(self, path, parts, upload_id): self.client.complete_multipart_upload( Bucket=self.bucket_name, - Key=self._path_to_key(path), + Key=self.path_to_key(path), MultipartUpload={"Parts": parts}, UploadId=upload_id, ) def put_object(self, path, body): - self.client.put_object(Bucket=self.bucket_name, Key=self._path_to_key(path), Body=body) + self.client.put_object(Bucket=self.bucket_name, Key=self.path_to_key(path), Body=body) def put_string(self, path, body): self.put_object(path, body) @@ -400,7 +406,7 @@ def get_object(self, path, start=None, stop=None): if start is None != stop is None: raise ValueError("Must specify both or neither start and stop arguments") - kwargs = {"Bucket": self.bucket_name, "Key": self._path_to_key(path)} + kwargs = {"Bucket": self.bucket_name, "Key": self.path_to_key(path)} if start is not None: kwargs["Range"] = f"bytes={start}-{stop}" @@ -408,7 +414,7 @@ def get_object(self, path, start=None, stop=None): return self.client.get_object(**kwargs)["Body"].read() def download_fileobj(self, path, fp): - self.client.download_fileobj(self.bucket_name, self._path_to_key(path), fp) + self.client.download_fileobj(self.bucket_name, self.path_to_key(path), fp) def _update_storage_class(self, path: str, desired_storage_class: str) -> None: self.copy( @@ -428,13 +434,13 @@ def __init__(self, bucket, client, bucket_name, rclone_params, external_host=Non self._server = ObjectStore.AWS self.remote_type = "s3" - def _path_to_key(self, path): + def path_to_key(self, path): return f"{self.bucket_name}/{path}" def restore_object(self, path, desired_storage_class, min_exp_days): return self.client.restore_object( Bucket=self.bucket_name, - Key=self._path_to_key(path), + Key=self.path_to_key(path), RestoreRequest={"Days": min_exp_days}, ) @@ -447,7 +453,7 @@ def __init__(self, bucket, client, bucket_name, rclone_params, external_host=Non self.remote_type = "google cloud storage" def _get_blob(self, path): - blob = self.gcs_bucket.get_blob(self._path_to_key(path)) + blob = self.gcs_bucket.get_blob(self.path_to_key(path)) if blob is None: raise ValueError() @@ -455,7 +461,7 @@ def _get_blob(self, path): return blob def check_key(self, path): - return self.gcs_bucket.blob(self._path_to_key(path)).exists() + return self.gcs_bucket.blob(self.path_to_key(path)).exists() def _head_object(self, path): """ @@ -473,21 +479,24 @@ def _put_archive_tag(self, path): blob.custom_time = datetime.now() blob.patch() - def put_media_id_tag(self, path): - pass # TODO: implement this + def put_media_id_tag(self, path, media_id): + blob = self._get_blob(path) + metadata = {MEDIA_ID_KEY : str(media_id)} + blob.metadata = metadata + blob.patch() def copy(self, source_path, dest_path, extra_args=None): self.gcs_bucket.copy_blob( blob=self._get_blob(path), destination_bucket=self.gcs_bucket, - new_name=self._path_to_key(dest_path), + new_name=self.path_to_key(dest_path), ) def delete_object(self, path): - self.gcs_bucket.delete_blob(self._path_to_key(path)) + self.gcs_bucket.delete_blob(self.path_to_key(path)) def get_download_url(self, path, expiration): - key = self._path_to_key(path) + key = self.path_to_key(path) blob = self.gcs_bucket.blob(key) return blob.generate_signed_url( version="v4", @@ -529,10 +538,10 @@ def complete_multipart_upload(self, path, parts, upload_id): logger.info(f"No need to complete upload for GCP store") def put_object(self, path, body): - self.gcs_bucket.blob(self._path_to_key(path)).upload_from_file(body) + self.gcs_bucket.blob(self.path_to_key(path)).upload_from_file(body) def put_string(self, path, body): - self.gcs_bucket.blob(self._path_to_key(path)).upload_from_string(body) + self.gcs_bucket.blob(self.path_to_key(path)).upload_from_string(body) def get_object(self, path, start=None, stop=None): return self._get_blob(path).download_as_bytes(start=start, end=stop) diff --git a/main/tests.py b/main/tests.py index d776e2804..b7ec16540 100644 --- a/main/tests.py +++ b/main/tests.py @@ -64,7 +64,7 @@ def create_test_bucket(organization): region='us-east-2', ) -def create_test_project(user, organization=None, backup_bucket=None): +def create_test_project(user, organization=None, backup_bucket=None, bucket=None): kwargs = { "name": "asdf", "creator": user, @@ -76,6 +76,9 @@ def create_test_project(user, organization=None, backup_bucket=None): if backup_bucket: kwargs["backup_bucket"] = backup_bucket + if bucket: + kwargs["bucket"] = bucket + return Project.objects.create(**kwargs) def create_test_membership(user, project): @@ -3398,43 +3401,33 @@ def test_backup_lifecycle(self): class ResourceWithBackupTestCase(ResourceTestCase): - """ This runs the same tests as `ResourceTestCase` but adds a project-specific backup bucket """ + """ This runs the same tests as `ResourceTestCase` but adds project-specific buckets """ def setUp(self): - super().setUp() - endpoint = os.getenv(f"OBJECT_STORAGE_HOST") - secure = "https" in endpoint - endpoint_stripped = re.sub("https?://", "", endpoint) - access_key = os.getenv(f"OBJECT_STORAGE_ACCESS_KEY") - secret_key = os.getenv(f"OBJECT_STORAGE_SECRET_KEY") - self.mc = Minio( - endpoint=endpoint_stripped, secure=secure, access_key=access_key, secret_key=secret_key - ) - self.backup_bucket_name = f"tator-backup-{uuid1()}" - self.mc.make_bucket(self.backup_bucket_name) - self.backup_bucket = Bucket.objects.create( - name=self.backup_bucket_name, - organization=self.organization, - access_key=access_key, - secret_key=secret_key, - endpoint_url=endpoint, - region=os.getenv(f"OBJECT_STORAGE_REGION_NAME"), - ) - self.project.backup_bucket = self.backup_bucket - self.project.save() + logging.disable(logging.CRITICAL) + self.user = create_test_user() + self.client.force_authenticate(self.user) + self.organization = create_test_organization() + self.affiliation = create_test_affiliation(self.user, self.organization) + self.store = get_tator_store() + self.backup_bucket = self.store.bucket + self.project = create_test_project( + self.user, self.organization, bucket=self.store.bucket, backup_bucket=self.store.bucket + ) + self.membership = create_test_membership(self.user, self.project) + self.entity_type = MediaType.objects.create( + name="video", + dtype='video', + project=self.project, + attribute_types=create_test_attribute_types(), + ) + self.file_entity_type = FileType.objects.create( + name="TestFileType", + project=self.project, + attribute_types=create_test_attribute_types(), + ) def tearDown(self): - # Delete objects from temporary backup bucket - objs_to_delete = [DeleteObject(x.object_name) for x in self.mc.list_objects(self.backup_bucket_name, "", recursive=True)] - - # `remove_objects` returns a generator, iterate over it to evaluate - _ = list(self.mc.remove_objects(self.backup_bucket_name, objs_to_delete)) - - # Delete temporary backup bucket - self.mc.remove_bucket(self.backup_bucket_name) - - # Clean up project self.project.delete() - self.backup_bucket.delete() self.organization.delete() class AttributeTestCase(APITestCase): diff --git a/main/views.py b/main/views.py index 4dd6f15b1..e439925a8 100644 --- a/main/views.py +++ b/main/views.py @@ -47,6 +47,11 @@ def dispatch(self, request, *args, **kwargs): else: out = "/accounts/login" + # Carry `next` query parameter over to the login view, if it exists + next_url = getattr(request, request.method).get("next") + if next_url: + out += f"?next={next_url}" + return redirect(out) class MainRedirect(View): diff --git a/scripts/packages/tator-js b/scripts/packages/tator-js index 943766043..d94823186 160000 --- a/scripts/packages/tator-js +++ b/scripts/packages/tator-js @@ -1 +1 @@ -Subproject commit 943766043d50004be3f37443fea8a2fcf6dfe4f9 +Subproject commit d94823186634c90bf86327329daa0263d77ea140 diff --git a/test/test_project-detail.py b/test/test_project-detail.py index f524d46f1..77486320e 100644 --- a/test/test_project-detail.py +++ b/test/test_project-detail.py @@ -207,7 +207,8 @@ def test_basic(request, page_factory, project): #video page.query_selector('text="Show file attributes"').click() # select image labels & video (some in the card list and some not) @todo add video to this project - test_string = page.query_selector_all('.entity-gallery-labels .entity-gallery-labels--checkbox-div checkbox-input[name="Test String"]') + attribute_selected_name = "Test String" + test_string = page.query_selector_all(f'.entity-gallery-labels .entity-gallery-labels--checkbox-div checkbox-input[name="{attribute_selected_name}"]') print(f'Label panel is open: found {len(test_string)} string labels....') test_string[0].click() # for images test_string[2].click() # for video @@ -262,7 +263,7 @@ def test_basic(request, page_factory, project): #video # attributeShown = page.locator('section-files .entity-gallery-card__attribute span[display="block"]').innerHTML() attributeShown = page.query_selector_all('.entity-gallery-card__attribute:not(.hidden)') attributeShownText = attributeShown[1].text_content() - assert attributeShownText == '' + assert attributeShownText == f'{attribute_selected_name}: ' # page.fill('.annotation__panel-group_bulk-edit text-input:not([hidden=""]) input', 'updated') @@ -280,7 +281,7 @@ def test_basic(request, page_factory, project): #video attributeShown = page.query_selector_all('.entity-gallery-card__attribute:not(.hidden)') attributeShownText = attributeShown[1].text_content() - assert attributeShownText == 'updated' + assert attributeShownText == f'{attribute_selected_name}: updated' print('Complete!') # # test download file is working diff --git a/ui/package.json b/ui/package.json index c73e37d18..d08bd6bbb 100644 --- a/ui/package.json +++ b/ui/package.json @@ -28,7 +28,7 @@ "construct-style-sheets-polyfill": "3.1.0", "d3": "7.3.0", "fetch-retry": "5.0.3", - "hls.js": "1.1.5", + "hls.js": "1.2.3", "isomorphic-fetch": "3.0.0", "js-yaml": "4.1.0", "jszip": "3.7.1", @@ -36,6 +36,7 @@ "mp4box": "0.4.9", "spark-md5": "3.0.2", "uuid": "8.3.2", - "webrtc-streamer": "0.6.5-9-g0031565" + "webrtc-streamer": "0.6.5-9-g0031565", + "underwater-image-color-correction": "1.0.3" } } diff --git a/ui/src/._. b/ui/src/._. new file mode 100644 index 000000000..f878225d2 Binary files /dev/null and b/ui/src/._. differ diff --git a/ui/src/js/analytics/collections/gallery.js b/ui/src/js/analytics/collections/gallery.js index 3e4651c81..4ba17439a 100644 --- a/ui/src/js/analytics/collections/gallery.js +++ b/ui/src/js/analytics/collections/gallery.js @@ -114,7 +114,10 @@ export class CollectionsGallery extends EntityCardSlideGallery { var stateTypes = this.collectionsData.getStateTypes(); + + // Label Values + this._cardAttributeLabels.init(this.modelData._project); this.currentLabelValues = {}; const labelValues = []; for (let idx = 0; idx < stateTypes.length; idx++) { diff --git a/ui/src/js/analytics/corrections/gallery.js b/ui/src/js/analytics/corrections/gallery.js index 5cb9fcc2d..624f686df 100644 --- a/ui/src/js/analytics/corrections/gallery.js +++ b/ui/src/js/analytics/corrections/gallery.js @@ -112,6 +112,8 @@ export class AnnotationsCorrectionsGallery extends EntityCardGallery { this.panelContainer._panelTop._panel.entityData.addEventListener("save", this.entityFormChange.bind(this)); this.panelContainer._panelTop._panel.mediaData.addEventListener("save", this.mediaFormChange.bind(this)); + this._cardAttributeLabels.init(this.modelData._project); + // Initialize for (let locTypeData of this.modelData._localizationTypes) { @@ -211,8 +213,11 @@ export class AnnotationsCorrectionsGallery extends EntityCardGallery { /** * Card labels / attributes of localization or media type */ + const builtInChosen = this._cardAttributeLabels._getValue(-1); this.cardLabelsChosenByType[entityTypeId] = this._cardAttributeLabels._getValue(entityTypeId); - this._bulkEdit._updateShownAttributes({typeId: entityTypeId, values: this.cardLabelsChosenByType[entityTypeId]} ); + const cardLabelsChosen = [...this.cardLabelsChosenByType[entityTypeId], ...builtInChosen]; + + this._bulkEdit._updateShownAttributes({ typeId: entityTypeId, values: this.cardLabelsChosenByType[entityTypeId] }); if (newCard) { card = document.createElement("entity-card"); @@ -309,7 +314,7 @@ export class AnnotationsCorrectionsGallery extends EntityCardGallery { idx: index, obj: cardObj, panelContainer : this.panelContainer, - cardLabelsChosen: this.cardLabelsChosenByType[entityTypeId], + cardLabelsChosen: cardLabelsChosen, memberships: this.modelData._memberships }); diff --git a/ui/src/js/analytics/localizations/gallery.js b/ui/src/js/analytics/localizations/gallery.js index 1c75325cb..c09fd5840 100644 --- a/ui/src/js/analytics/localizations/gallery.js +++ b/ui/src/js/analytics/localizations/gallery.js @@ -102,7 +102,10 @@ export class AnnotationsGallery extends EntityCardGallery { this.panelContainer._panelTop._panel.entityData.addEventListener("save", this.entityFormChange.bind(this)); this.panelContainer._panelTop._panel.mediaData.addEventListener("save", this.mediaFormChange.bind(this)); + + // Initialize labels selection + this._cardAttributeLabels.init(this.modelData._project); for (let locTypeData of this.modelData._localizationTypes){ this._cardAttributeLabels.add({ typeData: locTypeData, @@ -187,7 +190,9 @@ export class AnnotationsGallery extends EntityCardGallery { /** * Card labels / attributes of localization or media type */ + const builtInChosen = this._cardAttributeLabels._getValue(-1); this.cardLabelsChosenByType[entityTypeId] = this._cardAttributeLabels._getValue(entityTypeId); + const cardLabelsChosen = [...this.cardLabelsChosenByType[entityTypeId], ...builtInChosen]; if (newCard) { card = document.createElement("entity-card"); @@ -280,7 +285,7 @@ export class AnnotationsGallery extends EntityCardGallery { card.init({ obj: cardObj, panelContainer : this.panelContainer, - cardLabelsChosen: this.cardLabelsChosenByType[entityTypeId], + cardLabelsChosen: cardLabelsChosen, memberships: this.modelData._memberships }); diff --git a/ui/src/js/annotation/annotation-browser.js b/ui/src/js/annotation/annotation-browser.js index 0a87d68a2..e548a9529 100644 --- a/ui/src/js/annotation/annotation-browser.js +++ b/ui/src/js/annotation/annotation-browser.js @@ -9,6 +9,7 @@ export class AnnotationBrowser extends TatorElement { this._shadow.appendChild(this._panels); this._media = document.createElement("media-panel"); + this._media.style.display = "block"; this._panels.appendChild(this._media); this._framePanels = {}; @@ -41,7 +42,14 @@ export class AnnotationBrowser extends TatorElement { entity.addEventListener("close", evt => { this._media.style.display = "block"; for (const typeId in this._framePanels) { - this._framePanels[typeId].style.display = "block"; + + var count = this._framePanels[typeId].getEntityCount(); + if (count == 0) { + this._framePanels[typeId].style.display = "none"; + } + else { + this._framePanels[typeId].style.display = "block"; + } } }); @@ -60,6 +68,21 @@ export class AnnotationBrowser extends TatorElement { if (isFrameState && isInterpolated) { if (dataType.interpolation === "latest"){ const frame = document.createElement("frame-panel"); + + frame.style.display = "none"; + + frame.addEventListener("dataUpdated", () => { + if (this._media.style.display == "block" || this._openedTypeId == dataType.id) { + var count = frame.getEntityCount(); + if (count > 0) { + frame.style.display = "block"; + } + else { + frame.style.display = "none"; + } + } + }); + frame.setAttribute("media-id", this._mediaId); if (stateMediaIds) { @@ -130,9 +153,15 @@ export class AnnotationBrowser extends TatorElement { } _openForTypeId(typeId) { + this._openedTypeId = typeId; for (const key in this._framePanels) { if (key == typeId) { - this._framePanels[key].style.display = "block"; + if (this._framePanels[key].getEntityCount() > 0 ){ + this._framePanels[key].style.display = "block"; + } + else { + this._framePanels[key].style.display = "none"; + } } else { this._framePanels[key].style.display = "none"; } diff --git a/ui/src/js/annotation/annotation-common.js b/ui/src/js/annotation/annotation-common.js index 0538987e5..9d0a23602 100644 --- a/ui/src/js/annotation/annotation-common.js +++ b/ui/src/js/annotation/annotation-common.js @@ -2,6 +2,7 @@ export function handle_video_error(evt, root) { let msg_html = ""; let errorType = ""; + let exit = false; if (evt.detail.secureContext == false) { errorType = "secureContext"; @@ -29,6 +30,21 @@ export function handle_video_error(evt, root) msg_html += `
For full feature support, please utilize the latest versions of
${chrome_link} or ${edge_link}.`; msg_html += ""; sessionStorage.setItem(`handle_error__browser-support`, 'true'); + } else if (evt.detail.forceCompat == 1) + { + errorType = "videoDecoderPresent"; + msg_html += ""; + msg_html += `Compatibility mode is enabled. ` + msg_html += `
Not all video playback features will work optimally when bypassing the VideoDecoder API.`; + msg_html += ""; + } + else + { + exit = true; + } + if (exit == true) + { + return; } diff --git a/ui/src/js/annotation/annotation-data.js b/ui/src/js/annotation/annotation-data.js index 9ddd13b96..5ea22895a 100644 --- a/ui/src/js/annotation/annotation-data.js +++ b/ui/src/js/annotation/annotation-data.js @@ -100,9 +100,8 @@ export class AnnotationData extends HTMLElement { const mediaIds = dataType.dtype == "state" ? this._stateMediaIds : this._localizationMediaIds; let dataUrl = "/rest/" + dataEndpoint + "/" + this._projectId + "?media_id=" + mediaIds.join(',') + "&type=" + dataType.id.split("_")[1]; - if (dataEndpoint == "Localizations") + if (dataEndpoint == "Localizations" || dataEndpoint == "States") { - // TODO probably want this for States as well once it is supported there dataUrl += "&excludeParents=1"; } return dataUrl; @@ -170,6 +169,14 @@ export class AnnotationData extends HTMLElement { } } + updateAllTypes(callback, search) { + for (const key in this._dataTypes) + { + let dataType = this._dataTypes[key]; + this.updateType(dataType, callback, search); + } + } + /** * #TODO Update this to allow states */ diff --git a/ui/src/js/annotation/annotation-multi.js b/ui/src/js/annotation/annotation-multi.js index ba58041c1..ee81b6179 100644 --- a/ui/src/js/annotation/annotation-multi.js +++ b/ui/src/js/annotation/annotation-multi.js @@ -6,6 +6,8 @@ import { RATE_CUTOFF_FOR_ON_DEMAND } from "../annotator/video.js"; import { handle_video_error, handle_decoder_error, frameToTime, PlayInteraction } from "./annotation-common.js"; import { fetchRetry } from "../util/fetch-retry.js"; +let MAGIC_PAD = 5; // if videos are failing at the end jump back this number of frames + export class AnnotationMulti extends TatorElement { constructor() { super(); @@ -525,7 +527,18 @@ export class AnnotationMulti extends TatorElement { video.pushFrame(frameIdx,source,width,height); video.updateOffscreenBuffer(frameIdx,source,width,height); } - promises.push(video.seekFrame(this_frame, cb)); + if (this_frame < video.length) + { + promises.push(video.seekFrame(this_frame, cb)); + } + else + { + if (video.currentFrame() < video.length-MAGIC_PAD) + { + const seekPromise = video.seekFrame(video.length-MAGIC_PAD, cb); + promises.push(seekPromise); + } + } } Promise.allSettled(promises).then(() => { for (let idx = 0; idx < this._videos.length; idx++) @@ -572,8 +585,19 @@ export class AnnotationMulti extends TatorElement { video.pushFrame(frameIdx,source,width,height); video.updateOffscreenBuffer(frameIdx,source,width,height); } - const seekPromise = video.seekFrame(this_frame, cb, true); - seekPromiseList.push(seekPromise); + if (this_frame < video.length) + { + const seekPromise = video.seekFrame(this_frame, cb, true); + seekPromiseList.push(seekPromise); + } + else + { + if (video.currentFrame() < video.length-MAGIC_PAD) + { + const seekPromise = video.seekFrame(video.length-MAGIC_PAD, cb, true); + seekPromiseList.push(seekPromise); + } + } } Promise.allSettled(seekPromiseList).then(() => { @@ -821,6 +845,13 @@ export class AnnotationMulti extends TatorElement { this._currentFrameText.textContent = frame; this._currentTimeText.style.width = 10 * (time.length - 1) + 5 + "px"; this._currentFrameText.style.width = (15 * String(frame).length) + "px"; + let prime_fps = this._fps[this._longest_idx]; + // Update global renderer + for (let idx = 0; idx < this._videos.length; idx++) + { + let this_frame = Math.round(frame * (this._fps[idx]/prime_fps)); + this._multiRenderer.setFrameReq(this._videos[idx]._videoObject.id, this_frame); + } }); } @@ -939,7 +970,7 @@ export class AnnotationMulti extends TatorElement { this._playbackReadyId = 0; this._numVideos = val.media_files['ids'].length; this._frameOffsets = []; - let renderer = new MultiRenderer(); + this._multiRenderer = new MultiRenderer(); for (const vid_id of val.media_files['ids']) { const wrapper_div = document.createElement("div"); @@ -949,8 +980,8 @@ export class AnnotationMulti extends TatorElement { let roi_vid = document.createElement("video-canvas"); this._videoGridInfo[vid_id] = {row: Math.floor(idx / this._multi_layout[1])+1, col: (idx % this._multi_layout[1])+1, video: roi_vid}; - roi_vid.renderer = renderer; - renderer.addVideo(vid_id, roi_vid); + roi_vid.renderer = this._multiRenderer; + this._multiRenderer.addVideo(vid_id, roi_vid); if ('frameOffset' in val.media_files) { this._frameOffsets.push(val.media_files.frameOffset[idx]); @@ -969,7 +1000,7 @@ export class AnnotationMulti extends TatorElement { let allVideosReady = true; for (let vidIdx = 0; vidIdx < this._videos.length; vidIdx++) { - if (this._videos[vidIdx].bufferDelayRequired() && this._videos[vidIdx].onDemandBufferAvailable() != "yes") + if (this._videos[vidIdx].bufferDelayRequired() && this._videos[vidIdx].onDemandBufferAvailable() == false) { allVideosReady = false; } @@ -1459,7 +1490,7 @@ export class AnnotationMulti extends TatorElement { let notReady; for (let video of this._videos) { - notReady |= video.bufferDelayRequired() && video.onDemandBufferAvailable() != "yes"; + notReady |= video.bufferDelayRequired() && video.onDemandBufferAvailable() == false; } if (notReady) { @@ -1516,7 +1547,7 @@ export class AnnotationMulti extends TatorElement { { for (let idx = 0; idx < this._videos.length; idx++) { - if (this._videos[idx].onDemandBufferAvailable() != "yes") + if (this._videos[idx].onDemandBufferAvailable() == false) { this.handleNotReadyEvent(idx); return; @@ -1550,6 +1581,7 @@ export class AnnotationMulti extends TatorElement { console.log("Already handling a not ready event"); return; } + this._videos[videoIndex].onDemandDownloadPrefetch(-1); this._playInteraction.disable(); @@ -1585,7 +1617,7 @@ export class AnnotationMulti extends TatorElement { check_ready(this._videos[videoIndex].currentFrame())}, clock_check); return; } - if (this._videos[videoIndex].bufferDelayRequired() && this._videos[videoIndex].onDemandBufferAvailable() != "yes") + if (this._videos[videoIndex].bufferDelayRequired() && this._videos[videoIndex].onDemandBufferAvailable() == false) { not_ready = true; if (timeoutCounter >= timeouts[timeoutIndex]) { @@ -1631,7 +1663,7 @@ export class AnnotationMulti extends TatorElement { const buffer_required = this._videos[vidIdx].bufferDelayRequired(); const on_demand_available = this._videos[vidIdx].onDemandBufferAvailable(); console.info(`${vidIdx}: ${buffer_required} and ${on_demand_available}`); - if (buffer_required == true && on_demand_available != "yes") + if (buffer_required == true && on_demand_available == false) { allVideosReady = false; } @@ -1704,7 +1736,7 @@ export class AnnotationMulti extends TatorElement { for (let idx = 0; idx < this._videos.length; idx++) { - if (this._videos[idx].bufferDelayRequired() && this._videos[idx].onDemandBufferAvailable() != "yes") + if (this._videos[idx].bufferDelayRequired() && this._videos[idx].onDemandBufferAvailable() == false) { console.info(`Video ${idx} not yet ready, ignoring play request.`); this.handleNotReadyEvent(idx); @@ -1871,6 +1903,7 @@ export class AnnotationMulti extends TatorElement { this.setDefaultVideoSettings(0); } } + this._playInteraction.disable(); this.forcePlaybackDownload(); this.checkReady(); } @@ -1968,8 +2001,19 @@ export class AnnotationMulti extends TatorElement { video.pushFrame(frameIdx,source,width,height); video.updateOffscreenBuffer(frameIdx,source,width,height); } - p_list.push(video.seekFrame(Math.min(this_frame,video._numFrames-1), cb, true)); - idx++; + if (this_frame < video.length) + { + p_list.push(video.seekFrame(Math.min(this_frame,video._numFrames-1), cb, true)); + idx++; + } + else + { + if (video.currentFrame() < video.length-MAGIC_PAD) + { + const seekPromise = video.seekFrame(video.length-MAGIC_PAD, cb, true); + p_list.push(seekPromise); + } + } } let coupled_promise = new Promise((resolve,_) => { Promise.all(p_list).then(() =>{ diff --git a/ui/src/js/annotation/annotation-page.js b/ui/src/js/annotation/annotation-page.js index b959fdb65..4bc4440fa 100644 --- a/ui/src/js/annotation/annotation-page.js +++ b/ui/src/js/annotation/annotation-page.js @@ -91,7 +91,7 @@ export class AnnotationPage extends TatorPage { window.addEventListener("error", (evt) => { this._loading.style.display = "none"; - //window.alert("System error detected"); + //window.alert(evt.message); Utilities.warningAlert("System error detected","#ff3e1d", true); }); @@ -179,6 +179,12 @@ export class AnnotationPage extends TatorPage { } else if (data.media_files && 'streaming' in data.media_files) { data.media_files.streaming.sort((a, b) => {return b.resolution[0] - a.resolution[0];}); } + + // Update Title Bar to show media information + // Usability guidance from mozilla specifies order should go fine -> coarser + // e.g. filename | tool name | org name + // We have tator before tool name, but having file name first is probably helpful enough. + document.title = `${data.name} | ${document.title}` this._breadcrumbs.setAttribute("media-name", data.name); this._browser.mediaInfo = data; this._undo.mediaInfo = data; @@ -350,7 +356,7 @@ export class AnnotationPage extends TatorPage { this._prev.disabled = true; } else { - this._prev.addEventListener("click", () => { + this._prev.addEventListener("click", (evt) => { let url = baseUrl + prevData.prev; var searchParams = this._settings._queryParams(); searchParams.delete("selected_type"); @@ -362,7 +368,18 @@ export class AnnotationPage extends TatorPage { } searchParams = this._videoSettingsDialog.queryParams(searchParams); url += "?" + searchParams.toString(); - window.location.href = url; + // If the control key is pressed jump to a new tab. + if (evt.ctrlKey) + { + let a = document.createElement("a"); + a.target="_blank"; + a.href = url; + a.click(); + } + else + { + window.location.href = url; + } }); this._prev.addEventListener("mouseenter", this.showPrevPreview.bind(this)); this._prev.addEventListener("mouseout", this.removeNextPrevPreview.bind(this)); @@ -372,7 +389,7 @@ export class AnnotationPage extends TatorPage { this._next.disabled = true; } else { - this._next.addEventListener("click", () => { + this._next.addEventListener("click", (evt) => { let url = baseUrl + nextData.next; var searchParams = this._settings._queryParams(); searchParams.delete("selected_type"); @@ -384,7 +401,18 @@ export class AnnotationPage extends TatorPage { } searchParams = this._videoSettingsDialog.queryParams(searchParams); url += "?" + searchParams.toString(); - window.location.href = url; + // If the control key is pressed jump to a new tab. + if (evt.ctrlKey) + { + let a = document.createElement("a"); + a.target="_blank"; + a.href = url; + a.click(); + } + else + { + window.location.href = url; + } }); this._next.addEventListener("mouseenter", this.showNextPreview.bind(this)); this._next.addEventListener("mouseout", this.removeNextPrevPreview.bind(this)); @@ -1238,6 +1266,8 @@ export class AnnotationPage extends TatorPage { }) .then(response => response.json()) .then(applets => { + this._appletMap = {}; + for (let applet of applets) { if (applet.categories == null) { @@ -1268,6 +1298,8 @@ export class AnnotationPage extends TatorPage { const toolAppletPanel = document.createElement("tools-applet-panel"); toolAppletPanel.saveApplet(applet, this, canvas, canvasElement); } + + this._appletMap[applet.name] = applet; } }); } @@ -1302,7 +1334,7 @@ export class AnnotationPage extends TatorPage { .then(result => { var registeredAnnotatorAlgos = []; for (const alg of result) { - if (alg.categories.includes("annotator-view")) { + if (alg.categories.includes("annotator-view") && !alg.categories.includes("hidden")) { registeredAnnotatorAlgos.push(alg.name); if (alg.name == this._extend_track_algo_name) { menu.enableExtendAutoMethod(); @@ -1610,10 +1642,13 @@ export class AnnotationPage extends TatorPage { canvas.addEventListener("launchMenuApplet", evt => { var data = { + applet: this._appletMap[evt.detail.appletName], frame: evt.detail.frame, version: evt.detail.version, media: evt.detail.media, - projectId: evt.detail.projectId + projectId: evt.detail.projectId, + selectedTrack: evt.detail.selectedTrack, + selectedLocalization: evt.detail.selectedLocalization }; if (this._player.mediaType.dtype == "multi") { diff --git a/ui/src/js/annotation/annotation-player.js b/ui/src/js/annotation/annotation-player.js index 0b0424943..2f43e0ff5 100644 --- a/ui/src/js/annotation/annotation-player.js +++ b/ui/src/js/annotation/annotation-player.js @@ -203,6 +203,10 @@ export class AnnotationPlayer extends TatorElement { this._slider.onDemandLoaded(evt); }); + this._video.addEventListener("maxPlaybackRate", evt => { + this._rateControl.max = evt.detail.rate; + }); + this._video.addEventListener("videoLengthChanged", evt => { this._slider.setAttribute('max',evt.detail.length); @@ -840,7 +844,7 @@ export class AnnotationPlayer extends TatorElement { checkReady() { - if (this._video.bufferDelayRequired() && this._video.onDemandBufferAvailable() != "yes") + if (this._video.bufferDelayRequired() && this._video.onDemandBufferAvailable() == false) { this.handleNotReadyEvent(); } @@ -868,6 +872,7 @@ export class AnnotationPlayer extends TatorElement { console.log("Already handling a not ready event"); return; } + this._video.onDemandDownloadPrefetch(-1); this._playInteraction.disable(); const timeouts = [3000, 6000, 12000, 16000]; @@ -904,7 +909,7 @@ export class AnnotationPlayer extends TatorElement { check_ready(this._video.currentFrame())}, 100); return; } - if (this._video.onDemandBufferAvailable() != "yes") + if (this._video.onDemandBufferAvailable() == false) { not_ready = true; if (timeoutCounter >= timeouts[timeoutIndex]) { @@ -971,7 +976,7 @@ export class AnnotationPlayer extends TatorElement { } this._ratesAvailable = this._video.playbackRatesAvailable(); - if (this._video.bufferDelayRequired() && this._video.onDemandBufferAvailable() != "yes") + if (this._video.bufferDelayRequired() && this._video.onDemandBufferAvailable() == false) { this.handleNotReadyEvent(); return; @@ -1029,7 +1034,6 @@ export class AnnotationPlayer extends TatorElement { this.dispatchEvent(new Event("paused", {composed: true})); this._fastForward.removeAttribute("disabled"); this._rewind.removeAttribute("disabled"); - this._rateControl.setValue(this._rate); this.enableRateChange(); const paused = this.is_paused(); @@ -1066,7 +1070,9 @@ export class AnnotationPlayer extends TatorElement { this.pause(); this._video.setQuality(quality, buffer); } + this._playInteraction.disable(); this._video.onDemandDownloadPrefetch(this._video.currentFrame()); + this.checkReady(); this._video.refresh(true); } diff --git a/ui/src/js/annotation/attribute-panel.js b/ui/src/js/annotation/attribute-panel.js index 119b4308a..237aa1e54 100644 --- a/ui/src/js/annotation/attribute-panel.js +++ b/ui/src/js/annotation/attribute-panel.js @@ -831,7 +831,7 @@ export class AttributePanel extends TatorElement { let version = null; let foundVersion = false; for (let index = 0; index < this._versionList.length; index++) { - if (this._versionList[index].result.id == values.modified_by) { + if (this._versionList[index].result.id == values.version) { foundVersion = true; version = this._versionList[index].result.name; break; diff --git a/ui/src/js/annotation/entity-browser.js b/ui/src/js/annotation/entity-browser.js index d7e20f1f1..c0b291c17 100644 --- a/ui/src/js/annotation/entity-browser.js +++ b/ui/src/js/annotation/entity-browser.js @@ -261,7 +261,28 @@ export class EntityBrowser extends TatorElement { let tweakedObj = Object.assign({}, selector.data); delete tweakedObj.version; tweakedObj.attributes = values; - this._canvas.cloneToNewVersion(tweakedObj, this._data.getVersion().id); + if (endpoint == "Localization") + { + this._canvas.cloneToNewVersion(tweakedObj, this._data.getVersion().id); + } + else if (endpoint == "State") + { + let newObject = {}; + let state = tweakedObj; + newObject.parent = state.id; + newObject = Object.assign(newObject, values); + newObject.version = this._data.getVersion().id; + newObject.type = Number(state.meta.split("_")[1]); + newObject.media_ids = state.media; + newObject.frame = state.frame; + newObject.localization_ids = state.localizations; + console.info(JSON.stringify(newObject)); + this._undo.post("States", newObject, this._dataType); + } + else + { + console.error(`Unknown endpoint '${endpoint}'`); + } document.body.classList.remove("shortcuts-disabled"); saved = true; } diff --git a/ui/src/js/annotation/entity-button.js b/ui/src/js/annotation/entity-button.js index 72b37c04c..df43adf22 100644 --- a/ui/src/js/annotation/entity-button.js +++ b/ui/src/js/annotation/entity-button.js @@ -60,6 +60,13 @@ export class EntityButton extends TatorElement { const name = this._dataType.name; const count = evt.detail.data.length; this._text.textContent = count + " " + name; + + if (count == 0) { + this.style.display = "none"; + } + else { + this.style.display = null; + } } }); } @@ -72,6 +79,13 @@ export class EntityButton extends TatorElement { })); }); this._text.textContent = val.count + " " + val.name; + + if (val.count == 0) { + this.style.display = "none"; + } + else { + this.style.display = null; + } } } diff --git a/ui/src/js/annotation/entity-delete-confirm.js b/ui/src/js/annotation/entity-delete-confirm.js index 5a41a38c4..32ba8a307 100644 --- a/ui/src/js/annotation/entity-delete-confirm.js +++ b/ui/src/js/annotation/entity-delete-confirm.js @@ -36,7 +36,7 @@ export class EntityDeleteConfirm extends ModalDialog { cancel.addEventListener("click", this._closeCallback); - this._title.nodeValue = "Delete Localization?"; + this._title.nodeValue = "Delete Object?"; this._accept.addEventListener("click", () => { // when they accept only, save their preference to storage @@ -52,6 +52,11 @@ export class EntityDeleteConfirm extends ModalDialog { }); } + set objectName(val) + { + this._title.nodeValue = `Delete '${val}'?`; + } + confirm() { console.log("Checking session var... allowSessionDelete: " + sessionStorage.getItem('allowSessionDelete')); // Check for stored item and dispatch event to delete; or open confirm diff --git a/ui/src/js/annotation/entity-selector.js b/ui/src/js/annotation/entity-selector.js index ee46ca0b4..7c085514a 100644 --- a/ui/src/js/annotation/entity-selector.js +++ b/ui/src/js/annotation/entity-selector.js @@ -166,9 +166,10 @@ export class EntitySelector extends TatorElement { this._canvas.deleteLocalization(this._data[index]); } else { endpoint = "State"; + let had_parent = this._data[index].parent != null; this._undo.del(endpoint, this._data[index].id, this._dataType).then(() => { - if (this._dataType.delete_child_localizations) { - this._canvas.updateAllLocalizations(); + if (this._dataType.delete_child_localizations || had_parent == true) { + this._canvas._data.updateAllTypes(this._canvas.refresh.bind(this._canvas), null); } }); } @@ -252,6 +253,7 @@ export class EntitySelector extends TatorElement { set dataType(val) { this._dataType = val; + this._delConfirm.objectName = val.name; } set undoBuffer(val) { diff --git a/ui/src/js/annotation/frame-panel.js b/ui/src/js/annotation/frame-panel.js index 71d761e4e..f71e78e4c 100644 --- a/ui/src/js/annotation/frame-panel.js +++ b/ui/src/js/annotation/frame-panel.js @@ -68,7 +68,23 @@ export class FramePanel extends TatorElement { this._undo.post("States", body, val); } else { const state = data[index]; - this._undo.patch("State", state.id, {"attributes": values}, val); + if (this._data.getVersion().bases.indexOf(state.version) >= 0) + { + let newObject = {}; + newObject.parent = state.id; + newObject = Object.assign(newObject, values); + newObject.version = this._data.getVersion().id; + newObject.type = Number(state.meta.split("_")[1]); + newObject.media_ids = state.media; + newObject.frame = state.frame; + newObject.localization_ids = state.localizations; + console.info(JSON.stringify(newObject)); + this._undo.post("States", newObject, val); + } + else + { + this._undo.patch("State", state.id, {"attributes": values}, val); + } } } }); @@ -94,12 +110,14 @@ export class FramePanel extends TatorElement { return; } if (data) { + this._count = data.length; if (data.length > 0) { this._blockingWrites = true; const values = this._getInterpolated(data); this._attributes.setValues(values); this._blockingWrites = false; } + this.dispatchEvent(new Event("dataUpdated")); } } @@ -123,14 +141,22 @@ export class FramePanel extends TatorElement { } let attrs; let id; + let version; + let created_by; switch (this._method) { case "latest": attrs = data[beforeIdx].attributes; id = data[beforeIdx].id; + version = data[beforeIdx].version; + created_by = data[beforeIdx].created_by; break; //TODO: Implement other interpolation methods } - return {attributes: attrs, id: id}; + return {attributes: attrs, id: id, version: version,created_by:created_by}; + } + + getEntityCount() { + return this._count; } } diff --git a/ui/src/js/annotation/menu-applet-dialog.js b/ui/src/js/annotation/menu-applet-dialog.js index e5462ab4d..6d8e52134 100644 --- a/ui/src/js/annotation/menu-applet-dialog.js +++ b/ui/src/js/annotation/menu-applet-dialog.js @@ -9,9 +9,8 @@ export class MenuAppletDialog extends ModalDialog { this._div.style.margin = "10vh auto"; this._title.nodeValue = "Menu Applet"; - this._appletView = document.createElement("iframe"); - this._appletView.setAttribute("class", "d-flex col-12") - this._main.appendChild(this._appletView); + // All applets will be stored in this modal and only the active applet will not be hidden. + this._appletViews = {}; this._main.classList.remove("px-6"); this._main.classList.add("px-2"); @@ -36,8 +35,6 @@ export class MenuAppletDialog extends ModalDialog { this.dispatchEvent(new Event("close")); this._appletElement.accept(); }); - - this._appletView.addEventListener("load", this.initApplet.bind(this)); } /** @@ -106,7 +103,16 @@ export class MenuAppletDialog extends ModalDialog { * @param {Tator.Applet} applet */ saveApplet(applet) { + + var appletView = document.createElement("iframe"); + appletView.setAttribute("class", "d-flex col-12"); + appletView.style.display = "none"; + appletView.src = applet.html_file; + this._main.appendChild(appletView); + this._appletViews[applet.name] = appletView; this._applets[applet.name] = applet; + + // #TODO Potentially need a check to ensure the iframe gets loadeds for each saved applet } /** @@ -119,18 +125,24 @@ export class MenuAppletDialog extends ModalDialog { * projectId {int} */ setApplet(appletName, data) { - this._appletView.src = this._applets[appletName].html_file; + for (const key in this._appletViews) { + this._appletViews[key].style.display = "none"; + } this._appletData = data; + this._appletName = appletName; + this.initApplet(); + + this._appletViews[appletName].style.display = "flex"; } initApplet() { if (this._appletData == null) { return; } - this._appletElement = this._appletView.contentWindow.document.getElementById("mainApplet"); + this._appletElement = this._appletViews[this._appletName].contentWindow.document.getElementById("mainApplet"); var height = this._appletElement.getModalHeight(); if (height.includes("px")) { - this._appletView.style.height = height; + this._appletViews[this._appletName].style.height = height; } var title = this._appletElement.getModalTitle(); @@ -191,7 +203,7 @@ export class MenuAppletDialog extends ModalDialog { }); this._appletElement.addEventListener("updateHeight", (evt) => { - this._appletView.style.height = evt.detail.height; + this._appletViews[this._appletName].style.height = evt.detail.height; }); // Set the applet data diff --git a/ui/src/js/annotation/rate-control.js b/ui/src/js/annotation/rate-control.js index f9b11f57e..a3fb64d19 100644 --- a/ui/src/js/annotation/rate-control.js +++ b/ui/src/js/annotation/rate-control.js @@ -18,15 +18,10 @@ export class RateControl extends TatorElement { div.appendChild(select); this._select = select; - this._rates = [0.125, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 6, 8, 16, 32, 64]; - for (const rate of this._rates) - { - let option = document.createElement("option"); - option.setAttribute("value", rate); - option.textContent = `${rate}x`; - select.append(option); - } - select.selectedIndex = 4; //represents 1x + this._defaultAvailable = [0.125, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 6, 8, 16, 32, 64]; + this._max = Math.max(...this._defaultAvailable); + this._setRates(); + select.selectedIndex = this._rates.findIndex((val)=>{return val == 1}); select.addEventListener("change", evt => { const rate = Number(evt.target.value); @@ -85,6 +80,39 @@ export class RateControl extends TatorElement { })); } + set max(val) + { + this._max = val; + this._setRates(); + } + + get max() + { + return this._max; + } + + _setRates() + { + this._rates=[]; + for (let idx = 0; idx < this._defaultAvailable.length; idx++) + { + if (this._defaultAvailable[idx] <= this._max) + { + this._rates.push(this._defaultAvailable[idx]); + } + } + const oldIdx = this._select.selectedIndex; + this._select.options.length = 0; + for (const rate of this._rates) + { + let option = document.createElement("option"); + option.setAttribute("value", rate); + option.textContent = `${rate}x`; + this._select.append(option); + } + this._select.selectedIndex = oldIdx; + } + getIdx() { return this._select.selectedIndex; diff --git a/ui/src/js/annotator/annotation.js b/ui/src/js/annotator/annotation.js index 180dbca8e..1ddd55b05 100644 --- a/ui/src/js/annotator/annotation.js +++ b/ui/src/js/annotator/annotation.js @@ -5,6 +5,7 @@ import { DrawGL } from "./drawGL.js"; import { color } from "./drawGL_colors.js"; import { Utilities } from "../util/utilities.js"; import { handle_video_error } from "../annotation/annotation-common.js"; +const getColorFilterMatrix = require('underwater-image-color-correction'); var statusAnimator=null; @@ -1246,7 +1247,13 @@ export class AnnotationCanvas extends TatorElement shortcuts += ` (${choice})`; } } - this._contextMenuNone.addMenuEntry(appletName, this.contextMenuCallback.bind(this), shortcuts); + + if (categories.includes("track-applet")) { + this._contextMenuTrack.addMenuEntry(appletName, this.contextMenuCallback.bind(this)); + } + else { + this._contextMenuNone.addMenuEntry(appletName, this.contextMenuCallback.bind(this), shortcuts); + } } /** @@ -1306,6 +1313,19 @@ export class AnnotationCanvas extends TatorElement */ contextMenuCallback(menuText) { + + // It's possible that right clicking on a localization didn't actually set + // the active track. + // Handle case when localization is in a track + if (this.activeLocalization) { + if (this.activeLocalization.id in this._data._trackDb) + { + const track = this._data._trackDb[this.activeLocalization.id]; + this._activeTrack = track; + this._activeTrackFrame = this.currentFrame(); + } + } + if (this._appletLaunchOptions.includes(menuText)) { this.dispatchEvent(new CustomEvent("launchMenuApplet", { detail: { @@ -1314,6 +1334,8 @@ export class AnnotationCanvas extends TatorElement media: this._videoObject, projectId: this._data._projectId, version: this._data.getVersion(), + selectedTrack: this._activeTrack, + selectedLocalization: this.activeLocalization }, composed: true, })); @@ -1326,25 +1348,16 @@ export class AnnotationCanvas extends TatorElement algoName: menuText, frame: this.currentFrame(), mediaId: this._videoObject.id, - projectId: this._data._projectId + projectId: this._data._projectId, + version: this._data.getVersion(), + selectedTrack: this._activeTrack, + selectedLocalization: this.activeLocalization }, composed: true, })); return; } - // It's possible that right clicking on a localization didn't actually set - // the active track. - // Handle case when localization is in a track - if (this.activeLocalization) { - if (this.activeLocalization.id in this._data._trackDb) - { - const track = this._data._trackDb[this.activeLocalization.id]; - this._activeTrack = track; - this._activeTrackFrame = this.currentFrame(); - } - } - // Save the general data that will be passed along to the dialog window var objDescription = {}; objDescription.id = 'modifyTrack'; @@ -1824,6 +1837,27 @@ export class AnnotationCanvas extends TatorElement if (this._mouseMode == MouseMode.QUERY) { + if (event.ctrlKey && event.code == "Digit9") + { + this._effectManager.grayscale(); + return; + } + if (event.ctrlKey && event.code == "Digit8") + { + this._effectManager.grayscale({'color': color.BLACK, 'alpha': 128}); + document.body.style.cursor = "progress"; + setTimeout(()=>{ + this.underwaterCorrection(); + },33); + return; + } + if (event.ctrlKey && event.code == "Digit7") + { + setTimeout(()=>{ + this.refresh(true); + },0); + return; + } if (event.key == "1") { if (event.altKey == true) { @@ -1942,6 +1976,7 @@ export class AnnotationCanvas extends TatorElement { if (event.code == 'Delete' && this._determineCanEdit(this.activeLocalization)) { + this._delConfirm.objectName = this.getObjectDescription(this.activeLocalization).name; this._delConfirm.confirm() } @@ -2468,7 +2503,10 @@ export class AnnotationCanvas extends TatorElement { for (let typed_list of this._framedData.get(currentFrame)) { - localizations.push(...typed_list[1]); + if (typed_list[0] != "CFM") + { + localizations.push(...typed_list[1]); + } } } @@ -4569,6 +4607,12 @@ export class AnnotationCanvas extends TatorElement var typeDict = this._framedData.get(frameIdx); for (let typeid of typeDict.keys()) { + // Handle CFM if present + if (typeid == "CFM") + { + this._draw.setCFM(this._framedData.get(frameIdx).get('CFM')); + continue; + } var localList = typeDict.get(typeid); for (var localIdx = 0; localIdx < localList.length; localIdx++) @@ -4948,4 +4992,191 @@ export class AnnotationCanvas extends TatorElement return this._offscreen.convertToBlob(); } -} + + underwaterCorrection(skip_shader) + { + const begin = performance.now(); + // TODO this actually lets us due the entire GOP (this is in NV12 off the decoder!) + let frameData = this._videoElement[this._seek_idx]._buffer.codec_image_buffer; + let newFrame = new VideoFrame(new Uint8Array(frameData), {'format': frameData.format, + 'codedWidth': frameData.width, + 'codedHeight': frameData.height, + 'timestamp': frameData.timestamp}); + const width = frameData.width; + const height = frameData.height; + console.info(`Underwater correction using ${width}x${height} canvas`); + let temp = new OffscreenCanvas(width, height); + let tempCtx = temp.getContext("2d", {desynchronized:true}); + console.info(`Canvas creation (Remove this step) ${performance.now()-begin} ms`); + // Rasterize to RGBA / ImageData + tempCtx.drawImage(newFrame,0,0, width, height); + newFrame.close(); + + let imageData = tempCtx.getImageData(0,0,width,height); + // Get the Array Buffer + off to the races + let data = imageData.data; + + let histogram_input = this.getGOPTile(); + let filter = getColorFilterMatrix(histogram_input.data, width, height); + + console.info(`Underwater correction Matrix in ${performance.now()-begin} ms`); + console.info(`Color correction matrix: ${filter}`); + + if (skip_shader == true) + { + for (var i = 0; i < data.length; i += 4) + { + data[i] = Math.min(255, Math.max(0, data[i] * filter[0] + data[i + 1] * filter[1] + data[i + 2] * filter[2] + filter[4] * 255)) // Red + data[i + 1] = Math.min(255, Math.max(0, data[i + 1] * filter[6] + filter[9] * 255)) // Green + data[i + 2] = Math.min(255, Math.max(0, data[i + 2] * filter[12] + filter[14] * 255)) // Blue + } + } + else + { + console.info("Using OpenGL accelerated CFM") + this._draw.setCFM(filter); + } + + // update display, this function takes an ImageData too! + this.drawFrame(this.currentFrame(), imageData, this._dims[0], this._dims[1], true); + this._effectManager.clear(); + this._draw.disableCFM(); + console.info(`Underwater correction finished in ${performance.now()-begin} ms`); + document.body.style.cursor = null; + } + + loadPerFrameCFM() + { + let attachments = this._mediaInfo.media_files.attachment; + let found_it = -1; + if (attachments != null) + { + for (let idx = 0; idx < attachments.length; idx++) + { + if (attachments[idx].name == "cfm.bin") + { + found_it = idx; + } + } + } + if (found_it == -1) + { + window.alert("No per-frame color correction available for this video."); + return; + } + + fetch(attachments[found_it].path) + .then(response => {return response.arrayBuffer();}) + .then((buffer)=>{ + let cfm = new Float64Array(buffer); + const cfmLength = 4*5; + console.info(`Fetched color filter matrix for ${cfm.length/cfmLength} frames (size=${buffer.byteLength})!`); + for (let frameIdx = 0; frameIdx < cfm.length; frameIdx++) + { + if (this._framedData.has(frameIdx) != true) + { + this._framedData.set(frameIdx,new Map()); + } + let frameMap = this._framedData.get(frameIdx); + frameMap.set('CFM', cfm.slice(frameIdx*cfmLength, (frameIdx+1)*cfmLength)); + } + this.refresh(); + }); + } + underwaterCorrection_notile() + { + const begin = performance.now(); + // TODO this actually lets us due the entire GOP (this is in NV12 off the decoder!) + let frameData = this._videoElement[this._seek_idx]._buffer.codec_image_buffer; + let newFrame = new VideoFrame(new Uint8Array(frameData), {'format': frameData.format, + 'codedWidth': frameData.width, + 'codedHeight': frameData.height, + 'timestamp': frameData.timestamp}); + const width = frameData.width; + const height = frameData.height; + console.info(`Underwater correction using ${width}x${height} canvas`); + let temp = new OffscreenCanvas(width, height); + let tempCtx = temp.getContext("2d", {desynchronized:true}); + console.info(`Canvas creation (Remove this step) ${performance.now()-begin} ms`); + // Rasterize to RGBA / ImageData + tempCtx.drawImage(newFrame,0,0, width, height); + newFrame.close(); + + let imageData = tempCtx.getImageData(0,0,width,height); + // Get the Array Buffer + off to the races + let data = imageData.data; + + let filter = getColorFilterMatrix(data, width, height); + console.info(`Underwater correction Matrix in ${performance.now()-begin} ms`); + console.info(`Color correction matrix: ${filter}`); + if (skip_shader == true) + { + for (var i = 0; i < data.length; i += 4) + { + data[i] = Math.min(255, Math.max(0, data[i] * filter[0] + data[i + 1] * filter[1] + data[i + 2] * filter[2] + filter[4] * 255)) // Red + data[i + 1] = Math.min(255, Math.max(0, data[i + 1] * filter[6] + filter[9] * 255)) // Green + data[i + 2] = Math.min(255, Math.max(0, data[i + 2] * filter[12] + filter[14] * 255)) // Blue + } + } + else + { + console.info("Using OpenGL accelerated CFM") + this._draw.setCFM(filter); + } + + // update display, this function takes an ImageData too! + this.drawFrame(this.currentFrame(), imageData, this._dims[0], this._dims[1], true); + this._effectManager.clear(); + console.info(`Underwater correction finished in ${performance.now()-begin} ms`); + document.body.style.cursor = null; + } + + getGOPTile() + { + let frameData = this._videoElement[this._seek_idx]._buffer.codec_image_buffer; + let newFrame = new VideoFrame(new Uint8Array(frameData), {'format': frameData.format, + 'codedWidth': frameData.width, + 'codedHeight': frameData.height, + 'timestamp': frameData.timestamp}); + const width = frameData.width; + const height = frameData.height; + console.info(`Tile GOP using ${width}x${height} canvas`); + let temp = new OffscreenCanvas(width, height); + let tempCtx = temp.getContext("2d", {desynchronized:true}); + newFrame.close(); + // Rasterize to RGBA / ImageData + + let matches = this._videoElement[this._seek_idx]._buffer.images_near_cursor(25, 25); + let nearest_square = Math.floor(Math.sqrt(matches.length)); + console.info(`Found ${matches.length} near by frames. ${nearest_square}`); + const tileWidth = Math.round(width/nearest_square); + const tileHeight = Math.round(height/nearest_square); + let idx = 0; + for (let i = 0; i < nearest_square; i++) + { + for (let j = 0; j < nearest_square; j++) + { + frameData = this._videoElement[this._seek_idx]._buffer.get_image(matches[idx]); + newFrame = new VideoFrame(new Uint8Array(frameData), {'format': frameData.format, + 'codedWidth': frameData.width, + 'codedHeight': frameData.height, + 'timestamp': frameData.timestamp}); + tempCtx.drawImage(newFrame,i*tileWidth,j*tileHeight, tileWidth, tileHeight); + newFrame.close(); + idx++; + } + } + + return tempCtx.getImageData(0,0,width,height); + } + tileGOP() + { + const begin = performance.now(); + // TODO this actually lets us due the entire GOP (this is in NV12 off the decoder!) + let imageData = this.getGOPTile(); + // update display, this function takes an ImageData too! + this.drawFrame(this.currentFrame(), imageData, this._dims[0], this._dims[1], true); + console.info(`Tile GOP finished in ${performance.now()-begin} ms`); + document.body.style.cursor = null; + } +} \ No newline at end of file diff --git a/ui/src/js/annotator/download_manager.js b/ui/src/js/annotator/download_manager.js index c347ed39e..9a3d8bc8f 100644 --- a/ui/src/js/annotator/download_manager.js +++ b/ui/src/js/annotator/download_manager.js @@ -29,7 +29,14 @@ export class DownloadManager biasForTime(time, idx) { - return this._startBias.get(idx) + if (this._startBias.has(idx) == false) + { + return null; + } + else + { + return this._startBias.get(idx) + } } _onMessage(msg) @@ -47,7 +54,7 @@ export class DownloadManager return; } msg.data["buffer"].fileStart = msg.data["startByte"]; - console.info(`Converting ${msg.data["frameStart"]} to ${msg.data["frameStart"]/this._parent._fps}`); + //console.info(`Converting ${msg.data["frameStart"]} to ${msg.data["frameStart"]/this._parent._fps}`); msg.data["buffer"].frameStart = (msg.data["frameStart"]/this._parent._fps); this._parent._videoElement[this._parent._seek_idx].appendSeekBuffer(msg.data["buffer"], msg.data['time']); let seek_time = performance.now() - this._parent._seekStart; @@ -197,6 +204,17 @@ export class DownloadManager detail: {"value" : msg.data.firstFrame} })); } + // If the reported number of frames is different we have a problem to rectify + // Note: If .length is shorter than numFrames that might be intentional truncation + // only update the video length if the mp4 is shorter than expecations. + if (msg.data.numFrames < this._parent.length) + { + console.warn(`Video length was ${this._parent.length} but segment map reports ${msg.data.numFrames}.`); + this._parent._numFrames = msg.data.numFrames; + this._parent.dispatchEvent(new CustomEvent("videoLengthChanged", + {composed: true, + detail: {length:this._parent._numFrames}})); + } this._parent._videoVersion = msg.data["version"]; console.info(`Video buf${buf_idx} has start bias of ${this._startBias.get(buf_idx)} - buffer: ${this._parent._scrub_idx}`); console.info("Setting hi performance mode"); diff --git a/ui/src/js/annotator/drawGL.js b/ui/src/js/annotator/drawGL.js index 2df6412c8..5502385e5 100644 --- a/ui/src/js/annotator/drawGL.js +++ b/ui/src/js/annotator/drawGL.js @@ -91,6 +91,10 @@ const imageFsSource = `#version 300 es // Image resolution (useful for filters) uniform vec2 u_Resolution; + uniform int u_csmEnable; + uniform mat4 u_csm; + uniform vec4 u_csmOffset; + void main() { // special mode is -1, so we have plenty of room to spare if (texcoord.x >= -0.25) @@ -137,6 +141,10 @@ const imageFsSource = `#version 300 es else { pixelOutput = texture(imageTexture, texcoord); + if (u_csmEnable == 1) + { + pixelOutput = (pixelOutput * u_csm)+u_csmOffset; + } } } else @@ -294,6 +302,9 @@ export class DrawGL setViewport(canvas) { + // Support format changing prior to openGL call + this._formatCanvas = new OffscreenCanvas(canvas.width, canvas.height); + this._formatCtx = this._formatCanvas.getContext("2d", {desynchronized:true}); // Turn off default antialias as we control it ourselves var gl = this.viewport.getContext("webgl2", {antialias: false, depth: false @@ -367,19 +378,34 @@ export class DrawGL prepBackward() { - this.bufferDepth = 16; //More in rewind mode is helpful - this.frameBuffer = new FrameBuffer(this.bufferDepth, this._initTexture); + if (this._jumbo == undefined && this.bufferDepth != 16) + { + this.bufferDepth = 16; //More in rewind mode is helpful + this.frameBuffer = new FrameBuffer(this.bufferDepth, this._initTexture); + } } prepForward() { - this.bufferDepth = 6; //More in rewind mode is helpful + if (this._jumbo == undefined && this.bufferDepth != 6) + { + this.bufferDepth = 6; //More in rewind mode is helpful + this.frameBuffer = new FrameBuffer(this.bufferDepth, this._initTexture); + } + } + + jumboBufferMode() + { + this._jumbo = true; + this.bufferDepth = 128; //More for compatibility modes is nice. this.frameBuffer = new FrameBuffer(this.bufferDepth, this._initTexture); } // This takes image width and image height. resizeViewport(width, height) { + this._formatCanvas = new OffscreenCanvas(width, height); + this._formatCtx = this._formatCanvas.getContext("2d", {desynchronized:true}); var gl=this.gl; try { @@ -425,30 +451,76 @@ export class DrawGL gl.bindFramebuffer(gl.FRAMEBUFFER, null); // Load the uniform for the view screen size + shift - var viewShiftLoc = gl.getUniformLocation(this.imageShaderProg, "u_ViewShift"); - gl.uniform2fv(viewShiftLoc,[this.clientWidth/2, + this._viewShiftLoc = gl.getUniformLocation(this.imageShaderProg, "u_ViewShift"); + gl.uniform2fv(this._viewShiftLoc,[this.clientWidth/2, this.clientHeight/2]); // The scale is in terms of actual image pixels so that inputs are in image // pixels not device pixels. var viewScale = [2/((this.clientWidth)),2/((this.clientHeight))]; // This vector scales an image unit to a viewscale unit - var viewScaleLoc = gl.getUniformLocation(this.imageShaderProg, "u_ViewScale"); - gl.uniform2fv(viewScaleLoc,viewScale); + this._viewScaleLoc = gl.getUniformLocation(this.imageShaderProg, "u_ViewScale"); + gl.uniform2fv(this._viewScaleLoc,viewScale); var resolution = [this.clientWidth,this.clientHeight]; // This vector scales an image unit to a viewscale unit - var viewScaleLoc = gl.getUniformLocation(this.imageShaderProg, "u_Resolution"); - gl.uniform2fv(viewScaleLoc,resolution); + this._resolutionLoc = gl.getUniformLocation(this.imageShaderProg, "u_Resolution"); + gl.uniform2fv(this._resolutionLoc,resolution); this.viewFlip=this.clientHeight; - var viewFlipLoc = gl.getUniformLocation(this.imageShaderProg, "u_ViewFlip"); - gl.uniform1f(viewFlipLoc,this.viewFlip); + this._viewFlipLoc = gl.getUniformLocation(this.imageShaderProg, "u_ViewFlip"); + gl.uniform1f(this._viewFlipLoc,this.viewFlip); // Image texture is in slot 0 - var imageTexLoc = gl.getUniformLocation(this.imageShaderProg, + this._imageTexLoc = gl.getUniformLocation(this.imageShaderProg, "imageTexture"); - gl.uniform1i(imageTexLoc, 0); + gl.uniform1i(this._imageTexLoc, 0); + + this._csmEnableLoc = gl.getUniformLocation(this.imageShaderProg, "u_csmEnable"); + gl.uniform1i(this._csmEnableLoc, 0); + + this._csmLoc = gl.getUniformLocation(this.imageShaderProg, "u_csm"); + this._csmOffsetLoc = gl.getUniformLocation(this.imageShaderProg, "u_csmOffset"); + } + + _decomposeFilterMatrix(filter) + { + let rgba=[]; + let offset=[]; + for (let i = 0; i < filter.length; i++) + { + if ((i+1) % 5 == 0) + { + offset.push(filter[i]) + } + else + { + rgba.push(filter[i]); + } + } + return {'rgba': rgba, 'offset': offset}; + } + + // Set a color filter matrix for the video/image + // Layout: + // [ + // RedRed, RedGreen, RedBblue, RedAlpha, RedOffset, + // GreenRed, GreenGreen, GreenBlue, GreenAlpha, GOffset, + // BlueRed, BlueGreen, BlueBlue, BlueAlpha, BOffset, + // AlphaRed, AlphaGreen, AlphaBlue, AlphaAlpha, AOffset, + // ] + setCFM(filter) + { + let temp = this._decomposeFilterMatrix(filter); + var gl=this.gl; + gl.uniform1i(this._csmEnableLoc, 1); + gl.uniformMatrix4fv(this._csmLoc, false, temp.rgba); + gl.uniform4fv(this._csmOffsetLoc, temp.offset); + } + disableCFM() + { + var gl=this.gl; + gl.uniform1i(this._csmEnableLoc, 0); } // Constructs the vertices into the viewport @@ -582,7 +654,21 @@ export class DrawGL gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, frameInfo.tex); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frameData); + if ('format' in frameData && !('returnFrame' in frameData)) + { + // Convert to RGBA prior to push to GL + let newFrame = new VideoFrame(new Uint8Array(frameData), {'format': frameData.format, + 'codedWidth': frameData.width, + 'codedHeight': frameData.height, + 'timestamp': frameData.timestamp}); + this._formatCtx.drawImage(newFrame,0,0, this._formatCanvas.width, this._formatCanvas.height); + newFrame.close(); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, this._formatCanvas); + } + else + { + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frameData); + } }; reloadQuadVertices(dWidth, dHeight, uv) @@ -708,11 +794,9 @@ export class DrawGL if (this.lastDx != dx || this.lastDy != dy) { - var viewShiftLoc = gl.getUniformLocation(this.imageShaderProg, "u_ViewShift"); - var cHeight = this.clientHeight; var cWidth = this.clientWidth; - gl.uniform2fv(viewShiftLoc,[(cWidth/2)-dx, + gl.uniform2fv(this._viewShiftLoc,[(cWidth/2)-dx, (cHeight/2)-dy]); this.lastDx = dx; this.lastDy = dy; diff --git a/ui/src/js/annotator/multi-renderer.js b/ui/src/js/annotator/multi-renderer.js index 7c6b6a375..a436c3142 100644 --- a/ui/src/js/annotator/multi-renderer.js +++ b/ui/src/js/annotator/multi-renderer.js @@ -2,7 +2,7 @@ export class MultiRenderer { constructor() { this._videos = {}; this._callbacks = {}; - this._count = 0; + this._frameReq = {}; this._readyCount = 0; } @@ -10,7 +10,7 @@ export class MultiRenderer { { this._videos[video] = element; this._callbacks[video] = null; - this._count += 1; + this._frameReq[video] = 0; } notifyReady(video, callback) @@ -19,12 +19,27 @@ export class MultiRenderer { this.checkAll(); } + setFrameReq(video, frame) + { + this._frameReq[video] = frame; + } + checkAll() { // Check all the videos for ready, bail out if they aren't all set. for (let video in this._videos) { let callback = this._callbacks[video]; + // If we get to the end let it go. + if (this._frameReq[video] > this._videos[video].length-5) + { + if (callback != null) + { + this._videos[video].gotoFrame(this._videos[video].length-5, true); + } + this._callbacks[video] = null; + continue; + } if (callback == null || this._videos[video]._draw.canPlay() <= 0) { return; @@ -46,7 +61,10 @@ export class MultiRenderer { // upon successful drawing. for (let video in this._videos) { - this._callbacks[video].bind(this._videos[video])(domtime); + if (this._callbacks[video] != null) + { + this._callbacks[video].bind(this._videos[video])(domtime); + } } }); } diff --git a/ui/src/js/annotator/vid_downloader.js b/ui/src/js/annotator/vid_downloader.js index 7d3b7a6ce..1a49f896a 100644 --- a/ui/src/js/annotator/vid_downloader.js +++ b/ui/src/js/annotator/vid_downloader.js @@ -2,11 +2,12 @@ import { fetchRetry } from "../util/fetch-retry.js"; export class VideoDownloader { - constructor(media_files, blockSize, offsite_config, frameJump) + constructor(media_files, blockSize, offsite_config, frameJump, infoOnly) { this._media_files = media_files; this._blockSizes = []; this._frameJump = frameJump; + this._infoOnly = infoOnly; console.info(JSON.stringify(media_files)); for (let idx = 0; idx < media_files.length; idx++) { @@ -80,6 +81,10 @@ export class VideoDownloader buffersInitialized() { + if (this._infoOnly == true) + { + return true; + } // Processed all the segment files yet? if (this._infoObjectsInitialized < this._num_res) { @@ -151,6 +156,7 @@ export class VideoDownloader } var startBias = 0.0; var firstFrame = 0.0; + var numFrames = 0.0; if ('file' in data) { startBias = data.file.start; @@ -165,9 +171,14 @@ export class VideoDownloader break; } } - if (firstFrame != 0) + + for (let idx = data.segments.length-1; idx > 0; idx--) { - console.info(""); + if (data.segments[idx].name == "moof") + { + numFrames = data.segments[idx].frame_start+data.segments[idx].frame_samples; + break; + } } that._readyMessages.push( @@ -175,6 +186,7 @@ export class VideoDownloader "type": "ready", "startBias": startBias, "firstFrame": firstFrame, + "numFrames": numFrames, "version": version, "buf_idx": buf_idx }); @@ -189,10 +201,17 @@ export class VideoDownloader if (that._infoObjectsInitialized == that._num_res) { that._fileInfoRequested = true; - for (let buf_idx = 0; buf_idx < that._media_files.length; buf_idx++) + if (that._infoOnly != true) { - // Download the initial fragment info into each buffer - that.downloadNextSegment(buf_idx, 2); + for (let buf_idx = 0; buf_idx < that._media_files.length; buf_idx++) + { + // Download the initial fragment info into each buffer + that.downloadNextSegment(buf_idx, 2); + } + } + if (that._infoOnly == true) + { + that.sendReadyMessages(); } } @@ -577,6 +596,11 @@ export class VideoDownloader downloadForFrame(buf_idx, frame, time) { + // Skip seek requests in info only mode. + if (this._infoOnly == true) + { + return; + } var version = 1; try { @@ -814,7 +838,8 @@ onmessage = function(e) ref = new VideoDownloader(msg.media_files, 2*1024*1024, msg.offsite_config, - msg.frameJump); + msg.frameJump, + msg.infoOnly); } } else if (type == 'download') @@ -825,7 +850,10 @@ onmessage = function(e) ref.saveDownloadNextSegmentRequest(msg.buf_idx); } else { - ref.downloadNextSegment(msg.buf_idx); + if (ref._infoOnly != true) + { + ref.downloadNextSegment(msg.buf_idx); + } } } } diff --git a/ui/src/js/annotator/video-buffer-manager.js b/ui/src/js/annotator/video-buffer-manager.js new file mode 100644 index 000000000..941d5cd8f --- /dev/null +++ b/ui/src/js/annotator/video-buffer-manager.js @@ -0,0 +1,34 @@ +export var CTRL_SIZE = 4*8; // Keep 256-bit alignment of raw frame data + +export class VideoBufferManager +{ + // Size of a frame and how many to keep + constructor(size, depth) + { + // Allocate + this._slots=[] + for (let idx = 0; idx < depth; idx++) + { + let buffer = new SharedArrayBuffer(size+CTRL_SIZE); + let ctrl = new Uint32Array(buffer, 0, CTRL_SIZE); + Atomics.store(ctrl, 0, 0); + this._slots.push(buffer); + } + } + + getSlot() + { + for (let idx = 0; idx < this._slots.length; idx++) + { + let buffer = this._slots[idx]; + let ctrl = new Uint32Array(buffer, 0, CTRL_SIZE); + // Attempts to claim slot, if it isn't claimed sets it to 1, returns old value + if (Atomics.exchange(ctrl, 0, 1) == 0) + { + return buffer; + } + } + console.error("Consumed all the image slots, no where to put latest decoded frame."); + return null; + } +} \ No newline at end of file diff --git a/ui/src/js/annotator/video-codec-worker.js b/ui/src/js/annotator/video-codec-worker.js index 7be8fd2a5..632fec4ab 100644 --- a/ui/src/js/annotator/video-codec-worker.js +++ b/ui/src/js/annotator/video-codec-worker.js @@ -1,5 +1,6 @@ import * as MP4Box from "./mp4box.all.js"; +import { CTRL_SIZE, VideoBufferManager } from "./video-buffer-manager.js"; import { TatorTimeRanges } from "./video-codec.js"; const MAX_DECODED_FRAMES_PER_DECODER = 8; @@ -278,8 +279,11 @@ class TatorVideoBuffer { codedHeight: Number(this._trackHeight), optimizeForLatency: true}); } - console.info(JSON.stringify(info.tracks[0])); - console.info(`${this._name} is configuring decoder = ${JSON.stringify(this._encoderConfig.get(timestampOffset))}`); + if (this._sentReady == undefined) + { + console.info(JSON.stringify(info.tracks[0])); + console.info(`${this._name} is configuring decoder = ${JSON.stringify(this._encoderConfig.get(timestampOffset))}`); + } try { this._videoDecoder.configure(this._encoderConfig.get(timestampOffset)); @@ -292,10 +296,14 @@ class TatorVideoBuffer { error: this._frameError.bind(this)}); this._videoDecoder.configure(this._encoderConfig.get(timestampOffset)); } - console.info(`${this._name} decoder reports ${this._videoDecoder.state}`); - console.info(JSON.stringify(info)); + if (this._sentReady == undefined) + { + this._sentReady = true; + console.info(`${this._name} decoder reports ${this._videoDecoder.state}`); + console.info(JSON.stringify(info)); + } postMessage({"type": "ready", "data": info, "timestampOffset": timestampOffset}); @@ -555,6 +563,7 @@ class TatorVideoBuffer { const seek_value = this._pendingSeek; this._pendingSeek = null; this._setCurrentTime(seek_value, false); + break; } } } @@ -573,6 +582,7 @@ class TatorVideoBuffer { play() { this._setIdle(false); + this._frameIdx = 0; this._pendingSeek = null; //console.info(`PLAYING VIDEO ${this._current_cursor}`); if (this._videoDecoder.state == 'closed') @@ -667,7 +677,14 @@ class TatorVideoBuffer { { this._framesOut = 0; this._pendingEncodedFrames = []; - this._videoDecoder.reset(); + try + { + this._videoDecoder.reset(); + } + catch + { + console.warn("Attempted to reset a closed codec."); + } } _frameReady(frame) @@ -697,8 +714,16 @@ class TatorVideoBuffer { this._frameReturn(); return; } + console.info() + if (this._frameIdx % this.frameIncrement != 0) + { + frame.close(); + this._frameReturn(); + this._frameIdx = (this._frameIdx + 1) % this.frameIncrement; + return; + } this._current_cursor = (frame.timestamp / timeScale); - //console.info(`${performance.now()}: Sending ${this._ready_frames.length}`); + //console.info(`${performance.now()}: Sending ${frame.timestamp}`); postMessage({"type": "frame", "data": frame, "cursor": this._current_cursor, @@ -706,6 +731,7 @@ class TatorVideoBuffer { "timestampOffset": timestampOffset}, [frame] ); // transfer frame copy to primary UI thread + this._frameIdx = (this._frameIdx + 1) % this.frameIncrement; if (this._switchTape) { this._mp4FileMap.get(this._oldTape).stop(); @@ -727,18 +753,36 @@ class TatorVideoBuffer { const timestamp = frame.timestamp; // Make an ImageBitmap from the frame and release the memory // Send all decoded frames to draw UI - this._canvasCtx.drawImage(frame,0,0); - let image = this._canvas.transferToImageBitmap(); //GPU copy of frame - //console.info(`${performance.now()}: ${this._name}@${this._current_cursor}: Publishing @ ${frame.timestamp/timeScale}-${(frame.timestamp+frameDelta)/timeScale} KFO=${this.keyframeOnly}`); - frame.close(); - this._frameReturn(); - postMessage({"type": "image", - "data": image, - "timestamp": timestamp, - "timescale": timeScale, - "frameDelta": frameDelta, - "seconds": timestamp/timeScale}, - image); + if (this._bufferManager == null) + { + // Allocate enough static space for 55 frames + this._bufferManager = new VideoBufferManager(frame.allocationSize(), 55); + } + let slot = this._bufferManager.getSlot(); + if (slot == null) + { + // No slots, out of luck + frame.close(); + return; + } + let image = new Uint8Array(slot, CTRL_SIZE); + frame.copyTo(image).then(() => { + //console.info(`${performance.now()}: ${this._name}@${this._current_cursor}: Publishing @ ${frame.timestamp/timeScale}-${(frame.timestamp+frameDelta)/timeScale} KFO=${this.keyframeOnly}`); + const width = frame.displayWidth; + const height = frame.displayHeight; + const format = frame.format; + frame.close(); + this._frameReturn(); + postMessage({"type": "image", + "data": slot, + "width": width, + "height": height, + "format": format, + "timestamp": timestamp, + "timescale": timeScale, + "frameDelta": frameDelta, + "seconds": timestamp/timeScale}); + }); } } @@ -837,7 +881,7 @@ class TatorVideoBuffer { if (data.frameStart != undefined && data.fileStart != mp4File.nextParsePosition) { const timescale=this._timescaleMap.get(timestampOffset); - console.info(`Setting dts bias to SF=${data.frameStart} FS=${data.fileStart} (was ${mp4File.nextParsePosition}) BIAS=${mp4File.dtsBias} ${mp4File.dtsBias/timescale}`); + //console.info(`Setting dts bias to SF=${data.frameStart} FS=${data.fileStart} (was ${mp4File.nextParsePosition}) BIAS=${mp4File.dtsBias} ${mp4File.dtsBias/timescale}`); mp4File.lastBoxStartPosition = data.fileStart; mp4File.nextParsePosition = data.fileStart; mp4File.dtsBias = Math.round(data.frameStart * timescale); @@ -996,7 +1040,7 @@ class TatorVideoBuffer { tempFile.lastBoxStartPosition = data.fileStart; tempFile.nextParsePosition = data.fileStart; tempFile.dtsBias = Math.round(data.frameStart * this._timescaleMap.get(timestampOffset)); - console.info(`${this._name} TEMP Setting dts bias to FS=${data.fileStart} BIAS=${tempFile.dtsBias} ${tempFile.dtsBias/this._timescaleMap.get(timestampOffset)}`); + //console.info(`${this._name} TEMP Setting dts bias to FS=${data.fileStart} BIAS=${tempFile.dtsBias} ${tempFile.dtsBias/this._timescaleMap.get(timestampOffset)}`); tempFile.stop(); tempFile.appendBuffer(data); tempFile.seek(0); // Always go to 0 for this @@ -1142,4 +1186,8 @@ onmessage = function(e) { ref._clearAllPending(); } + else if (msg.type == "frameIncrement") + { + ref.frameIncrement = msg.value; + } } diff --git a/ui/src/js/annotator/video-codec.js b/ui/src/js/annotator/video-codec.js index 593de2d34..2320de03c 100644 --- a/ui/src/js/annotator/video-codec.js +++ b/ui/src/js/annotator/video-codec.js @@ -8,7 +8,7 @@ // operations. - +import { CTRL_SIZE } from "./video-buffer-manager"; // TimeRanges isn't user constructable so make our own export class TatorTimeRanges { @@ -155,6 +155,12 @@ class TatorVideoManager { this._codec_worker.postMessage({"type": "keyframeOnly", "value": val}); } + set frameIncrement(val) + { + this._frameIncrement = val; + this._codec_worker.postMessage({"type": "frameIncrement", "value": val}); + } + set scrubbing(val) { this._scrubbing = val; @@ -209,6 +215,16 @@ class TatorVideoManager { else if (msg.data.type == "error") { console.warn(msg.data); + if (msg.data.message.toLocaleString().indexOf("Unsupported configuration") > 0) + { + if (this._alertSent == undefined) + { + this._alertSent = true; + this._parent._canvas.dispatchEvent(new CustomEvent("codecNotSupported", + {composed: true, + detail: {"codec": this._codec_string}})); + } + } } else if (msg.data.type == "frameDelta") { @@ -259,6 +275,45 @@ class TatorVideoManager { return false; } + images_near_cursor(max_distance, limit) + { + let timestamps = this._hot_frames.keys() // make sure keys are sorted! + let matches=[]; + for (let timestamp of timestamps) + { + let image_timescale = this._hot_frames.get(timestamp).timescale; + let frame_delta = this._hot_frames.get(timestamp).frameDelta; + let cursor_in_ctx = this._current_cursor * image_timescale; + if (Math.abs(cursor_in_ctx - timestamp) <= (max_distance*frame_delta)) + { + matches.push(timestamp); + } + } + return matches; + } + + get_image(timestamp) + { + if (this._hot_frames.has(timestamp)) + { + let sab = this._hot_frames.get(timestamp); + let image = new Uint8Array(sab, CTRL_SIZE); + // Todo function this + image.timescale = sab.timescale; + image.frameDelta = sab.frameDelta; + image.time = sab.timestamp / sab.timescale; + image.width = sab.width; + image.height = sab.height; + image.format = sab.format; + image.timestamp = sab.timestamp; + return image; + } + else + { + return null; + } + } + // Returns true if the cursor is in the range of the hot frames time_is_hot(time) { @@ -278,9 +333,9 @@ class TatorVideoManager { cursor_in_image(image) { - const image_timescale = image.data.timescale; - const frame_delta = image.data.frameDelta; - const time = image.data.time; + const image_timescale = image.timescale; + const frame_delta = image.frameDelta; + const time = image.time; let time_in_ctx = time * image_timescale; let cursor_in_ctx = this._current_cursor * image_timescale; if (cursor_in_ctx >= time_in_ctx && cursor_in_ctx < time_in_ctx+frame_delta) @@ -294,17 +349,17 @@ class TatorVideoManager { } _returnFrame(frame) - { + { frame.close(); this._codec_worker.postMessage({"type": "frameReturn"}) - } + } _frameReady(msg) { // If there is a frame handler callback potentially avoid // internal buffering. msg.data.returnFrame = () => {this._returnFrame(msg.data);}; - //console.info(`${performance.now()} ${this._name} Frame @ ${msg.cursor} Ready`); + //console.info(`${performance.now()} ${this._name} Frame @ ${msg.data.timestamp}} Ready`); if (this.onFrame && this._playing == true) { this._current_cursor = msg.data.cursor; @@ -321,12 +376,24 @@ class TatorVideoManager { _imageReady(image) { //console.info(`${performance.now()}: GOT ${this._name}: GOT h=${image.height}`); + // Make the image a uint8 clamped array image.data.timescale = image.timescale; image.data.frameDelta = image.frameDelta; - image.data.time = image.timestamp / image.data.timescale; + image.data.time = image.timestamp / image.timescale; + image.data.width = image.width; + image.data.height = image.height; + image.data.format = image.format; + image.data.timestamp = image.timestamp; + if (this._hot_frames.has(image.timestamp)) + { + // Clear old one. + let sab = this._hot_frames.get(image.timestamp); + let ctrl = new Uint32Array(sab,0,CTRL_SIZE); + Atomics.store(ctrl, 0, 0); + } this._hot_frames.set(image.timestamp, image.data); //console.info(`${performance.now()}: ${this._name}: _imageReady() time=${image.data.time}: CiI=${this.cursor_in_image(image)} KFO=${this._keyframeOnly} SCRUBBING=${this._scrubbing} MUTE=${this._mute}`); - if ((this.cursor_in_image(image) || this._keyframeOnly == true) && this._mute == false) + if ((this.cursor_in_image(image.data) || this._keyframeOnly == true) && this._mute == false) { this._safeCall(this.oncanplay); } @@ -370,9 +437,10 @@ class TatorVideoManager { // The seek buffer can keep up to 10 frames pre-decoded ready to go in either direction // to support extra fast prev/next - _clean_hot() + _clean_hot(force) { - if (this._hot_frames.size < 25) + //console.info(`${this._name}: _clean_hot(): ${this._hot_frames.size}`); + if (this._hot_frames.size < 25 && force != true) { return; } @@ -385,13 +453,18 @@ class TatorVideoManager { for (let hot_frame of timestamps) { // Only keep a max of 100 frames in memory - if (Math.abs(hot_frame - cursor_in_ctx)/this._frameDeltaMap.get(search.key) >= 25) + if (Math.abs(hot_frame - cursor_in_ctx)/this._frameDeltaMap.get(search.key) >= 25 || force == true) { delete_elements.push(hot_frame); } } for (let key of delete_elements) { + let sab = this._hot_frames.get(key); + let ctrl = new Uint32Array(sab,0,CTRL_SIZE); + Atomics.store(ctrl, 0, 0); + //console.info(`${this._name}: Deleting ${key}!`); + delete this._hot_frames.get(key); this._hot_frames.delete(key); } } @@ -417,7 +490,16 @@ class TatorVideoManager { lastTimestamp = timestamp; } } - return this._hot_frames.get(lastTimestamp); + let sab = this._hot_frames.get(lastTimestamp); + let image = new Uint8Array(sab, CTRL_SIZE); + image.timescale = sab.timescale; + image.frameDelta = sab.frameDelta; + image.time = sab.timestamp / sab.timescale; + image.width = sab.width; + image.height = sab.height; + image.format = sab.format; + image.timestamp = sab.timestamp; + return image; } /////////////////////////////////////////////////////////// @@ -484,6 +566,7 @@ class TatorVideoManager { this._current_cursor = video_time+this._bias; } const is_hot = this._cursor_is_hot(); + this._clean_hot(); // clean hot prior to potentially getting more data back this._codec_worker.postMessage( {"type": "currentTime", "currentTime": this._current_cursor, diff --git a/ui/src/js/annotator/video-effects.js b/ui/src/js/annotator/video-effects.js index 847324cbc..a4c2d8b09 100644 --- a/ui/src/js/annotator/video-effects.js +++ b/ui/src/js/annotator/video-effects.js @@ -8,17 +8,21 @@ export class EffectManager this._draw = draw; } - grayOut() + grayOut(delay_ms) { + if (delay_ms == undefined) + { + delay_ms = 150; + } const frame = this._video.currentFrame(); const maxX = this._canvas.width; const maxY = this._canvas.height; this._idx = 1; let prog = ()=>{ - this.clear(); + this._draw.beginDraw(); //this._draw.fillPolygon([[0,0], [maxX,0],[maxX,maxY],[0,maxY]], 0, color.BLACK, 75, [1.0,Math.atan(this._idx/10)*0.0025,0,0]); - const delay = Math.floor(150 / 16); + const delay = Math.floor(delay_ms / 16); if (this._idx > delay) { this._draw.fillPolygon([[0,0], [maxX,0],[maxX,maxY],[0,maxY]], 0, color.BLACK, 10 + (75*Math.atan((this._idx-delay)/10))); @@ -30,10 +34,45 @@ export class EffectManager prog(); } + darken(req_color, alpha) + { + if (req_color == undefined) + { + req_color = color.BLACK; + } + if (alpha == undefined) + { + alpha = 128; + } + const frame = this._video.currentFrame(); + const maxX = this._canvas.width; + const maxY = this._canvas.height; + this.clear(); + this._draw.fillPolygon([[0,0], [maxX,0],[maxX,maxY],[0,maxY]], 0, req_color, alpha); + this._draw.dispImage(true,false, frame); + } + + grayscale(darken) + { + const frame = this._video.currentFrame(); + const maxX = this._canvas.width; + const maxY = this._canvas.height; + this.clear(); + this._draw.fillPolygon([[0,0], [maxX,0],[maxX,maxY],[0,maxY]], 0, color.BLACK, 255, [2.0,0,0,0]); + if (darken) + { + this._draw.fillPolygon([[0,0], [maxX,0],[maxX,maxY],[0,maxY]], 0, darken.color, darken.alpha); + } + this._draw.dispImage(true,false, frame); + } + clear() { - window.cancelAnimationFrame(this._animator); - this._animator = null; + if (this._animator != null) + { + window.cancelAnimationFrame(this._animator); + this._animator = null; + } this._draw.beginDraw(); } diff --git a/ui/src/js/annotator/video-simple.js b/ui/src/js/annotator/video-simple.js new file mode 100644 index 000000000..9fde82b36 --- /dev/null +++ b/ui/src/js/annotator/video-simple.js @@ -0,0 +1,519 @@ +// Module using WebCodecs API to decode video instead of MediaSource Extensions +// reference: https://developer.mozilla.org/en-US/docs/Web/API/WebCodecs_API + +// Attempt is made to partially implement the HTML5 MediaElement interface +// such that this is a drop-in replacement for frame accurate MSE applications +// +// @TODO: Supply a 'cv2.VideoDecode.read()' type interface for client-side decode +// operations. + +import Hls from "hls.js"; + +class SimpleVideoWrapper { + constructor(parent, name, path) + { + this._name = name; + this._parent = parent; + this._video = document.createElement("VIDEO"); + this._video.setAttribute("crossorigin", "anonymous"); + this.use_codec_buffer = true; + this._path = path; + this._bias = 0; + this._keyframeOnly = false; + this._scrubbing = false; + this._mute = false; + this._checked = false; + } + + // Simple pass through to the underlying video + get codec_image_buffer() + { + this._video.time = this._video.currentTime; + return this._video; + } + + init() + { + this._video.onloadeddata = () => { + this._video.onloadeddata = null; + }; + this._video.oncanplay = () => { + if (this.oncanplay && this._mute == false) + { + this.oncanplay(); + } + } + if (this._path) + { + this._video.src = this._path; + console.info(`${this._name} is initialized with ${this._path}`); + } + } + + set oncanplay(val) + { + this._oncanplay = val; + } + + get oncanplay() + { + return this._oncanplay; + } + get canplay() + { + for (let idx = 0; idx < this._video.buffered.length; idx++) + { + if (this._video.currentTime >= this._video.buffered.start(idx) && this._video.currentTime < this._video.buffered.end(idx)) + { + return true; + } + } + return false; + } + + set keyframeOnly(val) + { + // Simple mode doesn't support this + } + + set frameIncrement(val) + { + + } + + set scrubbing(val) + { + + } + + get keyframeOnly() + { + return false; + } + + clearPending() + { + + } + + // Returns true if the cursor is in the range of the hot frames + _cursor_is_hot() + { + let timestamps = this._hot_frames.keys() // make sure keys are sorted! + for (let timestamp of timestamps) + { + let image_timescale = this._hot_frames.get(timestamp).timescale; + let frame_delta = this._hot_frames.get(timestamp).frameDelta; + let cursor_in_ctx = this._current_cursor * image_timescale; + if (cursor_in_ctx >= timestamp && cursor_in_ctx < timestamp+frame_delta) + { + return true; + } + } + + return false; + } + + images_near_cursor(max_distance, limit) + { + + } + + get_image(timestamp) + { + + } + + // Returns true if the cursor is in the range of the hot frames + time_is_hot(time) + { + + } + + _returnFrame(frame) + { + + } + + _safeCall(func_ptr) + { + if (func_ptr) + { + func_ptr(); + } + else + { + console.info("Safe call can't call null function"); + } + } + + // The seek buffer can keep up to 10 frames pre-decoded ready to go in either direction + // to support extra fast prev/next + _clean_hot(force) + { + + } + + _closest_frame_to_cursor() + { + + } + + /////////////////////////////////////////////////////////// + // Public interface mirrors that of a standard HTML5 video + /////////////////////////////////////////////////////////// + + set bias(bias) + { + this._bias = bias; + } + + set mute(val) + { + this._mute = val; + } + + get mute() + { + return this._mute; + } + // Set the current video time + // + // Timing considerations: + // - This will either grab from pre-decoded frames and run very quickly or + // jump to the nearest preceding keyframe and decode new frames (slightly slower) + set currentTime(video_time) + { + // If we are approximating seeking, we should land on the nearest buffered time + if (this.summaryLevel) + { + // Round to the nearest Nth second based on the summary level + const approx = Math.floor((video_time + this._bias)/ this.summaryLevel)*this.summaryLevel; + let lastDistance = 40000000; + for (let idx = 0; idx < this.buffered.length; idx++) + { + if (idx == 0 && approx < this.buffered.start(idx)) + { + this._current_cursor = this.buffered.start(idx); + break; + } + const fromBufStart = Math.abs(approx - this.buffered.start(idx)); + //console.info(`${idx}: APPX=${approx} ${fromBufStart} ${this.buffered.start(idx)} SL=${this.summaryLevel}`); + if (approx >= this.buffered.start(idx) && approx < this.buffered.end(idx)) + { + this._current_cursor = approx; + break; + } + else if (lastDistance < fromBufStart) + { + // If we went past it pick the last good start + this._current_cursor = this.buffered.start(idx-1); + break; + } + else + { + lastDistance = fromBufStart; + } + } + //console.info(`${this._name}: SUMMARIZING ${video_time+this._bias} to ${this._current_cursor} via ${this.summaryLevel}`); + } + else + { + // Keep worker and manager up to date. + this._current_cursor = video_time+this._bias; + } + // If we didn't set up oncanplay, ignore the cursor change. + if (this.oncanplay != undefined && this._mute == false) + { + this._video.currentTime = this._current_cursor; + } + } + + /// Return a list of TimeRange objects representing the downloaded/playable regions of the + /// video data. + get buffered() + { + return this._video.buffered; + } + + // Returns the current video cursor position + get currentTime() + { + return this._current_cursor; + } + + // Append data to the mp4 file + // - This data should either be sequentially added or added on a segment boundary + // - Prior to adding video segments the mp4 header must be supplied first. + appendBuffer(data, timestampOffset) + { + + } + + // Append data to the mp4 file (seek alt) + // - This data should either be sequentially added or added on a segment boundary + // - Prior to adding video segments the mp4 header must be supplied first. + appendSeekBuffer(data, time, timestampOffset) + { + + } + + deleteUpTo(seconds) + { + + } + + pause() + { + + } + + play() + { + + } + + set named_idx(val) + { + this._named_idx = val; + } + + get named_idx() + { + return this._named_idx; + } +} + +export class TatorSimpleVideo { + constructor(id, path) + { + this._named_idx = id; + console.info("Created Simple Video Decoder"); + this._buffer = new SimpleVideoWrapper(this, `Simple Video Buffer ${id}`, path) + this._buffer.init(); + this._init = false; + this._compat = true; // Set to tell higher level code this is the simple player. + } + + hls(playlistUrl) { + this._hls = new Hls(); + + return new Promise((resolve) => { + this._hls.on(Hls.Events.MANIFEST_LOADING, () => { + console.info(`Parsed ${playlistUrl}`); + resolve(); + }); + this._hls.on(Hls.Events.MEDIA_ATTACHED, () => { + this._hls.loadSource(playlistUrl); + }); + this._hls.attachMedia(this._buffer._video); + }); + } + + getMediaElementCount() { + // 1 for seek video, 1 for onDemand video, 1 for all of the scrub + return 1; + } + + // Save off the initialization data for this mp4 file + saveBufferInitData(data) { + + } + + clearScrubBuffer() { + + } + + recreateOnDemandBuffers(callback) { + + } + + status() + { + + } + + pause(time) + { + + } + + play() + { + + } + + /** + * Return the source buffer associated with the given frame / buffer type. Or null if not present + * + * @param {float} time - Seconds timestamp of frame request + * @param {string} buffer - "play" | "scrub" + * @param {Direction} direction - Forward or backward class + * @param {float} maxTime - Maximum number of seconds in the video + * @returns Video element based on the provided time. Returns null if the given time does not + * match any of the video buffers. + */ + forTime(time, buffer, direction, maxTime) + { + for (let idx = 0; idx < this._buffer.buffered.length; idx++) + { + if (time >= this._buffer.buffered.start(idx) && time < this._buffer.buffered.end(idx)) + { + return this._buffer; + } + } + return null; + } + + // Returns the seek buffer if it is present, or + // The time buffer if in there + returnSeekIfPresent(time, direction) + { + return this.forTime(time, "seek", direction); + } + + playBuffer() + { + return this._buffer; + } + + /** + * Queues the requests to delete buffered onDemand video ranges + */ + resetOnDemandBuffer() + { + let p_func = (resolve, reject) => + { + this.reset().then(() => { + resolve(); + }); + }; + let p = new Promise(p_func); + return p; + } + + /** + * @returns {boolean} True if the onDemand buffer has no data + */ + isOnDemandBufferCleared() + { + + } + + /** + * @returns {boolean} True if the onDemand buffer is busy + */ + isOnDemandBufferBusy() + { + return false; + } + + /** + * If there are any pending deletes for the onDemand buffer, this will rotate through + * them and delete them + */ + cleanOnDemandBuffer() + { + + } + + /** + * Removes the given range from the play buffer + * @param {tuple} delete_range - start/end (seconds) + */ + deletePendingOnDemand(delete_range) + { + this._buffer.deleteUpTo(delete_range[1]); + } + + seekBuffer() + { + return this._buffer; + } + + currentIdx() + { + + } + + set named_idx(val) + { + this._named_idx = val; + this._buffer.named_idx = val; + } + + get named_idx() + { + return this._named_idx; + } + + error() + { + + } + + /** + * Used for initialization of the video object. + * @returns Promise that is resolved when the first video element is in the ready state or + * data has been loaded. This promise is rejected if an error occurs + * with the video element. + */ + loadedDataPromise(parent) + { + let p = new Promise((resolve, reject) => { + resolve(); + }); + return p; + } + + /** + * If there are any pending deletes for the seek buffer, this will rotate through them + * and delete them + */ + cleanSeekBuffer() + { + + } + + reset() + { + + } + + appendSeekBuffer(data, time, timestampOffset) + { + + } + + appendLatestBuffer(data, callback, timestampOffset) + { + + } + + /** + * Appends the video data to the onDemand buffer. + * After the buffer has been updated, the callback routine will be called. + * + * @param {bytes} data - Video segment + * @param {function} callback - Callback executed once the buffer has been updated + * @param {bool} force - Force update if true. False will yield updates only if init'd + */ + appendOnDemandBuffer(data, callback, force, timestampOffset) + { + //console.info(`${JSON.stringify(data)}`); + // Fail-safe, if we have a frame start this is the start of a new buffer + // and we need to clear everything we had. + + } + + /** + * Appends data to all buffers (generally init information) + * @param {*} data + * @param {*} callback + * @param {*} force + */ + appendAllBuffers(data, callback, force, timestampOffset) + { + + } +} diff --git a/ui/src/js/annotator/video.js b/ui/src/js/annotator/video.js index 9fd974672..239a79531 100644 --- a/ui/src/js/annotator/video.js +++ b/ui/src/js/annotator/video.js @@ -4,11 +4,11 @@ import { TatorVideoDecoder} from "./video-codec.js"; import { fetchRetry } from "../util/fetch-retry.js"; import { getCookie } from "../util/get-cookie.js"; import { PeriodicTaskProfiler } from "./periodic_task_profiler"; -import { VideoBufferDemux } from "./video_buffer_demux"; import { MotionComp } from "./motion_comp"; import { ConcatDownloadManager } from "./concat_download_manager.js"; import { color } from "./drawGL_colors.js"; import { EffectManager } from "./video-effects.js"; +import { TatorSimpleVideo } from "./video-simple.js"; // Video export class handles interactions between HTML presentation layer and the // javascript application. @@ -64,9 +64,33 @@ import { EffectManager } from "./video-effects.js"; /// performance in the teens. On an XPS 15 (2008 era), Chrome performed /// in the teens. +// Events of interest to watch for with addEventListener(): +// +// - videoError : Emitted if there is an issue with system configuration +// detail : {videoDecoderPresent: , +// forceCompat: , +// secureContext: } +// - codecNotSupported : emitted if the given codec will not play (FATAL) +// detail: {codec: } +// - frameChange : emitted when a frame changes +// detail : {frame: } +// - playbackReady : emitted when playback is ready to go +// - bufferLoaded +// detail : {percent_complete: <0 to 1, representing percentage of scrub buffer loaded>} +// - videoLengthChanged +// detail : {length: } +// - onDemandDetail +// detail : {ranges: } +// - playbackEnded : emitted if the video reaches the end of the video +// - temporarilyMaskEdits +// detail : {enabled: } +// - rateChange : emitted when the video playback rate changes +// detail : {rate: } +// - playbackStalled : emitted when playback stalls + // Constrain the video display FPS to not allow dropped frames during playback // -export var guiFPS=30; +export var guiFPS = 30; var Direction = { BACKWARDS:-1, STOPPED: 0, FORWARD: 1}; var State = {PLAYING: 0, IDLE: 1, LOADING: -1}; @@ -82,6 +106,8 @@ export class VideoCanvas extends AnnotationCanvas { // Set global variable to find us window.tator_video = this; + this._localMode = false; + this._forceCompat = false; this._ready = false; this._diagnosticMode = false; this._videoVersion = 1; @@ -113,6 +139,21 @@ export class VideoCanvas extends AnnotationCanvas { { this._summaryLevel = Number(searchParams.get("summaryLevel")); } + if (searchParams.has("localMode")) + { + this._localMode = Number(searchParams.get("localMode")); + } + if (searchParams.has("forceCompat")) + { + this._forceCompat = Number(searchParams.get("forceCompat")); + } + + this._networkSeekTimeout = 5000; + if (searchParams.has("seekTimeout")) + { + this._networkSeekTimeout = Number(searchParams.get("seekTimeout")); + } + this._lastDirection=Direction.FORWARD; this._direction=Direction.STOPPED; this._fpsDiag=0; @@ -311,7 +352,7 @@ export class VideoCanvas extends AnnotationCanvas { })); } - startDownload(streaming_files, offsite_config) + startDownload(streaming_files, offsite_config, info_only) { if (this._children) { @@ -327,7 +368,8 @@ export class VideoCanvas extends AnnotationCanvas { "hq_idx": this._seek_idx, "scrub_idx": this._scrub_idx, "offsite_config": offsite_config, - "frameJump": frameJump}); + "frameJump": frameJump, + "infoOnly": info_only}); } else if (streaming_files[0].hls) { @@ -367,7 +409,8 @@ export class VideoCanvas extends AnnotationCanvas { "hq_idx": this._seek_idx, "scrub_idx": this._scrub_idx, "offsite_config": offsite_config, - "frameJump": frameJump}); + "frameJump": frameJump, + "infoOnly": info_only}); } } @@ -429,10 +472,12 @@ export class VideoCanvas extends AnnotationCanvas { { let new_play_idx = this.find_closest(this._videoObject, quality); - if (buffer == undefined) { - this._play_idx = new_play_idx; - } - else if (buffer == "play") { + if (buffer == undefined || buffer == "play") { + // If we are switching buffer clean up memory of old quality + if (new_play_idx != this._play_idx) + { + this._videoElement[this._play_idx].reset(); + } this._play_idx = new_play_idx; } else if (buffer == "seek") { @@ -442,10 +487,18 @@ export class VideoCanvas extends AnnotationCanvas { if (new_play_idx != this._scrub_idx) { this.stopDownload(); this._videoElement[this._scrub_idx].clearScrubBuffer(); - this.startDownload(this._videoObject.media_files["streaming"], this._offsiteConfig); + this.startDownload(this._videoObject.media_files["streaming"], this._offsiteConfig, this._videoElement[0]._compat == true); } this._scrub_idx = new_play_idx; } + + // If we are going to scrub buffer, remove old on-demand buffer indication + if (this._play_idx == this._scrub_idx) + { + this.dispatchEvent(new CustomEvent("onDemandDetail", + {composed: true, + detail: {"ranges": []}})); + } console.log("Setting 1x-4x playback quality to: " + quality); // This try/catch exists only because setQuality is sometimes called and @@ -473,20 +526,28 @@ export class VideoCanvas extends AnnotationCanvas { let use_hls = (this._videoObject.media_files.streaming[0].hls ? true : false); let searchParams = new URLSearchParams(window.location.search); console.info(`VideoDecoder: ${'VideoDecoder' in window}; Secure Context: ${window.isSecureContext}`); - if ('VideoDecoder' in window == false || Number(searchParams.get('force_mse'))==1 || use_hls == true) + if ('VideoDecoder' in window == false || this._forceCompat == 1 || use_hls == true) { - // TODO: Can possibly make this a warning and fall back to compat mode. - // with some caveats on performance. - let decoder = 'VideoDecoder' in window; - if (Number(searchParams.get('force_mse'))==1) + this.dispatchEvent(new CustomEvent("videoError", + {detail: {"videoDecoderPresent":'VideoDecoder' in window == true, + "forceCompat": this._forceCompat, + "secureContext": window.isSecureContext} + })); + let path = this._videoObject.media_files.streaming[idx].path; + if (use_hls) { - decoder = false; + path = null; } - this.dispatchEvent(new CustomEvent("videoError", - {composed: true, - detail: {"videoDecoderPresent": decoder, - "secureContext": window.isSecureContext}})); - return new VideoBufferDemux(); // TODO, per above, Turn this into simple demuxer + let v = new TatorSimpleVideo(idx, path); + if (idx == this._scrub_idx) + { + this.dispatchEvent(new CustomEvent("bufferLoaded", + {composed: true, + detail: {"percent_complete":100.0} + })); + } + //window.alert(`VideoDecoder: ${'VideoDecoder' in window}; Secure Context: ${window.isSecureContext}`); + return v; } else { @@ -506,29 +567,30 @@ export class VideoCanvas extends AnnotationCanvas { p.onReady = null; } } - if (idx == this._play_idx) - { - p.onBuffered = () => { - if (idx == this._scrub_idx) - { - return; - } - const ranges = p.playBuffer().buffered; - let ranges_list = []; - for (let idx = 0; idx < ranges.length; idx++) + p.onBuffered = () => { + if (idx == this._scrub_idx) + { + return; + } + if (idx != this._play_idx) + { + return; + } + const ranges = p.playBuffer().buffered; + let ranges_list = []; + for (let idx = 0; idx < ranges.length; idx++) + { + let startFrame = this.timeToFrame(ranges.start(idx), null, idx); + let endFrame = this.timeToFrame(ranges.end(idx), null, idx); + if (this.currentFrame() >= startFrame && this.currentFrame() <= endFrame) { - let startFrame = this.timeToFrame(ranges.start(idx), null, idx); - let endFrame = this.timeToFrame(ranges.end(idx), null, idx); - if (this.currentFrame() >= startFrame && this.currentFrame() <= endFrame) - { - ranges_list.push([startFrame, endFrame]); - } + ranges_list.push([startFrame, endFrame]); } - this.dispatchEvent(new CustomEvent("onDemandDetail", - {composed: true, - detail: {"ranges": ranges_list}})); - }; - } + } + this.dispatchEvent(new CustomEvent("onDemandDetail", + {composed: true, + detail: {"ranges": ranges_list}})); + }; return p; } } @@ -760,7 +822,7 @@ export class VideoCanvas extends AnnotationCanvas { {composed: true, detail: {media: this._children[0]}})); // Clear the buffer in case this is a hot-swap - this.startDownload(streaming_files, offsite_config); + this.startDownload(streaming_files, offsite_config, false); this._draw.clear(); this._draw.resizeViewport(dims[0], dims[1]); this._fps=Math.round(1000*this._children[0].fps)/1000; @@ -817,16 +879,88 @@ export class VideoCanvas extends AnnotationCanvas { fps = videoObject.fps; numFrames = videoObject.num_frames; + this._videoElement = []; + let streaming_files = this._videoObject.media_files.streaming; + + if (this._localMode == 1) + { + dims = this.identify_qualities(videoObject, quality, scrubQuality, seekQuality, offsite_config); + this._draw.resizeViewport(dims[0], dims[1]); + this._fps=Math.round(1000*fps)/1000; + this._numFrames=numFrames-1; + this._numSeconds=fps*numFrames; + this._dims=dims; + this.resetRoi(); + + // For local we only use buffer 0 + this._scrub_idx = 0; + this._play_idx = 0; + this._seek_idx = 0; + this.startDownload([streaming_files[0]], offsite_config, true); + + let element = this.construct_demuxer(0, dims[1]); + element.named_idx = 0; + this._videoElement.push(element); + + document.onclick = () => { + document.onclick = null; + window.showOpenFilePicker().then((fileHandle) => { + fileHandle[0].getFile().then(file => { + console.info("Got file, making array buffer."); + file.arrayBuffer().then(buffer => { + const length = buffer.byteLength; + const chunk_size = 1*1024*1024; + console.info(`Appending ${length} to video buffer`); + + // Send it once to grease the wheels + let bufferToSend=buffer.slice(0,chunk_size); + bufferToSend.fileStart = 0; + this._videoElement[0].appendLatestBuffer(bufferToSend, ()=>{}, 0); + + // Send the file for real + for (let idx = 0; idx < length; idx+=chunk_size) + { + let bufferToSend=buffer.slice(idx,idx+chunk_size); + bufferToSend.fileStart = idx; + console.info(`${bufferToSend.byteLength} @ ${bufferToSend.fileStart}`); + this._videoElement[0].appendLatestBuffer(bufferToSend, ()=>{}, 0); + this._videoElement[0]._init = true; + } + + this.dispatchEvent(new CustomEvent("bufferLoaded", + {composed: true, + detail: {"percent_complete":1.00} + })); + this.dispatchEvent(new CustomEvent("playbackReady", + {composed: true, + detail: {playbackReadyId: 1}, + })); + }); + }) + })}; + return this._videoElement[this._scrub_idx].loadedDataPromise(this); + } + // Use the largest resolution to set the viewport dims = this.identify_qualities(videoObject, quality, scrubQuality, seekQuality, offsite_config); - this._videoElement = []; - let streaming_files = this._videoObject.media_files.streaming; for (let idx = 0; idx < streaming_files.length; idx++) { this._videoElement.push(this.construct_demuxer(idx, streaming_files[idx].resolution[0])); this._videoElement[idx].named_idx = idx; } + + if (this._videoElement[0]._compat) + { + this.dispatchEvent(new CustomEvent("maxPlaybackRate", { + detail: {rate: 4}, + composed: true + })); + + // Increase GPU buffer to ensure smoother playback + this._draw.jumboBufferMode(); + } + // Clear the buffer in case this is a hot-swap this._draw.clear(); @@ -843,7 +977,7 @@ export class VideoCanvas extends AnnotationCanvas { this.stopDownload(); var promise = this._videoElement[this._scrub_idx].loadedDataPromise(this); - this.startDownload(streaming_files, offsite_config); + this.startDownload(streaming_files, offsite_config, this._videoElement[0]._compat == true); if (fps < 20) { console.info("Disable safe mode for low FPS"); @@ -909,7 +1043,7 @@ export class VideoCanvas extends AnnotationCanvas { // Update the canvas (immediate) with the source material, centered on // the view screen (resets GPU-bound frame buffer) // holds the buffer - drawFrame(frameIdx, source, width, height) + drawFrame(frameIdx, source, width, height, skipOffscreen) { // Need to draw the image to the viewable size of the canvas // .width is actually the rendering width which may be different @@ -942,10 +1076,13 @@ export class VideoCanvas extends AnnotationCanvas { this._dirty=false; this.displayLatest(true); - this.updateOffscreenBuffer(frameIdx, - source, - width, - height); + if (skipOffscreen != true) + { + this.updateOffscreenBuffer(frameIdx, + source, + width, + height); + } } /** @@ -971,8 +1108,6 @@ export class VideoCanvas extends AnnotationCanvas { composed: true })); - this.updateVideoDiagnosticOverlay(null, this._dispFrame); - let ended = false; if (this._direction == Direction.FORWARD && this._dispFrame >= (this._numFrames - 1)) @@ -1115,13 +1250,22 @@ export class VideoCanvas extends AnnotationCanvas { frameToComps(frame, buf_idx) { const time = ((1/this._fps)*frame)+(1/(this._fps*4)); - const bias = this._dlWorker.biasForTime(time, buf_idx); + let bias = 0.0; + if (this._dlWorker) + { + bias = this._dlWorker.biasForTime(time, buf_idx); + } return {'time': time, 'bias': bias}; } timeToFrame(time, bias, buf_idx) { - let video_time = time - this._dlWorker.biasForTime(time, buf_idx); + let vid_bias = 0.0; + if (this._dlWorker) + { + vid_bias = this._dlWorker.biasForTime(time, buf_idx); + } + let video_time = time - vid_bias; if (bias) { video_time -= (1/(this._fps*4)); @@ -1321,10 +1465,10 @@ export class VideoCanvas extends AnnotationCanvas { console.warn("Network Seek expired"); that.refresh(false); reject(); - }, 5000); + }, that._networkSeekTimeout); } - if (downloadSeekFrame) + if (downloadSeekFrame && that._dlWorker) { that._dlWorker.postMessage( {"type": "seek", @@ -1408,10 +1552,15 @@ export class VideoCanvas extends AnnotationCanvas { this._calculateAudioEligibility(); // If we are playing trim the frame buffer to a quarter second to make the rate change // feel responsive. - this._motionComp.computePlaybackSchedule(this._fps,this._playbackRate); + let effectiveRate = this._playbackRate; + if (this._videoElement[this._play_idx].playBuffer().keyframeOnly == true) + { + effectiveRate = 1; + } + this._motionComp.computePlaybackSchedule(this._fps,effectiveRate); const oldLoad = this._loadFrame; this._loadFrame = this._draw.trimBuffer(Math.round(this._fps*0.5)); - console.info(`Load: ${oldLoad} to ${this._loadFrame}, dispFrame = ${this._dispFrame}`); + //console.info(`Load: ${oldLoad} to ${this._loadFrame}, dispFrame = ${this._dispFrame}`); if (this._frameCallbackActive == false) { clearTimeout(this._loaderTimeout); @@ -1540,9 +1689,6 @@ export class VideoCanvas extends AnnotationCanvas { this._networkUpdate = 0; this._audioCheck = 0; - this._motionComp.computePlaybackSchedule(this._fps,this._playbackRate); - - this._lastTime = performance.now(); this._animationIdx = 0; @@ -1557,6 +1703,13 @@ export class VideoCanvas extends AnnotationCanvas { if (this._videoElement[this._scrub_idx].playBuffer().use_codec_buffer && this._videoElement[this._scrub_idx]._compat != true && direction == Direction.FORWARD) { + let effectiveRate = this._playbackRate; + if (this._videoElement[this._scrub_idx].playBuffer().keyframeOnly == true) + { + effectiveRate = 1; + } + + this._motionComp.computePlaybackSchedule(this._fps,effectiveRate); // Cap effective decode rate around 240 fps // This was emperically gathered as a good cut off for 5 15fps playing back at 16x // Can fine tune this more if required @@ -1571,12 +1724,37 @@ export class VideoCanvas extends AnnotationCanvas { if (this._videoElement[this._scrub_idx].playBuffer().use_codec_buffer) { this._videoElement[this._scrub_idx].playBuffer().clearPending(); + this._videoElement[this._scrub_idx].playBuffer()._clean_hot(true); if (this._fps * this._playbackRate >= 16*15) { this._videoElement[this._scrub_idx].playBuffer().keyframeOnly = true; } } - this._loaderTimeout=setTimeout(()=>{this.loaderThread(true, "scrub-only");}, 0); + + let effectiveRate = this._playbackRate; + if (this._videoElement[this._scrub_idx].playBuffer().keyframeOnly == true) + { + effectiveRate = 1; + } + + // If we are in multi and going backwards cap rewind at 10hz + if (this._direction == Direction.BACKWARDS && this._renderer) + { + this._motionComp.computePlaybackSchedule(Math.min(10,this._fps),effectiveRate); + } + else + { + this._motionComp.computePlaybackSchedule(this._fps,effectiveRate); + } + + if (this._videoElement[this._scrub_idx]._compat == true) + { + this._loaderTimeout=setTimeout(()=>{this.loaderThread(true, "scrub");}, 0); + } + else + { + this._loaderTimeout=setTimeout(()=>{this.loaderThread(true, "scrub-only");}, 0); + } } this._sentPlaybackReady = false; // Kick off the loader @@ -1656,7 +1834,12 @@ export class VideoCanvas extends AnnotationCanvas { this._lastTime = null; this._animationIdx = 0; - this._motionComp.computePlaybackSchedule(this._fps,this._playbackRate); + let effectiveRate = this._playbackRate; + if (this._videoElement[this._play_idx].playBuffer().keyframeOnly == true) + { + effectiveRate = 1; + } + this._motionComp.computePlaybackSchedule(this._fps,effectiveRate); // Kick off the onDemand thread immediately this._onDemandDownloadTimeout = setTimeout(() => {this.onDemandDownload();}, 0); @@ -1666,6 +1849,11 @@ export class VideoCanvas extends AnnotationCanvas { // as the argument playerThread(domtime) { + if (this._playEffect == true) + { + this._effectManager.clear(); + document.body.style.cursor = null; + } /// This is the notional scheduled diagnostic interval var schedDiagInterval=5000.0; //console.info(`PLAYER @ ${performance.now()}`); @@ -1727,7 +1915,8 @@ export class VideoCanvas extends AnnotationCanvas { } else { - console.warn("Player Stalled."); + console.warn(`Player Stalled. BD=${this._draw.bufferDepth}`); + this._stallCount += 1; // Done playing, clear playback. if (this._audioEligible && this._audioPlayer.paused) { @@ -1862,6 +2051,8 @@ export class VideoCanvas extends AnnotationCanvas { let video = this._videoElement[index].playBuffer(); let frameProfiler = new PeriodicTaskProfiler("Frame Fetch"); + video.frameIncrement = frameIncrement; + // Clear any old frames this._pendingFrames = []; clearTimeout(this._pendingTimeout); @@ -1869,7 +2060,6 @@ export class VideoCanvas extends AnnotationCanvas { // on frame processing logic - let increment_clk = 0; let lastRate = this._playbackRate; video.onFrame = (frame, timescale, timestampOffset) => { this._playing = true; @@ -1877,11 +2067,8 @@ export class VideoCanvas extends AnnotationCanvas { frame.frameNumber = this.timeToFrame((frame.timestamp/timescale), null, video.named_idx); this._loadFrame = frame.frameNumber; this._fpsLoadDiag++; - if (increment_clk % frameIncrement != 0) - { - frame.returnFrame(); - } - else if (this._draw.canLoad() > 0 && this._pendingFrames.length == 0) + + if (this._draw.canLoad() > 0 && this._pendingFrames.length == 0) { this.pushFrame(frame.frameNumber, frame, frame.displayWidth, frame.displayHeight); frame.returnFrame(); @@ -1900,14 +2087,13 @@ export class VideoCanvas extends AnnotationCanvas { { frameIncrement = this._playbackRate / 16; } + video.frameIncrement = frameIncrement; lastRate = this._playbackRate; } - // Don't let increment clock blow up - increment_clk = (increment_clk + 1) % frameIncrement; frameProfiler.push(performance.now()-start) - // Kick off the player thread once we have 25 frames loaded + // Kick off the player thread once we have some frames loaded if (this._playerTimeout == null && this._draw.canPlay() > (this._draw.bufferDepth*0.75)) { this._playerTimeout = setTimeout(()=>{this.playerThread();}, 250); @@ -1927,6 +2113,12 @@ export class VideoCanvas extends AnnotationCanvas { } this._loaderBuffer = bufferName; let loader = () => {this.loaderThread(false, bufferName)}; + let readyThreshold = this._draw.bufferDepth*0.75; + if (this._stallCount > 0) + { + readyThreshold = 4; + } + // Loader thread that seeks to the current frame and continually kicks off seeking // to the next frame. // @@ -1935,7 +2127,7 @@ export class VideoCanvas extends AnnotationCanvas { { //console.info("Loader Full"); this._loaderTimeout = setTimeout(loader, 0); - if (this._playerTimeout == null && this._draw.canPlay() > (this._draw.bufferDepth*0.75)) + if (this._playerTimeout == null && this._draw.canPlay() > readyThreshold) { this._playerTimeout = setTimeout(()=>{this.playerThread();}, 250); } @@ -1985,7 +2177,7 @@ export class VideoCanvas extends AnnotationCanvas { this.seekFrame(this._loadFrame, pushAndGoToNextFrame, false, bufferName); - if (this._playerTimeout == null && this._draw.canPlay() > (this._draw.bufferDepth*0.75)) + if (this._playerTimeout == null && this._draw.canPlay() > readyThreshold) { this._playerTimeout = setTimeout(()=>{this.playerThread();}, 250); } @@ -2024,7 +2216,7 @@ export class VideoCanvas extends AnnotationCanvas { } let appendThreshold = this._calculateReadyThreshold(); let video = this.videoBuffer(frame, "play", true); - if (video == null) + if (video == null || this._videoElement[this._play_idx]._compat == true) { return false; } @@ -2080,7 +2272,12 @@ export class VideoCanvas extends AnnotationCanvas { // Returns true if on-demand buffer check + delay is required based on current settings. bufferDelayRequired() { - return (this._playbackRate <= RATE_CUTOFF_FOR_ON_DEMAND && this._play_idx != this._scrub_idx); + return (this._playbackRate <= RATE_CUTOFF_FOR_ON_DEMAND && this._play_idx != this._scrub_idx && this._videoElement[0]._compat != true); + } + + get length() + { + return this._numFrames; } onDemandDownloadPrefetch(reqFrame) @@ -2215,6 +2412,7 @@ export class VideoCanvas extends AnnotationCanvas { }) */; } + onDemandDownload(inhibited) { if (this._disableAutoDownloads) { @@ -2640,6 +2838,10 @@ export class VideoCanvas extends AnnotationCanvas { play() { + this._effectManager.grayOut(1000); + this._playEffect = true; + document.body.style.cursor = "progress"; + this._stallCount = 0; if (this._dispFrame >= (this._numFrames)) { return false; @@ -2653,7 +2855,7 @@ export class VideoCanvas extends AnnotationCanvas { } else { - if (this._play_idx == this._scrub_idx && this.videoBuffer(this.currentFrame(), "scrub") != null) + if (this._play_idx == this._scrub_idx && this.videoBuffer(this.currentFrame(), "scrub") != null || this._videoElement[this._play_idx]._compat == true) { this._playGenericScrub(Direction.FORWARD); } @@ -2771,11 +2973,23 @@ export class VideoCanvas extends AnnotationCanvas { // If we weren't already paused send the event if (currentDirection != Direction.STOPPED) { - this._pauseCb.forEach(cb => {cb();}); - this._direction=Direction.STOPPED; + this._pauseCb.forEach(cb => {cb();}); this._videoElement[this._play_idx].pause(this.frameToTime(this._dispFrame, this._play_idx)); this._videoElement[this._scrub_idx].pause(this.frameToTime(this._dispFrame, this._scrub_idx)); + this.updateVideoDiagnosticOverlay(null, this._dispFrame); + + // Reclaim memory from any pending frames + let pendingFrame = null; + if (this._pendingFrames) + { + pendingFrame = this._pendingFrames.shift(); + } + while (pendingFrame) + { + pendingFrame.returnFrame(); + pendingFrame = this._pendingFrames.shift(); + } // force a redraw at the currently displayed frame var finalPromise = new Promise((resolve, reject) => { diff --git a/ui/src/js/annotator/video_buffer_demux.js b/ui/src/js/annotator/video_buffer_demux.js deleted file mode 100644 index a27b1e50c..000000000 --- a/ui/src/js/annotator/video_buffer_demux.js +++ /dev/null @@ -1,792 +0,0 @@ -import Hls from "hls.js"; - -/// Support multiple off-screen videos at varying resolutions -/// the intention is this export class is used to store raw video -/// frames as they are downloaded. - -export class VideoBufferDemux { - constructor() { - this._bufferSize = 140 * 1024 * 1024; // 140Mb - this._numBuffers = 1; - - this._vidBuffers = []; - this._inUse = []; - this._full = []; - this._mediaSources = []; - this._sourceBuffers = []; - this._compat = false; - this._activeBuffers = 0; - - // Video, source, and buffer for seek track - this._seekVideo = document.createElement("VIDEO"); - this._seekVideo.setAttribute("crossorigin", "anonymous"); - console.log("MediaSource element created: VIDEO (seek)"); - this._seekReady = false; - this._pendingSeeks = []; - this._pendingSeekDeletes = []; - - this._mime_str = 'video/mp4; codecs="avc1.64001e"'; - - for (var idx = 0; idx < this._numBuffers; idx++) { - this._vidBuffers.push(document.createElement("VIDEO")); - this._vidBuffers[idx].setAttribute("crossorigin", "anonymous"); - console.log("MediaSource element created: VIDEO (scrub)"); - this._inUse.push(0); - this._sourceBuffers.push(null); - this._full.push(false); - } - - // Create another video buffer specifically used for onDemand playback - this._onDemandBufferIndex = this._numBuffers; - this._pendingOnDemandDeletes = []; - this.recreateOnDemandBuffers(() => { return; }); - - this._needNewScrubBuffer = true; - this._init = false; - this._dataLag = []; - let init_buffers = () => { - console.info("Init buffers"); - - // Initialize the seek buffer - this._seekBuffer = null; - this._seekSource = new MediaSource(); - this._seekVideo.src = URL.createObjectURL(this._seekSource); - this._seekSource.onsourceopen = () => { - this._seekSource.onsourceopen = null; - this._seekBuffer = this._seekSource.addSourceBuffer(this._mime_str); - if (this._pendingSeeks.length > 0) { - console.info("Applying pending seek data."); - var pending = this._pendingSeeks.shift(); - this.appendSeekBuffer(pending.data, pending.time); - } - }; - - // Initialize the playback buffers - let that = this; - var makeSourceBuffer = function (idx, event) { - var args = this; - var ms = args["ms"]; - var idx = args["idx"]; - ms.onsourceopen = null; - - // Need to add a source buffer for the video. - that._sourceBuffers[idx] = ms.addSourceBuffer(that._mime_str); - - // Reached the onDemand buffer, rest of the function isn't associated with it - if (idx == that._numBuffers) { - if (that._initData) { - that.appendOnDemandBuffer(that._initData, () => { }, true); - } - return; - } - - for (let idx = 0; idx < that._numBuffers; idx++) { - if (that._sourceBuffers[idx] == null) - return; - } - - if (that._initData) { - let handleDataLag = () => { - if (that._pendingSeeks.length > 0) { - var pending = that._pendingSeeks.shift(); - that.appendSeekBuffer(pending.data, pending.time); - } - let lag = that._dataLag.shift(); - if (lag) { - if (lag.callback && that._dataLag.length == 0) { - setTimeout(() => { that.appendLatestBuffer(lag.data, lag.callback, "handlingDataLog"); }, 0); - } - else { - setTimeout(() => { that.appendLatestBuffer(lag.data, handleDataLag, "handlingDataLog"); }, 0); - } - } - - else { - that._initData = undefined; - } - }; - that.appendAllBuffers(that._initData, () => { that._init = true; handleDataLag(); }, true); - } - - else { - that._init = true; - } - }; - - // This links the source element buffers with a paired video element and also - // a media source - for (var idx = 0; idx < this._numBuffers; idx++) { - var ms = new MediaSource(); - this._mediaSources[idx] = ms; - this._vidBuffers[idx].src = URL.createObjectURL(this._mediaSources[idx]); - ms.onsourceopen = makeSourceBuffer.bind({ "idx": idx, "ms": ms }); - } - }; - if (document.hidden == true) { - document.addEventListener("visibilitychange", () => { - if (document.hidden == false && this._init == false) { - init_buffers(); - } - }); - } - - else { - init_buffers(); - } - } - - getMediaElementCount() { - // 1 for seek video, 1 for onDemand video, numBuffers for scrub video - return this._numBuffers + 2; - } - - saveBufferInitData(data) { - this._ftypInfo = data; - } - - clearScrubBuffer() { - - if (this._ftypInfo == null) { - return; - } - - for (let idx = 0; idx < this._numBuffers; idx++) { - - this._vidBuffers[idx].pause(); - this._vidBuffers[idx].removeAttribute('src'); - this._vidBuffers[idx].load(); - - delete this._mediaSources[idx]; - delete this._sourceBuffers[idx]; - } - - this._numBuffers = 0; - this.appendNewScrubBuffer(() => { }, true); - } - - appendNewScrubBuffer(callback, skipInit) { - this._numBuffers += 1; - var idx = this._numBuffers - 1; - - this._vidBuffers.push(document.createElement("VIDEO")); - console.log("MediaSource element created: VIDEO (scrub)"); - this._vidBuffers[idx].setAttribute("crossorigin", "anonymous"); - this._inUse.push(0); - this._sourceBuffers.push(null); - this._full.push(false); - - var ms = new MediaSource(); - this._mediaSources[idx] = ms; - this._vidBuffers[idx].src = URL.createObjectURL(this._mediaSources[idx]); - ms.onsourceopen = () => { - ms.onsourceopen = null; - this._sourceBuffers[idx] = ms.addSourceBuffer(this._mime_str); - console.log("appendNewScrubBuffer - onsourceopen"); - if (skipInit != true) { - this._updateBuffers([idx], this._ftypInfo, callback); - }; - }; - } - - recreateOnDemandBuffers(callback) { - - if (this._onDemandVideo != null) { - this._onDemandVideo.pause(); - this._onDemandVideo.removeAttribute('src'); - this._onDemandVideo.load(); - } - - this._onDemandSource = new MediaSource(); - this._onDemandVideo = document.createElement("VIDEO"); - console.log("MediaSource element created: VIDEO (onDemand)"); - this._onDemandVideo.setAttribute("crossorigin", "anonymous"); - this._onDemandVideo.src = URL.createObjectURL(this._onDemandSource); - - this._onDemandSource.onsourceopen = () => { - if (this._onDemandSource.readyState == "open") { - this._onDemandSource.onsourceopen = null; - this._onDemandSourceBuffer = this._onDemandSource.addSourceBuffer(this._mime_str); - console.log("recreateOnDemandBuffers - onsourceopen"); - callback(); - } - }; - } - - status() { - console.info("Buffer Status"); - console.info(`Active Buffer Count = ${this._activeBuffers}`); - var bufferSizeMb = this._bufferSize / (1024 * 1024); - for (var idx = 0; idx < this._numBuffers; idx++) { - var mbInUse = this._inUse[idx] / (1024 * 1024); - console.info(`\t${idx} = ${mbInUse}/${bufferSizeMb} MB`); - if (this._vidBuffers[idx] == null) { - return; - } - var ranges = this._vidBuffers[idx].buffered; - if (ranges.length > 0) { - console.info("\tRanges:"); - for (var rIdx = 0; rIdx < ranges.length; rIdx++) { - console.info(`\t\t${rIdx}: ${ranges.start(rIdx)}:${ranges.end(rIdx)}`); - } - } - - else { - console.info("\tEmpty"); - } - - } - - console.info("Seek Buffer:"); - if (this._seekBuffer == null) { - return; - } - var ranges = this._seekBuffer.buffered; - if (ranges.length > 0) { - console.info("\tRanges:"); - for (var rIdx = 0; rIdx < ranges.length; rIdx++) { - console.info(`\t\t${rIdx}: ${ranges.start(rIdx)}:${ranges.end(rIdx)}`); - } - } - - else { - console.info("\tEmpty"); - } - } - - currentVideo() { - for (var idx = 0; idx < this._numBuffers; idx++) { - if (this._full[idx] != true) { - return this._vidBuffers[idx]; - } - } - return null; - } - - /** - * Return the source buffer associated with the given frame / buffer type. - * - * @param {float} time - Seconds timestamp of frame request - * @param {string} buffer - "play" | "scrub" - * @param {Direction} direction - Forward or backward class - * @param {float} maxTime - Maximum number of seconds in the video - * @returns Video element based on the provided time. Returns null if the given time does not - * match any of the video buffers. - */ - forTime(time, buffer, direction, maxTime) { - if (this._compat == true) { - return this._vidBuffers[0]; - } - - if (buffer == "play") { - const video = this.playBuffer(); - var ranges = video.buffered; - - // Note: The way it's setup right now, there should only be a continuous range - // But we'll keep the for loop for now. - for (var rangeIdx = 0; rangeIdx < ranges.length; rangeIdx++) { - var start = ranges.start(rangeIdx); - var end = ranges.end(rangeIdx); - - if (time >= start && time <= end) { - return video; - } - } - - /* - if (ranges.length > 0) - { - console.warn(`Playback buffer doesn't contain time (ranges/start/end/time) ${ranges.length} ${start} ${end} ${time}`); - } - */ - } - else if (buffer == "scrub") { - for (var idx = this._activeBuffers - 1; idx >= 0; idx--) { - var ranges = this._vidBuffers[idx].buffered; - for (var rangeIdx = 0; rangeIdx < ranges.length; rangeIdx++) { - var start = ranges.start(rangeIdx); - var end = ranges.end(rangeIdx); - if (time >= start && - time <= end) { - return this._vidBuffers[idx]; - } - } - } - } - - return null; - } - - // Returns the seek buffer if it is present, or - // The time buffer if in there - returnSeekIfPresent(time, direction) { - //let time_result= this.forTime(time, "scrub"); - //if (time_result) - //{ - // return time_result; - //} - for (let idx = 0; idx < this._seekVideo.buffered.length; idx++) { - // If the time is comfortably in the range don't bother getting - // additional data - let timeFromStart = time - this._seekVideo.buffered.start(idx); - let bufferedLength = (this._seekVideo.buffered.end(idx) - this._seekVideo.buffered.start(idx)) * 0.75; - if (timeFromStart <= bufferedLength && timeFromStart > 0) { - return this._seekVideo; - } - } - return null; - } - - playBuffer() { - return this._onDemandVideo; - } - - playSource() { - return this._onDemandSource; - } - - playSourceBuffer() { - return this._onDemandSourceBuffer; - } - - /** - * Queues the requests to delete buffered onDemand video ranges - */ - resetOnDemandBuffer() { - const video = this.playBuffer(); - this._pendingOnDemandDeletes = []; - for (var rangeIdx = 0; rangeIdx < video.buffered.length; rangeIdx++) { - let start = video.buffered.start(rangeIdx); - let end = video.buffered.end(rangeIdx); - this.deletePendingOnDemand([start, end]); - } - - let promise = new Promise((resolve, _) => { - let checkBuffer = () => { - if (!this.isOnDemandBufferCleared()) { - setTimeout(checkBuffer, 100); - } - else { - console.log(`resetOnDemandBuffer: length - ${video.buffered.length}`); - resolve(); - } - }; - - checkBuffer(); - }); - - return promise; - } - - /** - * @returns {boolean} True if the onDemand buffer has no data - */ - isOnDemandBufferCleared() { - return this.playBuffer().buffered.length == 0; - } - - /** - * @returns {boolean} True if the onDemand buffer is busy - */ - isOnDemandBufferBusy() { - return this.playSourceBuffer().updating; - } - - /** - * If there are any pending deletes for the onDemand buffer, this will rotate through - * them and delete them - */ - cleanOnDemandBuffer() { - if (this._pendingOnDemandDeletes.length > 0) { - var pending = this._pendingOnDemandDeletes.shift(); - this.deletePendingOnDemand(pending.delete_range); - } - } - - /** - * Removes the given range from the play buffer - * @param {tuple} delete_range - start/end (seconds) - */ - deletePendingOnDemand(delete_range) { - const buffer = this.playSourceBuffer(); - if (buffer.updating == false) { - buffer.onupdateend = () => { - buffer.onupdateend = null; - this.cleanOnDemandBuffer(); - }; - - buffer.remove(delete_range[0], delete_range[1]); - } - - else { - this._pendingOnDemandDeletes.push( - { "delete_range": delete_range }); - } - } - - seekBuffer() { - return this._seekVideo; - } - - currentIdx() { - for (var idx = 0; idx < this._numBuffers; idx++) { - if (this._full[idx] != true) { - return idx; - } - } - return null; - } - - error() { - var currentVid = this.currentVideo(); - if (currentVid) { - return currentVid.error; - } - - else { - return { code: 500, message: "All buffers full." }; - } - } - - /** - * Set to compatibility mode - */ - compat(videoUrl) { - this._vidBuffers[0].src = videoUrl; - this._vidBuffers[0].load(); - this._compat = true; - } - - hls(playlistUrl) { - this._hls = new Hls(); - - return new Promise((resolve) => { - this._hls.on(Hls.Events.MANIFEST_LOADING, () => { - console.info(`Parsed ${playlistUrl}`); - resolve(); - }); - this._hls.on(Hls.Events.MEDIA_ATTACHED, () => { - this._hls.loadSource(playlistUrl); - }); - this._hls.attachMedia(this._vidBuffers[0]); - - this._compat = true; - }); - } - - /** - * Pause each of the video elements - */ - pause() { - for (var idx = 0; idx < this._numBuffers; idx++) { - this._vidBuffers[idx].pause(); - } - this.playBuffer().pause(); - } - - /** - * Used for initialization of the video object. - * @returns Promise that is resolved when the first video element is in the ready state or - * data has been loaded. This promise is rejected if an error occurs - * with the video element. - */ - loadedDataPromise(video) { - var that = this; - var promise = new Promise( - function (resolve, reject) { - let loaded_data_callback = function () { - console.info("Called promise"); - // In version 2 buffers are immediately available - if (video._videoVersion >= 2) { - that._vidBuffers[0].onloadeddata = null; - resolve(); - } - - else { - // attempt to go to the frame that is requested to be loaded - console.log("Going to frame " + video._dispFrame); - video.gotoFrame(video._dispFrame).then(() => { - resolve(); - that._vidBuffers[0].onloadeddata = null; - }); - } - }; - that._vidBuffers[0].onloadeddata = loaded_data_callback; - that._vidBuffers[0].onerror = function () { - reject(); - that._vidBuffers[0].onerror = null; - }; - - if (that._vidBuffers[0].readyState == "open") { - resolve(); - } - }); - return promise; - } - - /** - * If there are any pending deletes for the seek buffer, this will rotate through them - * and delete them - */ - cleanSeekBuffer() { - if (this._pendingSeekDeletes.length > 0) { - var pending = this._pendingSeekDeletes.shift(); - this.deletePendingSeeks(pending.delete_range); - } - } - - /** - * Removes the given start/end time segment from the seek buffer - * @param {*} delete_range - */ - deletePendingSeeks(delete_range = undefined) { - // Add to the buffer directly else add to the pending - // seek to get it there next go around - if (this._seekReady) { - if (this._seekBuffer.updating == false) { - this._seekBuffer.onupdateend = () => { - - // Remove this handler - this._seekBuffer.onupdateend = null; - this.cleanSeekBuffer(); - }; - - if (delete_range) { - this._seekBuffer.remove(delete_range[0], delete_range[1]); - } - } - - else { - this._pendingSeekDeletes.push( - { 'delete_range': delete_range }); - } - } - } - appendSeekBuffer(data, time = undefined) { - // Add to the buffer directly else add to the pending - // seek to get it there next go around - if (this._seekReady) { - if (this._seekBuffer.updating == false) { - this._seekBuffer.onupdateend = () => { - - // Remove this handler - this._seekBuffer.onupdateend = null; - // Seek to the time requested now that it is loaded - if (time != undefined) { - this._seekVideo.currentTime = time; - } - }; - - // If this is a data request delete the stuff currently in the buffer - if (data != null) { - for (let idx = 0; idx < this._seekBuffer.buffered.length; idx++) { - let begin = this._seekBuffer.buffered.start(idx); - let end = this._seekBuffer.buffered.end(idx); - - // If the seek buffer has 3 seconds extra on either side - // of the request chop of 1 seconds on either side this - // means there is a maximum of ~4 second buffer in the - // hq seek buffer. - if (begin < time - 3) { - this._pendingSeekDeletes.push({ - "delete_range": [begin, - time - 1] - }); - } - if (end > time + 3) { - this._pendingSeekDeletes.push({ - "delete_range": [time + 1, - end] - }); - } - } - this._seekBuffer.appendBuffer(data); - } - } - - else { - this._pendingSeeks.push({ - 'data': data, - 'time': time - }); - } - - } - } - - appendLatestBuffer(data, callback) { - if (this._init == false) { - this._dataLag.push({ data: data, callback: null }); - setTimeout(callback, 100); - return; - } - - var latest = this.currentIdx(); - if (latest != null) { - var newSize = this._inUse[latest] + data.byteLength; - if (newSize > this._bufferSize) { - console.log(`${latest} is full, proceeding to next buffer`); - this._full[latest] = true; - this._needNewScrubBuffer = true; - this.appendLatestBuffer(data, callback); - } - - else { - // If we are 5% away from the end, start overlapping with a new buffer - // If this does not happen, we will get short segments of missing time. - if (newSize > (this._bufferSize * 0.95)) { - if (this._needNewScrubBuffer) { - this._needNewScrubBuffer = false; - this.appendNewScrubBuffer(() => { - this._updateBuffers([latest, latest + 1], data, callback); - }); - } - else { - this._updateBuffers([latest, latest + 1], data, callback); - } - } - - else { - this._updateBuffers([latest], data, callback); - } - } - } - - else { - console.error("No Buffers available!"); - } - - } - - /** - * Appends the video data to the onDemand buffer. - * After the buffer has been updated, the callback routine will be called. - * - * @param {bytes} data - Video segment - * @param {function} callback - Callback executed once the buffer has been updated - * @param {bool} force - Force update if true. False will yield updates only if init'd - */ - appendOnDemandBuffer(data, callback, force) { - if (this._init == false && force != true) { - console.info("Waiting for init... (onDemand)"); - return; - } - this._updateOnDemandBuffer(data, callback); - } - - _updateOnDemandBuffer(data, callback) { - - var that = this; - - // Callback wrapper function used to help keep track of how many buffers - // have been updated. - var semaphore = 1; - var wrapper = function () { - that.playSourceBuffer().onupdateend = null; - semaphore--; - if (semaphore == 0) { - callback(); - } - }; - - // Place the provided frame data into each of the buffers if it's safe to do so. - // Once the all the buffers have been updated, perform the callback - var error = this.playBuffer().error; - if (error) { - console.error("Error " + error.code + "; details: " + error.message); - updateStatus("Video Decode Error", "danger", -1); - throw `Video Decode Error: ${bufferType}`; - } - this.safeUpdate(this.playSourceBuffer(), data).then(wrapper); - } - - /** - * - * @param {array} buffersToUpdate - List of buffer indices to add data to - * @param {array} data - Array of video bytes to store - * @param {function} callback - Callback function - */ - _updateBuffers(buffersToUpdate, data, callback) { - var that = this; - this._activeBuffers = Math.max(...buffersToUpdate) + 1; - - // Callback wrapper function used to help keep track of how many buffers - // have been updated. - var semaphore = buffersToUpdate.length; - var wrapper = function () { - that._sourceBuffers[this].onupdateend = null; - semaphore--; - if (semaphore == 0) { - callback(); - } - }; - - // Place the provided frame data into each of the buffers if it's safe to do so. - // Once the all the buffers have been updated, perform the callback - for (var idx = 0; idx < buffersToUpdate.length; idx++) { - var bIdx = buffersToUpdate[idx]; - var error = this._vidBuffers[bIdx].error; - if (error) { - console.error("Error " + error.code + "; details: " + error.message); - updateStatus("Video Decode Error", "danger", -1); - throw `Video Decode Error: ${bufferType}`; - } - this.safeUpdate(this._sourceBuffers[bIdx], data).then(wrapper.bind(bIdx)); - this._inUse[bIdx] += data.byteLength; - } - } - - appendAllBuffers(data, callback, force) { - if (force == undefined) { - force = false; - } - if (this._init == false && force == false) { - console.info("Waiting for init... (appendAllBuffers)"); - this._initData = data; - setTimeout(callback, 0); - return; - } - var semaphore = this._numBuffers; - var wrapper = function () { - semaphore--; - if (semaphore == 0) { - callback(); - } - }; - - this.safeUpdate(this._seekBuffer, data).then(() => { - this._seekReady = true; - // Handle any pending seeks - if (this._pendingSeeks.length > 0) { - var pending = this._pendingSeeks.shift(); - this.appendSeekBuffer(pending.data, pending.time); - } - - // Now fill the rest of the buffers - for (var idx = 0; idx < this._numBuffers; idx++) { - this.safeUpdate(this._sourceBuffers[idx], data).then(wrapper); - this._inUse[idx] += data.byteLength; - } - }); - } - - // Source buffers need a mutex to protect them, return a promise when - // the update is finished. - safeUpdate(buffer, data) { - let promise = new Promise((resolve, reject) => { - if (buffer.updating) { - setTimeout(() => { - this.safeUpdate(buffer, data).then(resolve); - }, 100); - } - - else { - buffer.onupdateend = () => { - buffer.onupdateend = null; - resolve(); - }; - buffer.appendBuffer(data); - } - }); - return promise; - } -} diff --git a/ui/src/js/components/entity-gallery/entity-gallery-card.js b/ui/src/js/components/entity-gallery/entity-gallery-card.js index 044fe8fb7..6db8f8fe5 100644 --- a/ui/src/js/components/entity-gallery/entity-gallery-card.js +++ b/ui/src/js/components/entity-gallery/entity-gallery-card.js @@ -430,36 +430,43 @@ export class EntityCard extends TatorElement { let attrStyleDiv = document.createElement("div"); attrStyleDiv.setAttribute("class", `entity-gallery-card__attribute`); - let attrLabel = document.createElement("span"); + let attrLabel = document.createElement("div"); attrLabel.setAttribute("class", "f3 text-gray text-normal"); attrStyleDiv.appendChild(attrLabel); - let key; + let key = ""; + let keyString = "" + + // Assign key and output string based on attr info if (typeof attr == "string") { key = attr; + if (typeof obj[this._type][key] !== null && obj[key] !== "" && key !== "type") { if (key.indexOf("_by") > -1 && this._membershipMap.has(Number(obj[this._type][key]))) { // It is a user ID lookup const username = this._membershipMap.get(Number(obj[this._type][key])); - attrLabel.appendChild(document.createTextNode(`${username}`)); + keyString = `${key}: ${username}`; } else { - attrLabel.appendChild(document.createTextNode(`${obj[this._type][key]}`)); + keyString = `${key}: ${obj[this._type][key]}`; } } else if(key === "type" && typeof obj.entityType["dtype"] !== null && obj.entityType["dtype"] !== "") { - attrLabel.appendChild(document.createTextNode(`${obj.entityType["name"]}`)); + keyString = `${key}: ${obj.entityType["name"]}`; } else { - attrLabel.innerHTML = `<not set>`; + keyString = `${key}: <not set>`; } } else { key = attr.name; + if (obj.attributes !== null && typeof obj.attributes[key] !== "undefined" && obj.attributes[key] !== null && obj.attributes[key] !== "") { - attrLabel.appendChild(document.createTextNode(`${obj.attributes[key]}`)); + keyString = `${key}: ${obj.attributes[key]}`; } else { - attrLabel.innerHTML = `<not set>`; + keyString = `${key}: <not set>`; } } - + + // Update output based on key and keystring + attrLabel.innerHTML = keyString; attrStyleDiv.setAttribute("title", `${key}`); // add to the card & keep a list diff --git a/ui/src/js/components/entity-gallery/entity-gallery-labels.js b/ui/src/js/components/entity-gallery/entity-gallery-labels.js index 09abd2a92..0c421e4c3 100644 --- a/ui/src/js/components/entity-gallery/entity-gallery-labels.js +++ b/ui/src/js/components/entity-gallery/entity-gallery-labels.js @@ -62,6 +62,16 @@ export class EntityGalleryLabels extends TatorElement { { name: "Type", id: "type" } ] + + } + + set titleEntityTypeName(val) { + this._titleEntityTypeName = val; + this._titleText.textContent = `Select ${val} labels to display in the gallery`; + } + + init(projectId) { + this.projectId = projectId; this.add({ typeData: { id: -1, @@ -71,17 +81,12 @@ export class EntityGalleryLabels extends TatorElement { }); } - set titleEntityTypeName(val) { - this._titleText.textContent = `Select ${val} labels to display in the gallery`; - } - /** * Add a section of labels to main label div * @param {typeData} - object * */ - async add({ typeData, hideTypeName = false, checkedFirst = null, customBuiltIns = [] }) { - // console.log(typeData); + async add({ typeData, hideTypeName = false, checkedFirst = null, customBuiltIns = [] }) { let typeName = typeData.name ? typeData.name : ""; // don't re-add this type, or don't add if visible=false... @@ -109,8 +114,12 @@ export class EntityGalleryLabels extends TatorElement { let idText = document.createElement("text"); idText.setAttribute("class", "d-flex py-1 text-gray f3"); - idText.textContent = `Type ID: ${typeData.id}`; - _title.appendChild(idText); + if (typeData.id == -1) { + idText.textContent = `Type: Built in`; + } else { + idText.textContent = `Type ID: ${typeData.id}`; + } + _title.appendChild(idText); labelsMain.appendChild(_title); } @@ -142,16 +151,30 @@ export class EntityGalleryLabels extends TatorElement { // Save to refer to in get/set later this._selectionValues[typeData.id] = selectionBoxes; + // Check any project preferences, then also apply any cached (in that order) + // #todo get preferences list, this may need to be passed into this function + const projectPreference = []; + const cacheTypeList = this.getLocalStorage(typeData.id); + const listToApply = [...projectPreference, ...cacheTypeList]; + console.log(typeData.id); + console.log(listToApply); + this._setValue({ typeId: typeData.id, values: listToApply }); + + // Append to main box styleDiv.appendChild(selectionBoxes); + // Listen for changes selectionBoxes.addEventListener("change", (e) => { const builtIns = this._selectionValues[-1] ? this._selectionValues[-1].getValue() : []; const currentBoxes = typeData.id == -1 ? [] : e.target.getValue(); + const newValue = [...builtIns, ...currentBoxes]; + + this.setLocalStorage(typeData.id); this.dispatchEvent(new CustomEvent("labels-update", { detail: { - value: [...builtIns, ...currentBoxes], + value: newValue, typeId: typeData.id } })); @@ -166,22 +189,17 @@ export class EntityGalleryLabels extends TatorElement { _getValue(typeId) { if (this._selectionValues[typeId]) { + // gets value of the selection boxes (type: checkbox-list) for that type return this._selectionValues[typeId].getValue(); } else { return []; } } - _setValue({ typeId, values }){ - // # assumes values are in the accepted format for checkbox set - // - let valuesList = this._getValue(typeId); - for(let box in valuesList){ - if(values.contains(box.name)){ - box.checked = true; - } - } - return this._selectionValues[typeId].setValue(valuesList); + _setValue({ typeId, values }) { + console.log("LABELS SET VALUES"); + console.log(values); + this._selectionValues[typeId].updateValue(values); } /* @@ -233,10 +251,32 @@ export class EntityGalleryLabels extends TatorElement { // reset checked - only check the first one checkedValue = false; } - console.log(this.newList); + // console.log(this.newList); return this.newList; } + + getLocalStorage(typeId) { + const storageKey = `project-${this.projectId}__${this._titleEntityTypeName}-labels__type-${typeId}`; + const storedData = localStorage.getItem(storageKey); + console.log(`GET storedData for ${storageKey} = ${storedData}`); + + if (storedData) { + const data = JSON.parse(storedData); + return data.values; + } else { + return []; + } + + } + + setLocalStorage(typeId) { + const storageKey = `project-${this.projectId}__${this._titleEntityTypeName}-labels__type-${typeId}`; + const newValue = JSON.stringify({ values: this._getValue(typeId) }); + console.log(`SET storedData for ${storageKey} set to newValue = ${newValue}`); + + localStorage.setItem(storageKey, newValue); + } } customElements.define("entity-gallery-labels", EntityGalleryLabels); diff --git a/ui/src/js/components/inputs/feature/checkbox-set.js b/ui/src/js/components/inputs/feature/checkbox-set.js index 4d234d9d8..75637fe17 100644 --- a/ui/src/js/components/inputs/feature/checkbox-set.js +++ b/ui/src/js/components/inputs/feature/checkbox-set.js @@ -69,6 +69,8 @@ export class CheckboxSet extends TatorElement { } } + + _newInput(item){ let checkbox = document.createElement("checkbox-input"); checkbox.setAttribute("name", `${item.name}`); @@ -111,7 +113,7 @@ export class CheckboxSet extends TatorElement { } // Array of checked inputs hidden data - // @TODO this follows current patter for some checkboxes to store hidden data + // @TODO this follows current pattern for some checkboxes to store hidden data // should look into setting the data as value instead? or type to data and getValue = this? getData() { return this._inputs.filter(input => input.getChecked()).map(checked => checked.getData()); @@ -142,6 +144,20 @@ export class CheckboxSet extends TatorElement { return console.log("No matching input found"); } + /** + * + * @param {list} val + * Map(name: [name: "Atribute 1", checked: true ]) + * @returns updates the checkbox elements related to val.name to checked or unchecked + */ + updateValue(checkedList) { + for (let checkbox of this._inputs) { + const currentVal = decodeURI(checkbox._input.value); + console.log(`checkedList.includes(currentVal) => ${checkedList.includes(currentVal)}`) + checkbox._checked = checkedList.includes(currentVal); + } + } + removeInput({ value }) { for (let checkbox of this._inputs) { if (Number(checkbox._input.value) === Number(value)) { diff --git a/ui/src/js/components/inputs/feature/file-input.js b/ui/src/js/components/inputs/feature/file-input.js index 6dff49ea2..255ef84a1 100644 --- a/ui/src/js/components/inputs/feature/file-input.js +++ b/ui/src/js/components/inputs/feature/file-input.js @@ -163,9 +163,10 @@ export class FileInput extends TatorElement { this._hiddenInput.setValue(val); if (typeof val !== "undefined" && val !== null && val !== "") { - this._viewFile.setAttribute("href", `/media/${val}`); + this._viewFile.setAttribute("href", `${window.location.origin}${val}`); this._viewFile.classList.remove("hidden"); } else { + this._viewFile.setAttribute("href", ``); this._viewFile.classList.add("hidden"); } diff --git a/ui/src/js/components/inputs/link-input.js b/ui/src/js/components/inputs/link-input.js new file mode 100644 index 000000000..e65801605 --- /dev/null +++ b/ui/src/js/components/inputs/link-input.js @@ -0,0 +1,37 @@ +import { TatorElement } from "../tator-element.js"; + +export class LinkInput extends TatorElement { + constructor() { + super(); + + this.label = document.createElement("label"); + this.label.setAttribute("class", "d-flex flex-justify-between flex-items-center py-1"); + this._shadow.appendChild(this.label); + + this._labelText = document.createTextNode(""); + this.label.appendChild(this._labelText); + + this._link = document.createElement("a"); + this._link.setAttribute("target", "_blank"); + this._link.setAttribute("class", "col-8 text-underline text-purple"); + this.label.appendChild(this._link); + } + + static get observedAttributes() { + return ["label", "name", "href"]; + } + + attributeChangedCallback(name, oldValue, newValue) { + switch (name) { + case "name": + this._labelText.nodeValue = newValue; + break; + case "href": + this._link.setAttribute("href", newValue); + this._link.textContent = newValue; + break; + } + } +} + +customElements.define("link-input", LinkInput); diff --git a/ui/src/js/components/modal-dialog.js b/ui/src/js/components/modal-dialog.js index 19550fc1a..d4af3666b 100644 --- a/ui/src/js/components/modal-dialog.js +++ b/ui/src/js/components/modal-dialog.js @@ -59,32 +59,12 @@ export class ModalDialog extends TatorElement { } } - fadeOut(timeOut = 500) { - this._div.style.opacity = 1; - const interval = 100; - const percent = 1 - ((interval / timeOut) * 2); - - const opacityTurnDown = () => { - const currentOpacity = this._div.style.opacity; - const turnDown = currentOpacity * percent; - this._div.style.opacity = turnDown - console.log(turnDown); - if (turnDown == 0 || turnDown < .3) { - this.stopFade(); - } - } - + fadeOut(timeOut = 1500) { setTimeout(() => { - this.startFade = setInterval(opacityTurnDown, interval); + this._closeCallback(); }, timeOut) } - stopFade() { - this._closeCallback(); - this._div.style.opacity = 1; - if(this.startFade) clearInterval(this.startFade); - } - } if (!customElements.get("modal-dialog")) { diff --git a/ui/src/js/organization-settings/invitation-edit.js b/ui/src/js/organization-settings/invitation-edit.js index 39cda702b..167da3aed 100644 --- a/ui/src/js/organization-settings/invitation-edit.js +++ b/ui/src/js/organization-settings/invitation-edit.js @@ -34,9 +34,6 @@ export class InvitationEdit extends OrganizationTypeForm { } _getExistingForm(data) { - // console.log("Get existing form"); - // console.log(data); - let current = this.boxHelper.boxWrapDefault({ "children": "" }); @@ -56,23 +53,24 @@ export class InvitationEdit extends OrganizationTypeForm { this._permissionSelect.setValue(data.permission); this._permissionSelect.default = data.permission; this._permissionSelect.addEventListener("change", this._formChanged.bind(this)); - - // status - const statusOptions = [ - { "label": "Pending", "value": "Pending" }, - { "label": "Expired", "value": "Expired" }, - { "label": "Accepted", "value": "Accepted" }, - ]; - this._statusSelect = document.createElement("enum-input"); - this._statusSelect.setAttribute("name", "Status"); - this._statusSelect.choices = statusOptions; - this._statusSelect._select.required = true; - this._statusSelect.setValue(data.status); - this._statusSelect.default = data.status; - this._statusSelect.permission = "View Only"; - this._statusSelect.addEventListener("change", this._formChanged.bind(this)); this._form.appendChild(this._permissionSelect); + // status #Todo let user fix and Expired? + // const statusOptions = [ + // { "label": "Pending", "value": "Pending" }, + // { "label": "Expired", "value": "Expired" }, + // { "label": "Accepted", "value": "Accepted" }, + // ]; + // this._statusSelect = document.createElement("enum-input"); + // this._statusSelect.setAttribute("name", "Status"); + // this._statusSelect.choices = statusOptions; + // this._statusSelect._select.required = true; + // this._statusSelect.setValue(data.status); + // this._statusSelect.default = data.status; + // this._statusSelect.permission = "View Only"; + // this._statusSelect.addEventListener("change", this._formChanged.bind(this)); + // this._form.appendChild(this._statusSelect); + // status this._statusField = document.createElement("text-input"); this._statusField.setAttribute("name", "Status"); @@ -83,6 +81,16 @@ export class InvitationEdit extends OrganizationTypeForm { // this._statusField.addEventListener("change", this._formChanged.bind(this)); this._form.appendChild(this._statusField); + // + if (this.data.status == "Pending") { + const registrationLink = `${window.location.origin}/registration?registration_token=${this.data.registration_token}`; + this._regLinkDisplay = document.createElement("link-input"); + this._regLinkDisplay.setAttribute("name", "Registration Link"); + this._regLinkDisplay.setAttribute("href", registrationLink); + this._regLinkDisplay.permission = "View Only"; + this._form.appendChild(this._regLinkDisplay); + } + current.appendChild(this._form); return current; @@ -136,6 +144,10 @@ export class InvitationEdit extends OrganizationTypeForm { if (this._permissionSelect.changed()) { formData.permission = this._permissionSelect.getValue(); } + + // if (this._statusSelect.changed()) { + // formData.status = this._statusSelect.getValue(); + // } } return formData; @@ -175,11 +187,6 @@ export class InvitationEdit extends OrganizationTypeForm { this._data.id = data.id; this._data.organization = this.organization; - if (!this._emailEnabled) { - let link = data.message.replace('User can register at ', '') - emailLinksHTML += `
  • ${email} can register at ${link}
    • `; - } - return data.id; }) .then((id) => { @@ -199,6 +206,13 @@ export class InvitationEdit extends OrganizationTypeForm { // if we can can get the status from endpoint, it is hardcoded above as pending (assumed since it was just created) this._data = data; } + console.log(this._data); + + + if (!this._emailEnabled) { + const registrationLink = `${window.location.origin}/registration?registration_token=${this._data.registration_token}`; + emailLinksHTML += `
  • ${data.email} can register at ${registrationLink}
    • `; + } let form = document.createElement(this._getTypeClass()); this.sideNav.fillContainer({ diff --git a/ui/src/js/project-detail/download-button.js b/ui/src/js/project-detail/download-button.js index e200f0c50..2bf0ef715 100644 --- a/ui/src/js/project-detail/download-button.js +++ b/ui/src/js/project-detail/download-button.js @@ -29,35 +29,31 @@ export class DownloadButton extends TatorElement { this._button.addEventListener("click", () => { if (this.request) { - // Get the size first - let getSize = new Request(this.request.url, {headers: this.request.headers, method:"HEAD"}); - fetch(getSize).then(sizeRes => { // HEAD fails (403) on presigned urls, but the file still downloads. - // TODO: Update DownloadInfo to return object size - const name = this.getAttribute("name"); - const fileSize = parseInt(sizeRes.headers.get("content-length")); - console.log(`${name} is ${fileSize} bytes`); - const fileStream = streamSaver.createWriteStream(name, {size: fileSize}); - fetch(this.request) - .then(res => { - // https://github.com/jimmywarting/StreamSaver.js/blob/master/examples/fetch.html - const readableStream = res.body; + const fileSize = this.getAttribute("size"); + const name = this.getAttribute("name"); - if (window.WritableStream && readableStream.pipeTo) { - return readableStream.pipeTo(fileStream) - .then(() => console.log('done writing')) - } + console.log(`${name} is ${fileSize} bytes`); + const fileStream = streamSaver.createWriteStream(name, {size: fileSize}); + fetch(this.request) + .then(res => { + // https://github.com/jimmywarting/StreamSaver.js/blob/master/examples/fetch.html + const readableStream = res.body; - window.writer = fileStream.getWriter() + if (window.WritableStream && readableStream.pipeTo) { + return readableStream.pipeTo(fileStream) + .then(() => console.log('done writing')) + } - const reader = res.body.getReader() - const pump = () => reader.read() - .then(res => res.done - ? writer.close() - : writer.write(res.value).then(pump)) + window.writer = fileStream.getWriter() - pump() - }); - }); + const reader = res.body.getReader() + const pump = () => reader.read() + .then(res => res.done + ? writer.close() + : writer.write(res.value).then(pump)) + + pump() + }); } else if (this.hasAttribute("url") && this.hasAttribute("name")) { const link = document.createElement("a"); diff --git a/ui/src/js/project-detail/media-more.js b/ui/src/js/project-detail/media-more.js index 5886430f6..b6468353c 100644 --- a/ui/src/js/project-detail/media-more.js +++ b/ui/src/js/project-detail/media-more.js @@ -118,7 +118,7 @@ export class MediaMore extends TatorElement { } set media(val) { - const request = Utilities.getDownloadRequest(val); + const downloadInfo = Utilities.getDownloadInfo(val); this._media = val; // if (this._media && this._media.id) { @@ -126,11 +126,12 @@ export class MediaMore extends TatorElement { // this._moreIcon._svg.setAttribute("mediaId", `${this._media.id}`) // } - if (request == null) { + if (downloadInfo["request"] == null) { this._download.style.display = "none"; this.setAttribute("downloadPermission", "Download in menu disabled due to permissions.") } else { - this._download.request = request; + this._download.request = downloadInfo["request"]; + this._download.setAttribute("size", downloadInfo["size"]); } let hide = true; @@ -163,7 +164,7 @@ export class MediaMore extends TatorElement { attributeChangedCallback(name, oldValue, newValue) { switch (name) { case "name": - if (newValue === null) { + if (newValue !== null) { this._download.setAttribute("name", newValue); } else { this._download.setAttribute("name", ""); diff --git a/ui/src/js/project-detail/media-section.js b/ui/src/js/project-detail/media-section.js index 0b4a6e1b0..6a338ef7d 100644 --- a/ui/src/js/project-detail/media-section.js +++ b/ui/src/js/project-detail/media-section.js @@ -433,7 +433,7 @@ export class MediaSection extends TatorElement { } filenames.add(basename); - const request = Utilities.getDownloadRequest(media, headers); + const request = Utilities.getDownloadInfo(media, headers)["request"]; if (request !== null) { // Media objects with no downloadable files will return null. // Download media file. console.log("Downloading " + media.name + " from " + request.url + "..."); diff --git a/ui/src/js/project-detail/project-detail.js b/ui/src/js/project-detail/project-detail.js index d2cd0a9e6..11fbb4eeb 100644 --- a/ui/src/js/project-detail/project-detail.js +++ b/ui/src/js/project-detail/project-detail.js @@ -782,6 +782,8 @@ export class ProjectDetail extends TatorPage { var hiddenAlgos = ['tator_extend_track', 'tator_fill_track_gaps']; const hiddenAlgoCategories = ['annotator-view', 'disabled']; + this._cardAttributeLabels.init(projectId); + // // Set up attributes for bulk edit for (let mediaTypeData of mediaTypes) { diff --git a/ui/src/js/project-detail/section-files.js b/ui/src/js/project-detail/section-files.js index 77397b99e..e0b4c6e70 100644 --- a/ui/src/js/project-detail/section-files.js +++ b/ui/src/js/project-detail/section-files.js @@ -94,10 +94,14 @@ export class SectionFiles extends TatorElement { /** * Card labels / attributes of localization or media type + * - Sticky choices from localStorage or Preferences set these + * - But, can be overridden during a session + * - This always goes off of the selection in entity-gallery-labels */ + const builtInChosen = this._cardAttributeLabels._getValue(-1); this.cardLabelsChosenByType[entityTypeId] = this._cardAttributeLabels._getValue(entityTypeId); // this._bulkEdit._updateShownAttributes({ typeId: entityTypeId, values: this.cardLabelsChosenByType[entityTypeId] }); - + const cardLabelsChosen = [...this.cardLabelsChosenByType[entityTypeId], ...builtInChosen]; if (newCard) { card = document.createElement("entity-card"); @@ -170,13 +174,11 @@ export class SectionFiles extends TatorElement { } // this is data used later by label chooser, and bulk edit - console.log("this._memberships"); - console.log(this._memberships); card.init({ obj: cardObj, idx: index, mediaInit: true, - cardLabelsChosen: this.cardLabelsChosenByType[entityTypeId], + cardLabelsChosen, enableMultiselect: this.multiEnabled, memberships: this._memberships }); diff --git a/ui/src/js/project-settings/attributes/attributes-form.js b/ui/src/js/project-settings/attributes/attributes-form.js index 79dee39fe..3790a8c82 100644 --- a/ui/src/js/project-settings/attributes/attributes-form.js +++ b/ui/src/js/project-settings/attributes/attributes-form.js @@ -947,7 +947,14 @@ export class AttributesForm extends TatorElement { // if (dtype === "enum") { - if ((this.isClone || this._dtype.changed() || this._enumDefault.changed || this._choices.changed() || this._labels.changed()) && this._enumDefault.value !== null) { //&& this._enumDefault.value !== "" + if ( + (this.isClone + || this._dtype.changed() + || this._enumDefault.changed + || this._choices.changed() + || this._labels.changed()) + && this._choices.getValue().includes(this._enumDefault.value) + ) { formData["default"] = this._enumDefault.value; } diff --git a/ui/src/js/project-settings/attributes/attributes-main.js b/ui/src/js/project-settings/attributes/attributes-main.js index 4b3a31639..6bcc53c01 100644 --- a/ui/src/js/project-settings/attributes/attributes-main.js +++ b/ui/src/js/project-settings/attributes/attributes-main.js @@ -227,7 +227,7 @@ export class AttributesMain extends HTMLElement { this.loading.hideSpinner(); this.boxHelper._modalSuccess(currentMessage); } else if(status == 400) { - iconWrap.appendChild(warningIcon); + // iconWrap.appendChild(warningIcon); this.loading.hideSpinner(); this.boxHelper._modalError(`${currentMessage}`); } diff --git a/ui/src/js/project-settings/type-forms/algorithm-edit.js b/ui/src/js/project-settings/type-forms/algorithm-edit.js index 3e53154d6..cedc6d2c7 100644 --- a/ui/src/js/project-settings/type-forms/algorithm-edit.js +++ b/ui/src/js/project-settings/type-forms/algorithm-edit.js @@ -124,8 +124,14 @@ export class AlgorithmEdit extends TypeForm { this._manifestPath.setAttribute("for", "manifest"); this._manifestPath.setAttribute("type", "yaml"); this._manifestPath.projectId = this.projectId; - this._manifestPath.setValue(this.data.manifest); - this._manifestPath.default = this.data.manifest; + + if (this.data.manifest) { + this._manifestPath.setValue(`/media/${this.data.manifest}`); + this._manifestPath.default = `/media/${this.data.manifest}`; + } else { + this._manifestPath.default = null; + } + this._manifestPath._fetchCall = (bodyData) => { return fetch(`/rest/SaveAlgorithmManifest/${this.projectId}`, @@ -142,8 +148,9 @@ export class AlgorithmEdit extends TypeForm { ).then(resp => resp.json()).then( manifestData => { // console.log(manifestData); - this._manifestPath.setValue(manifestData.url); - Utilities.showSuccessIcon(`Manifest file uploaded to: ${manifestData.url}`); + const viewLink = `/media/${manifestData.url}`; + this._manifestPath.setValue(viewLink); + Utilities.showSuccessIcon(`Manifest file uploaded to: ${viewLink}`); } ); }; diff --git a/ui/src/js/project-settings/type-forms/applet-edit.js b/ui/src/js/project-settings/type-forms/applet-edit.js index 1a16eee0a..51cd970e5 100644 --- a/ui/src/js/project-settings/type-forms/applet-edit.js +++ b/ui/src/js/project-settings/type-forms/applet-edit.js @@ -23,6 +23,7 @@ export class AppletEdit extends TypeForm { "children": "" }); + // append input for name this._editName = document.createElement("text-input"); this._editName.permission = !this.userCantSaveCluster ? "Can Edit" : "Ready Only"; @@ -33,6 +34,14 @@ export class AppletEdit extends TypeForm { this._editName.addEventListener("change", this._formChanged.bind(this)); this._form.appendChild(this._editName); + // append link + if (this.appletId && this.appletId !== "New") { + this._linkToDashboard = document.createElement("link-input"); + this._linkToDashboard.setAttribute("name", "Link"); + this._linkToDashboard.setAttribute("href", `${window.location.origin}/${this.projectId}/dashboards/${this.appletId}`); + this._form.appendChild(this._linkToDashboard); + } + // description this._editDescription = document.createElement("text-input"); this._editDescription.permission = !this.userCantSaveCluster ? "Can Edit" : "Ready Only"; diff --git a/ui/src/js/third-party/js-zip.js b/ui/src/js/third-party/js-zip.js new file mode 100644 index 000000000..fd12de32d --- /dev/null +++ b/ui/src/js/third-party/js-zip.js @@ -0,0 +1,5 @@ +import JSZip from 'jszip'; + +export function GetJSZip() { + return JSZip(); + } \ No newline at end of file diff --git a/ui/src/js/util/tator-data.js b/ui/src/js/util/tator-data.js index c3b13f571..ae9cd9806 100644 --- a/ui/src/js/util/tator-data.js +++ b/ui/src/js/util/tator-data.js @@ -106,7 +106,6 @@ export class TatorData { .then(response => response.json()) .then(memberships => { this._memberships = memberships; - console.log(memberships); resolve(); }); }); diff --git a/ui/src/js/util/utilities.js b/ui/src/js/util/utilities.js index 1e1ae38ce..110deedc5 100644 --- a/ui/src/js/util/utilities.js +++ b/ui/src/js/util/utilities.js @@ -59,25 +59,27 @@ export class Utilities } } // Get the download request object - static getDownloadRequest(media_element, session_headers) + static getDownloadInfo(media_element, session_headers) { // Download original file if available. let url; let http_authorization; let hostname; let path; + let size; var media_files = media_element.media_files; const byRes = (a, b) => {return b.resolution[0] - a.resolution[0];}; if (media_files) { if (media_files.layout) { - return null; + return {"request": null, "size": -1}; } if (media_files.image) { media_files.image.sort(byRes); path = media_files.image[0].path; + size = media_files.image[0].size; http_authorization = media_files.image[0].http_auth; hostname = media_files.image[0].host; } @@ -85,6 +87,7 @@ export class Utilities { media_files.archival.sort(byRes); path = media_files.archival[0].path; + size = media_files.archival[0].size; http_authorization = media_files.archival[0].http_auth; hostname = media_files.archival[0].host; } @@ -92,6 +95,7 @@ export class Utilities { media_files.streaming.sort(byRes); path = media_files.streaming[0].path; + size = media_files.streaming[0].size; http_authorization = media_files.streaming[0].http_auth; hostname = media_files.streaming[0].host; } @@ -116,7 +120,7 @@ export class Utilities } } - return request; + return {"request": request, "size": size}; } // Returns a promise with the clients IP diff --git a/ui/webpack.common.js b/ui/webpack.common.js index a76a3be26..1cffb0562 100644 --- a/ui/webpack.common.js +++ b/ui/webpack.common.js @@ -18,7 +18,8 @@ module.exports = { util: "./src/js/util/index.js", registration: "./src/js/registration/index.js", components: "./src/js/components/index.js", - portal: "./src/js/analytics/index.js" + portal: "./src/js/analytics/index.js", + "third-party": "./src/js/third-party/index.js" }, output: { filename: "[name].js", diff --git a/ui/webpack.dev.js b/ui/webpack.dev.js index 1a77f9636..61a0b6f25 100644 --- a/ui/webpack.dev.js +++ b/ui/webpack.dev.js @@ -3,7 +3,7 @@ const common = require("./webpack.common.js"); module.exports = merge(common, { mode: "development", - devtool: "inline-source-map", + devtool: "eval-source-map", devServer: { static: "./dist", },