Merge branch 'rc/0.2.21' into stable

# Conflicts: # scripts/packages/tator-py
cvisionai · Oct 26, 2022 · 5f02222 · 5f02222
2 parents 3e804da + 5b71d19
commit 5f02222
Show file tree

Hide file tree

Showing 87 changed files with 2,802 additions and 1,363 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -287,7 +287,7 @@ workflows:
         filters:
           branches:
             only:
-            - master
+            - rc/0.2.21
     - cleanup-lightsail:
         requires:
         - rest-tests

diff --git a/Makefile b/Makefile
@@ -148,8 +148,7 @@ dashboard-token:
 	kubectl -n kube-system describe secret $$(kubectl -n kube-system get secret | grep tator-kubernetes-dashboard | awk '{print $$1}')
 
 .PHONY: tator-image
-tator-image:
-	$(MAKE) webpack
+tator-image: webpack
 	docker build --network host -t $(DOCKERHUB_USER)/tator_online:$(GIT_VERSION) -f containers/tator/Dockerfile . || exit 255
 	docker push $(DOCKERHUB_USER)/tator_online:$(GIT_VERSION)
 
@@ -182,9 +181,22 @@ experimental_docker:
 endif
 
 
+USE_VPL=$(shell python3 -c 'import yaml; a = yaml.load(open("helm/tator/values.yaml", "r"),$(YAML_ARGS)); print(a.get("enableVpl","False"))')
+ifeq ($(USE_VPL),True)
+.PHONY: client-vpl
+client-vpl:
+	docker build --platform linux/amd64 --network host -t $(SYSTEM_IMAGE_REGISTRY)/tator_client_vpl:$(GIT_VERSION) -f containers/tator_client/Dockerfile.vpl . || exit 255
+	docker push $(SYSTEM_IMAGE_REGISTRY)/tator_client_vpl:$(GIT_VERSION)
+else
+.PHONY: client-vpl
+client-vpl:
+	@echo "Skipping VPL Build"
+endif
+
 # Publish client image to dockerhub so it can be used cross-cluster
 .PHONY: client-image
 client-image: experimental_docker
+	make client-vpl
 	docker build --platform linux/amd64 --network host -t $(SYSTEM_IMAGE_REGISTRY)/tator_client_amd64:$(GIT_VERSION) -f containers/tator_client/Dockerfile . || exit 255
 	docker build --platform linux/aarch64 --network host -t $(SYSTEM_IMAGE_REGISTRY)/tator_client_aarch64:$(GIT_VERSION) -f containers/tator_client/Dockerfile_arm . || exit 255
 	docker push $(SYSTEM_IMAGE_REGISTRY)/tator_client_amd64:$(GIT_VERSION)
@@ -223,11 +235,11 @@ USE_MIN_JS=$(shell python3 -c 'import yaml; a = yaml.load(open("helm/tator/value
 ifeq ($(USE_MIN_JS),True)
 webpack:
 	@echo "Building webpack bundles for production, because USE_MIN_JS is true"
-	cd ui && python3 make_index_files.py && npm run build
+	cd ui && npm install && python3 make_index_files.py && npm run build
 else
 webpack:
 	@echo "Building webpack bundles for development, because USE_MIN_JS is false"
-	cd ui && python3 make_index_files.py && npm run buildDev
+	cd ui && npm install && python3 make_index_files.py && npm run buildDev
 endif
 
 .PHONY: migrate
@@ -308,13 +320,13 @@ js-bindings:
 	./codegen.py tator-openapi-schema.yaml
 	docker run -it --rm \
 		-v $(shell pwd)/scripts/packages/tator-js:/pwd \
-		openapitools/openapi-generator-cli:v5.2.1 \
+		openapitools/openapi-generator-cli:v6.1.0 \
 		generate -c /pwd/config.json \
 		-i /pwd/tator-openapi-schema.yaml \
 		-g javascript -o /pwd/pkg -t /pwd/templates
 	docker run -it --rm \
 		-v $(shell pwd)/scripts/packages/tator-js:/pwd \
-		openapitools/openapi-generator-cli:v5.2.1 \
+		openapitools/openapi-generator-cli:v6.1.0 \
 		chmod -R 777 /pwd/pkg
 	cp -r examples pkg/examples
 	cp -r utils pkg/src/utils

diff --git a/containers/tator/Dockerfile b/containers/tator/Dockerfile
@@ -1,7 +1,7 @@
 # Build librclone shared object
 FROM golang:latest as rclone_build
 WORKDIR /go
-RUN git clone https://github.com/rclone/rclone.git
+RUN git clone https://github.com/rclone/rclone.git --single-branch --depth 1 --branch v1.59.2
 
 WORKDIR /go/rclone/librclone/
 RUN go build --buildmode=c-shared -o librclone.so github.com/rclone/rclone/librclone

diff --git a/containers/tator_client/Dockerfile b/containers/tator_client/Dockerfile
@@ -1,4 +1,4 @@
-FROM cvisionai/svt_encoder:v0.0.7 as builder
+FROM cvisionai/svt_encoder:v0.0.8 as builder
 ENV LANG C.UTF-8
 RUN apt-get update && apt-get install -y --no-install-recommends \
         wget unzip \

diff --git a/containers/tator_client/Dockerfile.vpl b/containers/tator_client/Dockerfile.vpl
@@ -0,0 +1,44 @@
+FROM cvisionai/vpl_encoder:v0.0.1 as builder
+ENV LANG C.UTF-8
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        wget unzip \
+        && rm -rf /var/lib/apt/lists
+RUN mkdir -p /opt/cvision
+RUN wget http://zebulon.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip
+RUN unzip Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip
+RUN cp Bento4-SDK-1-6-0-632.x86_64-unknown-linux/bin/mp4dump /opt/cvision/bin
+RUN cp Bento4-SDK-1-6-0-632.x86_64-unknown-linux/bin/mp4info /opt/cvision/bin
+
+FROM ubuntu:20.04 AS cvtranscoder
+MAINTAINER CVision AI <info@cvisionai.com>
+# Install apt packages
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        python3 python3-pip \
+        python3-setuptools python3-dev gcc git vim curl unzip wget \
+        && rm -rf /var/lib/apt/lists
+COPY --from=builder /opt/cvision /opt/cvision
+ENV PATH="/opt/cvision/bin/:$PATH"
+RUN echo "/opt/cvision/lib" > /etc/ld.so.conf.d/cvision.conf
+RUN ldconfig
+
+# Install pip packages
+RUN pip3 --no-cache-dir --timeout=1000 install wheel
+RUN pip3 --no-cache-dir --timeout=1000 install pillow==9.0.0 imageio==2.14.0 progressbar2==4.0.0 boto3==1.20.41 pandas==1.4.0
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update &&  \
+                   apt-get install -y --no-install-recommends fastjar libsm6 libxext6 \
+		   libxrender-dev libx265-179 libx264-155 \
+                   libpng16-16 libfreetype6 python3-opencv \
+		   libx265-179 libx264-155 libx11-6 libxext6 libxfixes3 \
+		   libxcb1 libxcb-shm0 libxcb-shape0 libxcb-xfixes0 \
+		   libwayland-client0 libaom0 && rm -rf /var/lib/apt/lists
+
+# Copy over scripts
+COPY scripts/transcoder /scripts
+COPY scripts/packages/tator-py/dist/*.whl /tmp
+
+# Build tator-py
+RUN pip3 install tmp/*.whl
+
+WORKDIR /scripts
diff --git a/helm/tator/templates/tator.yaml b/helm/tator/templates/tator.yaml
@@ -24,6 +24,9 @@
 {{- $filebeatSettings := dict "Values" .Values "name" "prune-filebeat-cron" "app" "prune-filebeat" "selector" "webServer: \"yes\""  "command" "[python3]" "args" "[\"manage.py\", \"prunefilebeat\"]" "schedule" "40 4 * * *" }}
 {{include "tatorCron.template" $filebeatSettings }}
 ---
+{{- $fileSettings := dict "Values" .Values "name" "prune-files-cron" "app" "prune-files" "selector" "webServer: \"yes\""  "command" "[python3]" "args" "[\"manage.py\", \"prunefiles\"]" "schedule" "30 1 * * *" }}
+{{include "tatorCron.template" $fileSettings }}
+---
 {{- $backupSettings := dict "Values" .Values "name" "backup-cron" "app" "backup" "selector" "webServer: \"yes\"" "command" "[/bin/sh]" "args" "[\"-c\", \"pg_dump -Fc -h $POSTGRES_HOST -U $POSTGRES_USERNAME -d tator_online -f /backup/tator_online_$(date +%Y_%m_%d__%H_%M_%S)_$(GIT_VERSION).sql;\"]" "schedule" "40 6 * * *" }}
 {{include "dbCron.template" $backupSettings }}
 ---

diff --git a/main/backup.py b/main/backup.py
@@ -1,5 +1,6 @@
 from collections import defaultdict
 from ctypes import CDLL, c_char_p, c_int, Structure
+from enum import auto, Enum
 import logging
 import json
 import os
@@ -13,7 +14,11 @@
 
 logger = logging.getLogger(__name__)
 LIVE_STORAGE_CLASS = "STANDARD"
-ALL_STORE_TYPES = ["backup", "live"]
+
+
+class StoreType(Enum):
+    BACKUP = auto()
+    LIVE = auto()
 
 
 """
@@ -126,10 +131,9 @@ def _rpc(self, rclone_method: str, **kwargs):
         except:
             logger.error("Call to `RcloneRPC` failed")
             raise
-        else:
-            output = json.loads(resp.Output.value.decode("utf-8"))
-        finally:
-            TatorBackupManager.__rclone.RcloneFreeString(resp.Output)
+
+        output = json.loads(resp.Output.value.decode("utf-8"))
+        TatorBackupManager.__rclone.RcloneFreeString(resp.Output)
 
         status = resp.Status
         if status != 200:
@@ -150,11 +154,16 @@ def _get_bucket_info(self, project) -> dict:
 
         if project_id not in TatorBackupManager.__project_stores:
             TatorBackupManager.__project_stores[project_id] = {}
+
+            # Determine if the default bucket is being used for all StoreTypes or none
+            use_default_bucket = project.get_bucket() is None
+
             # Get the `TatorStore` object that connects to object storage for the given type
-            for store_type in ALL_STORE_TYPES:
-                is_backup = store_type == "backup"
-                project_bucket = project.get_bucket(backup=is_backup)
-                use_default_bucket = project_bucket is None
+            for store_type in StoreType:
+                is_backup = store_type == StoreType.BACKUP
+                project_bucket = (
+                    None if use_default_bucket else project.get_bucket(backup=is_backup)
+                )
                 try:
                     store = get_tator_store(project_bucket, backup=is_backup and use_default_bucket)
                 except:
@@ -164,13 +173,13 @@ def _get_bucket_info(self, project) -> dict:
                         exc_info=True,
                     )
                     break
-                else:
-                    if store:
-                        TatorBackupManager.__project_stores[project_id][store_type] = {
-                            "store": store,
-                            "remote_name": f"{project_id}_{store_type}",
-                            "bucket_name": store.bucket_name,
-                        }
+
+                if store:
+                    TatorBackupManager.__project_stores[project_id][store_type] = {
+                        "store": store,
+                        "remote_name": f"{project_id}_{store_type}",
+                        "bucket_name": store.bucket_name,
+                    }
 
             if failed:
                 TatorBackupManager.__project_stores.pop(project_id, None)
@@ -188,10 +197,10 @@ def _create_rclone_remote(self, remote_name, bucket_name, remote_type, rclone_pa
 
     @staticmethod
     def get_backup_store(store_info):
-        if "backup" in store_info:
-            return True, store_info["backup"]["store"]
-        if "live" in store_info:
-            return True, store_info["live"]["store"]
+        if StoreType.BACKUP in store_info:
+            return True, store_info[StoreType.BACKUP]["store"]
+        if StoreType.LIVE in store_info:
+            return True, store_info[StoreType.LIVE]["store"]
 
         return False, None
 
@@ -230,7 +239,7 @@ def get_store_info(self, project) -> bool:
                     )
                 except:
                     logger.error(
-                        f"Failed to create remote config for bucket {bucket_name} in project "
+                        f"Failed to create remote config for bucket {bucket_info['bucket_name']} in project "
                         f"{project.id}",
                         exc_info=True,
                     )
@@ -257,36 +266,53 @@ def backup_resources(self, resource_qs) -> Generator[tuple, None, None]:
         """
         successful_backups = set()
         for resource in resource_qs.iterator():
-            project = self.project_from_resource(resource)
+            success = True
             path = resource.path
-            success, store_info = self.get_store_info(project)
-            success = success and "backup" in store_info
+            try:
+                project = self.project_from_resource(resource)
+            except:
+                logger.warning(f"Could not get project from resource with path '{path}', skipping", exc_info=True)
+                success = False
+                project = None
 
             if success:
-                if store_info["backup"]["store"].check_key(path):
-                    logger.info(f"Resource {path} already backed up")
-                    continue
-
-                # Get presigned url from the live bucket, set to expire in 1h
-                download_url = store_info["live"]["store"].get_download_url(path, 3600)
-
-                # Perform the actual copy operation directly from the presigned url
-                try:
-                    self._rpc(
-                        "operations/copyurl",
-                        fs=f"{store_info['backup']['remote_name']}:",
-                        remote=f"{store_info['backup']['bucket_name']}/{path}",
-                        url=download_url,
-                    )
-                except:
-                    success = False
-                    logger.error(
-                        f"Backing up resource '{path}' with presigned url {download_url} failed",
-                        exc_info=True,
-                    )
+                success, store_info = self.get_store_info(project)
+                success = success and StoreType.BACKUP in store_info
 
             if success:
-                successful_backups.add(resource.id)
+                backup_info = store_info[StoreType.BACKUP]
+                backup_store = backup_info["store"]
+                live_info = store_info[StoreType.LIVE]
+                live_store = live_info["store"]
+
+                backup_size = backup_store.get_size(path)
+                live_size = live_store.get_size(path)
+                if backup_size < 0 or live_size != backup_size:
+                    # Get presigned url from the live bucket, set to expire in 1h
+                    download_url = live_store.get_download_url(path, 3600)
+
+                    # Get the destination key
+                    key = backup_store.path_to_key(path)
+
+                    # Perform the actual copy operation directly from the presigned url
+                    try:
+                        self._rpc(
+                            "operations/copyurl",
+                            fs=f"{backup_info['remote_name']}:",
+                            remote=f"{backup_info['bucket_name']}/{key}",
+                            url=download_url,
+                        )
+                    except:
+                        success = False
+                        logger.error(
+                            f"Backing up resource '{path}' with presigned url {download_url} failed",
+                            exc_info=True,
+                        )
+                else:
+                    logger.info(f"Resource {path} already backed up, updating its state.")
+
+                if success:
+                    successful_backups.add(resource.id)
 
             yield success, resource
 
@@ -314,7 +340,9 @@ def request_restore_resource(self, path, project, min_exp_days) -> bool:
             success, store = self.get_backup_store(store_info)
 
         if success:
-            live_storage_class = store_info["live"]["store"].get_live_sc() or LIVE_STORAGE_CLASS
+            live_storage_class = (
+                store_info[StoreType.LIVE]["store"].get_live_sc() or LIVE_STORAGE_CLASS
+            )
             response = store.head_object(path)
             if not response:
                 logger.warning(f"Object {path} not found, skipping")
@@ -365,7 +393,7 @@ def finish_restore_resource(self, path, project) -> bool:
         if success:
             # If no backup store is defined, use the live bucket
             success, backup_store = self.get_backup_store(store_info)
-            live_store = store_info["live"]["store"]
+            live_store = store_info[StoreType.LIVE]["store"]
 
         if success:
             live_storage_class = live_store.get_live_sc() or LIVE_STORAGE_CLASS
@@ -375,11 +403,11 @@ def finish_restore_resource(self, path, project) -> bool:
                 return success
 
             request_state = response.get("Restore", "")
-            if 'true' in request_state:
+            if "true" in request_state:
                 # There is an ongoing request and the object is not ready to be permanently restored
                 logger.info(f"Object {path} not in standard access yet, skipping")
                 success = False
-            elif 'false' in request_state:
+            elif "false" in request_state:
                 # The request has completed and the object is ready for restoration
                 logger.info(f"Object {path} restoration request is complete, restoring...")
             elif "StorageClass" not in response or response["StorageClass"] == live_storage_class:
@@ -411,7 +439,7 @@ def finish_restore_resource(self, path, project) -> bool:
 
                 # Perform the actual copy operation directly from the presigned url
                 try:
-                    live_info = store_info["live"]
+                    live_info = store_info[StoreType.LIVE]
                     self._rpc(
                         "operations/copyurl",
                         fs=f"{live_info['remote_name']}:",
@@ -449,10 +477,10 @@ def get_size(self, resource):
         success, store_info = self.get_store_info(project)
 
         if success:
-            if resource.backed_up:
-                store_type = "backup"
+            if resource.backed_up and StoreType.BACKUP in store_info:
+                store_type = StoreType.BACKUP
             else:
-                store_type = "live"
+                store_type = StoreType.LIVE
 
             size = store_info[store_type]["store"].get_size(resource.path)
         return size