From af987e691c032d4c3512e05ae43086bd529a8cb8 Mon Sep 17 00:00:00 2001 From: GoldenAnpu Date: Fri, 12 Jul 2024 13:13:23 +0200 Subject: [PATCH 1/4] Add Workflow --- dev_requirements.txt | 2 +- supervisely/serve/config.json | 2 +- supervisely/serve/src/main.py | 5 ++ supervisely/serve/src/workflow.py | 46 +++++++++++ supervisely/train/config.json | 2 +- supervisely/train/src/sly_train.py | 5 ++ supervisely/train/src/sly_train_globals.py | 2 + supervisely/train/src/test.py | 15 ++++ supervisely/train/src/workflow.py | 90 ++++++++++++++++++++++ 9 files changed, 166 insertions(+), 3 deletions(-) create mode 100644 supervisely/serve/src/workflow.py create mode 100644 supervisely/train/src/test.py create mode 100644 supervisely/train/src/workflow.py diff --git a/dev_requirements.txt b/dev_requirements.txt index 71e97944a78f..e8b05cf33723 100755 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,7 +1,7 @@ # git+https://github.com/supervisely/supervisely.git@test-branch # pip install -r requirements.txt -supervisely==6.73.122 +supervisely==6.73.123 opencv-python-headless==4.5.5.62 opencv-python==4.5.5.62 diff --git a/supervisely/serve/config.json b/supervisely/serve/config.json index 10191023d6fb..c13346ee557f 100644 --- a/supervisely/serve/config.json +++ b/supervisely/serve/config.json @@ -11,7 +11,7 @@ "serve" ], "description": "Deploy model as REST API service", - "docker_image": "supervisely/yolov5:1.0.7", + "docker_image": "supervisely/yolov5:1.0.8", "instance_version": "6.8.88", "entrypoint": "python -m uvicorn main:m.app --app-dir ./supervisely/serve/src --host 0.0.0.0 --port 8000 --ws websockets", "port": 8000, diff --git a/supervisely/serve/src/main.py b/supervisely/serve/src/main.py index 920cd902678d..9b23ada39e72 100644 --- a/supervisely/serve/src/main.py +++ b/supervisely/serve/src/main.py @@ -18,11 +18,14 @@ from utils.datasets import letterbox from pathlib import Path + root_source_path = str(Path(__file__).parents[3]) app_source_path = str(Path(__file__).parents[1]) load_dotenv(os.path.join(app_source_path, "local.env")) load_dotenv(os.path.expanduser("~/supervisely.env")) +from workflow import Workflow + model_weights_options = os.environ["modal.state.modelWeightsOptions"] pretrained_weights = os.environ["modal.state.selectedModel"].lower() custom_weights = os.environ["modal.state.weightsPath"] @@ -45,6 +48,8 @@ def load_on_device( self.local_weights_path = self.download(custom_weights) cfg_path_in_teamfiles = os.path.join(Path(custom_weights).parents[1], "opt.yaml") configs_local_path = self.download(cfg_path_in_teamfiles) + workflow = Workflow(self.api) + workflow.add_input(custom_weights) self.device = select_device(device) self.half = self.device.type != "cpu" # half precision only supported on CUDA diff --git a/supervisely/serve/src/workflow.py b/supervisely/serve/src/workflow.py new file mode 100644 index 000000000000..d23b9d6ca8d4 --- /dev/null +++ b/supervisely/serve/src/workflow.py @@ -0,0 +1,46 @@ +import supervisely as sly + + +def check_compatibility(func): + def wrapper(self, *args, **kwargs): + if self.is_compatible is None: + self.is_compatible = self.check_instance_ver_compatibility() + if not self.is_compatible: + return + return func(self, *args, **kwargs) + + return wrapper + + +class Workflow: + def __init__(self, api: sly.Api, min_instance_version: str = None): + self.is_compatible = None + self.api = api + self._min_instance_version = ( + "6.9.31" if min_instance_version is None else min_instance_version + ) + + def check_instance_ver_compatibility(self): + if not self.api.is_version_supported(self._min_instance_version): + sly.logger.info( + f"Supervisely instance version {self.api.instance_version} does not support workflow features." + ) + if not sly.is_community(): + sly.logger.info( + f"To use them, please update your instance to version {self._min_instance_version} or higher." + ) + return False + return True + + @check_compatibility + def add_input(self, checkpoint_url: str): + meta = {"customNodeSettings": {"title": "

Serve Custom Model

"}} + sly.logger.debug(f"Workflow Input: Checkpoint URL - {checkpoint_url}") + if checkpoint_url and self.api.file.exists(sly.env.team_id(), checkpoint_url): + self.api.app.workflow.add_input_file(checkpoint_url, model_weight=True, meta=meta) + else: + sly.logger.debug(f"Checkpoint {checkpoint_url} not found in Team Files. Cannot set workflow input") + + @check_compatibility + def add_output(self): + raise NotImplementedError("add_output is not implemented in this workflow") diff --git a/supervisely/train/config.json b/supervisely/train/config.json index bf435d07dd1f..c5aa41d076cf 100644 --- a/supervisely/train/config.json +++ b/supervisely/train/config.json @@ -10,7 +10,7 @@ "train" ], "description": "Dashboard to configure and monitor training", - "docker_image": "supervisely/yolov5:1.0.7", + "docker_image": "supervisely/yolov5:1.0.8", "min_instance_version": "6.8.70", "main_script": "supervisely/train/src/sly_train.py", "gui_template": "supervisely/train/src/gui.html", diff --git a/supervisely/train/src/sly_train.py b/supervisely/train/src/sly_train.py index c182ef4f2661..1b2b301cdbfd 100644 --- a/supervisely/train/src/sly_train.py +++ b/supervisely/train/src/sly_train.py @@ -40,6 +40,10 @@ def train(api: sly.Api, task_id, context, state, app_logger): project_dir = os.path.join(my_app.data_dir, "sly_project") sly.fs.mkdir(project_dir, remove_content_if_exists=True) # clean content for debug, has no effect in prod + # -------------------------------------- Add Workflow Input -------------------------------------- # + g.workflow.add_input(g.project_info, state) + # ----------------------------------------------- - ---------------------------------------------- # + # download and preprocess Sypervisely project (using cache) try: download_project( @@ -145,6 +149,7 @@ def train(api: sly.Api, task_id, context, state, app_logger): # upload artifacts directory to Team Files upload_artifacts(g.local_artifacts_dir, g.remote_artifacts_dir) set_task_output() + g.workflow.add_output(state, g.remote_artifacts_dir) except Exception as e: msg = f"Something went wrong. Find more info in the app logs." my_app.show_modal_window(f"{msg} {repr(e)}", level="error", log_message=False) diff --git a/supervisely/train/src/sly_train_globals.py b/supervisely/train/src/sly_train_globals.py index 3e0e1d16d6e3..d02b37f4ed75 100644 --- a/supervisely/train/src/sly_train_globals.py +++ b/supervisely/train/src/sly_train_globals.py @@ -6,6 +6,7 @@ from supervisely.nn.artifacts.yolov5 import YOLOv5 from supervisely.app.v1.app_service import AppService from dotenv import load_dotenv +from workflow import Workflow root_source_dir = str(Path(sys.argv[0]).parents[3]) sly.logger.info(f"Root source directory: {root_source_dir}") @@ -35,6 +36,7 @@ project_id = int(os.environ['modal.state.slyProjectId']) api: sly.Api = my_app.public_api +workflow = Workflow(api) task_id = my_app.task_id local_artifacts_dir = None diff --git a/supervisely/train/src/test.py b/supervisely/train/src/test.py new file mode 100644 index 000000000000..9d319f71e11e --- /dev/null +++ b/supervisely/train/src/test.py @@ -0,0 +1,15 @@ +import supervisely as sly +import os + +api = sly.Api.from_env() +api.task_id = 62688 +os.environ.setdefault("TEAM_ID", "451") +from workflow import Workflow + +workflow = Workflow(api) + +state = {"weightsInitialization": "custom"} + +team_files_dir = "/yolov5_train/Train dataset - Eschikon Wheat Segmentation (EWS)/62688" + +workflow.add_output(state, team_files_dir) \ No newline at end of file diff --git a/supervisely/train/src/workflow.py b/supervisely/train/src/workflow.py new file mode 100644 index 000000000000..f8c00de7d456 --- /dev/null +++ b/supervisely/train/src/workflow.py @@ -0,0 +1,90 @@ +# Description: This file contains versioning features and the Workflow class that is used to add input and output to the workflow. + +import supervisely as sly +import os + + +def check_compatibility(func): + def wrapper(self, *args, **kwargs): + if self.is_compatible is None: + self.is_compatible = self.check_instance_ver_compatibility() + if not self.is_compatible: + return + return func(self, *args, **kwargs) + + return wrapper + + +class Workflow: + def __init__(self, api: sly.Api, min_instance_version: str = None): + self.is_compatible = None + self.api = api + self._min_instance_version = ( + "6.9.31" if min_instance_version is None else min_instance_version + ) + + def check_instance_ver_compatibility(self): + if not self.api.is_version_supported(self._min_instance_version): + sly.logger.info( + f"Supervisely instance version {self.api.instance_version} does not support workflow and versioning features." + ) + if not sly.is_community(): + sly.logger.info( + f"To use them, please update your instance to version {self._min_instance_version} or higher." + ) + return False + return True + + @check_compatibility + def add_input(self, project_info: sly.ProjectInfo, state: dict): + project_version_id = self.api.project.version.create( + project_info, "Train YOLO v5", f"This backup was created automatically by Supervisely before the Train YOLO task with ID: {self.api.task_id}" + ) + if project_version_id is None: + project_version_id = project_info.version.get("id", None) if project_info.version else None + self.api.app.workflow.add_input_project(project_info.id, version_id=project_version_id) + if state["weightsInitialization"] is not None and state["weightsInitialization"] == "custom": + file_info = self.api.file.get_info_by_path(sly.env.team_id(), state["_weightsPath"]) + self.api.app.workflow.add_input_file(file_info, model_weight=True) + sly.logger.debug(f"Workflow Input: Project ID - {project_info.id}, Project Version ID - {project_version_id}, Input File - {True if file_info else False}") + + @check_compatibility + def add_output(self, state: dict, team_files_dir: str): + weights_dir_in_team_files = os.path.join(team_files_dir, "weights") + files_info = self.api.file.list(sly.env.team_id(), weights_dir_in_team_files, return_type="fileinfo") + best_filename_info = None + for file_info in files_info: + if "best" in file_info.name: + best_filename_info = file_info + break + if best_filename_info: + module_id = self.api.task.get_info_by_id(self.api.task_id).get("meta", {}).get("app", {}).get("id") + if state["weightsInitialization"] is not None and state["weightsInitialization"] == "custom": + model_name = "Custom Model" + else: + model_name = "YOLOv5" + + meta = { + "customNodeSettings": { + "title": f"

Train {model_name}

", + "mainLink": { + "url": f"/apps/{module_id}/sessions/{self.api.task_id}" if module_id else f"apps/sessions/{self.api.task_id}", + "title": "Show Results" + } + }, + "customRelationSettings": { + "icon": { + "icon": "zmdi-folder", + "color": "#FFA500", + "backgroundColor": "#FFE8BE" + }, + "title": "

Checkpoints

", + "mainLink": {"url": f"/files/{best_filename_info.id}/true", "title": "Open Folder"} + } + } + sly.logger.debug(f"Workflow Output: Team Files dir - {team_files_dir}, Best filename - {best_filename_info.name}") + sly.logger.debug(f"Workflow Output: meta \n {meta}") + self.api.app.workflow.add_output_file(best_filename_info, model_weight=True, meta=meta) + else: + sly.logger.debug(f"File with the best weighs not found in Team Files. Cannot set workflow output.") + From bd9ef56674bc19fe4ca8d86602fd98996da35949 Mon Sep 17 00:00:00 2001 From: GoldenAnpu Date: Fri, 12 Jul 2024 13:14:01 +0200 Subject: [PATCH 2/4] Remove unused test.py file --- supervisely/train/src/test.py | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 supervisely/train/src/test.py diff --git a/supervisely/train/src/test.py b/supervisely/train/src/test.py deleted file mode 100644 index 9d319f71e11e..000000000000 --- a/supervisely/train/src/test.py +++ /dev/null @@ -1,15 +0,0 @@ -import supervisely as sly -import os - -api = sly.Api.from_env() -api.task_id = 62688 -os.environ.setdefault("TEAM_ID", "451") -from workflow import Workflow - -workflow = Workflow(api) - -state = {"weightsInitialization": "custom"} - -team_files_dir = "/yolov5_train/Train dataset - Eschikon Wheat Segmentation (EWS)/62688" - -workflow.add_output(state, team_files_dir) \ No newline at end of file From 53686babf2d75f08c5b6f0f8b267c2f5517a6151 Mon Sep 17 00:00:00 2001 From: GoldenAnpu Date: Fri, 12 Jul 2024 14:02:10 +0200 Subject: [PATCH 3/4] Fix referenced var --- supervisely/train/src/workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/supervisely/train/src/workflow.py b/supervisely/train/src/workflow.py index f8c00de7d456..2465bb475959 100644 --- a/supervisely/train/src/workflow.py +++ b/supervisely/train/src/workflow.py @@ -43,6 +43,7 @@ def add_input(self, project_info: sly.ProjectInfo, state: dict): if project_version_id is None: project_version_id = project_info.version.get("id", None) if project_info.version else None self.api.app.workflow.add_input_project(project_info.id, version_id=project_version_id) + file_info = False if state["weightsInitialization"] is not None and state["weightsInitialization"] == "custom": file_info = self.api.file.get_info_by_path(sly.env.team_id(), state["_weightsPath"]) self.api.app.workflow.add_input_file(file_info, model_weight=True) From 267391fc2478b18759c19394f9ef31487ba38a54 Mon Sep 17 00:00:00 2001 From: GoldenAnpu Date: Fri, 12 Jul 2024 14:36:11 +0200 Subject: [PATCH 4/4] Add error handlings to the workflow --- supervisely/serve/src/workflow.py | 17 +++-- supervisely/train/src/workflow.py | 104 ++++++++++++++++-------------- 2 files changed, 67 insertions(+), 54 deletions(-) diff --git a/supervisely/serve/src/workflow.py b/supervisely/serve/src/workflow.py index d23b9d6ca8d4..48263f02cb32 100644 --- a/supervisely/serve/src/workflow.py +++ b/supervisely/serve/src/workflow.py @@ -34,13 +34,16 @@ def check_instance_ver_compatibility(self): @check_compatibility def add_input(self, checkpoint_url: str): - meta = {"customNodeSettings": {"title": "

Serve Custom Model

"}} - sly.logger.debug(f"Workflow Input: Checkpoint URL - {checkpoint_url}") - if checkpoint_url and self.api.file.exists(sly.env.team_id(), checkpoint_url): - self.api.app.workflow.add_input_file(checkpoint_url, model_weight=True, meta=meta) - else: - sly.logger.debug(f"Checkpoint {checkpoint_url} not found in Team Files. Cannot set workflow input") - + try: + meta = {"customNodeSettings": {"title": "

Serve Custom Model

"}} + sly.logger.debug(f"Workflow Input: Checkpoint URL - {checkpoint_url}") + if checkpoint_url and self.api.file.exists(sly.env.team_id(), checkpoint_url): + self.api.app.workflow.add_input_file(checkpoint_url, model_weight=True, meta=meta) + else: + sly.logger.debug(f"Checkpoint {checkpoint_url} not found in Team Files. Cannot set workflow input") + except Exception as e: + sly.logger.error(f"Failed to add input to the workflow: {e}") + @check_compatibility def add_output(self): raise NotImplementedError("add_output is not implemented in this workflow") diff --git a/supervisely/train/src/workflow.py b/supervisely/train/src/workflow.py index 2465bb475959..546cf0bd4fda 100644 --- a/supervisely/train/src/workflow.py +++ b/supervisely/train/src/workflow.py @@ -37,55 +37,65 @@ def check_instance_ver_compatibility(self): @check_compatibility def add_input(self, project_info: sly.ProjectInfo, state: dict): - project_version_id = self.api.project.version.create( - project_info, "Train YOLO v5", f"This backup was created automatically by Supervisely before the Train YOLO task with ID: {self.api.task_id}" - ) - if project_version_id is None: - project_version_id = project_info.version.get("id", None) if project_info.version else None - self.api.app.workflow.add_input_project(project_info.id, version_id=project_version_id) - file_info = False - if state["weightsInitialization"] is not None and state["weightsInitialization"] == "custom": - file_info = self.api.file.get_info_by_path(sly.env.team_id(), state["_weightsPath"]) - self.api.app.workflow.add_input_file(file_info, model_weight=True) - sly.logger.debug(f"Workflow Input: Project ID - {project_info.id}, Project Version ID - {project_version_id}, Input File - {True if file_info else False}") + try: + project_version_id = self.api.project.version.create( + project_info, "Train YOLO v5", f"This backup was created automatically by Supervisely before the Train YOLO task with ID: {self.api.task_id}" + ) + except Exception as e: + sly.logger.error(f"Failed to create a project version: {e}") + project_version_id = None + + try: + if project_version_id is None: + project_version_id = project_info.version.get("id", None) if project_info.version else None + self.api.app.workflow.add_input_project(project_info.id, version_id=project_version_id) + file_info = False + if state["weightsInitialization"] is not None and state["weightsInitialization"] == "custom": + file_info = self.api.file.get_info_by_path(sly.env.team_id(), state["_weightsPath"]) + self.api.app.workflow.add_input_file(file_info, model_weight=True) + sly.logger.debug(f"Workflow Input: Project ID - {project_info.id}, Project Version ID - {project_version_id}, Input File - {True if file_info else False}") + except Exception as e: + sly.logger.error(f"Failed to add input to the workflow: {e}") @check_compatibility def add_output(self, state: dict, team_files_dir: str): - weights_dir_in_team_files = os.path.join(team_files_dir, "weights") - files_info = self.api.file.list(sly.env.team_id(), weights_dir_in_team_files, return_type="fileinfo") - best_filename_info = None - for file_info in files_info: - if "best" in file_info.name: - best_filename_info = file_info - break - if best_filename_info: - module_id = self.api.task.get_info_by_id(self.api.task_id).get("meta", {}).get("app", {}).get("id") - if state["weightsInitialization"] is not None and state["weightsInitialization"] == "custom": - model_name = "Custom Model" - else: - model_name = "YOLOv5" - - meta = { - "customNodeSettings": { - "title": f"

Train {model_name}

", - "mainLink": { - "url": f"/apps/{module_id}/sessions/{self.api.task_id}" if module_id else f"apps/sessions/{self.api.task_id}", - "title": "Show Results" - } - }, - "customRelationSettings": { - "icon": { - "icon": "zmdi-folder", - "color": "#FFA500", - "backgroundColor": "#FFE8BE" + try: + weights_dir_in_team_files = os.path.join(team_files_dir, "weights") + files_info = self.api.file.list(sly.env.team_id(), weights_dir_in_team_files, return_type="fileinfo") + best_filename_info = None + for file_info in files_info: + if "best" in file_info.name: + best_filename_info = file_info + break + if best_filename_info: + module_id = self.api.task.get_info_by_id(self.api.task_id).get("meta", {}).get("app", {}).get("id") + if state["weightsInitialization"] is not None and state["weightsInitialization"] == "custom": + model_name = "Custom Model" + else: + model_name = "YOLOv5" + + meta = { + "customNodeSettings": { + "title": f"

Train {model_name}

", + "mainLink": { + "url": f"/apps/{module_id}/sessions/{self.api.task_id}" if module_id else f"apps/sessions/{self.api.task_id}", + "title": "Show Results" + } }, - "title": "

Checkpoints

", - "mainLink": {"url": f"/files/{best_filename_info.id}/true", "title": "Open Folder"} + "customRelationSettings": { + "icon": { + "icon": "zmdi-folder", + "color": "#FFA500", + "backgroundColor": "#FFE8BE" + }, + "title": "

Checkpoints

", + "mainLink": {"url": f"/files/{best_filename_info.id}/true", "title": "Open Folder"} + } } - } - sly.logger.debug(f"Workflow Output: Team Files dir - {team_files_dir}, Best filename - {best_filename_info.name}") - sly.logger.debug(f"Workflow Output: meta \n {meta}") - self.api.app.workflow.add_output_file(best_filename_info, model_weight=True, meta=meta) - else: - sly.logger.debug(f"File with the best weighs not found in Team Files. Cannot set workflow output.") - + sly.logger.debug(f"Workflow Output: Team Files dir - {team_files_dir}, Best filename - {best_filename_info.name}") + sly.logger.debug(f"Workflow Output: meta \n {meta}") + self.api.app.workflow.add_output_file(best_filename_info, model_weight=True, meta=meta) + else: + sly.logger.debug(f"File with the best weighs not found in Team Files. Cannot set workflow output.") + except Exception as e: + sly.logger.error(f"Failed to add output to the workflow: {e}")