Restructured API files (#2285)

* API mock

* WIP

* Minor change in timer

* Better types for iterator nodes

* Added range iterator

* Disable type check for collectors and iterators

* Restructured API files

* Fixed invalid replacement

* Reformat server.py

* Fixed group errors

* `api2` -> `api`

* Removed duplicated comment

---------

Co-authored-by: Joey Ballentine <34788790+joeyballentine@users.noreply.github.com>

backend/src/api/__init__.py

@@ -0,0 +1,6 @@
+from .api import *
+from .group import *
+from .input import *
+from .output import *
+from .settings import *
+from .types import *

backend/src/api.py → backend/src/api/api.py

@@ -20,20 +20,20 @@
 from sanic.log import logger
 
 import navi
-from base_types import InputId, OutputId
-from custom_types import NodeType, RunFn
-from node_check import (
+
+from .group import Group, GroupId, NestedGroup, NestedIdGroup
+from .input import BaseInput
+from .node_check import (
     NAME_CHECK_LEVEL,
     TYPE_CHECK_LEVEL,
     CheckFailedError,
     CheckLevel,
     check_naming_conventions,
     check_schema_types,
 )
-from nodes.base_input import BaseInput
-from nodes.base_output import BaseOutput
-from nodes.group import Group, GroupId, NestedGroup, NestedIdGroup
-from nodes.utils.exec_options import SettingsJson, get_execution_options
+from .output import BaseOutput
+from .settings import SettingsJson, get_execution_options
+from .types import InputId, NodeType, OutputId, RunFn
 
 KB = 1024**1
 MB = 1024**2

backend/src/nodes/group.py → backend/src/api/group.py

@@ -1,7 +1,7 @@
 from typing import Any, Dict, Generic, List, NewType, Optional, TypeVar, Union
 
-from base_types import InputId
-from nodes.base_input import BaseInput
+from .input import BaseInput
+from .types import InputId
 
 T = TypeVar("T")
 

backend/src/nodes/base_input.py → backend/src/api/input.py

@@ -5,7 +5,8 @@
 from typing import List, Literal, Optional, Type, TypedDict, Union
 
 import navi
-from base_types import InputId
+
+from .types import InputId
 
 InputKind = Literal[
     "number",

backend/src/node_check.py → backend/src/api/node_check.py

@@ -7,8 +7,8 @@
 from enum import Enum
 from typing import Any, Callable, Dict, List, NewType, Set, Union, cast, get_args
 
-from nodes.base_input import BaseInput
-from nodes.base_output import BaseOutput
+from .input import BaseInput
+from .output import BaseOutput
 
 _Ty = NewType("_Ty", object)
 

backend/src/nodes/base_output.py → backend/src/api/output.py

@@ -3,7 +3,8 @@
 from typing import Literal, Type, Union
 
 import navi
-from base_types import OutputId
+
+from .types import OutputId
 
 OutputKind = Literal["image", "large-image", "tagged", "generic"]
 

backend/src/api/types.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Literal, NewType
+
+NodeId = NewType("NodeId", str)
+InputId = NewType("InputId", int)
+OutputId = NewType("OutputId", int)
+
+
+RunFn = Callable[..., Any]
+
+NodeType = Literal["regularNode", "newIterator", "collector"]

backend/src/chain/cache.py

@@ -5,7 +5,9 @@
 
 from sanic.log import logger
 
-from .chain import Chain, Edge, FunctionNode, NewIteratorNode, NodeId
+from api import NodeId
+
+from .chain import Chain, Edge, FunctionNode, NewIteratorNode
 
 
 class CacheStrategy:

backend/src/chain/chain.py

@@ -2,8 +2,7 @@
 
 from typing import Callable, Dict, List, Set, TypeVar, Union
 
-from api import NodeData, registry
-from base_types import InputId, NodeId, OutputId
+from api import InputId, NodeData, NodeId, OutputId, registry
 
 K = TypeVar("K")
 V = TypeVar("V")

backend/src/chain/input.py

@@ -1,6 +1,6 @@
 from typing import Dict, List, Optional, Union
 
-from base_types import NodeId
+from api import NodeId
 
 
 class EdgeInput:

backend/src/chain/json.py

@@ -1,6 +1,6 @@
 from typing import List, Literal, Optional, Tuple, TypedDict, Union
 
-from base_types import NodeId
+from api import NodeId
 
 from .chain import (
     Chain,

backend/src/custom_types.py

@@ -1,9 +1,5 @@
 from __future__ import annotations
 
-from typing import Any, Awaitable, Callable, Literal, Union
-
-RunFn = Callable[..., Any]
-
-NodeType = Literal["regularNode", "newIterator", "collector"]
+from typing import Awaitable, Callable, Union
 
 UpdateProgressFn = Callable[[str, float, Union[float, None]], Awaitable[None]]

backend/src/events.py

@@ -1,8 +1,7 @@
 import asyncio
 from typing import Dict, Literal, Optional, TypedDict, Union
 
-from base_types import InputId, NodeId, OutputId
-from nodes.base_input import ErrorValue
+from api import ErrorValue, InputId, NodeId, OutputId
 
 # Data of events
 

backend/src/nodes/groups.py

@@ -4,9 +4,7 @@
 from typing import Iterable, List, Literal, Tuple, TypedDict, Union
 
 import navi
-from base_types import InputId
-from nodes.base_input import BaseInput
-from nodes.group import NestedGroup, group
+from api import BaseInput, InputId, NestedGroup, group
 
 InputValue = Union[int, str]
 EnumValues = Union[
@@ -218,3 +216,18 @@ def linked_inputs_group(*inputs: BaseInput):
     that the inputs are of the same class and use the same parameters.
     """
     return group("linked-inputs")(*inputs)
+
+
+def ncnn_file_inputs_group(param_input: BaseInput, bin_input: BaseInput):
+    """
+    This group wraps around 2 .param and .bin file inputs and synchronizes them in the UI.
+    """
+    return group("ncnn-file-inputs")(param_input, bin_input)
+
+
+def from_to_dropdowns_group(from_dd: BaseInput, to_dd: BaseInput):
+    """
+    This group wraps around 2 dropdown inputs that will be displayed as
+    `[From] -> [To]` in the UI.
+    """
+    return group("from-to-dropdowns")(from_dd, to_dd)

backend/src/nodes/properties/inputs/__init__.py

@@ -1,5 +1,3 @@
-from nodes.base_input import *
-
 from .file_inputs import *
 from .generic_inputs import *
 from .image_dropdown_inputs import *

backend/src/nodes/properties/inputs/file_inputs.py

@@ -3,7 +3,7 @@
 import os
 from typing import Literal, Union
 
-from nodes.base_input import BaseInput
+from api import BaseInput
 
 # pylint: disable=relative-beyond-top-level
 from ...impl.image_formats import get_available_image_formats

backend/src/nodes/properties/inputs/generic_inputs.py

@@ -9,7 +9,7 @@
 from sanic.log import logger
 
 import navi
-from nodes.base_input import BaseInput, InputConversion
+from api import BaseInput, InputConversion
 
 from ...impl.blend import BlendMode
 from ...impl.color.color import Color

backend/src/nodes/properties/inputs/ncnn_inputs.py

@@ -1,4 +1,4 @@
-from nodes.base_input import BaseInput
+from api import BaseInput
 
 from ...impl.ncnn.model import NcnnModelWrapper
 

backend/src/nodes/properties/inputs/numeric_inputs.py

@@ -2,7 +2,7 @@
 from typing import List, Literal, Tuple, Union
 
 import navi
-from nodes.base_input import BaseInput, InputConversion, InputKind
+from api import BaseInput, InputConversion, InputKind
 
 from ...utils.utils import round_half_up
 

backend/src/nodes/properties/inputs/numpy_inputs.py

@@ -6,7 +6,7 @@
 import numpy as np
 
 import navi
-from nodes.base_input import BaseInput, ErrorValue
+from api import BaseInput, ErrorValue
 
 from ...impl.color.color import Color
 from ...utils.format import format_color_with_channels, format_image_with_channels

backend/src/nodes/properties/inputs/onnx_inputs.py

@@ -1,5 +1,5 @@
 import navi
-from nodes.base_input import BaseInput
+from api import BaseInput
 
 from ...impl.onnx.model import OnnxModel, OnnxModels, OnnxRemBg, is_rembg_model
 from .generic_inputs import DropDownInput

backend/src/nodes/properties/inputs/pytorch_inputs.py

@@ -15,7 +15,7 @@
     torch = None
 
 import navi
-from nodes.base_input import BaseInput
+from api import BaseInput
 
 
 class ModelInput(BaseInput):

backend/src/nodes/properties/outputs/__init__.py

@@ -1,5 +1,3 @@
-from nodes.base_output import *
-
 from .file_outputs import *
 from .generic_outputs import *
 from .numpy_outputs import *

backend/src/nodes/properties/outputs/file_outputs.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import navi
-from nodes.base_output import BaseOutput
+from api import BaseOutput
 
 
 class DirectoryOutput(BaseOutput):

backend/src/nodes/properties/outputs/generic_outputs.py

@@ -3,7 +3,7 @@
 from typing import Union
 
 import navi
-from nodes.base_output import BaseOutput
+from api import BaseOutput
 
 from ...impl.color.color import Color
 from ...utils.format import format_color_with_channels

backend/src/nodes/properties/outputs/ncnn_outputs.py

@@ -1,5 +1,5 @@
 import navi
-from nodes.base_output import BaseOutput, OutputKind
+from api import BaseOutput, OutputKind
 
 from ...impl.ncnn.model import NcnnModelWrapper
 from ...utils.format import format_channel_numbers

backend/src/nodes/properties/outputs/numpy_outputs.py

@@ -5,7 +5,7 @@
 import numpy as np
 
 import navi
-from nodes.base_output import BaseOutput, OutputKind
+from api import BaseOutput, OutputKind
 
 from ...impl.image_utils import normalize, to_uint8
 from ...impl.pil_utils import InterpolationMethod, resize

backend/src/nodes/properties/outputs/onnx_outputs.py

@@ -1,5 +1,5 @@
 import navi
-from nodes.base_output import BaseOutput
+from api import BaseOutput
 
 from ...impl.onnx.model import OnnxModel
 

backend/src/nodes/properties/outputs/pytorch_outputs.py

@@ -1,7 +1,7 @@
 from typing import List
 
 import navi
-from nodes.base_output import BaseOutput, OutputKind
+from api import BaseOutput, OutputKind
 
 from ...impl.pytorch.types import PyTorchModel
 from ...utils.format import format_channel_numbers

backend/src/packages/chaiNNer_ncnn/ncnn/io/load_model.py

@@ -2,7 +2,7 @@
 
 from typing import Tuple
 
-from nodes.group import group
+from nodes.groups import ncnn_file_inputs_group
 from nodes.impl.ncnn.model import NcnnModel, NcnnModelWrapper
 from nodes.impl.ncnn.optimizer import NcnnOptimizer
 from nodes.properties.inputs import BinFileInput, ParamFileInput
@@ -21,7 +21,7 @@
     ),
     icon="NCNN",
     inputs=[
-        group("ncnn-file-inputs")(
+        ncnn_file_inputs_group(
             ParamFileInput(primary_input=True),
             BinFileInput(primary_input=True),
         )

backend/src/packages/chaiNNer_standard/image_utility/miscellaneous/change_colorspace.py

@@ -3,8 +3,7 @@
 import numpy as np
 
 import navi
-from nodes.group import group
-from nodes.groups import if_enum_group
+from nodes.groups import from_to_dropdowns_group, if_enum_group
 from nodes.impl.color.convert import (
     color_space_from_id,
     color_space_or_detector_from_id,
@@ -36,7 +35,7 @@
     icon="MdColorLens",
     inputs=[
         ImageInput(image_type=navi.Image(channels="Input1.channels")),
-        group("from-to-dropdowns")(
+        from_to_dropdowns_group(
             ColorSpaceDetectorInput(label="From").with_id(1),
             ColorSpaceInput(label="To").with_id(2),
         ),

backend/src/process.py

@@ -12,13 +12,11 @@
 
 from sanic.log import logger
 
-from api import Collector, Iterator, NodeData
-from base_types import InputId, NodeId, OutputId
+from api import BaseOutput, Collector, InputId, Iterator, NodeData, NodeId, OutputId
 from chain.cache import CacheStrategy, OutputCache, StaticCaching, get_cache_strategies
 from chain.chain import Chain, CollectorNode, FunctionNode, NewIteratorNode, Node
 from chain.input import EdgeInput, Input, InputMap
 from events import Event, EventQueue, InputsDict
-from nodes.base_output import BaseOutput
 from progress_controller import Aborted, ProgressController
 from util import timed_supplier
 

backend/src/server.py

@@ -18,7 +18,13 @@
 from sanic_cors import CORS
 
 import api
-from base_types import NodeId
+from api import (
+    ExecutionOptions,
+    Group,
+    JsonExecutionOptions,
+    NodeId,
+    set_execution_options,
+)
 from chain.cache import OutputCache
 from chain.chain import Chain, FunctionNode
 from chain.input import InputMap
@@ -28,12 +34,6 @@
 from dependencies.store import DependencyInfo, install_dependencies, installed_packages
 from events import EventQueue, ExecutionErrorData
 from gpu import get_nvidia_helper
-from nodes.group import Group
-from nodes.utils.exec_options import (
-    ExecutionOptions,
-    JsonExecutionOptions,
-    set_execution_options,
-)
 from process import Executor, NodeExecutionError, NodeOutput
 from progress_controller import Aborted
 from response import (

backend/src/settings.py

@@ -1,8 +1,7 @@
 from dataclasses import dataclass
 from typing import Any
 
-from api import SettingsParser
-from nodes.utils.exec_options import get_execution_options
+from api import SettingsParser, get_execution_options
 
 
 @dataclass(frozen=True)