Add internal API typehints

doc/changelog.rst

@@ -34,28 +34,30 @@ A full list of changes and detailed notes can be found below:
 - Relax the coloredlogs version
 - Update Fortran tutorials for SmartRedis
 - Add support for multiple network interface binding in Orchestrator and Colocated DBs
+- Add typehints and static analysis
 
 Detailed notes
 
+- Typehints have been added. A makefile target `make check-mypy` executes static
+  analysis with mypy. (PR295_, PR303_)
 - Simplify code in `random_permutations` parameter generation strategy (PR300_)
 - Remove wait time associated with Experiment launch summary (PR298_)
 - Update Redis conf file to conform with Redis v7.0.5 conf file (PR293_)
 - Migrate from redis-py-cluster to redis-py for cluster status checks (PR292_)
 - Update full test suite to no longer require a tensorflow wheel to be available
   at test time. (PR291_)
 - Correct spelling of colocated in doc strings (PR290_)
-- Deprecated launcher-specific orchestrators, constants, and ML utilities
-were removed. (PR289_)
+- Deprecated launcher-specific orchestrators, constants, and ML 
+  utilities were removed. (PR289_)
 - Relax the coloredlogs version to be greater than 10.0 (PR288_)
 - Update the Github Actions runner image from `macos-10.15`` to `macos-12``. The
-former began deprecation in May 2022 and was finally removed in May 2023. (PR285_)
-- The Fortran tutorials had not been fully updated to show how to handle return/error
-codes. These have now all been updated. (PR284_)
-- Orchestrator and Colocated DB now accept a list of interfaces to bind to. The
-argument name is still `interface` for backward compatibility reasons. (PR281_)
-- Typehints have been added to public APIs. A makefile target to execute static 
-analysis with mypy is available `make check-mypy`. (PR295_)
+  former began deprecation in May 2022 and was finally removed in May 2023. (PR285_)
+- The Fortran tutorials had not been fully updated to show how to handle 
+  return/error codes. These have now all been updated. (PR284_)
+- Orchestrator and Colocated DB now accept a list of interfaces to bind to. The 
+  argument name is still `interface` for backward compatibility reasons. (PR281_)
 
+.. _PR303: https://github.com/CrayLabs/SmartSim/pull/303
 .. _PR300: https://github.com/CrayLabs/SmartSim/pull/300
 .. _PR298: https://github.com/CrayLabs/SmartSim/pull/298
 .. _PR293: https://github.com/CrayLabs/SmartSim/pull/293

pyproject.toml

@@ -72,25 +72,6 @@ files = [
   "smartsim"
 ]
 plugins = []
-ignore_errors = true
-
-[[tool.mypy.overrides]]
-# Ignore packages that are not used or not typed
-module = [
-  "coloredlogs",
-  "smartredis",
-  "smartredis.error"
-]
-ignore_missing_imports = true
-ignore_errors = true
-
-[[tool.mypy.overrides]]
-module = [
-  "smartsim.database.*",
-  "smartsim.entity.*",
-  "smartsim.experiment"
-]
-
 ignore_errors=false
 
 # Strict fn defs
@@ -102,3 +83,18 @@ disallow_untyped_decorators = true
 # Safety/Upgrading Mypy
 warn_unused_ignores = true
 # warn_redundant_casts = true # not a per-module setting?
+
+[[tool.mypy.overrides]]
+# Ignore packages that are not used or not typed
+module = [
+  "coloredlogs",
+  "smartredis",
+  "smartredis.error",
+  "redis.cluster",
+  "keras",
+  "torch",
+  "smartsim._core.launcher.step.*", # hints incomplete
+  "smartsim.ml.torch.*",            # must solve/ignore inheritance issues
+]
+ignore_missing_imports = true
+ignore_errors = true

smartsim/_core/_cli/build.py

@@ -30,11 +30,12 @@
 from pathlib import Path
 
 import pkg_resources
+import typing as t
 from tabulate import tabulate
 
 from smartsim._core._cli.utils import color_bool, pip_install
 from smartsim._core._install import builder
-from smartsim._core._install.buildenv import BuildEnv, SetupError, Version_, Versioner
+from smartsim._core._install.buildenv import BuildEnv, SetupError, Version_, Versioner, DbEngine
 from smartsim._core._install.builder import BuildError
 from smartsim._core.config import CONFIG
 from smartsim._core.utils.helpers import installed_redisai_backends
@@ -48,7 +49,8 @@
 #       may be installed into a different directory.
 
 
-def _install_torch_from_pip(versions, device="cpu", verbose=False):
+
+def _install_torch_from_pip(versions: Versioner, device: str = "cpu", verbose: bool = False) -> None:
 
     packages = []
     end_point = None
@@ -76,7 +78,7 @@ def _install_torch_from_pip(versions, device="cpu", verbose=False):
 
 
 class Build:
-    def __init__(self):
+    def __init__(self) -> None:
         parser = argparse.ArgumentParser()
         parser.add_argument(
             "-v",
@@ -154,11 +156,13 @@ def __init__(self):
             else:
                 logger.info("Checking for build tools...")
                 self.build_env = BuildEnv()
-
+                
                 if self.verbose:
                     logger.info("Build Environment:")
                     env = self.build_env.as_dict()
-                    print(tabulate(env, headers=env.keys(), tablefmt="github"), "\n")
+                    env_vars = list(env.keys())
+                    print(tabulate(env, headers=env_vars, tablefmt="github"), "\n")
+
                 if self.keydb:
                     self.versions.REDIS = Version_("6.2.0")
                     self.versions.REDIS_URL = "https://github.com/EQ-Alpha/KeyDB"
@@ -170,10 +174,11 @@ def __init__(self):
                         )
 
                 if self.verbose:
-                    db_name = "KEYDB" if self.keydb else "REDIS"
+                    db_name: DbEngine = "KEYDB" if self.keydb else "REDIS"
                     logger.info("Version Information:")
                     vers = self.versions.as_dict(db_name=db_name)
-                    print(tabulate(vers, headers=vers.keys(), tablefmt="github"), "\n")
+                    version_names = list(vers.keys())
+                    print(tabulate(vers, headers=version_names, tablefmt="github"), "\n")
 
                 # REDIS/KeyDB
                 self.build_database()
@@ -201,7 +206,7 @@ def __init__(self):
 
         logger.info("SmartSim build complete!")
 
-    def build_database(self):
+    def build_database(self) -> None:
         # check database installation
         database_name = "KeyDB" if self.keydb else "Redis"
         database_builder = builder.DatabaseBuilder(
@@ -218,8 +223,14 @@ def build_database(self):
         logger.info(f"{database_name} build complete!")
 
     def build_redis_ai(
-        self, device, torch=True, tf=True, onnx=False, torch_dir=None, libtf_dir=None
-    ):
+        self,
+        device: str,
+        torch: bool = True,
+        tf: bool = True,
+        onnx: bool = False,
+        torch_dir: t.Union[str, Path, None] = None,
+        libtf_dir: t.Union[str, Path, None] = None
+    ) -> None:
 
         # make sure user isn't trying to do something silly on MacOS
         if self.build_env.PLATFORM == "darwin" and device == "gpu":
@@ -268,8 +279,8 @@ def build_redis_ai(
 
         rai_builder = builder.RedisAIBuilder(
             build_env=self.build_env(),
-            torch_dir=str(torch_dir) if torch_dir else None,
-            libtf_dir=str(libtf_dir) if libtf_dir else None,
+            torch_dir=str(torch_dir) if torch_dir else "",
+            libtf_dir=str(libtf_dir) if libtf_dir else "",
             build_torch=torch,
             build_tf=tf,
             build_onnx=onnx,
@@ -313,14 +324,14 @@ def build_redis_ai(
             )
             logger.info("ML Backends and RedisAI build complete!")
 
-    def infer_torch_device(self):
+    def infer_torch_device(self) -> str:
         backend_torch_path = f"{CONFIG.lib_path}/backends/redisai_torch"
         device = "cpu"
         if Path(f"{backend_torch_path}/lib/libtorch_cuda.so").is_file():
             device = "gpu"
         return device
 
-    def install_torch(self, device="cpu"):
+    def install_torch(self, device: str = "cpu") -> None:
         """Torch shared libraries installed by pip are used in the build
         for SmartSim backends so we download them here.
         """
@@ -353,7 +364,7 @@ def install_torch(self, device="cpu"):
                     logger.error(msg)  # error because this is usually fatal
             logger.info(f"Torch {self.versions.TORCH} installed in Python environment")
 
-    def check_onnx_install(self):
+    def check_onnx_install(self) -> None:
         """Check Python environment for ONNX installation"""
         if not self.versions.ONNX:
             py_version = sys.version_info
@@ -380,7 +391,7 @@ def check_onnx_install(self):
         except SetupError as e:
             logger.warning(str(e))
 
-    def check_tf_install(self):
+    def check_tf_install(self) -> None:
         """Check Python environment for TensorFlow installation"""
 
         try:
@@ -399,7 +410,7 @@ def check_tf_install(self):
         except SetupError as e:
             logger.warning(str(e))
 
-    def check_backends_install(self):
+    def check_backends_install(self) -> None:
         """Checks if backends have already been installed.
         Logs details on how to proceed forward
         if the RAI_PATH environment variable is set or if

smartsim/_core/_cli/clean.py

@@ -36,7 +36,7 @@
 
 
 class Clean:
-    def __init__(self, clean_all=False):
+    def __init__(self, clean_all: bool = False) -> None:
         parser = argparse.ArgumentParser(
             description="Remove previous ML runtime installation"
         )
@@ -52,7 +52,7 @@ def __init__(self, clean_all=False):
         clobber = args.clobber or clean_all
         self.clean(_all=clobber)
 
-    def clean(self, _all=False):
+    def clean(self, _all: bool = False) -> None:
         """Remove pre existing installations of ML runtimes
 
         :param _all: Remove all non-python dependencies

smartsim/_core/_cli/cli.py

@@ -34,7 +34,7 @@
 from smartsim._core._cli.utils import get_install_path
 
 
-def _usage():
+def _usage() -> str:
     usage = [
         "smart <command> [<args>]\n",
         "Commands:",
@@ -49,7 +49,7 @@ def _usage():
 
 
 class SmartCli:
-    def __init__(self):
+    def __init__(self) -> None:
         parser = argparse.ArgumentParser(
             description="SmartSim command line interface", usage=_usage()
         )
@@ -67,23 +67,23 @@ def __init__(self):
             sys.exit(0)
         getattr(self, args.command)()
 
-    def build(self):
+    def build(self) -> None:
         Build()
         sys.exit(0)
 
-    def clean(self):
+    def clean(self) -> None:
         Clean()
         sys.exit(0)
 
-    def clobber(self):
+    def clobber(self) -> None:
         Clean(clean_all=True)
         sys.exit(0)
 
-    def site(self):
+    def site(self) -> None:
         print(get_install_path())
         sys.exit(0)
 
-    def dbcli(self):
+    def dbcli(self) -> None:
         bin_path = get_install_path() / "_core" / "bin"
         for option in bin_path.iterdir():
             if option.name in ("redis-cli", "keydb-cli"):
@@ -93,5 +93,5 @@ def dbcli(self):
         sys.exit(1)
 
 
-def main():
+def main() -> None:
     SmartCli()

smartsim/_core/_cli/utils.py

@@ -25,6 +25,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import subprocess
+import typing as t
 from pathlib import Path
 
 from smartsim._core._install.buildenv import SetupError
@@ -36,7 +37,7 @@
 logger = get_logger("Smart", fmt=smart_logger_format)
 
 
-def get_install_path():
+def get_install_path() -> Path:
     try:
         import smartsim as _
     except (ImportError, ModuleNotFoundError):
@@ -50,27 +51,27 @@ def get_install_path():
     return package_path
 
 
-def color_bool(trigger=True):
+def color_bool(trigger: bool = True) -> str:
     _color = "green" if trigger else "red"
     return colorize(str(trigger), color=_color)
 
 
-def pip_install(packages, end_point=None, verbose=False):
+def pip_install(packages: t.List[str], end_point: t.Optional[str] = None, verbose: bool = False) -> None:
     """Install a pip package to be used in the SmartSim build
     Currently only Torch shared libraries are re-used for the build
     """
+    # form pip install command
+    cmd = ["python", "-m", "pip", "install"]
+    cmd.extend(packages)
     if end_point:
-        packages.append(f"-f {end_point}")
-    packages = " ".join(packages)
+        cmd.extend(["-f", end_point])
 
-    # form pip install command
-    cmd = ["python", "-m", "pip", "install", packages]
-    cmd = " ".join(cmd)
+    cmd_arg = " ".join(cmd)
 
     if verbose:
         logger.info(f"Installing packages {packages}...")
     proc = subprocess.Popen(
-        cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        cmd_arg, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
     )
     _, err = proc.communicate()
     returncode = int(proc.returncode)