From fb967d99bdfc0291b46dba551a3d1d1d8e218020 Mon Sep 17 00:00:00 2001 From: Matt Drozt Date: Wed, 22 Mar 2023 11:43:45 -0500 Subject: [PATCH] Dependency Updates (#256) Update supported versions of python to 3.8-3.10. Drop support for Redis AI 1.2.3. [ committed by: @MattToast ] [ reviewed by: @al-rigazzi @billschereriii @ashao ] --- .github/workflows/run_tests.yml | 25 +++++++------ README.md | 12 ++++--- doc/changelog.rst | 11 ++++-- doc/installation.rst | 13 ++++--- setup.cfg | 4 +-- setup.py | 8 ++--- smartsim/__init__.py | 4 +-- smartsim/_core/_cli/build.py | 39 ++++++++++---------- smartsim/_core/_cli/cli.py | 1 - smartsim/_core/_install/buildenv.py | 56 ++++++++++++++++++----------- smartsim/_core/_install/builder.py | 38 +++++++++++++++----- 11 files changed, 128 insertions(+), 83 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index d8f31bc2a..db1de5edc 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -51,13 +51,16 @@ jobs: matrix: os: [macos-10.15, ubuntu-20.04] # Operating systems compiler: [8] # GNU compiler version - rai: [1.2.3, 1.2.5, 1.2.7] # Redis AI versions - py_v: [3.7, 3.8, 3.9] # Python versions + rai: [1.2.5, 1.2.7] # Redis AI versions + py_v: [3.8, 3.9, '3.10'] # Python versions exclude: - - os: macos-10.15 # Do not build with Redis AI 1.2.5 on MacOS + # Do not build with Redis AI 1.2.5 on MacOS + - os: macos-10.15 + rai: 1.2.5 + # Do not build Redis AI 1.2.5 with py3.10 + # as wheels for dependecies are not availble + - py_v: '3.10' rai: 1.2.5 - - py_v: 3.7 # ONNX requires python >= 3.8 - rai: 1.2.7 env: SMARTSIM_REDISAI: ${{ matrix.rai }} @@ -107,17 +110,13 @@ jobs: python -m pip install git+https://github.com/CrayLabs/SmartRedis.git@develop#egg=smartredis python -m pip install .[dev,ml] - - name: Install ML Runtimes with Smart - if: contains( matrix.os, 'macos' ) - run: smart build --device cpu -v - - name: Install ML Runtimes with Smart (with pt, tf, and onnx support) - if: contains( matrix.os, 'ubuntu' ) && (matrix.py_v != 3.9 || matrix.rai != '1.2.3') + if: (matrix.py_v != '3.10') run: smart build --device cpu --onnx -v - - name: Install ML Runtimes with Smart excluding PyTorch for Ubuntu/Python3.9/RAI1.2.3 combo - if: contains( matrix.os, 'ubuntu' ) && matrix.py_v == 3.9 && matrix.rai == '1.2.3' - run: smart build --device cpu --no_pt --onnx -v + - name: Install ML Runtimes with Smart (with pt and tf support) + if: (matrix.py_v == '3.10') + run: smart build --device cpu -v - name: Run Pytest run: | diff --git a/README.md b/README.md index ddf8b9382..9e1902784 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ +
+ [![License](https://img.shields.io/github/license/CrayLabs/SmartSim)](https://github.com/CrayLabs/SmartSim/blob/master/LICENSE.md) ![GitHub last commit](https://img.shields.io/github/last-commit/CrayLabs/SmartSim) ![GitHub deployments](https://img.shields.io/github/deployments/CrayLabs/SmartSim/github-pages?label=doc%20build) @@ -27,6 +29,8 @@ [![codecov](https://codecov.io/gh/CrayLabs/SmartSim/branch/develop/graph/badge.svg?token=96HFI2F45E)](https://codecov.io/gh/CrayLabs/SmartSim) [![Downloads](https://static.pepy.tech/personalized-badge/smartsim?period=total&units=international_system&left_color=grey&right_color=orange&left_text=Downloads)](https://pepy.tech/project/smartsim) +
+ ------------ # SmartSim @@ -633,17 +637,17 @@ from C, C++, Fortran and Python with the SmartRedis Clients: - 1.2.3-1.2.4 + 1.2.7 PyTorch - 1.7.x + 1.11.x TensorFlow\Keras - 2.4.x-2.5.x + 2.8.x ONNX - 1.9.x + 1.11.x 1.2.5 PyTorch diff --git a/doc/changelog.rst b/doc/changelog.rst index b4c29f235..030e0d727 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -25,6 +25,7 @@ Description - Drop support for Ray - Allow for models to be launched independently as batch jobs - Update to current version of Redis +- Add support for Python 3.10, deprecate support for Python 3.7 and RedisAI 1.2.3 - Fix bug in colocated database entrypoint when loading PyTorch models - Add support for RedisAI 1.2.7, pyTorch 1.11.0, Tensorflow 2.8.0, ONNXRuntime 1.11.1 @@ -34,17 +35,23 @@ Detailed Notes We plan to release a separate add-on library to accomplish the same results. If you are interested in getting the Ray launch functionality back in your workflow, please get in touch with us! (PR263_) - Update from Redis version 6.0.8 to 7.0.5. (PR258_) +- Adds support for Python 3.10 without the ONNX machine learning backend. Deprecates support for + Python 3.7 as it will stop receiving security updates. Deprecates support for RedisAI 1.2.3. + Update the build process to be able to correctly fetch supported dependencies. If a user + attempts to build an unsupported dependency, an error message is shown highlighting the + discrepancy. (PR256_) - Models were given a `batch_settings` attribute. When launching a model through `Experiment.start` the `Experiment` will first check for a non-nullish value at that attribute. If the check is satisfied, the `Experiment` will attempt to wrap the underlying run command in a batch job using the object referenced at `Model.batch_settings` as the batch settings for the job. If the check is not satisfied, the `Model` is launched in the traditional manner as a job step. (PR245_) - Fix bug in colocated database entrypoint stemming from uninitialized variables. This bug affects PyTorch models being loaded into the database. (PR237_) -- The release of RedisAI 1.2.7 allows us to update support for recent versions of pyTorch, Tensorflow, and ONNX (PR234_) -- Make installation of correct Torch backend more reliable according to instruction from pyTorch +- The release of RedisAI 1.2.7 allows us to update support for recent versions of PyTorch, Tensorflow, and ONNX (PR234_) +- Make installation of correct Torch backend more reliable according to instruction from PyTorch .. _PR263: https://github.com/CrayLabs/SmartSim/pull/263 .. _PR258: https://github.com/CrayLabs/SmartSim/pull/258 +.. _PR256: https://github.com/CrayLabs/SmartSim/pull/256 .. _PR245: https://github.com/CrayLabs/SmartSim/pull/245 .. _PR237: https://github.com/CrayLabs/SmartSim/pull/237 .. _PR234: https://github.com/CrayLabs/SmartSim/pull/234 diff --git a/doc/installation.rst b/doc/installation.rst index 7a51de6d2..0204214fb 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -10,7 +10,7 @@ Prerequisites The base prerequisites to install SmartSim and SmartRedis are: - - Python 3.7-3.9 + - Python 3.8-3.10 - Pip - Cmake 3.13.x (or later) - C compiler @@ -22,7 +22,7 @@ The base prerequisites to install SmartSim and SmartRedis are: For most developer systems, many of these packages will already be installed. -GCC 5-9 is recommended. There are known bugs with GCC >= 10. +GCC 5-9 or GCC>=11 is recommended. There are known bugs with GCC 10. Git LFS can be installed through ``conda install git-lfs`` @@ -46,11 +46,11 @@ Supported Versions * - MacOS - x86_64 - Not supported - - 3.7 - 3.9 + - 3.8 - 3.10 * - Linux - x86_64 - Nvidia - - 3.7 - 3.9 + - 3.8 - 3.10 .. note:: @@ -69,7 +69,6 @@ the version of the ML libraries). +==================+==========+=============+===============+ | 1.2.7 (default) | 1.11.0 | 2.8.0 | 1.11.1 | | 1.2.5 | 1.9.0 | 2.6.0 | 1.9.0 | -| 1.2.3 | 1.7.0 | 2.5.2 | 1.9.0 | +------------------+----------+-------------+---------------+ TensorFlow_ 2.0 and Keras_ are supported through graph freezing_. @@ -240,9 +239,9 @@ pre-built wheels that SmartSim does. * - Platform - Python Versions * - MacOS - - 3.7 - 3.9 + - 3.8 - 3.10 * - Linux - - 3.7 - 3.9 + - 3.8 - 3.10 The Python client for SmartRedis is installed through diff --git a/setup.cfg b/setup.cfg index 08516f3fe..297531e03 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,9 +42,9 @@ contact_email = craylabs@hpe.com license = BSD 2-Clause License keywords = scientific, ai, workflow, hpc, analysis classifiers = - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 License :: OSI Approved :: BSD License Intended Audience :: Science/Research Topic :: Scientific/Engineering @@ -55,7 +55,7 @@ setup_requires = setuptools>=39.2 cmake>=3.13 include_package_data = True -python_requires = >=3.7 +python_requires = >=3.8,<3.11 diff --git a/setup.py b/setup.py index e5dd3c150..6758bec75 100644 --- a/setup.py +++ b/setup.py @@ -127,7 +127,7 @@ class BuildError(Exception): # see https://github.com/google/or-tools/issues/616 class InstallPlatlib(install): def finalize_options(self): - install.finalize_options(self) + super().finalize_options() if self.distribution.has_ext_modules(): self.install_lib = self.install_platlib @@ -144,7 +144,7 @@ def run(self): database_builder.cleanup() # run original build_py command - build_py.run(self) + super().run() # Tested with wheel v0.29.0 @@ -167,7 +167,7 @@ def has_ext_modules(_placeholder): "redis==3.5.3", "tqdm>=4.50.2", "filelock>=3.4.2", - "protobuf==3.20", + "protobuf~=3.20", ] # Add SmartRedis at specific version @@ -207,4 +207,4 @@ def has_ext_modules(_placeholder): "smart = smartsim._core._cli.__main__:main", ] } -) \ No newline at end of file +) diff --git a/smartsim/__init__.py b/smartsim/__init__.py index 29f5c88ab..4867687fb 100644 --- a/smartsim/__init__.py +++ b/smartsim/__init__.py @@ -29,8 +29,8 @@ # -*- coding: utf-8 -*- from .version import __version__ as __version__ -if sys.version_info < (3, 7): # pragma: no cover - sys.exit("Python 3.7 or greater must be used with SmartSim.") +if sys.version_info < (3, 8): # pragma: no cover + sys.exit("Python 3.8 or greater must be used with SmartSim.") # Main API module from .experiment import Experiment diff --git a/smartsim/_core/_cli/build.py b/smartsim/_core/_cli/build.py index 782def0c3..24a45d968 100644 --- a/smartsim/_core/_cli/build.py +++ b/smartsim/_core/_cli/build.py @@ -178,11 +178,6 @@ def __init__(self): # REDIS/KeyDB self.build_database() - if self.verbose: - logger.info("Version Information:") - vers = self.versions.as_dict() - print(tabulate(vers, headers=vers.keys(), tablefmt="github"), "\n") - # REDISAI self.build_redis_ai( str(args.device), @@ -227,23 +222,17 @@ def build_redis_ai( ): # make sure user isn't trying to do something silly on MacOS - if self.build_env.PLATFORM == "darwin": - if device == "gpu": - logger.error("SmartSim does not support GPU on MacOS") - sys.exit(1) - if onnx and self.versions.REDISAI < "1.2.6": - logger.error("RedisAI < 1.2.6 does not support ONNX on MacOS") - sys.exit(1) - if self.versions.REDISAI == "1.2.4" or self.versions.REDISAI == "1.2.5": - logger.error("RedisAI support for MacOS is broken in 1.2.4 and 1.2.5") - sys.exit(1) + if self.build_env.PLATFORM == "darwin" and device == "gpu": + raise BuildError("SmartSim does not support GPU on MacOS") # decide which runtimes to build print("\nML Backends Requested") - print("-----------------------") - print(f" PyTorch {self.versions.TORCH}: {color_bool(torch)}") - print(f" TensorFlow {self.versions.TENSORFLOW}: {color_bool(tf)}") - print(f" ONNX {self.versions.ONNX}: {color_bool(onnx)}\n") + backends_table = [ + ["PyTorch", self.versions.TORCH, color_bool(torch)], + ["TensorFlow", self.versions.TENSORFLOW, color_bool(tf)], + ["ONNX", self.versions.ONNX or "Unavailable", color_bool(onnx)], + ] + print(tabulate(backends_table, tablefmt="fancy_outline"), end="\n\n") print(f"Building for GPU support: {color_bool(device == 'gpu')}\n") self.check_backends_install() @@ -257,7 +246,6 @@ def build_redis_ai( if tf: self.check_tf_install() - cmd = [] # TORCH if torch: if torch_dir: @@ -367,6 +355,17 @@ def install_torch(self, device="cpu"): def check_onnx_install(self): """Check Python environment for ONNX installation""" + if not self.versions.ONNX: + py_version = sys.version_info + msg = ( + "An onnx wheel is not available for " + f"Python {py_version.major}.{py_version.minor}. " + "Instead consider using Python 3.8 or 3.9 with Onnx " + ) + if sys.platform == "linux": + msg += "1.2.5 or " + msg += "1.2.7." + raise SetupError(msg) try: if not self.build_env.check_installed("onnx", self.versions.ONNX): msg = ( diff --git a/smartsim/_core/_cli/cli.py b/smartsim/_core/_cli/cli.py index 7ce813d77..fd5c2c494 100644 --- a/smartsim/_core/_cli/cli.py +++ b/smartsim/_core/_cli/cli.py @@ -32,7 +32,6 @@ from smartsim._core._cli.build import Build from smartsim._core._cli.clean import Clean from smartsim._core._cli.utils import get_install_path -from smartsim._core._install.buildenv import Versioner def _usage(): diff --git a/smartsim/_core/_install/buildenv.py b/smartsim/_core/_install/buildenv.py index 4f120d116..c77422862 100644 --- a/smartsim/_core/_install/buildenv.py +++ b/smartsim/_core/_install/buildenv.py @@ -148,17 +148,6 @@ class RedisAIVersion(Version_): """ defaults = { - "1.2.3": { - "tensorflow": "2.5.2", - "onnx": "1.9.0", - "skl2onnx": "1.10.3", - "onnxmltools": "1.10.0", - "scikit-learn": "1.0.2", - "torch": "1.7.1", - "torch_cpu_suffix": "+cpu", - "torch_cuda_suffix": "+cu110", - "torchvision": "0.8.2", - }, "1.2.5": { "tensorflow": "2.6.2", "onnx": "1.9.0", @@ -182,10 +171,23 @@ class RedisAIVersion(Version_): "torchvision": "0.12.0", }, } - # deps are the same between the following versions - defaults["1.2.4"] = defaults["1.2.3"] + # Remove options with unsported wheels for python>=3.10 + if sys.version_info >= (3, 10): + defaults.pop("1.2.5") + defaults["1.2.7"].pop("onnx") + defaults["1.2.7"].pop("skl2onnx") + defaults["1.2.7"].pop("onnxmltools") + defaults["1.2.7"].pop("scikit-learn") + # Remove incompatible RAI versions for OSX + if sys.platform == "darwin": + defaults.pop("1.2.5", None) def __init__(self, vers): + min_rai_version = min(Version_(ver) for ver in self.defaults) + if min_rai_version > vers: + raise SetupError( + f"RedisAI version must be greater than or equal to {min_rai_version}" + ) if vers not in self.defaults: if vers.startswith("1.2"): # resolve to latest version for 1.2.x @@ -199,10 +201,20 @@ def __init__(self, vers): self.version = vers def __getattr__(self, name): - return self.defaults[self.version][name] + try: + return self.defaults[self.version][name] + except KeyError: + raise AttributeError( + f"'{type(self).__name__}' object has no attribute '{name}'\n\n" + "This is likely a problem with the SmartSim build process;" + "if this problem persists please log a new issue at " + "https://github.com/CrayLabs/SmartSim/issues " + "or get in contact with us at " + "https://www.craylabs.org/docs/community.html" + ) from None def get_defaults(self): - return self.defaults[self.version] + return self.defaults[self.version].copy() class Versioner: @@ -229,7 +241,7 @@ class Versioner: """ # compatible Python version - PYTHON_MIN = Version_("3.7.0") + PYTHON_MIN = Version_("3.8.0") # Versions SMARTSIM = Version_(get_env("SMARTSIM_VERSION", "0.4.1")) @@ -260,7 +272,10 @@ class Versioner: # TensorFlow and ONNX only use the defaults, but these are not built into # the RedisAI package and therefore the user is free to pick other versions. TENSORFLOW = Version_(REDISAI.tensorflow) - ONNX = Version_(REDISAI.onnx) + try: + ONNX = Version_(REDISAI.onnx) + except AttributeError: + ONNX = None def as_dict(self, db_name="REDIS"): packages = [ @@ -270,7 +285,6 @@ def as_dict(self, db_name="REDIS"): "REDISAI", "TORCH", "TENSORFLOW", - "ONNX", ] versions = [ self.SMARTSIM, @@ -279,8 +293,10 @@ def as_dict(self, db_name="REDIS"): self.REDISAI, self.TORCH, self.TENSORFLOW, - self.ONNX, ] + if self.ONNX: + packages.append("ONNX") + versions.append(self.ONNX) vers = {"Packages": packages, "Versions": versions} return vers @@ -516,7 +532,7 @@ def get_cudnn_env(): def check_build_dependency(self, command): # TODO expand this to parse and check versions. try: - out = subprocess.check_call( + subprocess.check_call( [command, "--version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, diff --git a/smartsim/_core/_install/builder.py b/smartsim/_core/_install/builder.py index 3306d133f..ee310d6be 100644 --- a/smartsim/_core/_install/builder.py +++ b/smartsim/_core/_install/builder.py @@ -253,7 +253,7 @@ def build_from_git(self, git_url, branch): database = Path(os.environ.get("REDIS_PATH", database_exe)).resolve() _ = expand_exe_path(str(database)) except (TypeError, FileNotFoundError) as e: - raise SSConfigError("Installation of redis-server failed!") from e + raise BuildError("Installation of redis-server failed!") from e # validate install -- redis-cli try: @@ -261,7 +261,7 @@ def build_from_git(self, git_url, branch): redis_cli = Path(os.environ.get("REDIS_CLI_PATH", redis_cli_exe)).resolve() _ = expand_exe_path(str(redis_cli)) except (TypeError, FileNotFoundError) as e: - raise SSConfigError("Installation of redis-cli failed!") from e + raise BuildError("Installation of redis-cli failed!") from e class RedisAIBuilder(Builder): @@ -284,7 +284,6 @@ def __init__( verbose=False, ): super().__init__(build_env, jobs=jobs, verbose=verbose) - self.rai_build_path = Path(self.build_dir, "RedisAI") # convert to int for RAI build script self.torch = 1 if build_torch else 0 @@ -293,6 +292,10 @@ def __init__( self.libtf_dir = libtf_dir self.torch_dir = torch_dir + @property + def rai_build_path(self): + return Path(self.build_dir, "RedisAI") + @property def is_built(self): server = self.lib_path.joinpath("backends").is_dir() @@ -378,7 +381,6 @@ def build_from_git(self, git_url, branch, device): :param device: cpu or gpu :type device: str """ - # delete previous build dir (should never be there) if self.rai_build_path.is_dir(): shutil.rmtree(self.rai_build_path) @@ -395,12 +397,32 @@ def build_from_git(self, git_url, branch, device): "clone", "--recursive", git_url, - "--branch", - branch, - "--depth=1", - "RedisAI", ] + # Circumvent a bad `get_deps.sh` script from RAI on 1.2.7 with ONNX + # TODO: Look for a better way to do this or wait for RAI patch + if sys.platform == "darwin" and branch == "v1.2.7" and self.onnx: + # Clone RAI patch commit for OSX + clone_cmd += ["RedisAI"] + checkout_osx_fix = [ + "git", + "checkout", + "634916c722e718cc6ea3fad46e63f7d798f9adc2", + ] + else: + # Clone RAI release commit + clone_cmd += [ + "--branch", + branch, + "--depth=1", + "RedisAI", + ] + checkout_osx_fix = [] + self.run_command(clone_cmd, out=subprocess.DEVNULL, cwd=self.build_dir) + if checkout_osx_fix: + self.run_command( + checkout_osx_fix, out=subprocess.DEVNULL, cwd=self.rai_build_path + ) # copy FindTensorFlow.cmake to RAI cmake dir self.copy_tf_cmake()