From 572e13629bc78dab2ae7a8e9027842d2468931de Mon Sep 17 00:00:00 2001 From: Andrew Shao Date: Mon, 8 Jul 2024 15:13:29 -0700 Subject: [PATCH 01/10] rename deps to install_requires --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index dd6de4587..8dc92f88f 100644 --- a/setup.py +++ b/setup.py @@ -165,7 +165,7 @@ def has_ext_modules(_placeholder): # Define needed dependencies for the installation -deps = [ +install_requires = [ "packaging>=24.0", "psutil>=5.7.2", "coloredlogs>=10.0", @@ -183,7 +183,7 @@ def has_ext_modules(_placeholder): ] # Add SmartRedis at specific version -deps.append("smartredis>={}".format(versions.SMARTREDIS)) +install_requires.append("smartredis>={}".format(versions.SMARTREDIS)) extras_require = { "dev": [ @@ -213,7 +213,7 @@ def has_ext_modules(_placeholder): # rest in setup.cfg setup( version=smartsim_version, - install_requires=deps, + install_requires=install_requires, cmdclass={ "build_py": SmartSimBuild, "install": InstallPlatlib, From ea842565697d57322b7f040710f419caad05f64a Mon Sep 17 00:00:00 2001 From: Andrew Shao Date: Mon, 8 Jul 2024 15:22:01 -0700 Subject: [PATCH 02/10] Delete requirements-doc.txt --- doc/requirements-doc.txt | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 doc/requirements-doc.txt diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt deleted file mode 100644 index 696881bef..000000000 --- a/doc/requirements-doc.txt +++ /dev/null @@ -1,18 +0,0 @@ -Sphinx==6.2.1 -breathe==4.35.0 -sphinx-fortran==1.1.1 -sphinx-book-theme==1.0.1 -sphinx-copybutton==0.5.2 -sphinx-tabs==3.4.4 -nbsphinx==0.9.3 -docutils==0.18.1 -torch==2.0.1 -tensorflow==2.13.1 -ipython -jinja2==3.1.2 -protobuf -numpy -sphinx-design -pypandoc -sphinx-autodoc-typehints -myst_parser From 2064d7d9d9c660941a3f5aca5cc9af0ab79f532f Mon Sep 17 00:00:00 2001 From: Andrew Shao Date: Fri, 12 Jul 2024 11:23:01 -0700 Subject: [PATCH 03/10] Modify setup.py for code insight --- setup.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/setup.py b/setup.py index 8dc92f88f..ed757fddb 100644 --- a/setup.py +++ b/setup.py @@ -165,25 +165,9 @@ def has_ext_modules(_placeholder): # Define needed dependencies for the installation -install_requires = [ - "packaging>=24.0", - "psutil>=5.7.2", - "coloredlogs>=10.0", - "tabulate>=0.8.9", - "redis>=4.5", - "tqdm>=4.50.2", - "filelock>=3.4.2", - "protobuf~=3.20", - "jinja2>=3.1.2", - "watchdog>=4.0.0", - "pydantic==1.10.14", - "pyzmq>=25.1.2", - "pygithub>=2.3.0", - "numpy<2" -] # Add SmartRedis at specific version -install_requires.append("smartredis>={}".format(versions.SMARTREDIS)) +# install_requires.append("smartredis>={}".format(versions.SMARTREDIS)) extras_require = { "dev": [ @@ -213,7 +197,23 @@ def has_ext_modules(_placeholder): # rest in setup.cfg setup( version=smartsim_version, - install_requires=install_requires, + install_requires = [ + "packaging>=24.0", + "psutil>=5.7.2", + "coloredlogs>=10.0", + "tabulate>=0.8.9", + "redis>=4.5", + "tqdm>=4.50.2", + "filelock>=3.4.2", + "protobuf~=3.20", + "jinja2>=3.1.2", + "watchdog>=4.0.0", + "pydantic==1.10.14", + "pyzmq>=25.1.2", + "pygithub>=2.3.0", + "numpy<2", + "smartredis<=0.5.3, + ], cmdclass={ "build_py": SmartSimBuild, "install": InstallPlatlib, From f6efd72e522aec27b901b3ecd93c8d7975babd58 Mon Sep 17 00:00:00 2001 From: Andrew Shao Date: Tue, 16 Jul 2024 15:17:29 -0700 Subject: [PATCH 04/10] Remove SmartRedis version from Versioner --- setup.py | 2 +- smartsim/_core/_install/buildenv.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index ed757fddb..8eb7fc63f 100644 --- a/setup.py +++ b/setup.py @@ -212,7 +212,7 @@ def has_ext_modules(_placeholder): "pyzmq>=25.1.2", "pygithub>=2.3.0", "numpy<2", - "smartredis<=0.5.3, + "smartredis<=0.5.3", ], cmdclass={ "build_py": SmartSimBuild, diff --git a/smartsim/_core/_install/buildenv.py b/smartsim/_core/_install/buildenv.py index edb1ff116..a066ab16a 100644 --- a/smartsim/_core/_install/buildenv.py +++ b/smartsim/_core/_install/buildenv.py @@ -242,7 +242,7 @@ class Versioner: ``smart build`` command to determine which dependency versions to look for and download. - Default versions for SmartSim, SmartRedis, Redis, and RedisAI are + Default versions for SmartSim, Redis, and RedisAI are all set here. Setting a default version for RedisAI also dictates default versions of the machine learning libraries. """ @@ -252,7 +252,6 @@ class Versioner: # Versions SMARTSIM = Version_(get_env("SMARTSIM_VERSION", "0.7.0")) - SMARTREDIS = Version_(get_env("SMARTREDIS_VERSION", "0.5.3")) SMARTSIM_SUFFIX = get_env("SMARTSIM_SUFFIX", "") # Redis @@ -284,7 +283,6 @@ class Versioner: def as_dict(self, db_name: DbEngine = "REDIS") -> t.Dict[str, t.Tuple[str, ...]]: pkg_map = { "SMARTSIM": self.SMARTSIM, - "SMARTREDIS": self.SMARTREDIS, db_name: self.REDIS, "REDISAI": self.REDISAI, "TORCH": self.TORCH, From 885dc15982465d1c2b040e4b8d8dd1020a0e2253 Mon Sep 17 00:00:00 2001 From: Andrew Shao Date: Tue, 16 Jul 2024 17:05:05 -0700 Subject: [PATCH 05/10] Migrate doc dependencies to setup.py --- doc/changelog.md | 9 +++++++++ docker/docs/dev/Dockerfile | 3 +-- setup.py | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/doc/changelog.md b/doc/changelog.md index d8a94449a..565b39e08 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -15,6 +15,7 @@ To be released at some future point in time Description +- Make dependencies more discoverable in setup.py - Pin NumPy version to 1.x - New launcher support for SGE (and similar derivatives) - Fix test outputs being created in incorrect directory @@ -24,6 +25,14 @@ Description Detailed Notes +- setup.py used to define dependencies in a way that was not amenable + to code scanning tools. Direct dependencies now appear directly + in the setup call and the definition of the SmartRedis version + has been removed + ([SmartSim-PR627](https://github.com/CrayLabs/SmartSim/pull/627)) +- The separate definition of dependencies for the docs in + requirements-doc.txt is now defined as an extra. + ([SmartSim-PR627](https://github.com/CrayLabs/SmartSim/pull/627)) - The new major version release of Numpy is incompatible with modules compiled against Numpy 1.x. For both SmartSim and SmartRedis we request a 1.x version of numpy. This is needed in SmartSim because diff --git a/docker/docs/dev/Dockerfile b/docker/docs/dev/Dockerfile index e9db9c342..dbac524bc 100644 --- a/docker/docs/dev/Dockerfile +++ b/docker/docs/dev/Dockerfile @@ -55,8 +55,7 @@ RUN git clone https://github.com/CrayLabs/SmartDashboard.git --branch develop -- && rm -rf ~/.cache/pip # Install docs dependencies and SmartSim -RUN python -m pip install -r doc/requirements-doc.txt \ - && NO_CHECKS=1 SMARTSIM_SUFFIX=dev python -m pip install . +RUN NO_CHECKS=1 SMARTSIM_SUFFIX=dev python -m pip install .[docs] # Note this is needed to ensure that the Sphinx builds. Can be removed with newer Tensorflow RUN python -m pip install typing_extensions==4.6.1 diff --git a/setup.py b/setup.py index 8eb7fc63f..7c05fc3c9 100644 --- a/setup.py +++ b/setup.py @@ -189,6 +189,24 @@ def has_ext_modules(_placeholder): "types-setuptools", "typing_extensions>=4.1.0", ], + "docs": [ + "Sphinx==6.2.1", + "breathe==4.35.0", + "sphinx-fortran==1.1.1", + "sphinx-book-theme==1.0.1", + "sphinx-copybutton==0.5.2", + "sphinx-tabs==3.4.4", + "nbsphinx==0.9.3", + "docutils==0.18.1", + "torch==2.0.1", + "tensorflow==2.13.1", + "ipython", + "jinja2==3.1.2", + "sphinx-design", + "pypandoc", + "sphinx-autodoc-typehints", + "myst_parser", + ] # see smartsim/_core/_install/buildenv.py for more details **versions.ml_extras_required(), } From 3b4065dbd837dce8b1a912553fc6e9141eb62993 Mon Sep 17 00:00:00 2001 From: Andrew Shao Date: Tue, 16 Jul 2024 17:07:59 -0700 Subject: [PATCH 06/10] Update PR number --- doc/changelog.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/changelog.md b/doc/changelog.md index 565b39e08..84be20798 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -29,10 +29,10 @@ Detailed Notes to code scanning tools. Direct dependencies now appear directly in the setup call and the definition of the SmartRedis version has been removed - ([SmartSim-PR627](https://github.com/CrayLabs/SmartSim/pull/627)) + ([SmartSim-PR635](https://github.com/CrayLabs/SmartSim/pull/635)) - The separate definition of dependencies for the docs in requirements-doc.txt is now defined as an extra. - ([SmartSim-PR627](https://github.com/CrayLabs/SmartSim/pull/627)) + ([SmartSim-PR635](https://github.com/CrayLabs/SmartSim/pull/635)) - The new major version release of Numpy is incompatible with modules compiled against Numpy 1.x. For both SmartSim and SmartRedis we request a 1.x version of numpy. This is needed in SmartSim because From 534fb6d40c5e05703fcec1a570f0c7d3303e8081 Mon Sep 17 00:00:00 2001 From: Andrew Shao Date: Wed, 17 Jul 2024 09:31:03 -0700 Subject: [PATCH 07/10] Add a missing comma --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7c05fc3c9..611d82c74 100644 --- a/setup.py +++ b/setup.py @@ -206,7 +206,7 @@ def has_ext_modules(_placeholder): "pypandoc", "sphinx-autodoc-typehints", "myst_parser", - ] + ], # see smartsim/_core/_install/buildenv.py for more details **versions.ml_extras_required(), } From 70d780f7ff14668b08f5cea85d0e745b3b23937f Mon Sep 17 00:00:00 2001 From: Andrew Shao Date: Wed, 17 Jul 2024 09:44:25 -0700 Subject: [PATCH 08/10] Update the readthedocs workflow --- .readthedocs.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index cecdfe3bf..88f270ba7 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -23,7 +23,7 @@ build: - git clone --depth 1 https://github.com/CrayLabs/SmartRedis.git smartredis - git clone --depth 1 https://github.com/CrayLabs/SmartDashboard.git smartdashboard post_create_environment: - - python -m pip install .[dev] + - python -m pip install .[dev,docs] - cd smartredis; python -m pip install . - cd smartredis/doc; doxygen Doxyfile_c; doxygen Doxyfile_cpp; doxygen Doxyfile_fortran - ln -s smartredis/examples ./examples @@ -37,7 +37,3 @@ build: sphinx: configuration: doc/conf.py fail_on_warning: true - -python: - install: - - requirements: doc/requirements-doc.txt \ No newline at end of file From 7d54163debf73b3c6d9ee07866ae4e6192a64f80 Mon Sep 17 00:00:00 2001 From: Andrew Shao Date: Wed, 17 Jul 2024 09:54:59 -0700 Subject: [PATCH 09/10] Constrain minimum SmartRedis version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 611d82c74..3cbcf92eb 100644 --- a/setup.py +++ b/setup.py @@ -230,7 +230,7 @@ def has_ext_modules(_placeholder): "pyzmq>=25.1.2", "pygithub>=2.3.0", "numpy<2", - "smartredis<=0.5.3", + "smartredis>=0.5,<0.6", ], cmdclass={ "build_py": SmartSimBuild, From fe4d2282bb5c18aafd5255e41fd7d07df699ff0d Mon Sep 17 00:00:00 2001 From: Andrew Shao Date: Thu, 18 Jul 2024 07:47:34 -0700 Subject: [PATCH 10/10] Fix spacing issues --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3cbcf92eb..cd8eabec1 100644 --- a/setup.py +++ b/setup.py @@ -215,7 +215,7 @@ def has_ext_modules(_placeholder): # rest in setup.cfg setup( version=smartsim_version, - install_requires = [ + install_requires=[ "packaging>=24.0", "psutil>=5.7.2", "coloredlogs>=10.0", @@ -230,7 +230,7 @@ def has_ext_modules(_placeholder): "pyzmq>=25.1.2", "pygithub>=2.3.0", "numpy<2", - "smartredis>=0.5,<0.6", + "smartredis>=0.5,<0.6", ], cmdclass={ "build_py": SmartSimBuild,