From 0c74aee4330ec19390faddaa744d0b05430fd556 Mon Sep 17 00:00:00 2001 From: Eric Gustin <34000337+EricGustin@users.noreply.github.com> Date: Thu, 27 Jan 2022 16:26:43 -0800 Subject: [PATCH] Add Build Matrix for CI (#130) Adds various dimensions to the CI build matrix for SmartSim. The build matrix now uses MacOS & Ubuntu, GNU8, RedisAI 1.2.3 & 1.2.5, and Python 3.7-3.9. The build matrix excludes building with RedisAI 1.2.5 when on MacOS as RedisAI temporarily removed support for MacOS in 1.2.4 and 1.2.5 [ committed by @EricGustin and @Spartee ] [ reviewed by @Spartee ] --- .github/workflows/run_local_tests.yml | 69 -------------------------- .github/workflows/run_tests.yml | 65 ++++++++++++++++++++++++ smartsim/_core/_install/builder.py | 71 +++++++++++++-------------- 3 files changed, 100 insertions(+), 105 deletions(-) delete mode 100644 .github/workflows/run_local_tests.yml create mode 100644 .github/workflows/run_tests.yml diff --git a/.github/workflows/run_local_tests.yml b/.github/workflows/run_local_tests.yml deleted file mode 100644 index 61fbbac32..000000000 --- a/.github/workflows/run_local_tests.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: run-tests-local - -on: [push, pull_request] - -env: - HOMEBREW_NO_ANALYTICS: "ON" # Make Homebrew installation a little quicker - HOMEBREW_NO_AUTO_UPDATE: "ON" - HOMEBREW_NO_BOTTLE_SOURCE_FALLBACK: "ON" - HOMEBREW_NO_GITHUB_API: "ON" - HOMEBREW_NO_INSTALL_CLEANUP: "ON" - - -jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-20.04, macos-10.15] - python-version: [3.7, 3.8, 3.9] - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Display Python version - run: | - python -c "import sys; print(sys.version)" - - - name: Install build-essentials - if: contains(matrix.os, 'ubuntu') - run: | - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt-get update - sudo apt-get install -y build-essential - sudo apt-get install -y wget - - - name: Install GNU make for MacOS - if: contains(matrix.os, 'macos') - run: brew install make || true - - - - name: Install SmartSim (with ML backends if not python 3.9) - if: matrix.python-version != '3.9' - run: | - echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH - python -m pip install -vvv .[dev,ml,ray] - - - name: Install SmartSim - if: matrix.python-version == '3.9' - run: | - echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH - python -m pip install -vvv .[dev,ray] - - - name: Install ML Runtimes with Smart (with pt and tf) - if: contains(matrix.os, 'macos') - run: smart build --device cpu -v - - - name: Install ML Runtimes with Smart (with pt tf and onnx support) - if: contains(matrix.os, 'ubuntu') - run: smart build --device cpu --onnx -v - - - name: Run Pytest - run: | - echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV - py.test -s --import-mode=importlib -o log_cli=true --ignore=test/full_wlm/ diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml new file mode 100644 index 000000000..921ecb65e --- /dev/null +++ b/.github/workflows/run_tests.yml @@ -0,0 +1,65 @@ +name: run-tests + +on: [push, pull_request] + +env: + HOMEBREW_NO_ANALYTICS: "ON" # Make Homebrew installation a little quicker + HOMEBREW_NO_AUTO_UPDATE: "ON" + HOMEBREW_NO_BOTTLE_SOURCE_FALLBACK: "ON" + HOMEBREW_NO_GITHUB_API: "ON" + HOMEBREW_NO_INSTALL_CLEANUP: "ON" + + +jobs: + run_tests: + name: Run tests with ${{ matrix.os }}, Python ${{ matrix.py_v}}, RedisAI ${{ matrix.rai }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-10.15, ubuntu-20.04] # Operating systems + compiler: [8] # GNU compiler version + rai: [1.2.3, 1.2.5] # Redis AI versions + py_v: [3.7, 3.8, 3.9] # Python versions + exclude: + - os: macos-10.15 # Do not build with Redis AI 1.2.5 on MacOS + rai: 1.2.5 + + env: + SMARTSIM_REDISAI: ${{ matrix.rai }} + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.py_v }} + + - name: Install build-essentials for Ubuntu + if: contains( matrix.os, 'ubuntu' ) + run: | + sudo add-apt-repository ppa:ubuntu-toolchain-r/test + sudo apt-get update + sudo apt-get install -y build-essential + sudo apt-get install -y wget + + - name: Install GNU make for MacOS and set GITHUB_PATH + if: "contains( matrix.os, 'macos' )" + run: | + brew install make || true + echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH + + - name: Install SmartSim (with ML backends) + run: python -m pip install .[dev,ml,ray] + + - name: Install ML Runtimes with Smart + if: contains( matrix.os, 'macos' ) + run: smart build --device cpu -v + + - name: Install ML Runtimes with Smart (with pt, tf, and onnx support) + if: contains( matrix.os, 'ubuntu' ) + run: smart build --device cpu --onnx -v + + - name: Run Pytest + run: | + echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV + py.test -s --import-mode=importlib -o log_cli=true diff --git a/smartsim/_core/_install/builder.py b/smartsim/_core/_install/builder.py index 99e7cdaff..0c7ee550e 100644 --- a/smartsim/_core/_install/builder.py +++ b/smartsim/_core/_install/builder.py @@ -68,12 +68,11 @@ def is_built(self): def build_from_git(self): raise NotImplementedError - @property - def make(self): - make_bin = shutil.which("make") - if make_bin: - return make_bin - raise BuildError("Could not find Make binary") + def binary_path(self, binary): + binary_ = shutil.which(binary) + if binary_: + return binary_ + raise BuildError(f"{binary} not found in PATH") def copy_file(self, src, dst, set_exe=False): shutil.copyfile(src, dst) @@ -98,7 +97,7 @@ def cleanup(self): if self.build_dir.is_dir(): shutil.rmtree(str(self.build_dir)) - def run_command(self, cmd, shell=True, out=None, cwd=None): + def run_command(self, cmd, shell=False, out=None, cwd=None): # option to manually disable output if necessary if not out: out = self.out @@ -120,10 +119,8 @@ def run_command(self, cmd, shell=True, out=None, cwd=None): class RedisBuilder(Builder): """Class to build Redis from Source - Supported build methods: - from git - See buildenv.py for buildtime configuration of Redis version and url. """ @@ -141,7 +138,6 @@ def is_built(self): def build_from_git(self, git_url, branch): """Build Redis from git - :param git_url: url from which to retrieve Redis :type git_url: str :param branch: branch to checkout @@ -160,7 +156,7 @@ def build_from_git(self, git_url, branch): # clone Redis clone_cmd = [ - "git", + self.binary_path("git"), "clone", git_url, "--branch", @@ -169,11 +165,17 @@ def build_from_git(self, git_url, branch): "1", "redis", ] - self.run_command(" ".join(clone_cmd), shell=True, cwd=self.build_dir) + self.run_command(clone_cmd, cwd=self.build_dir) # build Redis - cmd = [self.make, "-j", str(self.jobs), f"MALLOC={self.malloc}"] - self.run_command(" ".join(cmd), shell=True, cwd=str(redis_build_path)) + build_cmd = [ + self.binary_path("make"), + "--debug=all", + "-j", + str(self.jobs), + f"MALLOC={self.malloc}", + ] + self.run_command(build_cmd, cwd=str(redis_build_path)) # move redis binaries to smartsim/smartsim/_core/bin redis_src_dir = redis_build_path / "src" @@ -187,10 +189,8 @@ def build_from_git(self, git_url, branch): class RedisAIBuilder(Builder): """Class to build RedisAI from Source - Supported build method: - from git - See buildenv.py for buildtime configuration of RedisAI version and url. """ @@ -223,13 +223,11 @@ def is_built(self): def copy_tf_cmake(self): """Copy the FindTensorFlow.cmake file to the build directory as the version included in RedisAI is out of date for us. - Note: opt/cmake/modules removed in RedisAI v1.2.5 """ # remove the previous version - tf_cmake = self.rai_build_path.joinpath( - "opt/cmake/modules/FindTensorFlow.cmake" - ).resolve() + tf_cmake = self.rai_build_path / "opt/cmake/modules/FindTensorFlow.cmake" + tf_cmake.resolve() if tf_cmake.is_file(): tf_cmake.unlink() # copy ours in @@ -239,7 +237,6 @@ def copy_tf_cmake(self): def build_from_git(self, git_url, branch, device): """Build RedisAI from git - :param git_url: url from which to retrieve RedisAI :type git_url: str :param branch: branch to checkout @@ -258,6 +255,7 @@ def build_from_git(self, git_url, branch, device): # clone RedisAI clone_cmd = [ + self.binary_path("env"), "GIT_LFS_SKIP_SMUDGE=1", "git", "clone", @@ -268,34 +266,32 @@ def build_from_git(self, git_url, branch, device): "--depth=1", "RedisAI", ] - clone_cmd = " ".join(clone_cmd) - self.run_command( - clone_cmd, out=subprocess.DEVNULL, shell=True, cwd=self.build_dir - ) + self.run_command(clone_cmd, out=subprocess.DEVNULL, cwd=self.build_dir) # copy FindTensorFlow.cmake to RAI cmake dir self.copy_tf_cmake() # get RedisAI dependencies dep_cmd = [ + self.binary_path("env"), f"WITH_PT=0", # torch is always 0 because we never use the torch from RAI f"WITH_TF={self.tf}", f"WITH_TFLITE=0", # never build with TF lite (for now) f"WITH_ORT={self.onnx}", "VERBOSE=1", - "bash", - "get_deps.sh", + self.binary_path("bash"), + self.rai_build_path / "get_deps.sh", device, ] - dep_cmd = " ".join(dep_cmd) + self.run_command( dep_cmd, - shell=True, out=subprocess.DEVNULL, # suppress this as it's not useful cwd=self.rai_build_path, ) build_cmd = [ + self.binary_path("env"), f"WITH_PT={self.torch}", # but we built it in if the user specified it f"WITH_TF={self.tf}", f"WITH_TFLITE=0", # never build TF Lite @@ -311,12 +307,17 @@ def build_from_git(self, git_url, branch, device): if self.torch_dir: self.env["Torch_DIR"] = str(self.torch_dir) - build_cmd.extend([self.make, "-C", "opt", "-j", f"{self.jobs}", "build"]) - self.run_command( - " ".join(build_cmd), - cwd=self.rai_build_path, - shell=True, + build_cmd.extend( + [ + self.binary_path("make"), + "-C", + str(self.rai_build_path / "opt"), + "-j", + f"{self.jobs}", + "build", + ] ) + self.run_command(build_cmd, cwd=self.rai_build_path) self._install_backends(device) if self.torch: @@ -325,7 +326,6 @@ def build_from_git(self, git_url, branch, device): def _install_backends(self, device): """Move backend libraries to smartsim/_core/lib/ - :param device: cpu or cpu :type device: str """ @@ -341,7 +341,6 @@ def _install_backends(self, device): def _move_torch_libs(self): """Move pip install torch libraries - Since we use pip installed torch libraries for building RedisAI, we need to move them into the LD_runpath of redisai.so in the smartsim/_core/lib directory.