Skip to content

Commit

Permalink
Replace inline pip install with pip install from requirements*.txt (#…
Browse files Browse the repository at this point in the history
…21106)

### Description
Replace inline pip install with pip install from requirements*.txt



### Motivation and Context
so that CG can recognize

### Dependency

- [x] #21085
  • Loading branch information
jchen351 authored Jul 22, 2024
1 parent 17e9ea6 commit 4e75605
Show file tree
Hide file tree
Showing 4 changed files with 21 additions and 16 deletions.
27 changes: 12 additions & 15 deletions tools/ci_build/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,9 +261,6 @@ def convert_arg_line_to_args(self, arg_line):
"--wheel_name_suffix",
help="Suffix to append to created wheel names. This value is currently only used for nightly builds.",
)
parser.add_argument(
"--numpy_version", help="Installs a specific version of numpy before building the python binding."
)
parser.add_argument("--skip-keras-test", action="store_true", help="Skip tests with Keras if keras is installed")

# C-Sharp bindings
Expand Down Expand Up @@ -868,16 +865,6 @@ def update_submodules(source_dir):
run_subprocess(["git", "submodule", "update", "--init", "--recursive"], cwd=source_dir)


def install_python_deps(numpy_version=""):
dep_packages = ["setuptools", "wheel", "pytest"]
dep_packages.append(f"numpy=={numpy_version}" if numpy_version else "numpy>=1.16.6")
dep_packages.append("sympy>=1.10")
dep_packages.append("packaging")
dep_packages.append("cerberus")
dep_packages.append("psutil")
run_subprocess([sys.executable, "-m", "pip", "install", *dep_packages])


def setup_test_data(source_onnx_model_dir, dest_model_dir_name, build_dir, configs):
# create the symlink/shortcut of onnx models dir under build_dir
# currently, there're 2 sources of onnx models, one is build in OS image, another is
Expand Down Expand Up @@ -2146,7 +2133,14 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs):
numpy_init_version = numpy.__version__
pb_init_version = google.protobuf.__version__
run_subprocess(
[sys.executable, "-m", "pip", "install", "-r", "requirements-transformers-test.txt"],
[
sys.executable,
"-m",
"pip",
"install",
"-r",
"requirements/transformers-test/requirements.txt",
],
cwd=SCRIPT_DIR,
)
run_subprocess([sys.executable, "-m", "pytest", "transformers"], cwd=cwd)
Expand Down Expand Up @@ -2818,7 +2812,10 @@ def main():
run_subprocess([emsdk_file, "activate", emsdk_version], cwd=emsdk_dir)

if args.enable_pybind and is_windows():
install_python_deps(args.numpy_version)
run_subprocess(
[sys.executable, "-m", "pip", "install", "-r", "requirements/pybind/requirements.txt"],
cwd=SCRIPT_DIR,
)

if args.use_rocm and args.rocm_version is None:
args.rocm_version = ""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ jobs:
onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)-test \
/bin/bash -c "
set -ex; \
pip install -r /onnxruntime_src/tools/ci_build/requirements-transformers-test.txt; \
pip install -r /onnxruntime_src/tools/ci_build/requirements/transformers-test/requirements.txt; \
pytest /onnxruntime_src/onnxruntime/test/python/transformers/test_flash_attn_rocm.py -v -n 4 --reruns 1"
workingDirectory: $(Build.SourcesDirectory)
displayName: 'Run tranformers tests'
Expand Down
8 changes: 8 additions & 0 deletions tools/ci_build/requirements/pybind/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
setuptools
wheel
pytest
numpy>=1.19.0
sympy>=1.10
packaging
cerberus
psutil

0 comments on commit 4e75605

Please sign in to comment.