From 38b506e1b1375b0767d7118cfa448861d9b6b6eb Mon Sep 17 00:00:00 2001 From: Bharath Ramaswamy Date: Tue, 11 May 2021 00:34:13 -0700 Subject: [PATCH] Fixed packaging, updated package versions, etc 1. Fixed packaging issues with variants 2. Added missing packaging stage to all PR variant pipelines 3. Added Deprecated package, pinned Astroid package, update numpy versions in PR and/or dependency lists 4. Fixed install issues with torch variants Signed-off-by: Bharath Ramaswamy --- CMakeLists.txt | 2 +- Jenkins/Dockerfile.tf-cpu | 4 +- Jenkins/Dockerfile.tf-gpu | 4 +- Jenkins/Dockerfile.tf-torch-cpu | 4 +- Jenkins/Dockerfile.torch-cpu | 4 +- Jenkins/Dockerfile.torch-gpu | 4 +- Jenkins/Jenkinsfile | 6 +- TrainingExtensions/torch/CMakeLists.txt | 14 +++- packaging/INSTALL.txt | 61 +---------------- packaging/dependencies/reqs_pip_common.txt | 2 +- packaging/install.md | 2 +- packaging/package_aimet.cmake | 2 +- packaging/setup_aimet.py | 5 ++ packaging/setup_aimet_common.py | 4 +- packaging/setup_aimet_tensorflow.py | 77 +++++++++++++++++++--- packaging/setup_aimet_torch.py | 77 +++++++++++++++++++--- 16 files changed, 179 insertions(+), 93 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 17837475065..dff341ab4bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -195,7 +195,7 @@ if(NOT remote_url STREQUAL "") # Run the install target first COMMAND "${CMAKE_COMMAND}" --build . --target install # Now run the packaging target to generate wheel files - COMMAND ${CMAKE_COMMAND} -DAIMET_PACKAGE_PATH=${AIMET_PACKAGE_PATH} -DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DENABLE_TENSORFLOW=${ENABLE_TENSORFLOW} -DENABLE_TORCH=${ENABLE_TORCH} -P ${CMAKE_CURRENT_SOURCE_DIR}/packaging/package_aimet.cmake + COMMAND ${CMAKE_COMMAND} -DAIMET_PACKAGE_PATH=${AIMET_PACKAGE_PATH} -DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DENABLE_CUDA=${ENABLE_CUDA} -DENABLE_TENSORFLOW=${ENABLE_TENSORFLOW} -DENABLE_TORCH=${ENABLE_TORCH} -P ${CMAKE_CURRENT_SOURCE_DIR}/packaging/package_aimet.cmake ) else() message(STATUS "Repo Remote URL is blank. Skipping packageaimet target") diff --git a/Jenkins/Dockerfile.tf-cpu b/Jenkins/Dockerfile.tf-cpu index a1b9911a53f..64b15508f23 100644 --- a/Jenkins/Dockerfile.tf-cpu +++ b/Jenkins/Dockerfile.tf-cpu @@ -85,7 +85,7 @@ RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists RUN pip3 --no-cache-dir install --upgrade \ setuptools==41.0.1 \ pip==20.2.4 \ - numpy==1.16.4 \ + numpy==1.16.6 \ wheel==0.33.4 ENV DEBIAN_FRONTEND=noninteractive @@ -119,12 +119,14 @@ RUN apt-get update > /dev/null && \ # Python3 Packages RUN pip3 --no-cache-dir install \ + astroid==2.5.3 \ attrs==19.1.0 \ behave==1.2.6 \ blosc==1.8.1 \ cffi==1.12.3 \ click==7.0 \ cython==0.29.10 \ + Deprecated==1.2.12 \ docutils==0.16 \ h5py==2.9.0 \ ipykernel==4.8.2 \ diff --git a/Jenkins/Dockerfile.tf-gpu b/Jenkins/Dockerfile.tf-gpu index 4bd00ba988b..c4ba21d3b2d 100644 --- a/Jenkins/Dockerfile.tf-gpu +++ b/Jenkins/Dockerfile.tf-gpu @@ -85,7 +85,7 @@ RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists RUN pip3 --no-cache-dir install --upgrade \ setuptools==41.0.1 \ pip==20.2.4 \ - numpy==1.16.4 \ + numpy==1.16.6 \ wheel==0.33.4 ENV DEBIAN_FRONTEND=noninteractive @@ -119,12 +119,14 @@ RUN apt-get update > /dev/null && \ # Python3 Packages RUN pip3 --no-cache-dir install \ + astroid==2.5.3 \ attrs==19.1.0 \ behave==1.2.6 \ blosc==1.8.1 \ cffi==1.12.3 \ click==7.0 \ cython==0.29.10 \ + Deprecated==1.2.12 \ docutils==0.16 \ h5py==2.9.0 \ ipykernel==4.8.2 \ diff --git a/Jenkins/Dockerfile.tf-torch-cpu b/Jenkins/Dockerfile.tf-torch-cpu index 679fe9ca2f1..81f5b0c6142 100644 --- a/Jenkins/Dockerfile.tf-torch-cpu +++ b/Jenkins/Dockerfile.tf-torch-cpu @@ -81,7 +81,7 @@ RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists RUN pip3 --no-cache-dir install --upgrade \ setuptools==41.0.1 \ pip==20.2.4 \ - numpy==1.16.4 \ + numpy==1.16.6 \ wheel==0.33.4 ENV DEBIAN_FRONTEND=noninteractive @@ -115,12 +115,14 @@ RUN apt-get update > /dev/null && \ # Python3 Packages RUN pip3 --no-cache-dir install \ + astroid==2.5.3 \ attrs==19.1.0 \ behave==1.2.6 \ blosc==1.8.1 \ cffi==1.12.3 \ click==7.0 \ cython==0.29.10 \ + Deprecated==1.2.12 \ docutils==0.16 \ h5py==2.9.0 \ ipykernel==4.8.2 \ diff --git a/Jenkins/Dockerfile.torch-cpu b/Jenkins/Dockerfile.torch-cpu index f75b3133a5d..1285044da39 100644 --- a/Jenkins/Dockerfile.torch-cpu +++ b/Jenkins/Dockerfile.torch-cpu @@ -81,7 +81,7 @@ RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists RUN pip3 --no-cache-dir install --upgrade \ setuptools==41.0.1 \ pip==20.2.4 \ - numpy==1.16.4 \ + numpy==1.16.6 \ wheel==0.33.4 ENV DEBIAN_FRONTEND=noninteractive @@ -115,12 +115,14 @@ RUN apt-get update > /dev/null && \ # Python3 Packages RUN pip3 --no-cache-dir install \ + astroid==2.5.3 \ attrs==19.1.0 \ behave==1.2.6 \ blosc==1.8.1 \ cffi==1.12.3 \ click==7.0 \ cython==0.29.10 \ + Deprecated==1.2.12 \ docutils==0.16 \ h5py==2.9.0 \ ipykernel==4.8.2 \ diff --git a/Jenkins/Dockerfile.torch-gpu b/Jenkins/Dockerfile.torch-gpu index 7f303e6a974..52b5a660fb2 100644 --- a/Jenkins/Dockerfile.torch-gpu +++ b/Jenkins/Dockerfile.torch-gpu @@ -81,7 +81,7 @@ RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists RUN pip3 --no-cache-dir install --upgrade \ setuptools==41.0.1 \ pip==20.2.4 \ - numpy==1.16.4 \ + numpy==1.16.6 \ wheel==0.33.4 ENV DEBIAN_FRONTEND=noninteractive @@ -115,12 +115,14 @@ RUN apt-get update > /dev/null && \ # Python3 Packages RUN pip3 --no-cache-dir install \ + astroid==2.5.3 \ attrs==19.1.0 \ behave==1.2.6 \ blosc==1.8.1 \ cffi==1.12.3 \ click==7.0 \ cython==0.29.10 \ + Deprecated==1.2.12 \ docutils==0.16 \ h5py==2.9.0 \ ipykernel==4.8.2 \ diff --git a/Jenkins/Jenkinsfile b/Jenkins/Jenkinsfile index 4d7382407cf..d2e6ccd86b0 100755 --- a/Jenkins/Jenkinsfile +++ b/Jenkins/Jenkinsfile @@ -202,7 +202,7 @@ pipeline { echo 'Building code (and generating Docs and pip packages)...' script { //TODO Change option back to "-bp" as soon as issue is fixed - runStage(env.AIMET_VARIANT_TORCH_GPU, "-b") + runStage(env.AIMET_VARIANT_TORCH_GPU, "-bp") } } } @@ -319,7 +319,7 @@ pipeline { echo 'Building code (and generating Docs and pip packages)...' script { //TODO Change option back to "-bp" as soon as issue is fixed - runStage(env.AIMET_VARIANT_TF_CPU, "-b") + runStage(env.AIMET_VARIANT_TF_CPU, "-bp") } } } @@ -436,7 +436,7 @@ pipeline { echo 'Building code (and generating Docs and pip packages)...' script { //TODO Change option back to "-bp" as soon as issue is fixed - runStage(env.AIMET_VARIANT_TORCH_CPU, "-b") + runStage(env.AIMET_VARIANT_TORCH_CPU, "-bp") } } } diff --git a/TrainingExtensions/torch/CMakeLists.txt b/TrainingExtensions/torch/CMakeLists.txt index f46b9d9c0f2..0cf6828e84b 100644 --- a/TrainingExtensions/torch/CMakeLists.txt +++ b/TrainingExtensions/torch/CMakeLists.txt @@ -84,6 +84,14 @@ install(DIRECTORY ${CMAKE_BINARY_DIR}/artifacts/ PATTERN "__pycache*" EXCLUDE ) -install(FILES ${CMAKE_BINARY_DIR}/artifacts/easy-install.pth ${CMAKE_BINARY_DIR}/artifacts/site.py - DESTINATION ${AIMET_INSTALL_DIR}/lib/x86_64-linux-gnu - ) \ No newline at end of file +if(EXISTS ${CMAKE_BINARY_DIR}/artifacts/easy-install.pth) + install(FILES ${CMAKE_BINARY_DIR}/artifacts/easy-install.pth + DESTINATION ${AIMET_INSTALL_DIR}/lib/x86_64-linux-gnu + ) +endif() + +if(EXISTS ${CMAKE_BINARY_DIR}/artifacts/site.py) + install(FILES ${CMAKE_BINARY_DIR}/artifacts/site.py + DESTINATION ${AIMET_INSTALL_DIR}/lib/x86_64-linux-gnu + ) +endif() diff --git a/packaging/INSTALL.txt b/packaging/INSTALL.txt index 37c03a040b2..ae60cb7a742 100644 --- a/packaging/INSTALL.txt +++ b/packaging/INSTALL.txt @@ -38,62 +38,5 @@ # Installation instructions for AI Model Efficienty Toolkit package and dependencies # ================================================================================== -# NOTE: -# 1. Please pre-pend the "apt-get install" and "pip3 install" commands with "sudo -H" as appropriate -# 2. These instructions that pip packages will be installed in the path: /usr/local/lib/python3.6/dist-packages. If that is not the case, please modify it accordingly. - -# Install the following pre-requisite packages -apt-get update -apt-get install python3.6 python3.6-dev python3-pip -pip3 install --upgrade pip - -# ====================== -# Install AIMET packages -# ====================== -# Go to https://github.com/quic/aimet/releases and identify the release tag of the package you want to install -# NOTE: Python dependencies would automatically get installed. -# - -release_tag= -pip3 install https://github.com/quic/aimet/releases/download/${release_tag}/AimetCommon-${release_tag}-py3-none-any.whl -f https://download.pytorch.org/whl/torch_stable.html -pip3 install https://github.com/quic/aimet/releases/download/${release_tag}/AimetTorch-${release_tag}-py3-none-any.whl -pip3 install https://github.com/quic/aimet/releases/download/${release_tag}/AimetTensorflow-${release_tag}-py3-none-any.whl -pip3 install https://github.com/quic/aimet/releases/download/${release_tag}/Aimet-${release_tag}-py3-none-any.whl - -# ================================================================ -# Install common debian packages from the packages_common.txt file -# ================================================================ -apt-get update -cat /usr/local/lib/python3.6/dist-packages/aimet_common/bin/packages_common.txt | xargs apt-get --assume-yes install - -# Replace the Pillow package with Pillow-SIMD as follows: -pip3 uninstall -y pillow && pip3 install --no-cache-dir Pillow-SIMD==6.0.0.post0 - -# GPU preparation steps -wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.0.130-1_amd64.deb -apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub -dpkg -i cuda-repo-ubuntu1804_10.0.130-1_amd64.deb -apt-get update -wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb -apt-get --assume-yes install ./nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb -apt-get update - -# ====================================================== -# Install GPU packages from the packages_common.txt file -# ====================================================== -cat /usr/local/lib/python3.6/dist-packages/aimet_common/bin/packages_gpu.txt | xargs apt-get --assume-yes install - -# Post installation steps -ln -s /usr/local/cuda-10.0 /usr/local/cuda -ln -s /usr/lib/x86_64-linux-gnu/libjpeg.so /usr/lib - -# ===================== -# Setup the environment -# ===================== -# Set the common environment variables as follows -source /usr/local/lib/python3.6/dist-packages/aimet_common/bin/envsetup.sh - -# Add AIMET packages to the environment paths -export LD_LIBRARY_PATH=/usr/local/lib/python3.6/dist-packages/aimet_common/x86_64-linux-gnu:/usr/local/lib/python3.6/dist-packages/aimet_common:$LD_LIBRARY_PATH - -if [[ $PYTHONPATH = "" ]]; then export PYTHONPATH=/usr/local/lib/python3.6/dist-packages/aimet_common/x86_64-linux-gnu; else export PYTHONPATH=/usr/local/lib/python3.6/dist-packages/aimet_common/x86_64-linux-gnu:$PYTHONPATH; fi +# Please see install.md for the latest installation instructions. +# This file will be removed shortly. diff --git a/packaging/dependencies/reqs_pip_common.txt b/packaging/dependencies/reqs_pip_common.txt index f810b2a3bfb..8f86c4cac5d 100644 --- a/packaging/dependencies/reqs_pip_common.txt +++ b/packaging/dependencies/reqs_pip_common.txt @@ -1,5 +1,5 @@ ipython==7.16.1 -numpy==1.16.4 +numpy==1.16.6 scipy==1.2.1 protobuf==3.7.1 scikit-learn==0.21.0 diff --git a/packaging/install.md b/packaging/install.md index 4d51a702e49..8d69336f54a 100644 --- a/packaging/install.md +++ b/packaging/install.md @@ -47,7 +47,7 @@ release_tag= python3 -m pip install https://github.com/quic/aimet/releases/download/${release_tag}/AimetCommon-${AIMET_VARIANT}_${release_tag}-py3-none-any.whl # Install ONE of the following depending on the variant -python3 -m pip install https://github.com/quic/aimet/releases/download/${release_tag}/AimetTorch-${AIMET_VARIANT}_${release_tag}-py3-none-any.whl +python3 -m pip install https://github.com/quic/aimet/releases/download/${release_tag}/AimetTorch-${AIMET_VARIANT}_${release_tag}-py3-none-any.whl -f https://download.pytorch.org/whl/torch_stable.html # OR python3 -m pip install https://github.com/quic/aimet/releases/download/${release_tag}/AimetTensorflow-${AIMET_VARIANT}_${release_tag}-py3-none-any.whl diff --git a/packaging/package_aimet.cmake b/packaging/package_aimet.cmake index 798389132df..f73b15f36a5 100755 --- a/packaging/package_aimet.cmake +++ b/packaging/package_aimet.cmake @@ -141,7 +141,7 @@ foreach(package ${package_name_list}) file(COPY ${AIMET_PACKAGE_PATH}/lib/python/${package} DESTINATION ${build_packaging_dir}/) endif() - # Invoke the setup tools scrip to create the wheel packages. + # Invoke the setup tools script to create the wheel packages. execute_process(COMMAND python3 setup.py sdist bdist_wheel ${CUDA_OPTION} WORKING_DIRECTORY ${build_packaging_dir} OUTPUT_VARIABLE output_var) endforeach() diff --git a/packaging/setup_aimet.py b/packaging/setup_aimet.py index 0e448b85f5c..80e0eebcf70 100644 --- a/packaging/setup_aimet.py +++ b/packaging/setup_aimet.py @@ -36,6 +36,7 @@ """ Package generation file for top-level aimet package """ +import sys import os.path from setuptools import setup import setup_cfg # pylint: disable=import-error @@ -45,6 +46,10 @@ torch_dep_whl = package_url_base + "/AimetTorch-" + str(setup_cfg.version) + "-py3-none-any.whl" tf_dep_whl = package_url_base + "/AimetTensorflow-" + str(setup_cfg.version) + "-py3-none-any.whl" +if "--gpu" in sys.argv: + # There is NO common GPU dependency list, so just ignore the option if it was passed in + sys.argv.remove("--gpu") + dependency_list = [] if os.path.isfile(torch_dep_whl): dependency_list.append(torch_dep_whl) diff --git a/packaging/setup_aimet_common.py b/packaging/setup_aimet_common.py index cde14ae181b..39669dc0b9c 100755 --- a/packaging/setup_aimet_common.py +++ b/packaging/setup_aimet_common.py @@ -52,9 +52,7 @@ install_requires_list = [open(package_bin_dir + '/reqs_pip_common.txt').read()] package_dependency_files = ['reqs_pip_common.txt', 'reqs_deb_common.txt', 'INSTALL.txt', 'envsetup.sh', 'LICENSE.pdf'] if "--gpu" in sys.argv: - # Create common GPU dependency list - package_dependency_files.append(['reqs_pip_gpu.txt', 'reqs_deb_gpu.txt']) - install_requires_list.append(open(package_bin_dir + '/reqs_pip_gpu.txt').read()) + # There is NO common GPU dependency list, so just ignore the option if it was passed in sys.argv.remove("--gpu") # Loop over package artifacts folder diff --git a/packaging/setup_aimet_tensorflow.py b/packaging/setup_aimet_tensorflow.py index e9de955a637..094e85d1cc9 100644 --- a/packaging/setup_aimet_tensorflow.py +++ b/packaging/setup_aimet_tensorflow.py @@ -42,26 +42,87 @@ import setup_cfg # pylint: disable=import-error package_name = "aimet_tensorflow" -package_bin_dir = package_name + "/bin" -package_url_base = setup_cfg.remote_url + "/releases/download/" + str(setup_cfg.version) +def prepend_bin_path(dependency_list_file): + ''' + Append the common path to the dependency file + ''' + package_bin_dir = package_name + "/bin" + full_path = package_bin_dir + '/' + dependency_list_file + return full_path + + +def get_dependency_packages(dependency_list_file): + ''' + Read the dependency file and return a list of packages + ''' + + dependency_file_full_path = prepend_bin_path(dependency_list_file) + dependency_list_array = open(dependency_file_full_path).read().splitlines() + + dependency_packages_list = [] + for dependency_line in dependency_list_array: + if dependency_line.strip().startswith('#'): + # Skip the commented out lines + continue + # Pick up the package and version (== i.e. first column of file) + dependency_packages_list.append(dependency_line.lstrip().split()[0]) + + return dependency_packages_list + +def get_dependency_urls(dependency_list_file): + ''' + Read the dependency file and return a list of package source URLs + ''' + + url_delimiter = '-f ' + + dependency_file_full_path = prepend_bin_path(dependency_list_file) + dependency_list_array = open(dependency_file_full_path).read().splitlines() + + dependency_urls_list = [] + for dependency_line in dependency_list_array: + if dependency_line.strip().startswith('#'): + # Skip the commented out lines + continue + + # Extract any package custom URLs from the requirements file + # Ex. torch==1.7.1+cu110 -f https://download.pytorch.org/whl/torch_stable.html + url_delimiter = '-f ' + # Split each line using the URL option as delimiter + dependency_line_split_list = dependency_line.split(url_delimiter) + # The split list will have at least 2 elements if a URL was present + if len(dependency_line_split_list) > 1: + # If the URL exists, remove all whitespaces + dependency_url = dependency_line_split_list[1].strip() + # Add to the list only if not already present + if dependency_url not in dependency_urls_list: + dependency_urls_list.append(dependency_url) + + return dependency_urls_list + + +package_url_base = setup_cfg.remote_url + "/releases/download/"+str(setup_cfg.version) common_dep_whl = package_url_base + "/AimetCommon-" + str(setup_cfg.version) + "-py3-none-any.whl" +dependency_url_list = [common_dep_whl] # Obtain package contents; exclude build and other files packages_found = find_packages() + find_namespace_packages(exclude=['*bin', 'pyenv3*', 'build', 'dist', '*bin', '*x86*']) # Create common dependency list package_dependency_files = ['reqs_pip_tf_common.txt'] -install_requires_list = [open(package_bin_dir + '/reqs_pip_tf_common.txt').read()] +install_requires_list = get_dependency_packages('reqs_pip_tf_common.txt') if "--gpu" in sys.argv: # Create Tensorflow GPU dependency list - package_dependency_files.extend(['reqs_pip_tf_gpu.txt', 'reqs_deb_tf_gpu.txt']) - install_requires_list.append(open(package_bin_dir + '/reqs_pip_tf_gpu.txt').read()) + package_dependency_files.extend(['bin/reqs_pip_tf_gpu.txt', 'bin/reqs_deb_tf_gpu.txt']) + install_requires_list.extend(get_dependency_packages('reqs_pip_tf_gpu.txt')) + dependency_url_list.extend(get_dependency_urls('reqs_pip_tf_gpu.txt')) sys.argv.remove("--gpu") else: # Create Tensorflow CPU dependency list - package_dependency_files.extend(['reqs_pip_tf_cpu.txt']) - install_requires_list.append(open(package_bin_dir + '/reqs_pip_tf_cpu.txt').read()) + package_dependency_files.extend(['bin/reqs_pip_tf_cpu.txt']) + install_requires_list.extend(get_dependency_packages('reqs_pip_tf_cpu.txt')) + dependency_url_list.extend(get_dependency_urls('reqs_pip_tf_cpu.txt')) # Loop over package artifacts folder required_package_data = ['acceptance_tests/*.*'] @@ -83,7 +144,7 @@ long_description=open('README.txt').read(), package_data={package_name:required_package_data}, install_requires=install_requires_list, - dependency_links=[common_dep_whl], + dependency_links=dependency_url_list, include_package_data=True, zip_safe=True, platforms='x86', diff --git a/packaging/setup_aimet_torch.py b/packaging/setup_aimet_torch.py index 0d03d2d051a..ec858941a29 100644 --- a/packaging/setup_aimet_torch.py +++ b/packaging/setup_aimet_torch.py @@ -41,29 +41,90 @@ from setuptools import setup, find_packages, find_namespace_packages import setup_cfg # pylint: disable=import-error -package_url_base = setup_cfg.remote_url + "/releases/download/"+str(setup_cfg.version) +package_name = "aimet_torch" + +def prepend_bin_path(dependency_list_file): + ''' + Append the common path to the dependency file + ''' + package_bin_dir = package_name + "/bin" + full_path = package_bin_dir + '/' + dependency_list_file + return full_path + + +def get_dependency_packages(dependency_list_file): + ''' + Read the dependency file and return a list of packages + ''' + + dependency_file_full_path = prepend_bin_path(dependency_list_file) + dependency_list_array = open(dependency_file_full_path).read().splitlines() + + dependency_packages_list = [] + for dependency_line in dependency_list_array: + if dependency_line.strip().startswith('#'): + # Skip the commented out lines + continue + # Pick up the package and version (== i.e. first column of file) + dependency_packages_list.append(dependency_line.lstrip().split()[0]) + + return dependency_packages_list +def get_dependency_urls(dependency_list_file): + ''' + Read the dependency file and return a list of package source URLs + ''' + + dependency_file_full_path = prepend_bin_path(dependency_list_file) + dependency_list_array = open(dependency_file_full_path).read().splitlines() + + dependency_urls_list = [] + for dependency_line in dependency_list_array: + if dependency_line.strip().startswith('#'): + # Skip the commented out lines + continue + + # Extract any package custom URLs from the requirements file + # Ex. torch==1.7.1+cu110 -f https://download.pytorch.org/whl/torch_stable.html + url_delimiter = '-f ' + # Split each line using the URL option as delimiter + dependency_line_split_list = dependency_line.split(url_delimiter) + # The split list will have at least 2 elements if a URL was present + if len(dependency_line_split_list) > 1: + # If the URL exists, remove all whitespaces + dependency_url = dependency_line_split_list[1].strip() + # Add to the list only if not already present + if dependency_url not in dependency_urls_list: + dependency_urls_list.append(dependency_url) + + return dependency_urls_list + + +package_url_base = setup_cfg.remote_url + "/releases/download/"+str(setup_cfg.version) common_dep_whl = package_url_base + "/AimetCommon-" + str(setup_cfg.version) + "-py3-none-any.whl" +dependency_url_list = [common_dep_whl] # Obtain package contents; exclude build and other files packages_found = find_packages() + find_namespace_packages(exclude=['*bin', 'pyenv3*', 'build', 'dist', '*bin', '*x86*']) # Create common dependency list -package_dependency_files = [] -install_requires_list = [open('bin/reqs_pip_torch_common.txt').read()] +package_dependency_files = ['reqs_pip_torch_common.txt'] +install_requires_list = get_dependency_packages('reqs_pip_torch_common.txt') if "--gpu" in sys.argv: # Create Torch GPU dependency list package_dependency_files.extend(['bin/reqs_pip_torch_gpu.txt', 'bin/reqs_deb_torch_gpu.txt']) - install_requires_list.append(open('bin/reqs_pip_torch_gpu.txt').read()) + install_requires_list.extend(get_dependency_packages('reqs_pip_torch_gpu.txt')) + dependency_url_list.extend(get_dependency_urls('reqs_pip_torch_gpu.txt')) sys.argv.remove("--gpu") else: # Create Torch CPU dependency list package_dependency_files.extend(['bin/reqs_pip_torch_cpu.txt']) - install_requires_list.append(open('bin/reqs_pip_torch_cpu.txt').read()) + install_requires_list.extend(get_dependency_packages('reqs_pip_torch_cpu.txt')) + dependency_url_list.extend(get_dependency_urls('reqs_pip_torch_cpu.txt')) # Loop over package artifacts folder required_package_data = ['acceptance_tests/*.*'] -for path, _, filenames in os.walk('aimet_torch'): +for path, _, filenames in os.walk(package_name): required_package_data += [os.path.join(path, filename) for filename in filenames if filename.endswith(tuple(package_dependency_files))] required_package_data = ['/'.join(files.split('/')[1:]) for files in required_package_data] @@ -79,9 +140,9 @@ license='NOTICE.txt', description='AIMET PyTorch Package', long_description=open('README.txt').read(), - package_data={'aimet_torch':required_package_data}, + package_data={package_name:required_package_data}, install_requires=install_requires_list, - dependency_links=[common_dep_whl], + dependency_links=dependency_url_list, include_package_data=True, zip_safe=True, platforms='x86',