Fixed packaging, updated package versions, etc

1. Fixed packaging issues with variants
2. Added missing packaging stage to all PR variant pipelines
3. Added Deprecated package, pinned Astroid package, update numpy versions in PR and/or dependency lists
4. Fixed install issues with torch variants

Signed-off-by: Bharath Ramaswamy <quic_bharathr@quicinc.com>

CMakeLists.txt

@@ -195,7 +195,7 @@ if(NOT remote_url STREQUAL "")
     # Run the install target first
     COMMAND "${CMAKE_COMMAND}" --build . --target install
     # Now run the packaging target to generate wheel files
-    COMMAND ${CMAKE_COMMAND} -DAIMET_PACKAGE_PATH=${AIMET_PACKAGE_PATH} -DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DENABLE_TENSORFLOW=${ENABLE_TENSORFLOW} -DENABLE_TORCH=${ENABLE_TORCH} -P ${CMAKE_CURRENT_SOURCE_DIR}/packaging/package_aimet.cmake
+    COMMAND ${CMAKE_COMMAND} -DAIMET_PACKAGE_PATH=${AIMET_PACKAGE_PATH} -DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DENABLE_CUDA=${ENABLE_CUDA} -DENABLE_TENSORFLOW=${ENABLE_TENSORFLOW} -DENABLE_TORCH=${ENABLE_TORCH} -P ${CMAKE_CURRENT_SOURCE_DIR}/packaging/package_aimet.cmake
   )
 else()
   message(STATUS "Repo Remote URL is blank. Skipping packageaimet target")

Jenkins/Dockerfile.tf-cpu

@@ -85,7 +85,7 @@ RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists
 RUN pip3 --no-cache-dir install --upgrade \
         setuptools==41.0.1 \
         pip==20.2.4 \
-        numpy==1.16.4 \
+        numpy==1.16.6 \
         wheel==0.33.4
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -119,12 +119,14 @@ RUN apt-get update > /dev/null && \
 
 # Python3 Packages
 RUN pip3 --no-cache-dir install \
+        astroid==2.5.3 \
         attrs==19.1.0 \
         behave==1.2.6 \
         blosc==1.8.1 \
         cffi==1.12.3 \
         click==7.0 \
         cython==0.29.10 \
+        Deprecated==1.2.12 \
         docutils==0.16 \
         h5py==2.9.0 \
         ipykernel==4.8.2 \

Jenkins/Dockerfile.tf-gpu

@@ -85,7 +85,7 @@ RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists
 RUN pip3 --no-cache-dir install --upgrade \
         setuptools==41.0.1 \
         pip==20.2.4 \
-        numpy==1.16.4 \
+        numpy==1.16.6 \
         wheel==0.33.4
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -119,12 +119,14 @@ RUN apt-get update > /dev/null && \
 
 # Python3 Packages
 RUN pip3 --no-cache-dir install \
+        astroid==2.5.3 \
         attrs==19.1.0 \
         behave==1.2.6 \
         blosc==1.8.1 \
         cffi==1.12.3 \
         click==7.0 \
         cython==0.29.10 \
+        Deprecated==1.2.12 \
         docutils==0.16 \
         h5py==2.9.0 \
         ipykernel==4.8.2 \

Jenkins/Dockerfile.tf-torch-cpu

@@ -81,7 +81,7 @@ RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists
 RUN pip3 --no-cache-dir install --upgrade \
         setuptools==41.0.1 \
         pip==20.2.4 \
-        numpy==1.16.4 \
+        numpy==1.16.6 \
         wheel==0.33.4
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -115,12 +115,14 @@ RUN apt-get update > /dev/null && \
 
 # Python3 Packages
 RUN pip3 --no-cache-dir install \
+        astroid==2.5.3 \
         attrs==19.1.0 \
         behave==1.2.6 \
         blosc==1.8.1 \
         cffi==1.12.3 \
         click==7.0 \
         cython==0.29.10 \
+        Deprecated==1.2.12 \
         docutils==0.16 \
         h5py==2.9.0 \
         ipykernel==4.8.2 \

Jenkins/Dockerfile.torch-cpu

@@ -81,7 +81,7 @@ RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists
 RUN pip3 --no-cache-dir install --upgrade \
         setuptools==41.0.1 \
         pip==20.2.4 \
-        numpy==1.16.4 \
+        numpy==1.16.6 \
         wheel==0.33.4
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -115,12 +115,14 @@ RUN apt-get update > /dev/null && \
 
 # Python3 Packages
 RUN pip3 --no-cache-dir install \
+        astroid==2.5.3 \
         attrs==19.1.0 \
         behave==1.2.6 \
         blosc==1.8.1 \
         cffi==1.12.3 \
         click==7.0 \
         cython==0.29.10 \
+        Deprecated==1.2.12 \
         docutils==0.16 \
         h5py==2.9.0 \
         ipykernel==4.8.2 \

Jenkins/Dockerfile.torch-gpu

@@ -81,7 +81,7 @@ RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists
 RUN pip3 --no-cache-dir install --upgrade \
         setuptools==41.0.1 \
         pip==20.2.4 \
-        numpy==1.16.4 \
+        numpy==1.16.6 \
         wheel==0.33.4
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -115,12 +115,14 @@ RUN apt-get update > /dev/null && \
 
 # Python3 Packages
 RUN pip3 --no-cache-dir install \
+        astroid==2.5.3 \
         attrs==19.1.0 \
         behave==1.2.6 \
         blosc==1.8.1 \
         cffi==1.12.3 \
         click==7.0 \
         cython==0.29.10 \
+        Deprecated==1.2.12 \
         docutils==0.16 \
         h5py==2.9.0 \
         ipykernel==4.8.2 \

Jenkins/Jenkinsfile

@@ -202,7 +202,7 @@ pipeline {
                                 echo 'Building code (and generating Docs and pip packages)...'
                                 script {
                                     //TODO Change option back to "-bp" as soon as issue is fixed
-                                    runStage(env.AIMET_VARIANT_TORCH_GPU, "-b")
+                                    runStage(env.AIMET_VARIANT_TORCH_GPU, "-bp")
                                 }
                             }
                         }
@@ -319,7 +319,7 @@ pipeline {
                                 echo 'Building code (and generating Docs and pip packages)...'
                                 script {
                                     //TODO Change option back to "-bp" as soon as issue is fixed
-                                    runStage(env.AIMET_VARIANT_TF_CPU, "-b")
+                                    runStage(env.AIMET_VARIANT_TF_CPU, "-bp")
                                 }
                             }
                         }
@@ -436,7 +436,7 @@ pipeline {
                                 echo 'Building code (and generating Docs and pip packages)...'
                                 script {
                                     //TODO Change option back to "-bp" as soon as issue is fixed
-                                    runStage(env.AIMET_VARIANT_TORCH_CPU, "-b")
+                                    runStage(env.AIMET_VARIANT_TORCH_CPU, "-bp")
                                 }
                             }
                         }

TrainingExtensions/torch/CMakeLists.txt

@@ -84,6 +84,14 @@ install(DIRECTORY ${CMAKE_BINARY_DIR}/artifacts/
       PATTERN "__pycache*" EXCLUDE
       )
 
-install(FILES ${CMAKE_BINARY_DIR}/artifacts/easy-install.pth ${CMAKE_BINARY_DIR}/artifacts/site.py
-      DESTINATION ${AIMET_INSTALL_DIR}/lib/x86_64-linux-gnu
-      )
+if(EXISTS ${CMAKE_BINARY_DIR}/artifacts/easy-install.pth)
+      install(FILES ${CMAKE_BINARY_DIR}/artifacts/easy-install.pth 
+            DESTINATION ${AIMET_INSTALL_DIR}/lib/x86_64-linux-gnu
+            )
+endif()
+
+if(EXISTS ${CMAKE_BINARY_DIR}/artifacts/site.py)
+      install(FILES ${CMAKE_BINARY_DIR}/artifacts/site.py
+            DESTINATION ${AIMET_INSTALL_DIR}/lib/x86_64-linux-gnu
+            )
+endif()

packaging/INSTALL.txt

@@ -38,62 +38,5 @@
 # Installation instructions for AI Model Efficienty Toolkit package and dependencies
 # ==================================================================================
 
-# NOTE:
-# 1. Please pre-pend the "apt-get install" and "pip3 install" commands with "sudo -H" as appropriate
-# 2. These instructions that pip packages will be installed in the path: /usr/local/lib/python3.6/dist-packages. If that is not the case, please modify it accordingly.
-
-# Install the following pre-requisite packages
-apt-get update
-apt-get install python3.6 python3.6-dev python3-pip
-pip3 install --upgrade pip
-
-# ======================
-# Install AIMET packages
-# ======================
-# Go to https://github.com/quic/aimet/releases and identify the release tag of the package you want to install
-# NOTE: Python dependencies would automatically get installed.
-#
-
-release_tag=<release_tag>
-pip3 install https://github.com/quic/aimet/releases/download/${release_tag}/AimetCommon-${release_tag}-py3-none-any.whl -f https://download.pytorch.org/whl/torch_stable.html
-pip3 install https://github.com/quic/aimet/releases/download/${release_tag}/AimetTorch-${release_tag}-py3-none-any.whl
-pip3 install https://github.com/quic/aimet/releases/download/${release_tag}/AimetTensorflow-${release_tag}-py3-none-any.whl
-pip3 install https://github.com/quic/aimet/releases/download/${release_tag}/Aimet-${release_tag}-py3-none-any.whl
-
-# ================================================================
-# Install common debian packages from the packages_common.txt file
-# ================================================================
-apt-get update
-cat /usr/local/lib/python3.6/dist-packages/aimet_common/bin/packages_common.txt | xargs apt-get --assume-yes install
-
-# Replace the Pillow package with Pillow-SIMD as follows:
-pip3 uninstall -y pillow && pip3 install --no-cache-dir Pillow-SIMD==6.0.0.post0
-
-# GPU preparation steps
-wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.0.130-1_amd64.deb
-apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
-dpkg -i cuda-repo-ubuntu1804_10.0.130-1_amd64.deb
-apt-get update
-wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb
-apt-get --assume-yes install ./nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb
-apt-get update
-
-# ======================================================
-# Install GPU packages from the packages_common.txt file
-# ======================================================
-cat /usr/local/lib/python3.6/dist-packages/aimet_common/bin/packages_gpu.txt | xargs apt-get --assume-yes install
-
-# Post installation steps
-ln -s /usr/local/cuda-10.0 /usr/local/cuda
-ln -s /usr/lib/x86_64-linux-gnu/libjpeg.so /usr/lib
-
-# =====================
-# Setup the environment
-# =====================
-# Set the common environment variables as follows
-source /usr/local/lib/python3.6/dist-packages/aimet_common/bin/envsetup.sh 
-
-# Add AIMET packages to the environment paths
-export LD_LIBRARY_PATH=/usr/local/lib/python3.6/dist-packages/aimet_common/x86_64-linux-gnu:/usr/local/lib/python3.6/dist-packages/aimet_common:$LD_LIBRARY_PATH
-
-if [[ $PYTHONPATH = "" ]]; then export PYTHONPATH=/usr/local/lib/python3.6/dist-packages/aimet_common/x86_64-linux-gnu; else export PYTHONPATH=/usr/local/lib/python3.6/dist-packages/aimet_common/x86_64-linux-gnu:$PYTHONPATH; fi
+# Please see install.md for the latest installation instructions.
+# This file will be removed shortly.

packaging/dependencies/reqs_pip_common.txt

@@ -1,5 +1,5 @@
 ipython==7.16.1
-numpy==1.16.4
+numpy==1.16.6
 scipy==1.2.1
 protobuf==3.7.1
 scikit-learn==0.21.0

packaging/install.md

@@ -47,7 +47,7 @@ release_tag=<release_tag>
 python3 -m pip install https://github.com/quic/aimet/releases/download/${release_tag}/AimetCommon-${AIMET_VARIANT}_${release_tag}-py3-none-any.whl
 
 # Install ONE of the following depending on the variant
-python3 -m pip install https://github.com/quic/aimet/releases/download/${release_tag}/AimetTorch-${AIMET_VARIANT}_${release_tag}-py3-none-any.whl
+python3 -m pip install https://github.com/quic/aimet/releases/download/${release_tag}/AimetTorch-${AIMET_VARIANT}_${release_tag}-py3-none-any.whl -f https://download.pytorch.org/whl/torch_stable.html
 # OR
 python3 -m pip install https://github.com/quic/aimet/releases/download/${release_tag}/AimetTensorflow-${AIMET_VARIANT}_${release_tag}-py3-none-any.whl
 

packaging/package_aimet.cmake

@@ -141,7 +141,7 @@ foreach(package ${package_name_list})
     file(COPY ${AIMET_PACKAGE_PATH}/lib/python/${package} DESTINATION ${build_packaging_dir}/)
   endif()
 
-  # Invoke the setup tools scrip to create the wheel packages. 
+  # Invoke the setup tools script to create the wheel packages.
   execute_process(COMMAND python3 setup.py sdist bdist_wheel ${CUDA_OPTION} WORKING_DIRECTORY ${build_packaging_dir} OUTPUT_VARIABLE output_var)
 
 endforeach()

packaging/setup_aimet.py

@@ -36,6 +36,7 @@
 
 """ Package generation file for top-level aimet package """
 
+import sys
 import os.path
 from setuptools import setup
 import setup_cfg # pylint: disable=import-error
@@ -45,6 +46,10 @@
 torch_dep_whl = package_url_base + "/AimetTorch-" + str(setup_cfg.version) + "-py3-none-any.whl"
 tf_dep_whl = package_url_base + "/AimetTensorflow-" + str(setup_cfg.version) + "-py3-none-any.whl"
 
+if "--gpu" in sys.argv:
+    # There is NO common GPU dependency list, so just ignore the option if it was passed in
+    sys.argv.remove("--gpu")
+
 dependency_list = []
 if os.path.isfile(torch_dep_whl):
     dependency_list.append(torch_dep_whl)

packaging/setup_aimet_common.py

@@ -52,9 +52,7 @@
 install_requires_list = [open(package_bin_dir + '/reqs_pip_common.txt').read()]
 package_dependency_files = ['reqs_pip_common.txt', 'reqs_deb_common.txt', 'INSTALL.txt', 'envsetup.sh', 'LICENSE.pdf']
 if "--gpu" in sys.argv:
-    # Create common GPU dependency list
-    package_dependency_files.append(['reqs_pip_gpu.txt', 'reqs_deb_gpu.txt'])
-    install_requires_list.append(open(package_bin_dir + '/reqs_pip_gpu.txt').read())
+    # There is NO common GPU dependency list, so just ignore the option if it was passed in
     sys.argv.remove("--gpu")
 
 # Loop over package artifacts folder

packaging/setup_aimet_tensorflow.py

@@ -42,26 +42,87 @@
 import setup_cfg # pylint: disable=import-error
 
 package_name = "aimet_tensorflow"
-package_bin_dir = package_name + "/bin"
-package_url_base = setup_cfg.remote_url + "/releases/download/" + str(setup_cfg.version)
 
+def prepend_bin_path(dependency_list_file):
+    '''
+    Append the common path to the dependency file
+    '''
+    package_bin_dir = package_name + "/bin"
+    full_path = package_bin_dir + '/' + dependency_list_file
+    return full_path
+
+
+def get_dependency_packages(dependency_list_file):
+    '''
+    Read the dependency file and return a list of packages
+    '''
+
+    dependency_file_full_path = prepend_bin_path(dependency_list_file)
+    dependency_list_array = open(dependency_file_full_path).read().splitlines()
+
+    dependency_packages_list = []
+    for dependency_line in dependency_list_array:
+        if dependency_line.strip().startswith('#'):
+            # Skip the commented out lines
+            continue
+        # Pick up the package and version (<package>==<ver> i.e. first column of file)
+        dependency_packages_list.append(dependency_line.lstrip().split()[0])
+
+    return dependency_packages_list
+
+def get_dependency_urls(dependency_list_file):
+    '''
+    Read the dependency file and return a list of package source URLs
+    '''
+
+    url_delimiter = '-f '
+
+    dependency_file_full_path = prepend_bin_path(dependency_list_file)
+    dependency_list_array = open(dependency_file_full_path).read().splitlines()
+
+    dependency_urls_list = []
+    for dependency_line in dependency_list_array:
+        if dependency_line.strip().startswith('#'):
+            # Skip the commented out lines
+            continue
+
+        # Extract any package custom URLs from the requirements file
+        # Ex. torch==1.7.1+cu110 -f https://download.pytorch.org/whl/torch_stable.html
+        url_delimiter = '-f '
+        # Split each line using the URL option as delimiter
+        dependency_line_split_list = dependency_line.split(url_delimiter)
+        # The split list will have at least 2 elements if a URL was present
+        if len(dependency_line_split_list) > 1:
+            # If the URL exists, remove all whitespaces
+            dependency_url = dependency_line_split_list[1].strip()
+            # Add to the list only if not already present
+            if dependency_url not in dependency_urls_list:
+                dependency_urls_list.append(dependency_url)
+
+    return dependency_urls_list
+
+
+package_url_base = setup_cfg.remote_url + "/releases/download/"+str(setup_cfg.version)
 common_dep_whl = package_url_base + "/AimetCommon-" + str(setup_cfg.version) + "-py3-none-any.whl"
+dependency_url_list = [common_dep_whl]
 
 # Obtain package contents; exclude build and other files
 packages_found = find_packages() + find_namespace_packages(exclude=['*bin', 'pyenv3*', 'build', 'dist', '*bin', '*x86*'])
 
 # Create common dependency list
 package_dependency_files = ['reqs_pip_tf_common.txt']
-install_requires_list = [open(package_bin_dir + '/reqs_pip_tf_common.txt').read()]
+install_requires_list = get_dependency_packages('reqs_pip_tf_common.txt')
 if "--gpu" in sys.argv:
     # Create Tensorflow GPU dependency list
-    package_dependency_files.extend(['reqs_pip_tf_gpu.txt', 'reqs_deb_tf_gpu.txt'])
-    install_requires_list.append(open(package_bin_dir + '/reqs_pip_tf_gpu.txt').read())
+    package_dependency_files.extend(['bin/reqs_pip_tf_gpu.txt', 'bin/reqs_deb_tf_gpu.txt'])
+    install_requires_list.extend(get_dependency_packages('reqs_pip_tf_gpu.txt'))
+    dependency_url_list.extend(get_dependency_urls('reqs_pip_tf_gpu.txt'))
     sys.argv.remove("--gpu")
 else:
     # Create Tensorflow CPU dependency list
-    package_dependency_files.extend(['reqs_pip_tf_cpu.txt'])
-    install_requires_list.append(open(package_bin_dir + '/reqs_pip_tf_cpu.txt').read())
+    package_dependency_files.extend(['bin/reqs_pip_tf_cpu.txt'])
+    install_requires_list.extend(get_dependency_packages('reqs_pip_tf_cpu.txt'))
+    dependency_url_list.extend(get_dependency_urls('reqs_pip_tf_cpu.txt'))
 
 # Loop over package artifacts folder
 required_package_data = ['acceptance_tests/*.*']
@@ -83,7 +144,7 @@
     long_description=open('README.txt').read(),
     package_data={package_name:required_package_data},
     install_requires=install_requires_list,
-    dependency_links=[common_dep_whl],
+    dependency_links=dependency_url_list,
     include_package_data=True,
     zip_safe=True,
     platforms='x86',