From d352f1290bb2d03e16462478d5dd4e0ad9421666 Mon Sep 17 00:00:00 2001 From: dhruvesh09 <30351549+dhruvesh09@users.noreply.github.com> Date: Thu, 24 Sep 2020 14:10:37 -0700 Subject: [PATCH] Deletes unnecessary license gathering scripts from the release process. * Update RELEASE.md * Update RELEASE.md * Update README.md * Update version.py * Update dependencies.py * Update RELEASE.md * Update RELEASE.md * Update Dockerfile * Update build_docker_image.sh * Delete license.sh * Delete third_party_licenses.csv * Update test_utils.py --- tfx/orchestration/test_utils.py | 4 - tfx/tools/docker/Dockerfile | 6 -- tfx/tools/docker/build_docker_image.sh | 3 +- tfx/tools/docker/license.sh | 106 ---------------------- tfx/tools/docker/third_party_licenses.csv | 39 -------- 5 files changed, 1 insertion(+), 157 deletions(-) delete mode 100755 tfx/tools/docker/license.sh delete mode 100644 tfx/tools/docker/third_party_licenses.csv diff --git a/tfx/orchestration/test_utils.py b/tfx/orchestration/test_utils.py index ba9255f919..1a39a40b1b 100644 --- a/tfx/orchestration/test_utils.py +++ b/tfx/orchestration/test_utils.py @@ -75,10 +75,6 @@ def build_and_push_docker_image(container_image: str, repo_base: str): path=repo_base, dockerfile='tfx/tools/docker/Dockerfile', tag=container_image, - buildargs={ - # Skip license gathering for tests. - 'gather_third_party_licenses': 'false', - }, ) logging.info('Pushing image %s', container_image) diff --git a/tfx/tools/docker/Dockerfile b/tfx/tools/docker/Dockerfile index 728e41df4d..9477a743cd 100644 --- a/tfx/tools/docker/Dockerfile +++ b/tfx/tools/docker/Dockerfile @@ -44,12 +44,6 @@ RUN patch `python -c 'import googleapiclient; print(googleapiclient.__path__[0]) patch `python -c 'import apitools; print(apitools.__path__[0])'`/base/py/base_api.py \ /tfx-src/tfx/tools/docker/patches/base_api.patch -ARG gather_third_party_licenses=true -RUN if [ "${gather_third_party_licenses}" = "true" ]; then \ - /tfx-src/tfx/tools/docker/license.sh \ - /tfx-src/tfx/tools/docker/third_party_licenses.csv \ - /third_party/licenses; \ - fi # TODO(b/139532550): Consolidate container entrypoint with Kubeflow runner. ENTRYPOINT ["python", "/tfx-src/tfx/scripts/run_executor.py"] diff --git a/tfx/tools/docker/build_docker_image.sh b/tfx/tools/docker/build_docker_image.sh index e7dcdcc0cf..bcc79aa9ca 100755 --- a/tfx/tools/docker/build_docker_image.sh +++ b/tfx/tools/docker/build_docker_image.sh @@ -19,7 +19,6 @@ set -u -x DOCKER_IMAGE_REPO=${DOCKER_IMAGE_REPO:-"tensorflow/tfx"} DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"latest"} DOCKER_FILE=${DOCKER_FILE:-"Dockerfile"} -GATHER_THIRD_PARTY_LICENSES=${GATHER_THIRD_PARTY_LICENSES:-"true"} # Run docker build command. -docker build -t ${DOCKER_IMAGE_REPO}:${DOCKER_IMAGE_TAG} -f tfx/tools/docker/${DOCKER_FILE} . --build-arg gather_third_party_licenses=${GATHER_THIRD_PARTY_LICENSES} "$@" +docker build -t ${DOCKER_IMAGE_REPO}:${DOCKER_IMAGE_TAG} -f tfx/tools/docker/${DOCKER_FILE} . "$@" diff --git a/tfx/tools/docker/license.sh b/tfx/tools/docker/license.sh deleted file mode 100755 index 442ae615eb..0000000000 --- a/tfx/tools/docker/license.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -e -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# A script to gather licenses of locally-installed python packages. -# There are two sources of licenses -# - Locally available LICENSE files which are installed with the package. -# - A license table (3 columns: name,license_link,license_type) csv file -# which can be used to download license files. -# -# These license files will be copied into specified directory. -# Usage: -# license.sh third_party_licenses.csv /usr/licenses - -set -e -set +x - -# Get the list of python packages installed locally. -OLD_IFS="$IFS" -IFS=$'\n' -INSTALLED_PACKAGES_WITH_VERSION=($(pip freeze --exclude-editable | grep -v "pkg-resources")) - -# Get the list of python packages tracked in the given CSV file. -REGISTERED_PACKAGES=() -while IFS=, read -r col1 col2 col3 -do - REGISTERED_PACKAGES+=($col1) -done < $1 -IFS="$OLD_IFS" - -mkdir -p $2 - -GPL_PACKAGES=() -REMOTE_LICENSE_PACKAGES=() -# Copy locally available LICENSE files and gather packages without them. -for i in "${INSTALLED_PACKAGES_WITH_VERSION[@]}"; do - # `i` looks like "pkg_name==0.0.1". - pkg=${i%==*} - pkg_base_dir=$(python -m pip show ${pkg} | sed -n 's/^Location: //p') - # LICENSE files are located under .../site-packages/pkg_name-0.0.1.dist-info/. - pkg_without_underscore=${i//-/_} - pkg_dist_info_dir="${pkg_base_dir}/${pkg_without_underscore/==/-}.dist-info" - set +e - license_files=$(ls ${pkg_dist_info_dir}/LICENSE* 2> /dev/null) - if [[ $? -ne 0 ]]; then - REMOTE_LICENSE_PACKAGES+=("$pkg") - else - echo "License found: ${license_files}" - cat ${license_files} > $2/$pkg.LICENSE - fi - set -e - - license=$(python -m pip show ${pkg} | sed -n 's/^License: //p') - if [[ "${license}" == *GPL* ]]; then - GPL_PACKAGES+=("$pkg") - fi -done - -# Collect missing packages. -MISSING=() -for i in "${REMOTE_LICENSE_PACKAGES[@]}"; do - skip= - for j in "${REGISTERED_PACKAGES[@]}"; do - # PyPI package name is case-insensitive so cast to lower case. - [[ ${i,,} == ${j,,} ]] && { skip=1; break; } - done - [[ -n $skip ]] || MISSING+=("$i") -done - -if [ -n "$MISSING" ]; then - echo "The following packages are not found for licenses tracking." - echo "Please add an entry in $1 for each of them." - echo ${MISSING[@]} - exit 1 -fi - -# Download licenses from the url in the csv file. -# This should be done before guessing with github URL to override the problem of -# packages with wrong(but working) Home-page URLs. -while IFS=, read -r col1 col2 col3 -do - if [[ " ${REMOTE_LICENSE_PACKAGES[@]} " =~ " ${col1} " ]]; then - echo "Downloading license for ${col1} from ${col2}" - curl --fail -sSL -o $2/$col1.LICENSE $col2 - fi -done < $1 -IFS="$OLD_IFS" - -# Download source code for GPL packages. -mkdir -p $2/source -for i in "${GPL_PACKAGES[@]}"; do - echo "Downloading source of the GPL-licensed package: ${i}" - python -m pip install -t "$2/source/${i}" ${i} -done diff --git a/tfx/tools/docker/third_party_licenses.csv b/tfx/tools/docker/third_party_licenses.csv deleted file mode 100644 index c8087efc8e..0000000000 --- a/tfx/tools/docker/third_party_licenses.csv +++ /dev/null @@ -1,39 +0,0 @@ -Send2Trash,https://raw.githubusercontent.com/hsoft/send2trash/master/LICENSE,3-Clause BSD -apache-beam,https://raw.githubusercontent.com/apache/beam/master/LICENSE,Apache 2.0 -avro-python3,https://raw.githubusercontent.com/apache/avro/master/LICENSE.txt,Apache 2.0 -chardet,https://raw.githubusercontent.com/chardet/chardet/master/LICENSE,LGPL 2.1 -defusedxml,https://raw.githubusercontent.com/tiran/defusedxml/master/LICENSE,PSF -google-apitools,https://raw.githubusercontent.com/google/apitools/master/LICENSE,Apache 2.0 -grpcio,https://raw.githubusercontent.com/grpc/grpc/master/LICENSE,Apache 2.0 -grpcio-gcp,https://raw.githubusercontent.com/grpc/grpc/master/LICENSE,Apache 2.0 -h5py,https://raw.githubusercontent.com/h5py/h5py/master/licenses/license.txt -ipykernel,https://raw.githubusercontent.com/ipython/ipykernel/master/COPYING.md,3-Clause BSD -ipython-genutils,https://raw.githubusercontent.com/ipython/ipython_genutils/master/COPYING.md,BSD -jsonschema,https://raw.githubusercontent.com/Julian/jsonschema/master/COPYING,MIT -jupyter,https://raw.githubusercontent.com/jupyter/notebook/6.0.2/LICENSE,BSD -jupyter-client,https://raw.githubusercontent.com/jupyter/jupyter_client/master/COPYING.md,BSD -jupyter-console,https://raw.githubusercontent.com/jupyter/jupyter_console/master/COPYING.md,BSD -jupyter-core,https://raw.githubusercontent.com/jupyter/jupyter_core/master/COPYING.md,BSD -keras-tuner,https://raw.githubusercontent.com/keras-team/keras-tuner/master/LICENSE,Apache 2.0 -mistune,https://raw.githubusercontent.com/lepture/mistune/master/LICENSE,3-Clause BSD -ml-metadata,https://raw.githubusercontent.com/google/ml-metadata/master/LICENSE,Apache 2.0 -mock,https://raw.githubusercontent.com/testing-cabal/mock/master/LICENSE.txt,2-Clause BSD -monotonic,https://raw.githubusercontent.com/atdt/monotonic/master/LICENSE,Apache 2.0 -nbformat,https://raw.githubusercontent.com/jupyter/nbformat/master/COPYING.md,BSD-4-Clause -oauth2client,https://raw.githubusercontent.com/google/oauth2client/master/LICENSE,Apache 2.0 -pbr,https://raw.githubusercontent.com/openstack/pbr/16f19608c446db67e2ad0950a3209078ec6be18b/LICENSE,Apache 2.0 -pickleshare,https://raw.githubusercontent.com/pickleshare/pickleshare/master/LICENSE,MIT -protobuf,https://raw.githubusercontent.com/protocolbuffers/protobuf/master/LICENSE,BSD-3-clause -pyarrow,https://raw.githubusercontent.com/apache/arrow/master/LICENSE.txt,Apache 2.0 -pyrsistent,https://raw.githubusercontent.com/tobgu/pyrsistent/master/LICENCE.mit,MIT -pyzmq,https://raw.githubusercontent.com/zeromq/pyzmq/master/COPYING.BSD,LGPL+BSD -scikit-learn,https://raw.githubusercontent.com/scikit-learn/scikit-learn/master/COPYING,BSD -scipy,https://raw.githubusercontent.com/scipy/scipy/master/LICENSE.txt,BSD -tensorboard-plugin-wit,https://raw.githubusercontent.com/tensorflow/tensorboard/master/LICENSE,Apache 2.0 -tensorflow-estimator,https://raw.githubusercontent.com/tensorflow/estimator/master/LICENSE,Apache 2.0 -tensorflow-serving-api,https://raw.githubusercontent.com/tensorflow/serving/master/LICENSE,Apache 2.0 -termcolor,https://raw.githubusercontent.com/hfeeki/termcolor/master/COPYING.txt,MIT -terminaltables,https://raw.githubusercontent.com/Robpol86/terminaltables/master/LICENSE,MIT -tqdm,https://raw.githubusercontent.com/tqdm/tqdm/master/LICENCE,MIT -traitlets,https://raw.githubusercontent.com/ipython/traitlets/master/COPYING.md,BSD -webencodings,https://raw.githubusercontent.com/gsnedders/python-webencodings/master/LICENSE,BSD