Skip to content

Commit

Permalink
backend: Pin python dependencies
Browse files Browse the repository at this point in the history 
Builds/tests failed due to broken transident dependencies. This pins the
python dependencies to prevent this in the future.
  • Loading branch information
@discordianfish
discordianfish committed Mar 4, 2020
1 parent 00a2cad commit 0ab6975
Show file tree
Hide file tree
Showing 8 changed files with 299 additions and 13 deletions.
3 changes: 2 additions & 1 deletion backend/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ FROM python:3.5 as compiler
RUN apt-get update -y && \
apt-get install --no-install-recommends -y -q default-jdk python3-setuptools python3-dev
RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py
RUN python3 -m pip install tfx==0.21.0 tensorflow_transform==0.21.0
COPY backend/requirements.txt .
RUN python3 -m pip install -r requirements.txt

WORKDIR /go/src/github.com/kubeflow/pipelines
COPY sdk sdk
Expand Down
8 changes: 8 additions & 0 deletions backend/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,11 @@ If a new external Go dependency is added, or an existing one has its version
bumped in the `go.mod` file, ensure the BUILD files pick this up by updating the
WORKSPACE go_repository rules using the following command: `bazel run
//:gazelle -- update-repos --from_file=go.mod`

## Updating python dependencies

[pip-tools](https://github.com/jazzband/pip-tools) is used to manage python
dependencies. To update dependencies, edit [requirements.in](requirements.in)
and run `./update_requirements.sh` to update and pin the transitive
dependencies.

1 change: 1 addition & 0 deletions backend/requirements.in
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
tfx==0.21.0
134 changes: 134 additions & 0 deletions backend/requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
#
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile requirements.in
#
absl-py==0.8.1 # via ml-metadata, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx, tfx-bsl
apache-beam[gcp]==2.17.0 # via tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx, tfx-bsl
astor==0.8.1 # via tensorflow
attrs==19.3.0 # via jsonschema
avro-python3==1.9.1 # via apache-beam, tensorflow-data-validation, tensorflow-model-analysis, tfx, tfx-bsl
backcall==0.1.0 # via ipython
bleach==3.1.1 # via nbconvert
cachetools==3.1.1 # via apache-beam, google-auth
certifi==2019.11.28 # via requests
chardet==3.0.4 # via requests
click==7.0 # via tfx
crcmod==1.7 # via apache-beam
decorator==4.4.1 # via ipython, traitlets
defusedxml==0.6.0 # via nbconvert
dill==0.3.0 # via apache-beam
docker==4.2.0 # via tfx
docopt==0.6.2 # via hdfs
entrypoints==0.3 # via nbconvert
fastavro==0.21.24 # via apache-beam
fasteners==0.15 # via google-apitools
future==0.18.2 # via apache-beam
gast==0.2.2 # via tensorflow
google-api-core[grpc]==1.16.0 # via google-cloud-bigtable, google-cloud-core, google-cloud-datastore, google-cloud-pubsub
google-api-python-client==1.7.11 # via tfx
google-apitools==0.5.28 # via apache-beam
google-auth-httplib2==0.0.3 # via google-api-python-client
google-auth-oauthlib==0.4.1 # via tensorboard
google-auth==1.11.2 # via google-api-core, google-api-python-client, google-auth-httplib2, google-auth-oauthlib, tensorboard
google-cloud-bigquery==1.17.1 # via apache-beam
google-cloud-bigtable==1.0.0 # via apache-beam
google-cloud-core==1.3.0 # via apache-beam, google-cloud-bigquery, google-cloud-bigtable, google-cloud-datastore
google-cloud-datastore==1.7.4 # via apache-beam
google-cloud-pubsub==1.0.2 # via apache-beam
google-pasta==0.1.8 # via tensorflow
google-resumable-media==0.4.1 # via google-cloud-bigquery
googleapis-common-protos[grpc]==1.51.0 # via google-api-core, grpc-google-iam-v1, tensorflow-metadata
grpc-google-iam-v1==0.12.3 # via google-cloud-bigtable, google-cloud-pubsub
grpcio==1.27.2 # via apache-beam, google-api-core, googleapis-common-protos, grpc-google-iam-v1, tensorboard, tensorflow, tensorflow-serving-api, tfx
h5py==2.10.0 # via keras-applications
hdfs==2.5.8 # via apache-beam
httplib2==0.12.0 # via apache-beam, google-api-python-client, google-apitools, google-auth-httplib2, oauth2client
idna==2.9 # via requests
importlib-metadata==1.5.0 # via jsonschema
ipykernel==5.1.4 # via ipywidgets, jupyter, jupyter-console, notebook, qtconsole
ipython-genutils==0.2.0 # via nbformat, notebook, qtconsole, traitlets
ipython==7.9.0 # via ipykernel, ipywidgets, jupyter-console, tensorflow-data-validation
ipywidgets==7.5.1 # via jupyter, tensorflow-model-analysis
jedi==0.16.0 # via ipython
jinja2==2.11.1 # via nbconvert, notebook, tfx
joblib==0.14.1 # via scikit-learn, tensorflow-data-validation
jsonschema==3.2.0 # via nbformat
jupyter-client==6.0.0 # via ipykernel, jupyter-console, notebook, qtconsole
jupyter-console==6.1.0 # via jupyter
jupyter-core==4.6.3 # via jupyter-client, nbconvert, nbformat, notebook, qtconsole
jupyter==1.0.0 # via tensorflow-model-analysis
keras-applications==1.0.8 # via tensorflow
keras-preprocessing==1.1.0 # via tensorflow
markdown==3.2.1 # via tensorboard
markupsafe==1.1.1 # via jinja2
mistune==0.8.4 # via nbconvert
ml-metadata==0.21.1 # via tfx
mock==2.0.0 # via apache-beam
monotonic==1.5 # via fasteners
nbconvert==5.6.1 # via jupyter, notebook
nbformat==5.0.4 # via ipywidgets, nbconvert, notebook
notebook==6.0.3 # via jupyter, widgetsnbextension
numpy==1.18.1 # via h5py, keras-applications, keras-preprocessing, opt-einsum, pandas, pyarrow, scikit-learn, scipy, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx-bsl
oauth2client==3.0.0 # via apache-beam, google-apitools
oauthlib==3.1.0 # via requests-oauthlib
opt-einsum==3.1.0 # via tensorflow
pandas==0.25.3 # via tensorflow-data-validation
pandocfilters==1.4.2 # via nbconvert
parso==0.6.1 # via jedi
pbr==5.4.4 # via mock
pexpect==4.8.0 # via ipython
pickleshare==0.7.5 # via ipython
prometheus-client==0.7.1 # via notebook
prompt-toolkit==2.0.10 # via ipython, jupyter-console
protobuf==3.11.3 # via apache-beam, google-api-core, google-cloud-bigquery, googleapis-common-protos, ml-metadata, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-metadata, tensorflow-model-analysis, tensorflow-serving-api, tensorflow-transform, tfx, tfx-bsl
ptyprocess==0.6.0 # via pexpect, terminado
pyarrow==0.15.1 # via apache-beam, tensorflow-data-validation, tfx, tfx-bsl
pyasn1-modules==0.2.8 # via google-auth, oauth2client
pyasn1==0.4.8 # via oauth2client, pyasn1-modules, rsa
pydot==1.4.1 # via apache-beam, tensorflow-transform
pygments==2.5.2 # via ipython, jupyter-console, nbconvert, qtconsole
pymongo==3.10.1 # via apache-beam
pyparsing==2.4.6 # via pydot
pyrsistent==0.15.7 # via jsonschema
python-dateutil==2.8.1 # via apache-beam, jupyter-client, pandas
pytz==2019.3 # via apache-beam, google-api-core, pandas
pyyaml==3.13 # via tfx
pyzmq==19.0.0 # via jupyter-client, notebook
qtconsole==4.6.0 # via jupyter
requests-oauthlib==1.3.0 # via google-auth-oauthlib
requests==2.23.0 # via docker, google-api-core, hdfs, requests-oauthlib, tensorboard
rsa==4.0 # via google-auth, oauth2client
scikit-learn==0.21.3 # via tensorflow-data-validation
scipy==1.4.1 # via scikit-learn, tensorflow, tensorflow-model-analysis
send2trash==1.5.0 # via notebook
six==1.14.0 # via absl-py, bleach, docker, fasteners, google-api-core, google-api-python-client, google-apitools, google-auth, google-pasta, google-resumable-media, grpcio, h5py, hdfs, jsonschema, keras-preprocessing, ml-metadata, mock, oauth2client, prompt-toolkit, protobuf, pyarrow, pyrsistent, python-dateutil, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx, tfx-bsl, traitlets, websocket-client
tensorboard==2.1.0 # via tensorflow
tensorflow-data-validation==0.21.2 # via tfx
tensorflow-estimator==2.1.0 # via tensorflow
tensorflow-metadata==0.21.1 # via tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx-bsl
tensorflow-model-analysis==0.21.3 # via tfx
tensorflow-serving-api==2.1.0 # via tfx, tfx-bsl
tensorflow-transform==0.21.0 # via tensorflow-data-validation, tfx
tensorflow==2.1.0 # via ml-metadata, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-serving-api, tensorflow-transform, tfx, tfx-bsl
termcolor==1.1.0 # via tensorflow
terminado==0.8.3 # via notebook
testpath==0.4.4 # via nbconvert
tfx-bsl==0.21.2 # via tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx
tfx==0.21.0 # via -r requirements.in (line 1)
tornado==6.0.3 # via ipykernel, jupyter-client, notebook, terminado
traitlets==4.3.3 # via ipykernel, ipython, ipywidgets, jupyter-client, jupyter-core, nbconvert, nbformat, notebook, qtconsole
uritemplate==3.0.1 # via google-api-python-client
urllib3==1.25.8 # via requests
wcwidth==0.1.8 # via prompt-toolkit
webencodings==0.5.1 # via bleach
websocket-client==0.57.0 # via docker
werkzeug==1.0.0 # via tensorboard
wheel==0.34.2 # via tensorboard, tensorflow
widgetsnbextension==3.5.1 # via ipywidgets
wrapt==1.12.0 # via tensorflow
zipp==1.2.0 # via importlib-metadata

# The following packages are considered to be unsafe in a requirements file:
# setuptools
5 changes: 5 additions & 0 deletions backend/update_requirements.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/bash
set -euo pipefail
IMAGE=${1:-"python:3.5"}
exec docker run -v "$PWD:/src" -w /src --rm "$IMAGE" bash \
-c 'pip3 install pip-tools==4.5.0 > /dev/null 2>&1 && pip-compile -v requirements.in'
14 changes: 2 additions & 12 deletions test/sample-test/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,8 @@ RUN apt-get install --no-install-recommends -y -q default-jre default-jdk python

RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py

RUN pip3 install wheel
RUN pip3 install junit-xml
RUN pip3 install kubernetes==9.0.0
RUN pip3 install minio
RUN pip3 install setuptools==40.5.0
RUN pip3 install papermill==1.0.1
RUN pip3 install ipykernel==5.1.0
RUN pip3 install google-api-python-client==1.7.0
RUN pip3 install google-cloud-storage==1.17.0
RUN pip3 install fire==0.2.1
RUN pip3 install yamale==2.0
RUN pip3 install tfx==0.21.0 tensorflow_transform==0.21.0
COPY ./test/sample-test/requirements.txt /python/src/github.com/kubeflow/pipelines/test/sample-test/requirements.txt
RUN pip3 install -r /python/src/github.com/kubeflow/pipelines/test/sample-test/requirements.txt

# Install python client, including DSL compiler.
COPY ./sdk/python /sdk/python
Expand Down
13 changes: 13 additions & 0 deletions test/sample-test/requirements.in
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
wheel
junit-xml
kubernetes==9.0.0
minio
setuptools==40.5.0
papermill==1.0.1
ipykernel==5.1.0
google-api-python-client==1.7.8
google-cloud-storage==1.17.0
fire==0.2.1
yamale==2.0
tfx==0.21.0
tensorflow_transform==0.21.0
134 changes: 134 additions & 0 deletions test/sample-test/requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
#
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile requirements.in
#
absl-py==0.8.1 # via ml-metadata, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx, tfx-bsl
apache-beam[gcp]==2.17.0 # via tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx, tfx-bsl
astor==0.8.1 # via tensorflow
attrs==19.3.0 # via jsonschema
avro-python3==1.9.1 # via apache-beam, tensorflow-data-validation, tensorflow-model-analysis, tfx, tfx-bsl
backcall==0.1.0 # via ipython
bleach==3.1.1 # via nbconvert
cachetools==3.1.1 # via apache-beam, google-auth
certifi==2019.11.28 # via requests
chardet==3.0.4 # via requests
click==7.0 # via tfx
crcmod==1.7 # via apache-beam
decorator==4.4.1 # via ipython, traitlets
defusedxml==0.6.0 # via nbconvert
dill==0.3.0 # via apache-beam
docker==4.2.0 # via tfx
docopt==0.6.2 # via hdfs
entrypoints==0.3 # via nbconvert
fastavro==0.21.24 # via apache-beam
fasteners==0.15 # via google-apitools
future==0.18.2 # via apache-beam
gast==0.2.2 # via tensorflow
google-api-core[grpc]==1.16.0 # via google-cloud-bigtable, google-cloud-core, google-cloud-datastore, google-cloud-pubsub
google-api-python-client==1.7.8 # via tfx
google-apitools==0.5.28 # via apache-beam
google-auth-httplib2==0.0.3 # via google-api-python-client
google-auth-oauthlib==0.4.1 # via tensorboard
google-auth==1.11.2 # via google-api-core, google-api-python-client, google-auth-httplib2, google-auth-oauthlib, tensorboard
google-cloud-bigquery==1.17.1 # via apache-beam
google-cloud-bigtable==1.0.0 # via apache-beam
google-cloud-core==1.3.0 # via apache-beam, google-cloud-bigquery, google-cloud-bigtable, google-cloud-datastore
google-cloud-datastore==1.7.4 # via apache-beam
google-cloud-pubsub==1.0.2 # via apache-beam
google-pasta==0.1.8 # via tensorflow
google-resumable-media==0.4.1 # via google-cloud-bigquery
googleapis-common-protos[grpc]==1.51.0 # via google-api-core, grpc-google-iam-v1, tensorflow-metadata
grpc-google-iam-v1==0.12.3 # via google-cloud-bigtable, google-cloud-pubsub
grpcio==1.27.2 # via apache-beam, google-api-core, googleapis-common-protos, grpc-google-iam-v1, tensorboard, tensorflow, tensorflow-serving-api, tfx
h5py==2.10.0 # via keras-applications
hdfs==2.5.8 # via apache-beam
httplib2==0.12.0 # via apache-beam, google-api-python-client, google-apitools, google-auth-httplib2, oauth2client
idna==2.9 # via requests
importlib-metadata==1.5.0 # via jsonschema
ipykernel==5.1.0 # via ipywidgets, jupyter, jupyter-console, notebook, qtconsole
ipython-genutils==0.2.0 # via nbformat, notebook, qtconsole, traitlets
ipython==7.9.0 # via ipykernel, ipywidgets, jupyter-console, tensorflow-data-validation
ipywidgets==7.5.1 # via jupyter, tensorflow-model-analysis
jedi==0.16.0 # via ipython
jinja2==2.11.1 # via nbconvert, notebook, tfx
joblib==0.14.1 # via scikit-learn, tensorflow-data-validation
jsonschema==3.2.0 # via nbformat
jupyter-client==6.0.0 # via ipykernel, jupyter-console, notebook, qtconsole
jupyter-console==6.1.0 # via jupyter
jupyter-core==4.6.3 # via jupyter-client, nbconvert, nbformat, notebook, qtconsole
jupyter==1.0.0 # via tensorflow-model-analysis
keras-applications==1.0.8 # via tensorflow
keras-preprocessing==1.1.0 # via tensorflow
markdown==3.2.1 # via tensorboard
markupsafe==1.1.1 # via jinja2
mistune==0.8.4 # via nbconvert
ml-metadata==0.21.1 # via tfx
mock==2.0.0 # via apache-beam
monotonic==1.5 # via fasteners
nbconvert==5.6.1 # via jupyter, notebook
nbformat==5.0.4 # via ipywidgets, nbconvert, notebook
notebook==6.0.3 # via jupyter, widgetsnbextension
numpy==1.18.1 # via h5py, keras-applications, keras-preprocessing, opt-einsum, pandas, pyarrow, scikit-learn, scipy, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx-bsl
oauth2client==3.0.0 # via apache-beam, google-apitools
oauthlib==3.1.0 # via requests-oauthlib
opt-einsum==3.1.0 # via tensorflow
pandas==0.25.3 # via tensorflow-data-validation
pandocfilters==1.4.2 # via nbconvert
parso==0.6.1 # via jedi
pbr==5.4.4 # via mock
pexpect==4.8.0 # via ipython
pickleshare==0.7.5 # via ipython
prometheus-client==0.7.1 # via notebook
prompt-toolkit==2.0.10 # via ipython, jupyter-console
protobuf==3.11.3 # via apache-beam, google-api-core, google-cloud-bigquery, googleapis-common-protos, ml-metadata, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-metadata, tensorflow-model-analysis, tensorflow-serving-api, tensorflow-transform, tfx, tfx-bsl
ptyprocess==0.6.0 # via pexpect, terminado
pyarrow==0.15.1 # via apache-beam, tensorflow-data-validation, tfx, tfx-bsl
pyasn1-modules==0.2.8 # via google-auth, oauth2client
pyasn1==0.4.8 # via oauth2client, pyasn1-modules, rsa
pydot==1.4.1 # via apache-beam, tensorflow-transform
pygments==2.5.2 # via ipython, jupyter-console, nbconvert, qtconsole
pymongo==3.10.1 # via apache-beam
pyparsing==2.4.6 # via pydot
pyrsistent==0.15.7 # via jsonschema
python-dateutil==2.8.1 # via apache-beam, jupyter-client, pandas
pytz==2019.3 # via apache-beam, google-api-core, pandas
pyyaml==3.13 # via tfx
pyzmq==19.0.0 # via jupyter-client, notebook
qtconsole==4.6.0 # via jupyter
requests-oauthlib==1.3.0 # via google-auth-oauthlib
requests==2.23.0 # via docker, google-api-core, hdfs, requests-oauthlib, tensorboard
rsa==4.0 # via google-auth, oauth2client
scikit-learn==0.21.3 # via tensorflow-data-validation
scipy==1.4.1 # via scikit-learn, tensorflow, tensorflow-model-analysis
send2trash==1.5.0 # via notebook
six==1.14.0 # via absl-py, bleach, docker, fasteners, google-api-core, google-api-python-client, google-apitools, google-auth, google-pasta, google-resumable-media, grpcio, h5py, hdfs, jsonschema, keras-preprocessing, ml-metadata, mock, oauth2client, prompt-toolkit, protobuf, pyarrow, pyrsistent, python-dateutil, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx, tfx-bsl, traitlets, websocket-client
tensorboard==2.1.0 # via tensorflow
tensorflow-data-validation==0.21.2 # via tfx
tensorflow-estimator==2.1.0 # via tensorflow
tensorflow-metadata==0.21.1 # via tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx-bsl
tensorflow-model-analysis==0.21.3 # via tfx
tensorflow-serving-api==2.1.0 # via tfx, tfx-bsl
tensorflow-transform==0.21.0 # via tensorflow-data-validation, tfx
tensorflow==2.1.0 # via ml-metadata, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-serving-api, tensorflow-transform, tfx, tfx-bsl
termcolor==1.1.0 # via tensorflow
terminado==0.8.3 # via notebook
testpath==0.4.4 # via nbconvert
tfx-bsl==0.21.2 # via tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx
tfx==0.21.0 # via -r requirements.in (line 1)
tornado==6.0.3 # via ipykernel, jupyter-client, notebook, terminado
traitlets==4.3.3 # via ipykernel, ipython, ipywidgets, jupyter-client, jupyter-core, nbconvert, nbformat, notebook, qtconsole
uritemplate==3.0.1 # via google-api-python-client
urllib3==1.25.8 # via requests
wcwidth==0.1.8 # via prompt-toolkit
webencodings==0.5.1 # via bleach
websocket-client==0.57.0 # via docker
werkzeug==1.0.0 # via tensorboard
wheel==0.34.2 # via tensorboard, tensorflow
widgetsnbextension==3.5.1 # via ipywidgets
wrapt==1.12.0 # via tensorflow
zipp==1.2.0 # via importlib-metadata

# The following packages are considered to be unsafe in a requirements file:
# setuptools

0 comments on commit 0ab6975

Please sign in to comment.