diff --git a/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb b/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb index a6f8027e98b..d7302bfbcec 100644 --- a/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb +++ b/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb @@ -24,15 +24,64 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "scrolled": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting https://storage.googleapis.com/ml-pipeline/release/0.1.3-rc.2/kfp.tar.gz\n", + "\u001b[?25l Downloading https://storage.googleapis.com/ml-pipeline/release/0.1.3-rc.2/kfp.tar.gz (68kB)\n", + "\u001b[K 100% |████████████████████████████████| 71kB 9.4MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied, skipping upgrade: urllib3>=1.15 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (1.22)\n", + "Requirement already satisfied, skipping upgrade: six>=1.10 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (1.11.0)\n", + "Requirement already satisfied, skipping upgrade: certifi in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (2018.10.15)\n", + "Requirement already satisfied, skipping upgrade: python-dateutil in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (2.7.3)\n", + "Requirement already satisfied, skipping upgrade: PyYAML in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (3.13)\n", + "Requirement already satisfied, skipping upgrade: google-cloud-storage==1.13.0 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (1.13.0)\n", + "Requirement already satisfied, skipping upgrade: kubernetes==8.0.0 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (8.0.0)\n", + "Requirement already satisfied, skipping upgrade: google-api-core<2.0.0dev,>=0.1.1 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage==1.13.0->kfp==0.1) (1.5.2)\n", + "Requirement already satisfied, skipping upgrade: google-cloud-core<0.29dev,>=0.28.0 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage==1.13.0->kfp==0.1) (0.28.1)\n", + "Requirement already satisfied, skipping upgrade: google-resumable-media>=0.3.1 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage==1.13.0->kfp==0.1) (0.3.1)\n", + "Requirement already satisfied, skipping upgrade: requests-oauthlib in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (1.0.0)\n", + "Requirement already satisfied, skipping upgrade: adal>=1.0.2 in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (1.2.0)\n", + "Requirement already satisfied, skipping upgrade: requests in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (2.18.4)\n", + "Requirement already satisfied, skipping upgrade: google-auth>=1.0.1 in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (1.6.1)\n", + "Requirement already satisfied, skipping upgrade: setuptools>=21.0.0 in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (38.4.0)\n", + "Requirement already satisfied, skipping upgrade: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (0.54.0)\n", + "Requirement already satisfied, skipping upgrade: protobuf>=3.4.0 in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=0.1.1->google-cloud-storage==1.13.0->kfp==0.1) (3.6.1)\n", + "Requirement already satisfied, skipping upgrade: pytz in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=0.1.1->google-cloud-storage==1.13.0->kfp==0.1) (2018.7)\n", + "Requirement already satisfied, skipping upgrade: googleapis-common-protos!=1.5.4,<2.0dev,>=1.5.3 in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=0.1.1->google-cloud-storage==1.13.0->kfp==0.1) (1.5.5)\n", + "Requirement already satisfied, skipping upgrade: oauthlib>=0.6.2 in /opt/conda/lib/python3.6/site-packages (from requests-oauthlib->kubernetes==8.0.0->kfp==0.1) (2.1.0)\n", + "Requirement already satisfied, skipping upgrade: cryptography>=1.1.0 in /opt/conda/lib/python3.6/site-packages (from adal>=1.0.2->kubernetes==8.0.0->kfp==0.1) (2.1.4)\n", + "Requirement already satisfied, skipping upgrade: PyJWT>=1.0.0 in /opt/conda/lib/python3.6/site-packages (from adal>=1.0.2->kubernetes==8.0.0->kfp==0.1) (1.6.4)\n", + "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->kubernetes==8.0.0->kfp==0.1) (3.0.4)\n", + "Requirement already satisfied, skipping upgrade: idna<2.7,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->kubernetes==8.0.0->kfp==0.1) (2.6)\n", + "Requirement already satisfied, skipping upgrade: cachetools>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.0.1->kubernetes==8.0.0->kfp==0.1) (3.0.0)\n", + "Requirement already satisfied, skipping upgrade: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.0.1->kubernetes==8.0.0->kfp==0.1) (0.2.2)\n", + "Requirement already satisfied, skipping upgrade: rsa>=3.1.4 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.0.1->kubernetes==8.0.0->kfp==0.1) (4.0)\n", + "Requirement already satisfied, skipping upgrade: asn1crypto>=0.21.0 in /opt/conda/lib/python3.6/site-packages (from cryptography>=1.1.0->adal>=1.0.2->kubernetes==8.0.0->kfp==0.1) (0.24.0)\n", + "Requirement already satisfied, skipping upgrade: cffi>=1.7 in /opt/conda/lib/python3.6/site-packages (from cryptography>=1.1.0->adal>=1.0.2->kubernetes==8.0.0->kfp==0.1) (1.11.4)\n", + "Requirement already satisfied, skipping upgrade: pyasn1<0.5.0,>=0.4.1 in /opt/conda/lib/python3.6/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes==8.0.0->kfp==0.1) (0.4.4)\n", + "Requirement already satisfied, skipping upgrade: pycparser in /opt/conda/lib/python3.6/site-packages (from cffi>=1.7->cryptography>=1.1.0->adal>=1.0.2->kubernetes==8.0.0->kfp==0.1) (2.18)\n", + "Building wheels for collected packages: kfp\n", + " Running setup.py bdist_wheel for kfp ... \u001b[?25ldone\n", + "\u001b[?25h Stored in directory: /tmp/pip-ephem-wheel-cache-bctiow25/wheels/f9/43/15/db34c8d6291d495360ac6eea475e6bf473750320547f837caf\n", + "Successfully built kfp\n", + "Installing collected packages: kfp\n", + " Found existing installation: kfp 0.1\n", + " Uninstalling kfp-0.1:\n", + " Successfully uninstalled kfp-0.1\n", + "Successfully installed kfp-0.1\n" + ] + } + ], "source": [ "# Install Pipeline SDK\n", - "!pip3 install https://storage.googleapis.com/ml-pipeline/release/0.1.1/kfp.tar.gz --upgrade\n", - " \n" + "!pip3 install https://storage.googleapis.com/ml-pipeline/release/0.1.3-rc.2/kfp.tar.gz --upgrade" ] }, { @@ -97,7 +146,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -117,13 +166,13 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "Run link here" + "Run link here" ], "text/plain": [ "" @@ -148,7 +197,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -302,13 +351,13 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "Run link here" + "Run link here" ], "text/plain": [ "" @@ -346,9 +395,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: google-api-python-client in /opt/conda/lib/python3.6/site-packages (1.7.4)\n", + "Requirement already satisfied: google-auth>=1.4.1 in /opt/conda/lib/python3.6/site-packages (from google-api-python-client) (1.6.1)\n", + "Requirement already satisfied: httplib2<1dev,>=0.9.2 in /opt/conda/lib/python3.6/site-packages (from google-api-python-client) (0.11.3)\n", + "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /opt/conda/lib/python3.6/site-packages (from google-api-python-client) (3.0.0)\n", + "Requirement already satisfied: six<2dev,>=1.6.1 in /opt/conda/lib/python3.6/site-packages (from google-api-python-client) (1.11.0)\n", + "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /opt/conda/lib/python3.6/site-packages (from google-api-python-client) (0.0.3)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.4.1->google-api-python-client) (0.2.2)\n", + "Requirement already satisfied: cachetools>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.4.1->google-api-python-client) (3.0.0)\n", + "Requirement already satisfied: rsa>=3.1.4 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.4.1->google-api-python-client) (4.0)\n", + "Requirement already satisfied: pyasn1<0.5.0,>=0.4.1 in /opt/conda/lib/python3.6/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.4.1->google-api-python-client) (0.4.4)\n" + ] + } + ], "source": [ "# in order to run it locally we need a python package\n", "!pip3 install google-api-python-client" @@ -356,7 +422,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -421,9 +487,75 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Build a Pipeline Step With the Above Function\n", + "### Build a Pipeline Step With the Above Function(Note: run either of the two options below)\n", + "#### Option One: Specify the dependency directly\n", + "Now that we've tested the function locally, we want to build a component that can run as a step in the pipeline. " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2018-11-26 19:29:13:INFO:Build an image that is based on gcr.io/ml-pipeline-dogfood/pusherbase:dev and push the image to gcr.io/ml-pipeline-dogfood/pusher:dev\n", + "2018-11-26 19:29:13:INFO:Checking path: gs://ngao-bugbash...\n", + "2018-11-26 19:29:13:INFO:Generate entrypoint and serialization codes.\n", + "2018-11-26 19:29:13:INFO:Generate build files.\n", + "2018-11-26 19:29:13:INFO:Start a kaniko job for build.\n", + "2018-11-26 19:29:18:INFO:5 seconds: waiting for job to complete\n", + "2018-11-26 19:29:23:INFO:10 seconds: waiting for job to complete\n", + "2018-11-26 19:29:28:INFO:15 seconds: waiting for job to complete\n", + "2018-11-26 19:29:33:INFO:20 seconds: waiting for job to complete\n", + "2018-11-26 19:29:38:INFO:25 seconds: waiting for job to complete\n", + "2018-11-26 19:29:43:INFO:30 seconds: waiting for job to complete\n", + "2018-11-26 19:29:48:INFO:35 seconds: waiting for job to complete\n", + "2018-11-26 19:29:53:INFO:40 seconds: waiting for job to complete\n", + "2018-11-26 19:29:58:INFO:45 seconds: waiting for job to complete\n", + "2018-11-26 19:30:03:INFO:50 seconds: waiting for job to complete\n", + "2018-11-26 19:30:08:INFO:55 seconds: waiting for job to complete\n", + "2018-11-26 19:30:13:INFO:60 seconds: waiting for job to complete\n", + "2018-11-26 19:30:18:INFO:65 seconds: waiting for job to complete\n", + "2018-11-26 19:30:23:INFO:70 seconds: waiting for job to complete\n", + "2018-11-26 19:30:28:INFO:75 seconds: waiting for job to complete\n", + "2018-11-26 19:30:33:INFO:80 seconds: waiting for job to complete\n", + "2018-11-26 19:30:38:INFO:85 seconds: waiting for job to complete\n", + "2018-11-26 19:30:43:INFO:90 seconds: waiting for job to complete\n", + "2018-11-26 19:30:48:INFO:95 seconds: waiting for job to complete\n", + "2018-11-26 19:30:53:INFO:100 seconds: waiting for job to complete\n", + "2018-11-26 19:30:58:INFO:105 seconds: waiting for job to complete\n", + "2018-11-26 19:31:03:INFO:110 seconds: waiting for job to complete\n", + "2018-11-26 19:31:08:INFO:115 seconds: waiting for job to complete\n", + "2018-11-26 19:31:13:INFO:120 seconds: waiting for job to complete\n", + "2018-11-26 19:31:18:INFO:125 seconds: waiting for job to complete\n", + "2018-11-26 19:31:24:INFO:130 seconds: waiting for job to complete\n", + "2018-11-26 19:31:29:INFO:135 seconds: waiting for job to complete\n", + "2018-11-26 19:31:34:INFO:140 seconds: waiting for job to complete\n", + "2018-11-26 19:31:39:INFO:145 seconds: waiting for job to complete\n", + "2018-11-26 19:31:39:INFO:Kaniko job complete.\n", + "2018-11-26 19:31:39:INFO:Build component complete.\n" + ] + } + ], + "source": [ + "from kfp import compiler\n", "\n", - "Now that we've tested the function locally, we want to build a component that can run as a step in the pipeline. First we need to build a base docker container image. We need TensorFlow and google-api-python-client packages." + "# The return value \"DeployerOp\" represents a step that can be used directly in a pipeline function\n", + "DeployerOp = compiler.build_python_component(\n", + " component_func=deploy_model,\n", + " staging_gcs_path=OUTPUT_DIR,\n", + " dependency=[kfp.compiler.VersionedDependency(name='google-api-python-client', version='1.7.0')],\n", + " target_image=TARGET_IMAGE)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Option Two: build a base docker container image with both tensorflow and google api client packages" ] }, { @@ -547,7 +679,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [