feat(components): Add v1beta1 Katib launcher and samples (#4798)

* Add Katib v1beta1 launcher and 2 examples

* Generate Notebook output

* Modify README

* Fix links

* Fix comments in mpi-job example

* Change Experiment spec structure to JsonObject

* Remove generated file

* Use component.yaml in MPI example

* Use built-in Katib API client serialization

* Update notebook example

* Assign current status to None

* Fix Copyright

components/kubeflow/katib-launcher/Dockerfile

@@ -1,23 +1,23 @@
-# Copyright 2019 The Kubeflow Authors
+# Copyright 2020 The Kubeflow Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM ubuntu:16.04
 
-RUN apt-get update -y && \
-    apt-get install --no-install-recommends -y -q ca-certificates python-dev python-setuptools wget && \
-    easy_install pip && \
-    pip install pyyaml==3.12 kubernetes
+FROM python:3.6
 
-ADD build /ml
+ENV APP_HOME /app
+WORKDIR ${APP_HOME}
 
-ENTRYPOINT ["python", "/ml/launch_experiment.py"]
+ADD . ${APP_HOME}
+RUN pip install --no-cache-dir -r requirements.txt
+
+ENTRYPOINT ["python", "src/launch_experiment.py"]

components/kubeflow/katib-launcher/OWNERS

@@ -1,4 +1,5 @@
 approvers:
+  - andreyvelich
+  - gaocegege
   - hougangliu
-reviewers:
-  - hougangliu
+  - johnugeorge

components/kubeflow/katib-launcher/build_image.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -e
-# Copyright 2019 Google LLC
+# Copyright 2020 The Kubeflow Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,44 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-while getopts ":hp:t:i:" opt; do
-  case "${opt}" in
-    h) echo "-p: project name"
-       echo "-t: tag name"
-       echo "-i: image name. If provided, project name and tag name are not necessary"
-       exit
-      ;;
-    p) PROJECT_ID=${OPTARG}
-      ;;
-    t) TAG_NAME=${OPTARG}
-      ;;
-    i) LAUNCHER_IMAGE_NAME=${OPTARG}
-      ;;
-    \? ) echo "Usage: cmd [-p] project [-t] tag [-i] image"
-      exit
-      ;;
-  esac
-done
+IMAGE="docker.io/kubeflowkatib/kubeflow-pipelines-launcher"
 
-mkdir -p ./build
-rsync -arvp ./src/ ./build/
-rsync -arvp ../common/ ./build/
+echo "Releasing image for the Katib Pipeline Launcher..."
+echo -e "Image: ${IMAGE}\n"
 
-LOCAL_LAUNCHER_IMAGE_NAME=ml-pipeline-kubeflow-experiment
-
-docker build -t ${LOCAL_LAUNCHER_IMAGE_NAME} .
-if [ -z "${TAG_NAME}" ]; then
-  TAG_NAME=$(date +v%Y%m%d)-$(git describe --tags --always --dirty)-$(git diff | shasum -a256 | cut -c -6)
-fi
-if [ -z "${LAUNCHER_IMAGE_NAME}" ]; then
-  if [ -z "${PROJECT_ID}" ]; then
-    PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)")
-  fi
-  docker tag ${LOCAL_LAUNCHER_IMAGE_NAME} gcr.io/${PROJECT_ID}/${LOCAL_LAUNCHER_IMAGE_NAME}:${TAG_NAME}
-  docker push gcr.io/${PROJECT_ID}/${LOCAL_LAUNCHER_IMAGE_NAME}:${TAG_NAME}
-else
-  docker tag ${LOCAL_LAUNCHER_IMAGE_NAME} ${LAUNCHER_IMAGE_NAME}:${TAG_NAME}
-  docker push ${LAUNCHER_IMAGE_NAME}:${TAG_NAME}
-fi
-
-rm -rf ./build
+docker build . -f Dockerfile -t ${IMAGE}
+docker push ${IMAGE}

components/kubeflow/katib-launcher/component.yaml

@@ -1,38 +1,22 @@
-name: Kubeflow - Launch Experiment
-description: Kubeflow Experiment launcher
+name: Katib - Launch Experiment
+description: Katib Experiment launcher
 inputs:
-- {name: Experiment Name,           type: String,                       description: 'Experiment name.'}
-- {name: Experiment Namespace,      type: String,   default: kubeflow,  description: 'Experiment namespace.'}
-- {name: Experiment Version,        type: String,   default: v1alpha3,  description: 'Experiment version.'}
-- {name: Max Trial Count,           type: Integer,                      description: 'How many trials can be executed at most.'}
-- {name: Max Failed Trial Count,    type: Integer,  default: 3,         description: 'How many trials can fail at most.'}
-- {name: Parallel Trial Count,      type: Integer,  default: 3,         description: 'How many trials can be running in parallel at most.'}
-- {name: Objective,                 type: JSON,                         description: 'Experiment objective.'}
-- {name: Algorithm,                 type: JSON,                         description: 'Experiment algorithm.'}
-- {name: Trial Template,            type: JSON,                         description: 'Experiment trialTemplate.'}
-- {name: Parameters,                type: JSON,                         description: 'Experiment Parameter configuration.'}
-- {name: Metrics Collector,         type: JSON,   default: '{}',        description: 'Experiment metricsCollector.'}
-- {name: Experiment Timeout Minutes, type: Integer,  default: 1440,     description: 'Time in minutes to wait for the Experiment to complete.'}
-- {name: Delete Finished Experiment, type: Bool,     default: 'True',   description: 'Whether to delete the experiment after it is finished.'}
+- {name: Experiment Name,            type: String,       default: '',        description: 'Experiment name'}
+- {name: Experiment Namespace,       type: String,       default: anonymous, description: 'Experiment namespace'}
+- {name: Experiment Spec,            type: JsonObject,   default: '{}',      description: 'Experiment specification in dict format'}
+- {name: Experiment Timeout Minutes, type: Integer,      default: 1440,      description: 'Time in minutes to wait for the Experiment to complete'}
+- {name: Delete Finished Experiment, type: Bool,         default: 'True',    description: 'Whether to delete the Experiment after it is finished'}
 outputs:
-- {name: Best Parameter Set,        type: JSON,                         description: 'The parameter set of the best Experiment trial.'}
+- {name: Best Parameter Set,         type: JSON,                             description: 'The hyperparameter set of the best Experiment Trial'}
 implementation:
   container:
-    image: liuhougangxa/katib-experiment-launcher:latest
-    command: [python, /ml/launch_experiment.py]
+    image: docker.io/kubeflowkatib/kubeflow-pipelines-launcher
+    command: [python, src/launch_experiment.py]
     args: [
-      --name,                     {inputValue: Experiment Name},
-      --namespace,                {inputValue: Experiment Namespace},
-      --version,                  {inputValue: Experiment Version},
-      --maxTrialCount,            {inputValue: Max Trial Count},
-      --maxFailedTrialCount,      {inputValue: Max Failed Trial Count},
-      --parallelTrialCount,       {inputValue: Parallel Trial Count},
-      --objectiveConfig,          {inputValue: Objective},
-      --algorithmConfig,          {inputValue: Algorithm},
-      --trialTemplate,            {inputValue: Trial Template},
-      --parameters,               {inputValue: Parameters},
-      --metricsCollector,         {inputValue: Metrics Collector},
-      --experimentTimeoutMinutes, {inputValue: Experiment Timeout Minutes},
-      --deleteAfterDone,          {inputValue: Delete Finished Experiment},
-      --outputFile,               {outputPath: Best Parameter Set},
+      --experiment-name,            {inputValue: Experiment Name},
+      --experiment-namespace,       {inputValue: Experiment Namespace},
+      --experiment-spec,            {inputValue: Experiment Spec},
+      --experiment-timeout-minutes, {inputValue: Experiment Timeout Minutes},
+      --delete-after-done,          {inputValue: Delete Finished Experiment},
+      --output-file,                {outputPath: Best Parameter Set},
     ]

components/kubeflow/katib-launcher/requirements.txt

@@ -0,0 +1,2 @@
+kubernetes==10.0.1
+kubeflow-katib==0.10.1

components/kubeflow/katib-launcher/src/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2019 Google LLC
+# Copyright 2020 The Kubeflow Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.