Merge pull request #1 from jlewi/image

Docker image to use in our E2E tests

images/Dockerfile

@@ -0,0 +1,117 @@
+# Docker image for running E2E tests using Argo.
+
+FROM python:2.7-slim
+MAINTAINER Jeremy Lewi
+
+# Never prompt the user for choices on installation/configuration of packages
+ENV DEBIAN_FRONTEND noninteractive
+ENV TERM linux
+
+# Define en_US.
+ENV LANGUAGE=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    LC_ALL=en_US.UTF-8 \
+    LC_CTYPE=en_US.UTF-8 \
+    LC_MESSAGES=en_US.UTF-8 \
+    LC_ALL=en_US.UTF-8
+
+
+# buildDeps should be packages needed only to build some other packages as
+# these packages are purged in a later step.
+#
+# gcc & python-dev are needed so we can install crcmod for gsutil
+RUN set -ex \
+    && apt-get update -yqq \
+    && apt-get install -yqq --no-install-recommends \
+        curl \
+        locales \
+        wget \
+        ca-certificates \
+        git \
+        zip \
+        unzip \
+        gcc python-dev \
+        python-setuptools \
+    && apt-get clean \
+    && rm -rf \
+        /var/lib/apt/lists/* \
+        /tmp/* \
+        /var/tmp/* \
+        /usr/share/man \
+        /usr/share/doc \
+        /usr/share/doc-base
+
+# Set the locale
+RUN sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \
+    && locale-gen \
+    && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8
+
+# Install go
+RUN cd /tmp && \
+    wget -O /tmp/go.tar.gz https://redirector.gvt1.com/edgedl/go/go1.9.2.linux-amd64.tar.gz && \
+    tar -C /usr/local -xzf go.tar.gz 
+
+# Install gcloud
+ENV PATH=/usr/local/go/bin:/google-cloud-sdk/bin:/workspace:${PATH} \
+    CLOUDSDK_CORE_DISABLE_PROMPTS=1
+
+RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz && \
+    tar xzf google-cloud-sdk.tar.gz -C / && \
+    rm google-cloud-sdk.tar.gz && \
+    /google-cloud-sdk/install.sh \
+        --disable-installation-options \
+        --bash-completion=false \
+        --path-update=false \
+        --usage-reporting=false && \
+    gcloud components install alpha beta kubectl
+
+# Install CRCMOD for gsutil
+RUN easy_install -U pip && \
+    pip install -U crcmod
+
+# Install Helm
+RUN wget -O /tmp/get_helm.sh \
+    https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get && \
+    chmod 700 /tmp/get_helm.sh && \
+    /tmp/get_helm.sh && \
+    rm /tmp/get_helm.sh
+
+# Initialize helm
+RUN helm init --client-only
+
+# Install  Node.js
+RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - \
+    && apt-get install -y nodejs
+
+# Install yarn
+RUN curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+    && echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
+    && apt-get update -yqq \
+    && apt-get install -yqq --no-install-recommends yarn
+
+
+# Install glide
+RUN cd /tmp && \
+    wget -O glide-v0.13.0-linux-amd64.tar.gz \
+            https://github.com/Masterminds/glide/releases/download/v0.13.0/glide-v0.13.0-linux-amd64.tar.gz && \
+    tar -xvf glide-v0.13.0-linux-amd64.tar.gz && \
+    mv ./linux-amd64/glide /usr/local/bin/
+
+# Install ksonnet
+RUN curl -o /usr/local/bin/ks -L \
+    https://github.com/ksonnet/ksonnet/releases/download/v0.8.0/ks-linux-amd64 && \
+    chmod a+x /usr/local/bin/ks
+
+
+# Install various python libraries.
+RUN  pip install --upgrade six pyyaml google-api-python-client \
+     google-cloud-storage google-auth-httplib2 pylint kubernetes==4.0.0 mock retrying \
+     jinja2
+
+COPY bootstrap.sh /usr/local/bin
+RUN chmod a+x /usr/local/bin/bootstrap.sh
+
+COPY checkout.sh /usr/local/bin
+RUN chmod a+x /usr/local/bin/checkout.sh
+
+ENTRYPOINT ["/usr/local/bin/bootstrap.sh"]

images/Makefile

@@ -0,0 +1,35 @@
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Requirements:
+#   https://github.com/mattrobenolt/jinja2-cli
+#   pip install jinja2-clie
+IMG = gcr.io/mlkube-testing/test-worker
+TAG := $(shell date +v%Y%m%d)-$(shell git describe --tags --always --dirty)-$(shell git diff | sha256sum | cut -c -6)
+DIR := ${CURDIR}
+
+all: build
+
+# To build without the cache set the environment variable
+# export DOCKER_BUILD_OPTS=--no-cache
+build:
+	@echo {\"image\": \"$(IMG):$(TAG)\"} > version.json
+	docker build ${DOCKER_BUILD_OPTS} -t $(IMG):$(TAG) .
+	docker tag $(IMG):$(TAG) $(IMG):latest
+	@echo Built $(IMG):$(TAG) and tagged with latest
+
+push: build
+	gcloud docker -- push $(IMG):$(TAG)
+	gcloud docker -- push $(IMG):latest
+	@echo Pushed $(IMG) with :latest and :$(TAG) tags

images/bootstrap.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# This script is used to bootstrap our prow jobs.
+# The point of this script is to check out the kubeflow/kubeflow repo
+# at the commit corresponding to the Prow job. We can then
+# invoke the launcher script at that commit to submit and
+# monitor an Argo workflow
+set -xe
+
+mkdir -p /src
+
+# TODO(jlewi): We should eventually move the code for running the workflow from
+# kubeflow/kubeflow into kubeflow/testing
+git clone https://github.com/kubeflow/kubeflow.git /src/google_kubeflow
+
+cd /src/google_kubeflow
+
+echo Job Name = ${JOB_NAME}
+
+# See https://github.com/kubernetes/test-infra/tree/master/prow#job-evironment-variables
+if [ ! -z ${PULL_NUMBER} ]; then
+ git fetch origin  pull/${PULL_NUMBER}/head:pr
+ git checkout ${PULL_PULL_SHA}
+else
+ if [ ! -z ${PULL_BASE_SHA} ]; then
+ 	# Its a post submit; checkout the commit to test.
+  	git checkout ${PULL_BASE_SHA}
+ fi
+fi
+
+# Update submodules.
+git submodule init
+git submodule update
+
+# Print out the commit so we can tell from logs what we checked out.
+echo Repo is at `git describe --tags --always --dirty`
+git submodule
+git status
+
+export PYTHONPATH=$PYTHONPATH:/src/google_kubeflow/tensorflow_k8s
+cd /src/google_kubeflow
+# Invoke the script to run the workflow
+python -m testing.run_e2e_workflow  \
+  --project=mlkube-testing \
+  --zone=us-east1-d \
+  --cluster=kubeflow-testing \
+  --bucket=kubernetes-jenkins

images/checkout.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+#
+# This script is used as the first step in our Argo workflows to check out the code
+# corresponding the prow job.
+#
+# TODO(jlewi): Eliminate code duplication with bootstraph.sh my moving shared code into
+# a bash script that can be sourced from multiple scripts.
+#!/bin/bash
+set -xe
+SRC_DIR=$1
+
+# Print out env for debugging.
+env | sort
+
+git clone https://github.com/${REPO_OWNER}/${REPO_NAME}.git /tmp/src
+
+# Some git operations are really slow when using NFS.
+# We observed clone times increasing from O(30) seconds to O(4 minutes)
+# when we switched to NFS.
+# As a workaround we clone into a local directory and then move the files onto
+# NFS. Copying to NFS is still a bottleneck and increases the run time to O(1. 5 minutes).
+# clone --recurse-submodules https://github.com/kubeflow/kubeflow.git /tmp/src",
+cd /tmp/src
+
+# We need to set the preloadindex option; to try to speedup git ops like describe
+# and status when using an NFS filesystem.
+# See: https://stackoverflow.com/questions/4994772/ways-to-improve-git-status-performance
+# unfortunately this doesn't seem to help with sub modules.
+git config core.preloadindex true
+
+# See https://github.com/kubernetes/test-infra/tree/master/prow#job-evironment-variables
+if [ ! -z ${PULL_NUMBER} ]; then
+ git fetch origin  pull/${PULL_NUMBER}/head:pr
+ if [ ! -z ${PULL_PULL_SHA} ]; then
+	git checkout ${PULL_PULL_SHA}
+ else
+ 	# Checkout the latest commit for this PR since no commit specified.
+ 	git checkout pr
+ fi
+else
+ if [ ! -z ${PULL_BASE_SHA} ]; then
+  # Its a post submit; checkout the commit to test.
+    git checkout ${PULL_BASE_SHA}
+ fi
+fi
+
+# Update submodules.
+git submodule init
+git submodule update
+
+# TODO(jlewi): As noted above the git operations below are really
+# slow when using NFS.
+# Print out the git version in the logs
+git describe --tags --always --dirty
+git status
+
+# Move it to NFS
+mkdir -p + ${SRC_DIR}
+
+# The period is needed because we want to copy the contents of the src directory
+# into srcDir not srcDir/src/.
+cp -r /tmp/src/.  ${SRC_DIR}
+
+# Make the files world readable/writable.
+# This is a hack to make it easy to modify the files from jupyterhub which is using
+# a different user/group id.
+chmod -R a+rwx  ${SRC_DIR}

images/version.json

@@ -0,0 +1 @@
+{"image": "gcr.io/mlkube-testing/test-worker:v20180131-434dfe6-dirty-098026"}