From ce89d13b2fc46ca654b6e417bc195d3b2a0379ff Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Fri, 2 Feb 2018 05:52:41 -0800 Subject: [PATCH] This change was not properly merged and ended up deleting existing files. (#7) Revert "Merge pull request #2 from jlewi/py" This reverts commit 9273125fb4a708540b1527d8f3cb77368225d6a2, reversing changes made to b018ac67abfeebb9027fcfd97f51f7b9f49371a6. --- images/Dockerfile | 117 ++++++++++++++++++++++++++++++++++++++++++++ images/Makefile | 35 +++++++++++++ images/bootstrap.sh | 47 ++++++++++++++++++ images/checkout.sh | 67 +++++++++++++++++++++++++ images/version.json | 1 + 5 files changed, 267 insertions(+) create mode 100644 images/Dockerfile create mode 100755 images/Makefile create mode 100755 images/bootstrap.sh create mode 100644 images/checkout.sh create mode 100644 images/version.json diff --git a/images/Dockerfile b/images/Dockerfile new file mode 100644 index 00000000000..f12f7661eeb --- /dev/null +++ b/images/Dockerfile @@ -0,0 +1,117 @@ +# Docker image for running E2E tests using Argo. + +FROM python:2.7-slim +MAINTAINER Jeremy Lewi + +# Never prompt the user for choices on installation/configuration of packages +ENV DEBIAN_FRONTEND noninteractive +ENV TERM linux + +# Define en_US. +ENV LANGUAGE=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + LC_CTYPE=en_US.UTF-8 \ + LC_MESSAGES=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 + + +# buildDeps should be packages needed only to build some other packages as +# these packages are purged in a later step. +# +# gcc & python-dev are needed so we can install crcmod for gsutil +RUN set -ex \ + && apt-get update -yqq \ + && apt-get install -yqq --no-install-recommends \ + curl \ + locales \ + wget \ + ca-certificates \ + git \ + zip \ + unzip \ + gcc python-dev \ + python-setuptools \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /usr/share/man \ + /usr/share/doc \ + /usr/share/doc-base + +# Set the locale +RUN sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \ + && locale-gen \ + && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 + +# Install go +RUN cd /tmp && \ + wget -O /tmp/go.tar.gz https://redirector.gvt1.com/edgedl/go/go1.9.2.linux-amd64.tar.gz && \ + tar -C /usr/local -xzf go.tar.gz + +# Install gcloud +ENV PATH=/usr/local/go/bin:/google-cloud-sdk/bin:/workspace:${PATH} \ + CLOUDSDK_CORE_DISABLE_PROMPTS=1 + +RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz && \ + tar xzf google-cloud-sdk.tar.gz -C / && \ + rm google-cloud-sdk.tar.gz && \ + /google-cloud-sdk/install.sh \ + --disable-installation-options \ + --bash-completion=false \ + --path-update=false \ + --usage-reporting=false && \ + gcloud components install alpha beta kubectl + +# Install CRCMOD for gsutil +RUN easy_install -U pip && \ + pip install -U crcmod + +# Install Helm +RUN wget -O /tmp/get_helm.sh \ + https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get && \ + chmod 700 /tmp/get_helm.sh && \ + /tmp/get_helm.sh && \ + rm /tmp/get_helm.sh + +# Initialize helm +RUN helm init --client-only + +# Install Node.js +RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - \ + && apt-get install -y nodejs + +# Install yarn +RUN curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ + && echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \ + && apt-get update -yqq \ + && apt-get install -yqq --no-install-recommends yarn + + +# Install glide +RUN cd /tmp && \ + wget -O glide-v0.13.0-linux-amd64.tar.gz \ + https://github.com/Masterminds/glide/releases/download/v0.13.0/glide-v0.13.0-linux-amd64.tar.gz && \ + tar -xvf glide-v0.13.0-linux-amd64.tar.gz && \ + mv ./linux-amd64/glide /usr/local/bin/ + +# Install ksonnet +RUN curl -o /usr/local/bin/ks -L \ + https://github.com/ksonnet/ksonnet/releases/download/v0.8.0/ks-linux-amd64 && \ + chmod a+x /usr/local/bin/ks + + +# Install various python libraries. +RUN pip install --upgrade six pyyaml google-api-python-client \ + google-cloud-storage google-auth-httplib2 pylint kubernetes==4.0.0 mock retrying \ + jinja2 + +COPY bootstrap.sh /usr/local/bin +RUN chmod a+x /usr/local/bin/bootstrap.sh + +COPY checkout.sh /usr/local/bin +RUN chmod a+x /usr/local/bin/checkout.sh + +ENTRYPOINT ["/usr/local/bin/bootstrap.sh"] \ No newline at end of file diff --git a/images/Makefile b/images/Makefile new file mode 100755 index 00000000000..c86ac6d665f --- /dev/null +++ b/images/Makefile @@ -0,0 +1,35 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requirements: +# https://github.com/mattrobenolt/jinja2-cli +# pip install jinja2-clie +IMG = gcr.io/mlkube-testing/test-worker +TAG := $(shell date +v%Y%m%d)-$(shell git describe --tags --always --dirty)-$(shell git diff | sha256sum | cut -c -6) +DIR := ${CURDIR} + +all: build + +# To build without the cache set the environment variable +# export DOCKER_BUILD_OPTS=--no-cache +build: + @echo {\"image\": \"$(IMG):$(TAG)\"} > version.json + docker build ${DOCKER_BUILD_OPTS} -t $(IMG):$(TAG) . + docker tag $(IMG):$(TAG) $(IMG):latest + @echo Built $(IMG):$(TAG) and tagged with latest + +push: build + gcloud docker -- push $(IMG):$(TAG) + gcloud docker -- push $(IMG):latest + @echo Pushed $(IMG) with :latest and :$(TAG) tags diff --git a/images/bootstrap.sh b/images/bootstrap.sh new file mode 100755 index 00000000000..9d8bd240f9c --- /dev/null +++ b/images/bootstrap.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# This script is used to bootstrap our prow jobs. +# The point of this script is to check out the kubeflow/kubeflow repo +# at the commit corresponding to the Prow job. We can then +# invoke the launcher script at that commit to submit and +# monitor an Argo workflow +set -xe + +mkdir -p /src + +# TODO(jlewi): We should eventually move the code for running the workflow from +# kubeflow/kubeflow into kubeflow/testing +git clone https://github.com/kubeflow/kubeflow.git /src/google_kubeflow + +cd /src/google_kubeflow + +echo Job Name = ${JOB_NAME} + +# See https://github.com/kubernetes/test-infra/tree/master/prow#job-evironment-variables +if [ ! -z ${PULL_NUMBER} ]; then + git fetch origin pull/${PULL_NUMBER}/head:pr + git checkout ${PULL_PULL_SHA} +else + if [ ! -z ${PULL_BASE_SHA} ]; then + # Its a post submit; checkout the commit to test. + git checkout ${PULL_BASE_SHA} + fi +fi + +# Update submodules. +git submodule init +git submodule update + +# Print out the commit so we can tell from logs what we checked out. +echo Repo is at `git describe --tags --always --dirty` +git submodule +git status + +export PYTHONPATH=$PYTHONPATH:/src/google_kubeflow/tensorflow_k8s +cd /src/google_kubeflow +# Invoke the script to run the workflow +python -m testing.run_e2e_workflow \ + --project=mlkube-testing \ + --zone=us-east1-d \ + --cluster=kubeflow-testing \ + --bucket=kubernetes-jenkins diff --git a/images/checkout.sh b/images/checkout.sh new file mode 100644 index 00000000000..4c890289d40 --- /dev/null +++ b/images/checkout.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# This script is used as the first step in our Argo workflows to check out the code +# corresponding the prow job. +# +# TODO(jlewi): Eliminate code duplication with bootstraph.sh my moving shared code into +# a bash script that can be sourced from multiple scripts. +#!/bin/bash +set -xe +SRC_DIR=$1 + +# Print out env for debugging. +env | sort + +git clone https://github.com/${REPO_OWNER}/${REPO_NAME}.git /tmp/src + +# Some git operations are really slow when using NFS. +# We observed clone times increasing from O(30) seconds to O(4 minutes) +# when we switched to NFS. +# As a workaround we clone into a local directory and then move the files onto +# NFS. Copying to NFS is still a bottleneck and increases the run time to O(1. 5 minutes). +# clone --recurse-submodules https://github.com/kubeflow/kubeflow.git /tmp/src", +cd /tmp/src + +# We need to set the preloadindex option; to try to speedup git ops like describe +# and status when using an NFS filesystem. +# See: https://stackoverflow.com/questions/4994772/ways-to-improve-git-status-performance +# unfortunately this doesn't seem to help with sub modules. +git config core.preloadindex true + +# See https://github.com/kubernetes/test-infra/tree/master/prow#job-evironment-variables +if [ ! -z ${PULL_NUMBER} ]; then + git fetch origin pull/${PULL_NUMBER}/head:pr + if [ ! -z ${PULL_PULL_SHA} ]; then + git checkout ${PULL_PULL_SHA} + else + # Checkout the latest commit for this PR since no commit specified. + git checkout pr + fi +else + if [ ! -z ${PULL_BASE_SHA} ]; then + # Its a post submit; checkout the commit to test. + git checkout ${PULL_BASE_SHA} + fi +fi + +# Update submodules. +git submodule init +git submodule update + +# TODO(jlewi): As noted above the git operations below are really +# slow when using NFS. +# Print out the git version in the logs +git describe --tags --always --dirty +git status + +# Move it to NFS +mkdir -p + ${SRC_DIR} + +# The period is needed because we want to copy the contents of the src directory +# into srcDir not srcDir/src/. +cp -r /tmp/src/. ${SRC_DIR} + +# Make the files world readable/writable. +# This is a hack to make it easy to modify the files from jupyterhub which is using +# a different user/group id. +chmod -R a+rwx ${SRC_DIR} diff --git a/images/version.json b/images/version.json new file mode 100644 index 00000000000..00e7c2ffb18 --- /dev/null +++ b/images/version.json @@ -0,0 +1 @@ +{"image": "gcr.io/mlkube-testing/test-worker:v20180131-434dfe6-dirty-098026"}