Refactor Makefile/Dockerfile to remove volume binding in favor of bui…

…ld stages (#1189)

.argo-ci/ci.yaml

@@ -22,16 +22,13 @@ spec:
             value: "{{item}}"
         withItems:
         - make controller-image executor-image
-        - make cli-linux
-        - make cli-darwin
+        - make release-clis
       - name: test
         template: ci-builder
         arguments:
           parameters:
           - name: cmd
-            value: "{{item}}"
-        withItems:
-        - dep ensure && make lint test verify-codegen
+            value: dep ensure && make lint test verify-codegen
 
   - name: ci-builder
     inputs:
@@ -67,10 +64,11 @@ spec:
       env:
       - name: DOCKER_HOST
         value: 127.0.0.1
+      - name: DOCKER_BUILDKIT
+        value: "1"
     sidecars:
     - name: dind
-      image: docker:17.10-dind
+      image: docker:18.09-dind
       securityContext:
         privileged: true
       mirrorVolumeMounts: true
-

.dockerignore

@@ -1,4 +1,4 @@
-*
-!dist
-dist/pkg
-!Gopkg.*
+# Prevent vendor directory from being copied to ensure we are not not pulling unexpected cruft from 
+# a user's workspace, and are only building off of what is locked by dep.
+vendor
+dist

CONTRIBUTING.md

@@ -15,17 +15,17 @@ Go to https://github.com/argoproj/
 
 ## How to suggest a new feature
 
-Go to https://groups.google.com/forum/#!forum/argoproj
-* Create a new topic to discuss your feature.
+Go to https://github.com/argoproj/
+* Open an issue and discuss it.
 
 ## How to setup your dev environment
 
 ### Requirements
-* Golang 1.10
+* Golang 1.11
 * Docker
 * dep v0.5
    * Mac Install: `brew install dep`
-* gometalinter v2.0.5
+* gometalinter v2.0.12
 
 ### Quickstart
 ```
@@ -36,9 +36,16 @@ $ make
 ```
 
 ### Build workflow-controller and executor images
-The following will build the workflow-controller and executor images tagged with the `latest` tag, then push to a personal dockerhub repository:
+The following will build the release versions of workflow-controller and executor images tagged
+with the `latest` tag, then push to a personal dockerhub repository, `mydockerrepo`:
+```
+$ make controller-image executor-image IMAGE_TAG=latest IMAGE_NAMESPACE=mydockerrepo DOCKER_PUSH=true
+```
+Building release versions of the images will be slow during development, since the build happens
+inside a docker build context, which cannot re-use the golang build cache between builds. To build
+images quicker (for development purposes), images can be built by adding DEV_IMAGE=true.
 ```
-$ make controller-image executor-image IMAGE_TAG=latest IMAGE_NAMESPACE=jessesuen DOCKER_PUSH=true
+$ make controller-image executor-image IMAGE_TAG=latest IMAGE_NAMESPACE=mydockerrepo DOCKER_PUSH=true DEV_IMAGE=true
 ```
 
 ### Build argo cli
@@ -49,6 +56,6 @@ $ ./dist/argo version
 
 ### Deploying controller with alternative controller/executor images
 ```
-$ helm install argo/argo --set images.namespace=jessesuen --set
+$ helm install argo/argo --set images.namespace=mydockerrepo --set
 images.controller workflow-controller:latest
 ```

Dockerfile

@@ -0,0 +1,87 @@
+####################################################################################################
+# Builder image
+# Initial stage which pulls prepares build dependencies and CLI tooling we need for our final image
+# Also used as the image in CI jobs so needs all dependencies
+####################################################################################################
+FROM golang:1.11.4 as builder
+
+RUN apt-get update && apt-get install -y \
+    git \
+    make \
+    wget \
+    gcc \
+    zip && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+WORKDIR /tmp
+
+# Install docker
+ENV DOCKER_CHANNEL stable
+ENV DOCKER_VERSION 18.09.1
+RUN wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz" && \
+    tar --extract --file docker.tgz --strip-components 1 --directory /usr/local/bin/ && \
+    rm docker.tgz
+
+# Install dep
+ENV DEP_VERSION=0.5.0
+RUN wget https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -O /usr/local/bin/dep && \
+    chmod +x /usr/local/bin/dep
+
+# Install gometalinter
+ENV GOMETALINTER_VERSION=2.0.12
+RUN curl -sLo- https://github.com/alecthomas/gometalinter/releases/download/v${GOMETALINTER_VERSION}/gometalinter-${GOMETALINTER_VERSION}-linux-amd64.tar.gz | \
+    tar -xzC "$GOPATH/bin" --exclude COPYING --exclude README.md --strip-components 1 -f- && \
+    ln -s $GOPATH/bin/gometalinter $GOPATH/bin/gometalinter.v2
+
+
+####################################################################################################
+# Argo Build stage which performs the actual build of Argo binaries
+####################################################################################################
+FROM builder as argo-build
+
+# A dummy directory is created under $GOPATH/src/dummy so we are able to use dep
+# to install all the packages of our dep lock file
+COPY Gopkg.toml ${GOPATH}/src/dummy/Gopkg.toml
+COPY Gopkg.lock ${GOPATH}/src/dummy/Gopkg.lock
+
+RUN cd ${GOPATH}/src/dummy && \
+    dep ensure -vendor-only && \
+    mv vendor/* ${GOPATH}/src/ && \
+    rmdir vendor
+
+# Perform the build
+WORKDIR /go/src/github.com/argoproj/argo
+COPY . .
+ARG MAKE_TARGET="controller executor cli-linux-amd64"
+RUN make $MAKE_TARGET
+
+
+####################################################################################################
+# argoexec
+####################################################################################################
+FROM debian:9.6-slim as argoexec
+# NOTE: keep the version synced with https://storage.googleapis.com/kubernetes-release/release/stable.txt
+ENV KUBECTL_VERSION=1.13.1
+RUN apt-get update && \
+    apt-get install -y curl jq procps git tar mime-support && \
+    rm -rf /var/lib/apt/lists/* && \
+    curl -L -o /usr/local/bin/kubectl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \
+    chmod +x /usr/local/bin/kubectl
+COPY --from=argo-build /go/src/github.com/argoproj/argo/dist/argoexec /usr/local/bin/
+
+
+####################################################################################################
+# workflow-controller
+####################################################################################################
+FROM scratch as workflow-controller
+COPY --from=argo-build /go/src/github.com/argoproj/argo/dist/workflow-controller /bin/
+ENTRYPOINT [ "workflow-controller" ]
+
+
+####################################################################################################
+# argocli
+####################################################################################################
+FROM scratch as argocli
+COPY --from=argo-build /go/src/github.com/argoproj/argo/dist/argo-linux-amd64 /bin/argo
+ENTRYPOINT [ "argo" ]

Dockerfile.argoexec-dev

@@ -0,0 +1,12 @@
+####################################################################################################
+# argoexec-dev
+####################################################################################################
+FROM debian:9.6-slim as argoexec-dev
+# NOTE: keep the version synced with https://storage.googleapis.com/kubernetes-release/release/stable.txt
+ENV KUBECTL_VERSION=1.13.1
+RUN apt-get update && \
+    apt-get install -y curl jq procps git tar mime-support && \
+    rm -rf /var/lib/apt/lists/* && \
+    curl -L -o /usr/local/bin/kubectl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \
+    chmod +x /usr/local/bin/kubectl
+COPY argoexec /usr/local/bin/

Dockerfile.workflow-controller-dev

@@ -0,0 +1,6 @@
+####################################################################################################
+# workflow-controller-dev
+####################################################################################################
+FROM scratch as workflow-controller-dev
+COPY workflow-controller /bin/
+ENTRYPOINT [ "workflow-controller" ]