Merge pull request #151 from docker/develop

Merging the develop changes into master

.gitignore

@@ -1,3 +1,6 @@
 build
 debbuild
 rpmbuild
+tmp
+artifacts
+sources

Jenkinsfile

@@ -1,5 +1,23 @@
 #!groovy
 
+
+def genBranch(String arch) {
+	return [
+		"${arch}": { ->
+			stage("Build engine image on ${arch}") {
+				wrappedNode(label: "linux&&${arch}", cleanWorkspace: true) {
+					try {
+						checkout scm
+						sh("git clone https://github.com/moby/moby.git engine")
+						sh('make ENGINE_DIR=$(pwd)/engine image')
+					} finally {
+						sh('make ENGINE_DIR=$(pwd)/engine clean-image clean-engine')
+					}
+				}
+		}
+	}]
+}
+
 test_steps = [
 	'deb': { ->
 		stage('Ubuntu Xenial Debian Package') {
@@ -33,4 +51,16 @@ test_steps = [
 	},
 ]
 
+arches = [
+	"x86_64",
+	// "s390x",
+	"ppc64le",
+	"aarch64",
+	"armhf"
+]
+
+arches.each {
+	test_steps << genBranch(it)
+}
+
 parallel(test_steps)

Makefile

@@ -3,14 +3,28 @@ ENGINE_DIR:=$(CURDIR)/../engine
 CLI_DIR:=$(CURDIR)/../cli
 VERSION?=0.0.0-dev
 DOCKER_GITCOMMIT:=abcdefg
+ARCH=$(shell uname -m)
+STATIC_VERSION=$(shell static/gen-static-ver $(ENGINE_DIR) $(VERSION))
 GO_VERSION:=1.10.3
 
+# Taken from: https://www.cmcrossroads.com/article/printing-value-makefile-variable
+print-%  : ; @echo $($*)
+
 .PHONY: help
 help: ## show make targets
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf " \033[36m%-20s\033[0m  %s\n", $$1, $$2}' $(MAKEFILE_LIST)
 
+.PHONY: clean-engine
+clean-engine:
+	rm -rf $(ENGINE_DIR)
+
+.PHONY: clean-image
+clean-image:
+	$(MAKE) ENGINE_DIR=$(ENGINE_DIR) -C image clean
+
+
 .PHONY: clean
-clean: ## remove build artifacts
+clean: clean-image ## remove build artifacts
 	$(MAKE) -C rpm clean
 	$(MAKE) -C deb clean
 	$(MAKE) -C static clean
@@ -44,6 +58,9 @@ image: ## build static-compiled packages
 		$(MAKE) -C $@ VERSION=$(VERSION) ENGINE_DIR=$(ENGINE_DIR) CLI_DIR=$(CLI_DIR) GO_VERSION=$(GO_VERSION) $${p}; \
 	done
 
+engine-$(ARCH).tar:
+	$(MAKE) -C image $@
+
 .PHONY: release
 release:
 	$(MAKE) -C image $@

common/dockerd.json

@@ -0,0 +1,12 @@
+{
+    "image": "docker.io/${ENGINE_IMAGE}",
+    "imagePath": "/var/lib/docker-engine/engine.tar",
+    "namespace":"docker",
+    "args": [
+        "-s", "overlay",
+        "--containerd", "/run/containerd/containerd.sock",
+        "--default-runtime", "containerd",
+        "--add-runtime", "containerd=runc"
+    ],
+    "scope": "ce"
+}

containerd.mk

@@ -0,0 +1,17 @@
+# Common things for containerd functionality
+
+CONTAINERD_PROXY_COMMIT=82ae3d13e91d062dd4853379fe018638023c8da2
+CONTAINERD_SHIM_PROCESS_IMAGE=docker.io/docker/containerd-shim-process:ff98a47
+
+# If containerd is running use that socket instead
+ifeq ($(shell systemctl status containerd 2>/dev/null >/dev/null && echo -n "yes"), "yes")
+CONTAINERD_SOCK:=/var/run/containerd/containerd.sock
+else
+CONTAINERD_SOCK:=/var/run/docker/containerd/docker-containerd.sock
+endif
+CTR=docker run \
+	--rm -i \
+	-v $(CONTAINERD_SOCK):/ours/containerd.sock \
+	-v $(CURDIR)/artifacts:/artifacts \
+	docker:18.06.0-ce \
+	docker-containerd-ctr -a /ours/containerd.sock

deb/Makefile

@@ -1,26 +1,37 @@
+include ../containerd.mk
+
 SHELL:=/bin/bash
 ARCH:=$(shell uname -m)
-ENGINE_DIR:=$(CURDIR)/../../engine
 CLI_DIR:=$(CURDIR)/../../cli
-GITCOMMIT?=$(shell cd $(ENGINE_DIR) && git rev-parse --short HEAD)
+GITCOMMIT?=$(shell cd $(CLI_DIR) && git rev-parse --short HEAD)
 VERSION?=0.0.0-dev
+GO_BASE_IMAGE=golang
 GO_VERSION:=1.10.3
-DEB_VERSION=$(shell ./gen-deb-ver $(ENGINE_DIR) "$(VERSION)")
+GO_IMAGE=$(GO_BASE_IMAGE):$(GO_VERSION)
+DEB_VERSION=$(shell ./gen-deb-ver $(CLI_DIR) "$(VERSION)")
 CHOWN:=docker run --rm -v $(CURDIR):/v -w /v alpine chown
 EPOCH?=2
 
-BUILD=docker build --build-arg GO_VERSION=$(GO_VERSION) -t debbuild-$@/$(ARCH) -f $(CURDIR)/$@/Dockerfile.$(ARCH) .
+COMMON_FILES=common
+BUILD?=docker build \
+	--build-arg GO_IMAGE=$(GO_IMAGE) \
+	--build-arg COMMON_FILES=$(COMMON_FILES) \
+	-t debbuild-$@/$(ARCH) \
+	-f $(CURDIR)/$@/Dockerfile .
 RUN=docker run --rm -i \
 	-e EPOCH='$(EPOCH)' \
 	-e DEB_VERSION=$(word 1, $(DEB_VERSION)) \
 	-e VERSION=$(word 2, $(DEB_VERSION)) \
 	-e DOCKER_GITCOMMIT=$(GITCOMMIT) \
 	-v $(CURDIR)/debbuild/$@:/build \
-	-v $(ENGINE_DIR):/engine \
-	-v $(CLI_DIR):/cli \
-	-v $(CURDIR)/systemd:/root/build-deb/systemd \
 	debbuild-$@/$(ARCH)
 
+SOURCE_FILES=containerd-proxy.tgz cli.tgz containerd-shim-process.tar docker.service dockerd.json engine.tar
+SOURCES=$(addprefix sources/, $(SOURCE_FILES))
+ENGINE_IMAGE=docker/engine-community
+
+IMAGE_TAG=nightly
+
 .PHONY: help
 help: ## show make targets
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf " \033[36m%-20s\033[0m  %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -29,6 +40,15 @@ help: ## show make targets
 clean: ## remove build artifacts
 	[ ! -d debbuild ] || $(CHOWN) -R $(shell id -u):$(shell id -g) debbuild
 	$(RM) -r debbuild
+	[ ! -d sources ] || $(CHOWN) -R $(shell id -u):$(shell id -g) sources
+	$(RM) -r sources
+	[ ! -d artifacts ] || $(CHOWN) -R $(shell id -u):$(shell id -g) artifacts
+	$(RM) -r artifacts
+	-docker rm docker2oci
+
+engine-$(ARCH).tar:
+	$(MAKE) -C ../image image-linux
+	docker save -o $@ $$(cat ../image/image-linux)
 
 .PHONY: deb
 deb: ubuntu debian raspbian ## build all deb packages
@@ -42,50 +62,110 @@ debian: debian-stretch debian-jessie ## build all debian deb packages
 .PHONY: raspbian
 raspbian: raspbian-stretch debian-jessie ## build all raspbian deb packages
 
-.PHONY: ubuntu-xenial
-ubuntu-xenial: ## build ubuntu xenial deb packages
+.PHONY: ubuntu-bionic
+ubuntu-bionic: ## build ubuntu bionic deb packages
+ubuntu-bionic: $(SOURCES)
 	$(BUILD)
 	$(RUN)
 	$(CHOWN) -R $(shell id -u):$(shell id -g) debbuild/$@
 
-.PHONY: ubuntu-trusty
-ubuntu-trusty: ## build ubuntu trusty deb packages
+.PHONY: ubuntu-xenial
+ubuntu-xenial: ## build ubuntu xenial deb packages
+ubuntu-xenial: $(SOURCES)
 	$(BUILD)
 	$(RUN)
 	$(CHOWN) -R $(shell id -u):$(shell id -g) debbuild/$@
 
-.PHONY: ubuntu-bionic
-ubuntu-bionic: ## build ubuntu bionic deb packages
+.PHONY: ubuntu-trusty
+ubuntu-trusty: ## build ubuntu trusty deb packages
+ubuntu-trusty: $(SOURCES)
 	$(BUILD)
 	$(RUN)
 	$(CHOWN) -R $(shell id -u):$(shell id -g) debbuild/$@
 
 .PHONY: debian-buster
 debian-buster: ## build debian buster deb packages
+debian-buster: $(SOURCES)
 	$(BUILD)
 	$(RUN)
 	$(CHOWN) -R $(shell id -u):$(shell id -g) debbuild/$@
 
 .PHONY: debian-jessie
 debian-jessie: ## build debian jessie deb packages
+debian-jessie: $(SOURCES)
 	$(BUILD)
 	$(RUN)
 	$(CHOWN) -R $(shell id -u):$(shell id -g) debbuild/$@
 
 .PHONY: debian-stretch
 debian-stretch: ## build debian stretch deb packages
+debian-stretch: $(SOURCES)
 	$(BUILD)
 	$(RUN)
 	$(CHOWN) -R $(shell id -u):$(shell id -g) debbuild/$@
 
 .PHONY: raspbian-jessie
 raspbian-jessie: ## build raspbian jessie deb packages
+raspbian-jessie: $(SOURCES)
 	$(BUILD)
 	$(RUN)
 	$(CHOWN) -R $(shell id -u):$(shell id -g) debbuild/$@
 
 .PHONY: raspbian-stretch
 raspbian-stretch: ## build raspbian stretch deb packages
+raspbian-stretch: $(SOURCES)
 	$(BUILD)
 	$(RUN)
 	$(CHOWN) -R $(shell id -u):$(shell id -g) debbuild/$@
+
+sources/cli.tgz:
+	mkdir -p $(@D)
+	docker run --rm -i -w /v \
+		-v $(CLI_DIR):/cli \
+		-v $(CURDIR)/$(@D):/v \
+		alpine \
+		tar -C / -c -z -f /v/cli.tgz --exclude .git cli
+
+sources/containerd-proxy.tgz:
+	mkdir -p tmp/
+	curl -fL -o tmp/containerd-proxy.tgz "https://github.com/crosbymichael/containerd-proxy/archive/$(CONTAINERD_PROXY_COMMIT).tar.gz"
+	tar xzf tmp/containerd-proxy.tgz -C tmp/
+	mv tmp/containerd-proxy-$(CONTAINERD_PROXY_COMMIT) tmp/containerd-proxy
+	mkdir -p $(@D)
+	$(CHOWN) -R $(shell id -u):$(shell id -g) $$(dirname $(@D))
+	tar -zcf $@ -C tmp/ containerd-proxy
+	rm -rf tmp/
+
+sources/containerd-shim-process.tar:
+	$(CTR) content fetch $(CONTAINERD_SHIM_PROCESS_IMAGE)
+	$(CTR) image export artifacts/containerd-shim-process.tar $(CONTAINERD_SHIM_PROCESS_IMAGE)
+	mkdir -p $(@D)
+	cp artifacts/containerd-shim-process.tar $@
+	$(CHOWN) -R $(shell id -u):$(shell id -g) $$(dirname $(@D))
+
+sources/docker.service: ../systemd/docker.service
+	mkdir -p $(@D)
+	cp $< $@
+
+sources/dockerd.json: ../common/dockerd.json
+	mkdir -p $(@D)
+	sed -e 's!$${ENGINE_IMAGE}!$(ENGINE_IMAGE)!' -e 's/$${IMAGE_TAG}/$(IMAGE_TAG)/' $< > $@
+
+# TODO: Eventually clean this up when we release an image with a manifest
+DOCKER2OCI=artifacts/docker2oci
+$(DOCKER2OCI):
+	-$(CHOWN) -R $(shell id -u):$(shell id -g) $(@D)
+	docker run --name docker2oci $(GO_IMAGE) sh -c 'go get github.com/coolljt0725/docker2oci'
+	mkdir -p $(@D)
+	docker cp docker2oci:/go/bin/docker2oci "$@"
+	docker rm -f docker2oci
+	$(CHOWN) -R $(shell id -u):$(shell id -g) $(@D)
+
+# offline bundle
+sources/engine.tar: $(DOCKER2OCI)
+	$(MAKE) -C ../image ENGINE_IMAGE=$(ENGINE_IMAGE) image-linux
+	mkdir -p artifacts
+	docker save -o artifacts/docker-engine.tar $$(cat ../image/image-linux)
+	./$(DOCKER2OCI) -i artifacts/docker-engine.tar artifacts/engine-image
+	mkdir -p $(@D)
+	tar c -C artifacts/engine-image . > $@

deb/build-deb

@@ -13,18 +13,9 @@ if [[ -z "$DEB_VERSION" ]]; then
     exit 1
 fi
 
-(
-    set -e
-    cd engine
-    # I want to rip this install-binaries script out so badly
-    for component in tini "proxy dynamic" "runc all" "containerd dynamic";do
-        TMP_GOPATH="/go" hack/dockerfile/install/install.sh $component
-    done
-)
-
 echo VERSION AAA $VERSION
 
-VERSION=${VERSION:-$( cat engine/VERSION )}
+VERSION=${VERSION:-$( cat cli/VERSION )}
 
 echo VERSION bbb $VERSION
 
@@ -40,7 +31,7 @@ EOF
 # The space above at the start of the line for the debMaintainer is very important
 
 # Give the script a git commit because it wants it
-export DOCKER_GITCOMMIT=${DOCKER_GITCOMMIT-$($GIT_COMMAND rev-parse --short HEAD)}
+export DOCKER_GITCOMMIT=${DOCKER_GITCOMMIT-$(cd cli; $GIT_COMMAND rev-parse --short HEAD)}
 
 echo VERSION BBB $VERSION
 dpkg-buildpackage -uc -us -I.git

deb/common/control

@@ -2,22 +2,28 @@ Source: docker-ce
 Section: admin
 Priority: optional
 Maintainer: Docker <support@docker.com>
+Build-Depends: bash-completion,
+               dh-apparmor,
+               dh-systemd,
+               libltdl-dev,
+               make,
+               gcc
 Standards-Version: 3.9.6
-Homepage: https://dockerproject.org
+Homepage: https://docker.com
 Vcs-Browser: https://github.com/docker/docker
 Vcs-Git: git://github.com/docker/docker.git
 
 Package: docker-ce
 Architecture: linux-any
-Depends: iptables, ${misc:Depends}, ${perl:Depends}, ${shlibs:Depends}
-Recommends: aufs-tools,
+Depends: docker-ce-cli, containerd.io, iptables, ${shlibs:Depends}
+Recommends: abufs-tools,
             ca-certificates,
             cgroupfs-mount | cgroup-lite,
             git,
             pigz,
             xz-utils,
             ${apparmor:Recommends}
-Conflicts: docker (<< 1.5~), docker.io, lxc-docker, lxc-docker-virtual-package, docker-engine, docker-engine-cs, docker-ee
+Conflicts: docker (<< 1.5~), docker.io, lxc-docker, lxc-docker-virtual-package, docker-engine, docker-engine-cs
 Replaces: docker-engine
 Description: Docker: the open-source application container engine
  Docker is an open source project to build, ship and run any application as a
@@ -29,3 +35,19 @@ Description: Docker: the open-source application container engine
  language, framework or packaging system. That makes them great building blocks
  for deploying and scaling web apps, databases, and backend services without
  depending on a particular stack or provider.
+
+Package: docker-ce-cli
+Architecture: linux-any
+Depends: ${shlibs:Depends}
+Conflicts: docker (<< 1.5~), docker.io, lxc-docker, lxc-docker-virtual-package, docker-engine, docker-engine-cs
+Replaces:
+Description: Docker CLI: the open-source application container engine
+ Docker is an open source project to build, ship and run any application as a
+ lightweight container
+ .
+ Docker containers are both hardware-agnostic and platform-agnostic. This means
+ they can run anywhere, from your laptop to the largest EC2 compute instance and
+ everything in between - and they don't require you to use a particular
+ language, framework or packaging system. That makes them great building blocks
+ for deploying and scaling web apps, databases, and backend services without
+ depending on a particular stack or provider.