Add convenience container-runtime flag for kubeadm

hack/jenkins/minikube_set_pending.sh

@@ -27,7 +27,7 @@
 set -e
 set +x
 
-for job in "Linux-KVM-Kubeadm" "OSX-Virtualbox-Kubeadm" "OSX-Virtualbox" "OSX-XHyve" "OSX-Hyperkit" "Linux-Virtualbox" "Linux-KVM" "Linux-KVM-Alt" "Linux-None" "Windows-Virtualbox" "Linux-Container"; do
+for job in "Linux-KVM-Kubeadm" "OSX-Virtualbox-Kubeadm" "OSX-Virtualbox" "OSX-XHyve" "OSX-Hyperkit" "Linux-Virtualbox" "Linux-KVM" "Linux-KVM-Alt" "Linux-None" "Windows-Virtualbox" "Windows-Kubeadm-CRI-O" "Linux-Container"; do
   target_url="https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${job}.txt"
   curl "https://api.github.com/repos/kubernetes/minikube/statuses/${ghprbActualCommit}?access_token=$access_token" \
     -H "Content-Type: application/json" \

hack/jenkins/windows_integration_test_virtualbox_kubeadm_crio.ps1

@@ -0,0 +1,35 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mkdir -p out
+gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
+gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
+gsutil.cmd cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
+
+
+./out/minikube-windows-amd64.exe delete
+Remove-Item -Recurse -Force C:\Users\jenkins\.minikube
+
+out/e2e-windows-amd64.exe --% -minikube-start-args="--vm-driver=virtualbox --container-runtime=cri-o" -minikube-args="--v=10 --logtostderr --bootstrapper=kubeadm" -test.v -test.timeout=30m -binary=out/minikube-windows-amd64.exe
+
+$env:result=$lastexitcode
+# If the last exit code was 0->success, x>0->error
+If($env:result -eq 0){$env:status="success"}
+Else {$env:status="failure"}
+
+$env:target_url="https://storage.googleapis.com/minikube-builds/logs/$env:MINIKUBE_LOCATION/Windows-Kubeadm-CRI-O.txt"
+$json = "{`"state`": `"$env:status`", `"description`": `"Jenkins`", `"target_url`": `"$env:target_url`", `"context`": `"Windows-Kubeadm-CRI-O`"}"
+Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing
+
+Exit $env:result

pkg/minikube/assets/vm_assets.go

@@ -20,7 +20,7 @@ import (
 	"bytes"
 	"io"
 	"os"
-	"path/filepath"
+	"path"
 
 	"github.com/pkg/errors"
 )
@@ -65,7 +65,7 @@ type FileAsset struct {
 }
 
 func NewMemoryAssetTarget(d []byte, targetPath, permissions string) *MemoryAsset {
-	return NewMemoryAsset(d, filepath.Dir(targetPath), filepath.Base(targetPath), permissions)
+	return NewMemoryAsset(d, path.Dir(targetPath), path.Base(targetPath), permissions)
 }
 
 func NewFileAsset(assetName, targetDir, targetName, permissions string) (*FileAsset, error) {

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -21,12 +21,13 @@ import (
 	"crypto"
 	"fmt"
 	"os"
-	"path/filepath"
+	"path"
 	"strings"
 	"time"
 
 	"github.com/docker/machine/libmachine"
 	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
 	download "github.com/jimmidyson/go-download"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -178,6 +179,35 @@ func (k *KubeadmBootstrapper) SetupCerts(k8s bootstrapper.KubernetesConfig) erro
 	return bootstrapper.SetupCerts(k.c, k8s)
 }
 
+// SetContainerRuntime possibly sets the container runtime, if it hasn't already
+// been specified by the extra-config option.  It has a set of defaults known to
+// work for a particular runtime.
+func SetContainerRuntime(cfg map[string]string, runtime string) map[string]string {
+	if _, ok := cfg["container-runtime"]; ok {
+		glog.Infoln("Container runtime already set through extra options, ignoring --container-runtime flag.")
+		return cfg
+	}
+
+	if runtime == "" {
+		glog.Infoln("Container runtime flag provided with no value, using defaults.")
+		return cfg
+	}
+
+	switch runtime {
+	case "crio", "cri-o":
+		cfg["container-runtime"] = "remote"
+		cfg["container-runtime-endpoint"] = "/var/run/crio.sock"
+		cfg["image-service-endpoint"] = "/var/run/crio.sock"
+		cfg["runtime-request-timeout"] = "15m"
+	default:
+		cfg["container-runtime"] = runtime
+	}
+
+	return cfg
+}
+
+// NewKubeletConfig generates a new systemd unit containing a configured kubelet
+// based on the options present in the KubernetesConfig.
 func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
@@ -189,9 +219,19 @@ func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 		return "", errors.Wrap(err, "generating extra configuration for kubelet")
 	}
 
+	extraOpts = SetContainerRuntime(extraOpts, k8s.ContainerRuntime)
 	extraFlags := convertToFlags(extraOpts)
 	b := bytes.Buffer{}
-	if err := kubeletSystemdTemplate.Execute(&b, map[string]string{"ExtraOptions": extraFlags}); err != nil {
+	opts := struct {
+		ExtraOptions     string
+		FeatureGates     string
+		ContainerRuntime string
+	}{
+		ExtraOptions:     extraFlags,
+		FeatureGates:     k8s.FeatureGates,
+		ContainerRuntime: k8s.ContainerRuntime,
+	}
+	if err := kubeletSystemdTemplate.Execute(&b, opts); err != nil {
 		return "", err
 	}
 
@@ -262,17 +302,35 @@ sudo systemctl start kubelet
 	return nil
 }
 
+func parseFeatureGates(featureGates string) (map[string]string, error) {
+	if featureGates == "" {
+		return nil, nil
+	}
+	fgMap := map[string]string{}
+	fg := strings.Split(featureGates, ",")
+	for _, f := range fg {
+		kv := strings.SplitN(f, "=", 2)
+		if len(kv) != 2 {
+			return nil, fmt.Errorf("Invalid feature gate format: %s", f)
+		}
+		fgMap[kv[0]] = kv[1]
+	}
+
+	return fgMap, nil
+}
+
 func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return "", errors.Wrap(err, "parsing kubernetes version")
 	}
 
 	// generates a map of component to extra args for apiserver, controller-manager, and scheduler
-	extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version)
+	extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version, k8s.FeatureGates)
 	if err != nil {
 		return "", errors.Wrap(err, "generating extra component config for kubeadm")
 	}
+
 	opts := struct {
 		CertDir           string
 		ServiceCIDR       string
@@ -303,7 +361,7 @@ func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 
 func maybeDownloadAndCache(binary, version string) (string, error) {
 	targetDir := constants.MakeMiniPath("cache", version)
-	targetFilepath := filepath.Join(targetDir, binary)
+	targetFilepath := path.Join(targetDir, binary)
 
 	_, err := os.Stat(targetFilepath)
 	// If it exists, do no verification and continue

pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package kubeadm
 
 import (
+	"reflect"
 	"testing"
 
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
@@ -88,11 +89,111 @@ etcd:
   dataDir: /data
 nodeName: extra-args-minikube
 apiServerExtraArgs:
-  fail-no-swap: true
+  fail-no-swap: "true"
 controllerManagerExtraArgs:
-  kube-api-burst: 32
+  kube-api-burst: "32"
 schedulerExtraArgs:
-  scheduler-name: mini-scheduler
+  scheduler-name: "mini-scheduler"
+`,
+		},
+		{
+			description: "two extra args for one component",
+			cfg: bootstrapper.KubernetesConfig{
+				NodeIP:            "192.168.1.101",
+				KubernetesVersion: "v1.8.0-alpha.0",
+				NodeName:          "extra-args-minikube",
+				ExtraOptions: util.ExtraOptionSlice{
+					util.ExtraOption{
+						Component: Apiserver,
+						Key:       "fail-no-swap",
+						Value:     "true",
+					},
+					util.ExtraOption{
+						Component: Apiserver,
+						Key:       "kube-api-burst",
+						Value:     "32",
+					},
+				},
+			},
+			expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: 192.168.1.101
+  bindPort: 8443
+kubernetesVersion: v1.8.0-alpha.0
+certificatesDir: /var/lib/localkube/certs/
+networking:
+  serviceSubnet: 10.0.0.0/24
+etcd:
+  dataDir: /data
+nodeName: extra-args-minikube
+apiServerExtraArgs:
+  fail-no-swap: "true"
+  kube-api-burst: "32"
+`,
+		},
+		{
+			description: "enable feature gates",
+			cfg: bootstrapper.KubernetesConfig{
+				NodeIP:            "192.168.1.101",
+				KubernetesVersion: "v1.8.0-alpha.0",
+				NodeName:          "extra-args-minikube",
+				FeatureGates:      "HugePages=true,OtherFeature=false",
+			},
+			expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: 192.168.1.101
+  bindPort: 8443
+kubernetesVersion: v1.8.0-alpha.0
+certificatesDir: /var/lib/localkube/certs/
+networking:
+  serviceSubnet: 10.0.0.0/24
+etcd:
+  dataDir: /data
+nodeName: extra-args-minikube
+apiServerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
+controllerManagerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
+schedulerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
+`,
+		},
+		{
+			description: "enable feature gates and extra config",
+			cfg: bootstrapper.KubernetesConfig{
+				NodeIP:            "192.168.1.101",
+				KubernetesVersion: "v1.8.0-alpha.0",
+				NodeName:          "extra-args-minikube",
+				FeatureGates:      "HugePages=true,OtherFeature=false",
+				ExtraOptions: util.ExtraOptionSlice{
+					util.ExtraOption{
+						Component: Apiserver,
+						Key:       "fail-no-swap",
+						Value:     "true",
+					},
+				},
+			},
+			expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: 192.168.1.101
+  bindPort: 8443
+kubernetesVersion: v1.8.0-alpha.0
+certificatesDir: /var/lib/localkube/certs/
+networking:
+  serviceSubnet: 10.0.0.0/24
+etcd:
+  dataDir: /data
+nodeName: extra-args-minikube
+apiServerExtraArgs:
+  fail-no-swap: "true"
+  feature-gates: "HugePages=true,OtherFeature=false"
+controllerManagerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
+schedulerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
 `,
 		},
 		{
@@ -132,3 +233,54 @@ schedulerExtraArgs:
 		})
 	}
 }
+
+func TestParseFeatureGates(t *testing.T) {
+	tests := []struct {
+		description string
+		fg          string
+		expected    map[string]string
+		shouldErr   bool
+	}{
+		{
+			description: "no feature gates",
+		},
+		{
+			description: "one feature gate",
+			fg:          "AppArmor=true",
+			expected: map[string]string{
+				"AppArmor": "true",
+			},
+		},
+		{
+			description: "two feature gates",
+			fg:          "AppArmor=true,HugePages=true",
+			expected: map[string]string{
+				"AppArmor":  "true",
+				"HugePages": "true",
+			},
+		},
+		{
+			description: "missing value pair",
+			fg:          "AppArmor=true,HugePages",
+			shouldErr:   true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			actual, err := parseFeatureGates(test.fg)
+			t.Logf("%+v", actual)
+			if err == nil && test.shouldErr {
+				t.Errorf("Expected error but got none: fg: %v", actual)
+				return
+			}
+			if err != nil && !test.shouldErr {
+				t.Errorf("Unexpected error: %s", err)
+				return
+			}
+			if !reflect.DeepEqual(actual, test.expected) {
+				t.Errorf("Actual not equal expected: Actual: %v Expected: %v", actual, test.expected)
+			}
+		})
+	}
+}