k8s_master.yaml

#cloud-config
ssh_authorized_keys:
 - %PUB_KEY%
write-files:
  - path: /etc/kubernetes/ssl/openssl.cnf
    permissions: '0644'
    content: |
      [req]
      req_extensions = v3_req
      distinguished_name = req_distinguished_name
      [req_distinguished_name]
      [ v3_req ]
      basicConstraints = CA:FALSE
      keyUsage = nonRepudiation, digitalSignature, keyEncipherment
      subjectAltName = @alt_names
      [alt_names]
      DNS.1 = %HOSTNAME%
      DNS.2 = kubernetes
      DNS.3 = kubernetes.default
      DNS.4 = kubernetes.default.svc
      DNS.5 = kubernetes.default.svc.%K8S_DOMAIN%
      IP.1 = %K8S_SERVICE_IP%
  - path: /etc/kubernetes/ssl/make-ca-cert.sh
    permissions: '0755'
    content: |
      #!/bin/bash -e
      CA_KUBE=ca.pem
      CA_KUBE_KEY=ca-key.pem
      KUBE_SSL_CONF=openssl.cnf
      # Set permissions for the /etc/kubernetes/ssl directory
      chmod 700 /etc/kubernetes/ssl
      # inherit default 077 umask for the new files
      docker run --rm -v /etc/kubernetes/ssl:/etc/kubernetes/ssl %K8S_IMAGE% setfacl -d -m user::rwx /etc/kubernetes/ssl
      # Kubernetes CA
      openssl genrsa -out $CA_KUBE_KEY 2048
      openssl req -x509 -new -nodes -key $CA_KUBE_KEY -days 10000 -out $CA_KUBE -subj "/CN=kube-ca"
      # Kubernetes keypairs
      openssl genrsa -out apiserver-key.pem 2048
      openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config $KUBE_SSL_CONF
      openssl x509 -req -in apiserver.csr -CA $CA_KUBE -CAkey $CA_KUBE_KEY -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile $KUBE_SSL_CONF
      openssl genrsa -out worker-key.pem 2048
      openssl req -new -key worker-key.pem -out worker.csr -subj "/CN=kube-worker"
      openssl x509 -req -in worker.csr -CA $CA_KUBE -CAkey $CA_KUBE_KEY -CAcreateserial -out worker.pem -days 365
      openssl genrsa -out admin-key.pem 2048
      openssl req -new -key admin-key.pem -out admin.csr -subj "/CN=kube-admin"
      openssl x509 -req -in admin.csr -CA $CA_KUBE -CAkey $CA_KUBE_KEY -CAcreateserial -out admin.pem -days 365
  - path: /etc/kubernetes/mk-kubeconfig.sh
    permissions: '0700'
    content: |
      #!/bin/bash -e
      CERT_DIR=/etc/kubernetes/ssl
      KUBELET_CERT=$(cat $CERT_DIR/worker.pem | base64 -w0)
      KUBELET_KEY=$(cat $CERT_DIR/worker-key.pem | base64 -w0)
      KUBELET_CA_CERT=$(cat $CERT_DIR/ca.pem | base64 -w0)
      eval "echo \"$(cat /etc/kubernetes/kubeconfig.yaml.in)\"" | etcdctl set /kubeconfig > /dev/null
  - path: /etc/kubernetes/kubeconfig.yaml.in
    permissions: '0600'
    content: |
      apiVersion: v1
      kind: Config
      clusters:
      - name: local
        cluster:
          certificate-authority-data: ${KUBELET_CA_CERT}
      users:
      - name: kubelet
        user:
          client-certificate-data: ${KUBELET_CERT}
          client-key-data: ${KUBELET_KEY}
      contexts:
      - context:
          cluster: local
          user: kubelet
        name: kubelet-context
      current-context: kubelet-context
  - path: /etc/kubernetes/skydns.yaml
    permissions: '0644'
    content: |
      apiVersion: v1
      kind: Service
      metadata:
        name: kube-dns
        namespace: kube-system
        labels:
          k8s-app: kube-dns
          kubernetes.io/cluster-service: "true"
          kubernetes.io/name: "KubeDNS"
      spec:
        selector:
          k8s-app: kube-dns
        clusterIP: %DNS_SERVICE_IP%
        ports:
        - name: dns
          port: 53
          protocol: UDP
        - name: dns-tcp
          port: 53
          protocol: TCP
      ---
      apiVersion: extensions/v1beta1
      kind: Deployment
      metadata:
        name: kube-dns-v20
        namespace: kube-system
        labels:
          k8s-app: kube-dns
          version: v20
          kubernetes.io/cluster-service: "true"
      spec:
        strategy:
          type: RollingUpdate
          rollingUpdate:
            # Ensure we have at least 1 alive pod during update (don't kill old pod until new pod is up and running)
            maxSurge: 1
            maxUnavailable: 0
        replicas: 1
        selector:
          matchLabels:
            k8s-app: kube-dns
            version: v20
        template:
          metadata:
            labels:
              k8s-app: kube-dns
              version: v20
            annotations:
              scheduler.alpha.kubernetes.io/critical-pod: ''
              scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
          spec:
            containers:
            - name: kubedns
              image: gcr.io/google_containers/kubedns-amd64:1.8
              resources:
                # TODO: Set memory limits when we've profiled the container for large
                # clusters, then set request = limit to keep this container in
                # guaranteed class. Currently, this container falls into the
                # "burstable" category so the kubelet doesn't backoff from restarting it.
                limits:
                  memory: 170Mi
                requests:
                  cpu: 100m
                  memory: 70Mi
              livenessProbe:
                httpGet:
                  path: /healthz-kubedns
                  port: 8080
                  scheme: HTTP
                initialDelaySeconds: 60
                timeoutSeconds: 5
                successThreshold: 1
                failureThreshold: 5
              readinessProbe:
                httpGet:
                  path: /readiness
                  port: 8081
                  scheme: HTTP
                # we poll on pod startup for the Kubernetes master service and
                # only setup the /readiness HTTP server once that's available.
                initialDelaySeconds: 3
                timeoutSeconds: 5
              args:
              # command = "/kube-dns"
              - --domain=%K8S_DOMAIN%.
              - --dns-port=10053
              ports:
              - containerPort: 10053
                name: dns-local
                protocol: UDP
              - containerPort: 10053
                name: dns-tcp-local
                protocol: TCP
            - name: dnsmasq
              image: gcr.io/google_containers/kube-dnsmasq-amd64:1.4
              livenessProbe:
                httpGet:
                  path: /healthz-dnsmasq
                  port: 8080
                  scheme: HTTP
                initialDelaySeconds: 60
                timeoutSeconds: 5
                successThreshold: 1
                failureThreshold: 5
              args:
              - --cache-size=1000
              - --no-resolv
              - --server=127.0.0.1#10053
              - --log-facility=-
              ports:
              - containerPort: 53
                name: dns
                protocol: UDP
              - containerPort: 53
                name: dns-tcp
                protocol: TCP
            - name: healthz
              image: gcr.io/google_containers/exechealthz-amd64:1.2
              resources:
                limits:
                  memory: 50Mi
                requests:
                  cpu: 10m
                  # Note that this container shouldn't really need 50Mi of memory. The
                  # limits are set higher than expected pending investigation on #29688.
                  # The extra memory was stolen from the kubedns container to keep the
                  # net memory requested by the pod constant.
                  memory: 50Mi
              args:
              - --cmd=nslookup kubernetes.default.svc.%K8S_DOMAIN% 127.0.0.1 >/dev/null
              - --url=/healthz-dnsmasq
              - --cmd=nslookup kubernetes.default.svc.%K8S_DOMAIN% 127.0.0.1:10053 >/dev/null
              - --url=/healthz-kubedns
              - --port=8080
              - --quiet
              ports:
              - containerPort: 8080
                protocol: TCP
            dnsPolicy: Default  # Don't use cluster DNS.
  - path: /etc/kubernetes/manifests/kube-apiserver.yaml
    permissions: '0644'
    content: |
      apiVersion: v1
      kind: Pod
      metadata:
        name: kube-apiserver
        namespace: kube-system
      spec:
        hostNetwork: true
        containers:
        - name: kube-apiserver
          image: %K8S_IMAGE%
          command:
          - /hyperkube
          - apiserver
          - --bind-address=0.0.0.0
          - --etcd_servers=%ETCD_ENDPOINTS%
          - --allow-privileged=true
          - --service-cluster-ip-range=%SERVICE_IP_RANGE%
          - --secure_port=443
          # - --advertise-address=%HOSTNAME%
          - --admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
          - --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem
          - --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem
          - --client-ca-file=/etc/kubernetes/ssl/ca.pem
          - --service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem
          ports:
          - containerPort: 443
            hostPort: 443
            name: https
          - containerPort: 8080
            hostPort: 8080
            name: local
          volumeMounts:
          - mountPath: /etc/kubernetes/ssl
            name: ssl-certs-kubernetes
            readOnly: true
          - mountPath: /etc/ssl/certs
            name: ssl-certs-host
            readOnly: true
        volumes:
        - hostPath:
            path: /etc/kubernetes/ssl
          name: ssl-certs-kubernetes
        - hostPath:
            path: /usr/share/ca-certificates
          name: ssl-certs-host
  - path: /etc/kubernetes/manifests/kube-proxy.yaml
    permissions: '0644'
    content: |
      apiVersion: v1
      kind: Pod
      metadata:
        name: kube-proxy
        namespace: kube-system
      spec:
        hostNetwork: true
        containers:
        - name: kube-proxy
          image: %K8S_IMAGE%
          command:
          - /hyperkube
          - proxy
          - --master=http://127.0.0.1:8080
          securityContext:
            privileged: true
          volumeMounts:
          - mountPath: /etc/ssl/certs
            name: ssl-certs-host
            readOnly: true
        volumes:
        - hostPath:
            path: /usr/share/ca-certificates
          name: ssl-certs-host
  - path: /etc/kubernetes/manifests/kube-podmaster.yaml
    permissions: '0644'
    content: |
      apiVersion: v1
      kind: Pod
      metadata:
        name: kube-podmaster
        namespace: kube-system
      spec:
        hostNetwork: true
        containers:
        - name: scheduler-elector
          image: gcr.io/google_containers/podmaster:1.1
          command:
          - /podmaster
          - --etcd-servers=%ETCD_ENDPOINTS%
          - --key=scheduler
          - --whoami=%HOSTNAME%
          - --source-file=/src/manifests/kube-scheduler.yaml
          - --dest-file=/dst/manifests/kube-scheduler.yaml
          volumeMounts:
          - mountPath: /src/manifests
            name: manifest-src
            readOnly: true
          - mountPath: /dst/manifests
            name: manifest-dst
        - name: controller-manager-elector
          image: gcr.io/google_containers/podmaster:1.1
          command:
          - /podmaster
          - --etcd-servers=%ETCD_ENDPOINTS%
          - --key=controller
          - --whoami=%HOSTNAME%
          - --source-file=/src/manifests/kube-controller-manager.yaml
          - --dest-file=/dst/manifests/kube-controller-manager.yaml
          terminationMessagePath: /dev/termination-log
          volumeMounts:
          - mountPath: /src/manifests
            name: manifest-src
            readOnly: true
          - mountPath: /dst/manifests
            name: manifest-dst
        volumes:
        - hostPath:
            path: /srv/kubernetes/manifests
          name: manifest-src
        - hostPath:
            path: /etc/kubernetes/manifests
          name: manifest-dst
  - path: /srv/kubernetes/manifests/kube-controller-manager.yaml
    permissions: '0644'
    content: |
      apiVersion: v1
      kind: Pod
      metadata:
        name: kube-controller-manager
        namespace: kube-system
      spec:
        containers:
        - name: kube-controller-manager
          image: %K8S_IMAGE%
          command:
          - /hyperkube
          - controller-manager
          - --master=http://127.0.0.1:8080
          - --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem
          - --root-ca-file=/etc/kubernetes/ssl/ca.pem
          - --enable-hostpath-provisioner=%K8S_AUTO_HOSTPATH_PROVISIONER%
          livenessProbe:
            httpGet:
              host: 127.0.0.1
              path: /healthz
              port: 10252
            initialDelaySeconds: 15
            timeoutSeconds: 1
          volumeMounts:
          - mountPath: /etc/kubernetes/ssl
            name: ssl-certs-kubernetes
            readOnly: true
          - mountPath: /etc/ssl/certs
            name: ssl-certs-host
            readOnly: true
        hostNetwork: true
        volumes:
        - hostPath:
            path: /etc/kubernetes/ssl
          name: ssl-certs-kubernetes
        - hostPath:
            path: /usr/share/ca-certificates
          name: ssl-certs-host
  - path: /srv/kubernetes/manifests/kube-scheduler.yaml
    permissions: '0644'
    content: |
      apiVersion: v1
      kind: Pod
      metadata:
        name: kube-scheduler
        namespace: kube-system
      spec:
        hostNetwork: true
        containers:
        - name: kube-scheduler
          image: %K8S_IMAGE%
          command:
          - /hyperkube
          - scheduler
          - --master=http://127.0.0.1:8080
          livenessProbe:
            httpGet:
              host: 127.0.0.1
              path: /healthz
              port: 10251
            initialDelaySeconds: 15
            timeoutSeconds: 1
hostname: %HOSTNAME%
coreos:
  etcd2:
    # generate a new token for each unique cluster from https://discovery.etcd.io/new?size=3
    # specify the initial size of your cluster with ?size=X
    discovery: %DISCOVERY%
    advertise-client-urls: http://%H:2379,http://%H:4001
    initial-advertise-peer-urls: http://%H:2380
    # listen on both the official ports and the legacy ports
    # legacy ports can be omitted if your application doesn't depend on them
    listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001
    listen-peer-urls: http://%H:2380
#  fleet:
#    metadata: "role=master"
  units:
    - name: tmp-hostpath_pv.mount
      command: %K8S_HOSTPATH_PROVISIONER_MOUNT_POINT%
      content: |
        [Mount]
        What=/data/k8s
        Where=/tmp/hostpath_pv
        Type=none
        Options=bind
    - name: systemd-networkd.service
      command: restart
    - name: etcd2.service
      command: start
#    - name: fleet.service
#      command: start
    - name: flanneld.service
      command: start
      drop-ins:
        - name: 50-network-config.conf
          content: |
            [Unit]
            Requires=etcd2.service
            [Service]
            ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{"Network":"%POD_NETWORK%", "Backend": {"Type": "%FLANNEL_TYPE%"}}'
    - name: docker.service
      drop-ins:
        - name: 40-flannel.conf
          content: |
            [Unit]
            Requires=flanneld.service
            After=flanneld.service
    - name: generate-keys.service
      command: start
      content: |
        [Unit]
        ConditionPathExists=!/etc/kubernetes/ssl/.certs.lock
        Requires=network-online.target docker.service
        After=network-online.target docker.service
        [Service]
        WorkingDirectory=/etc/kubernetes/ssl
        ExecStart=/etc/kubernetes/ssl/make-ca-cert.sh
        ExecStartPost=/usr/bin/touch /etc/kubernetes/ssl/.certs.lock
        # wait for etcd ready
        ExecStartPost=/usr/bin/bash -c "while true; do etcdctl ls / > /dev/null && exit 0; sleep 1; done"
        ExecStartPost=/etc/kubernetes/mk-kubeconfig.sh
        Type=oneshot
        RemainAfterExit=true
    - name: kubernetes-download@.service
      content: |
        [Unit]
        Description=Download Kubernetes %i binary
        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
        Requires=network-online.target
        After=network-online.target
        [Service]
        ExecStartPre=-/usr/bin/mkdir -p /opt/bin
        ExecStart=/usr/bin/curl -s -L -o /opt/bin/%i -z /opt/bin/%i https://storage.googleapis.com/kubernetes-release/release/%K8S_RELEASE%/bin/linux/amd64/%i
        ExecStartPost=/usr/bin/chmod +x /opt/bin/%i
        RemainAfterExit=yes
        Type=oneshot
    - name: kubelet.service
      command: start
      content: |
        [Unit]
        Requires=docker.service generate-keys.service kubernetes-download@kubelet.service
        After=docker.service generate-keys.service kubernetes-download@kubelet.service
        [Service]
        ExecStart=/opt/bin/kubelet \
          --api_servers=http://127.0.0.1:8080 \
          --register-node=true \
          --register-schedulable=true \
          --node-labels=kubernetes.io/role=master \
          --allow-privileged=true \
          --pod-manifest-path=/etc/kubernetes/manifests \
          --hostname-override=%H \
          --cluster_dns=%DNS_SERVICE_IP% \
          --cluster_domain=%K8S_DOMAIN% \
          --cadvisor-port=0
        Restart=always
        RestartSec=10
        [Install]
        WantedBy=multi-user.target
    - name: kube-system-ns.service
      command: start
      content: |
        [Unit]
        ConditionPathExists=!/etc/kubernetes/.kube-system-ns.lock
        Requires=kubelet.service
        After=kubelet.service
        [Service]
        # Wait until apiserver start
        ExecStartPre=/usr/bin/bash -c "while true; do curl -v http://127.0.0.1:8080/api/v1/namespaces/kube-system -o /dev/null 2>&1 | grep '< HTTP/1.1' > /dev/null && exit 0; sleep 1; done"
        ExecStart=/usr/bin/bash -c 'curl -H "Content-Type: application/json" -XPOST -d\'{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"kube-system"}}\' "http://127.0.0.1:8080/api/v1/namespaces"'
        ExecStartPost=/usr/bin/touch /etc/kubernetes/.kube-system-ns.lock
        Type=oneshot
        RemainAfterExit=true
    - name: skydns.service
      command: start
      content: |
        [Unit]
        ConditionPathExists=!/etc/kubernetes/.skydns.lock
        Requires=kubernetes-download@kubectl.service
        After=kubernetes-download@kubectl.service
        [Service]
        # Wait until kubectl can access apiserver
        ExecStartPre=/usr/bin/bash -c "while true; do /opt/bin/kubectl get nodes && exit 0; sleep 1; done"
        ExecStart=/opt/bin/kubectl create -f /etc/kubernetes/skydns.yaml
        ExecStartPost=/usr/bin/touch /etc/kubernetes/.skydns.lock
        RemainAfterExit=yes
        Type=oneshot
  update:
    reboot-strategy: off