diff --git a/bindata/etcd/pod.yaml b/bindata/etcd/pod.yaml index ecc80e5fc..b7a524ff9 100644 --- a/bindata/etcd/pod.yaml +++ b/bindata/etcd/pod.yaml @@ -21,8 +21,8 @@ spec: #!/bin/sh set -euo pipefail - cp /etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt /etc/kubernetes/etcd-backup-dir/system:etcd-peer-NODE_NAME.crt - cp /etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key /etc/kubernetes/etcd-backup-dir/system:etcd-peer-NODE_NAME.key + cp /etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt /etc/kubernetes/etcd-backup-dir/system:etcd-peer-NODE_NAME.crt + cp /etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key /etc/kubernetes/etcd-backup-dir/system:etcd-peer-NODE_NAME.key resources: requests: memory: 60Mi @@ -37,6 +37,33 @@ spec: - mountPath: /etc/kubernetes/static-pod-certs name: cert-dir containers: + # The etcdctl container should always be first. It is intended to be used + # to open a remote shell via `oc rsh` that is ready to run `etcdctl`. + - name: etcdctl + image: ${IMAGE} + imagePullPolicy: IfNotPresent + terminationMessagePolicy: FallbackToLogsOnError + command: + - "/bin/bash" + - "-c" + - "trap: TERM INT; sleep infinity & wait" + resources: + requests: + memory: 60Mi + cpu: 30m + volumeMounts: + - mountPath: /etc/kubernetes/manifests + name: static-pod-dir + - mountPath: /etc/kubernetes/etcd-backup-dir + name: etcd-backup-dir + - mountPath: /etc/kubernetes/static-pod-resources + name: resource-dir + - mountPath: /etc/kubernetes/static-pod-certs + name: cert-dir + - mountPath: /var/lib/etcd/ + name: data-dir + env: +${COMPUTED_ENV_VARS} - name: etcd image: ${IMAGE} imagePullPolicy: IfNotPresent @@ -48,18 +75,14 @@ spec: #!/bin/sh set -euo pipefail - ETCDCTL="etcdctl --cacert=/etc/kubernetes/static-pod-resources/configmaps/etcd-serving-ca/ca-bundle.crt \ - --cert=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ - --key=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ - --endpoints=${ALL_ETCD_ENDPOINTS}" - ${ETCDCTL} member list || true + etcdctl member list || true # this has a non-zero return code if the command is non-zero. If you use an export first, it doesn't and you # will succeed when you should fail. ETCD_INITIAL_CLUSTER=$(discover-etcd-initial-cluster \ - --cacert=/etc/kubernetes/static-pod-resources/configmaps/etcd-serving-ca/ca-bundle.crt \ - --cert=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ - --key=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ + --cacert=/etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt \ + --cert=/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ + --key=/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ --endpoints=${ALL_ETCD_ENDPOINTS} \ --data-dir=/var/lib/etcd/member \ --target-peer-url-host=${NODE_NODE_ENVVAR_NAME_ETCD_DNS_NAME} \ @@ -76,13 +99,13 @@ spec: set -x exec etcd \ --initial-advertise-peer-urls=https://${NODE_NODE_ENVVAR_NAME_IP}:2380 \ - --cert-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-serving/etcd-serving-NODE_NAME.crt \ - --key-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-serving/etcd-serving-NODE_NAME.key \ - --trusted-ca-file=/etc/kubernetes/static-pod-resources/configmaps/etcd-serving-ca/ca-bundle.crt \ + --cert-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-serving/etcd-serving-NODE_NAME.crt \ + --key-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-serving/etcd-serving-NODE_NAME.key \ + --trusted-ca-file=/etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt \ --client-cert-auth=true \ - --peer-cert-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ - --peer-key-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ - --peer-trusted-ca-file=/etc/kubernetes/static-pod-resources/configmaps/etcd-peer-client-ca/ca-bundle.crt \ + --peer-cert-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ + --peer-key-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ + --peer-trusted-ca-file=/etc/kubernetes/static-pod-certs/configmaps/etcd-peer-client-ca/ca-bundle.crt \ --peer-client-cert-auth=true \ --advertise-client-urls=https://${NODE_NODE_ENVVAR_NAME_IP}:2379 \ --listen-client-urls=https://${LISTEN_ON_ALL_IPS}:2379 \ @@ -135,12 +158,12 @@ ${COMPUTED_ENV_VARS} --endpoints https://${NODE_NODE_ENVVAR_NAME_ETCD_DNS_NAME}:9978 \ --metrics-addr https://${LISTEN_ON_ALL_IPS}:9979 \ --listen-addr ${LOCALHOST_IP}:9977 \ - --key /etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ - --key-file /etc/kubernetes/static-pod-resources/secrets/etcd-all-serving-metrics/etcd-serving-metrics-NODE_NAME.key \ - --cert /etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ - --cert-file /etc/kubernetes/static-pod-resources/secrets/etcd-all-serving-metrics/etcd-serving-metrics-NODE_NAME.crt \ - --cacert /etc/kubernetes/static-pod-resources/configmaps/etcd-peer-client-ca/ca-bundle.crt \ - --trusted-ca-file /etc/kubernetes/static-pod-resources/configmaps/etcd-metrics-proxy-serving-ca/ca-bundle.crt + --key /etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ + --key-file /etc/kubernetes/static-pod-certs/secrets/etcd-all-serving-metrics/etcd-serving-metrics-NODE_NAME.key \ + --cert /etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ + --cert-file /etc/kubernetes/static-pod-certs/secrets/etcd-all-serving-metrics/etcd-serving-metrics-NODE_NAME.crt \ + --cacert /etc/kubernetes/static-pod-certs/configmaps/etcd-peer-client-ca/ca-bundle.crt \ + --trusted-ca-file /etc/kubernetes/static-pod-certs/configmaps/etcd-metrics-proxy-serving-ca/ca-bundle.crt env: ${COMPUTED_ENV_VARS} resources: diff --git a/bindata/etcd/restore-pod.yaml b/bindata/etcd/restore-pod.yaml index 9e7bdb1e5..8211f9544 100644 --- a/bindata/etcd/restore-pod.yaml +++ b/bindata/etcd/restore-pod.yaml @@ -61,13 +61,13 @@ spec: set -x exec etcd \ --initial-advertise-peer-urls=https://${NODE_NODE_ENVVAR_NAME_IP}:2380 \ - --cert-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-serving/etcd-serving-NODE_NAME.crt \ - --key-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-serving/etcd-serving-NODE_NAME.key \ - --trusted-ca-file=/etc/kubernetes/static-pod-resources/configmaps/etcd-serving-ca/ca-bundle.crt \ + --cert-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-serving/etcd-serving-NODE_NAME.crt \ + --key-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-serving/etcd-serving-NODE_NAME.key \ + --trusted-ca-file=/etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt \ --client-cert-auth=true \ - --peer-cert-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ - --peer-key-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ - --peer-trusted-ca-file=/etc/kubernetes/static-pod-resources/configmaps/etcd-peer-client-ca/ca-bundle.crt \ + --peer-cert-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ + --peer-key-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ + --peer-trusted-ca-file=/etc/kubernetes/static-pod-certs/configmaps/etcd-peer-client-ca/ca-bundle.crt \ --peer-client-cert-auth=true \ --advertise-client-urls=https://${NODE_NODE_ENVVAR_NAME_IP}:2379 \ --listen-client-urls=https://${LISTEN_ON_ALL_IPS}:2379 \ @@ -97,8 +97,6 @@ ${COMPUTED_ENV_VARS} name: static-pod-dir - mountPath: /etc/kubernetes/etcd-backup-dir name: etcd-backup-dir - - mountPath: /etc/kubernetes/static-pod-resources - name: resource-dir - mountPath: /etc/kubernetes/static-pod-certs name: cert-dir - mountPath: /var/lib/etcd/ @@ -116,9 +114,6 @@ ${COMPUTED_ENV_VARS} - hostPath: path: /etc/kubernetes/static-pod-resources/etcd-member name: etcd-backup-dir - - hostPath: - path: /etc/kubernetes/static-pod-resources/etcd-pod-REVISION - name: resource-dir - hostPath: path: /etc/kubernetes/static-pod-resources/etcd-certs name: cert-dir diff --git a/go.mod b/go.mod index 1af428405..8c49a4151 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/openshift/api v0.0.0-20200210091934-a0e53e94816b github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160 github.com/openshift/client-go v0.0.0-20200116152001-92a2713fa240 - github.com/openshift/library-go v0.0.0-20200226171210-caa110959f91 + github.com/openshift/library-go v0.0.0-20200227110433-19fff9ed3c27 github.com/prometheus/client_golang v1.1.0 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 178490aa5..184750c8d 100644 --- a/go.sum +++ b/go.sum @@ -320,8 +320,8 @@ github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160 h1:V4 github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc= github.com/openshift/client-go v0.0.0-20200116152001-92a2713fa240 h1:XYfJWv2Ch+qInGLDEedHRtDsJwnxyU1L8U7SY56NcA8= github.com/openshift/client-go v0.0.0-20200116152001-92a2713fa240/go.mod h1:4riOwdj99Hd/q+iAcJZfNCsQQQMwURnZV6RL4WHYS5w= -github.com/openshift/library-go v0.0.0-20200226171210-caa110959f91 h1:LMDLwcePKeCUGiMeTqBLdDJhtGivKPRfH0iI/Qbwwis= -github.com/openshift/library-go v0.0.0-20200226171210-caa110959f91/go.mod h1:0rRwY0q5NuKHdiP88Pe5+OVNU8mi0mv5XQ7f7nUbYVc= +github.com/openshift/library-go v0.0.0-20200227110433-19fff9ed3c27 h1:llVjJ8iap5T6aC8y7VaeaMVet0QeYgOP7ZW+JZB2I3U= +github.com/openshift/library-go v0.0.0-20200227110433-19fff9ed3c27/go.mod h1:0rRwY0q5NuKHdiP88Pe5+OVNU8mi0mv5XQ7f7nUbYVc= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= diff --git a/pkg/operator/etcd_assets/bindata.go b/pkg/operator/etcd_assets/bindata.go index ed8a80e35..e542bbab4 100644 --- a/pkg/operator/etcd_assets/bindata.go +++ b/pkg/operator/etcd_assets/bindata.go @@ -898,8 +898,8 @@ spec: #!/bin/sh set -euo pipefail - cp /etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt /etc/kubernetes/etcd-backup-dir/system:etcd-peer-NODE_NAME.crt - cp /etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key /etc/kubernetes/etcd-backup-dir/system:etcd-peer-NODE_NAME.key + cp /etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt /etc/kubernetes/etcd-backup-dir/system:etcd-peer-NODE_NAME.crt + cp /etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key /etc/kubernetes/etcd-backup-dir/system:etcd-peer-NODE_NAME.key resources: requests: memory: 60Mi @@ -914,6 +914,33 @@ spec: - mountPath: /etc/kubernetes/static-pod-certs name: cert-dir containers: + # The etcdctl container should always be first. It is intended to be used + # to open a remote shell via ` + "`" + `oc rsh` + "`" + ` that is ready to run ` + "`" + `etcdctl` + "`" + `. + - name: etcdctl + image: ${IMAGE} + imagePullPolicy: IfNotPresent + terminationMessagePolicy: FallbackToLogsOnError + command: + - "/bin/bash" + - "-c" + - "trap: TERM INT; sleep infinity & wait" + resources: + requests: + memory: 60Mi + cpu: 30m + volumeMounts: + - mountPath: /etc/kubernetes/manifests + name: static-pod-dir + - mountPath: /etc/kubernetes/etcd-backup-dir + name: etcd-backup-dir + - mountPath: /etc/kubernetes/static-pod-resources + name: resource-dir + - mountPath: /etc/kubernetes/static-pod-certs + name: cert-dir + - mountPath: /var/lib/etcd/ + name: data-dir + env: +${COMPUTED_ENV_VARS} - name: etcd image: ${IMAGE} imagePullPolicy: IfNotPresent @@ -925,18 +952,14 @@ spec: #!/bin/sh set -euo pipefail - ETCDCTL="etcdctl --cacert=/etc/kubernetes/static-pod-resources/configmaps/etcd-serving-ca/ca-bundle.crt \ - --cert=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ - --key=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ - --endpoints=${ALL_ETCD_ENDPOINTS}" - ${ETCDCTL} member list || true + etcdctl member list || true # this has a non-zero return code if the command is non-zero. If you use an export first, it doesn't and you # will succeed when you should fail. ETCD_INITIAL_CLUSTER=$(discover-etcd-initial-cluster \ - --cacert=/etc/kubernetes/static-pod-resources/configmaps/etcd-serving-ca/ca-bundle.crt \ - --cert=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ - --key=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ + --cacert=/etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt \ + --cert=/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ + --key=/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ --endpoints=${ALL_ETCD_ENDPOINTS} \ --data-dir=/var/lib/etcd/member \ --target-peer-url-host=${NODE_NODE_ENVVAR_NAME_ETCD_DNS_NAME} \ @@ -953,13 +976,13 @@ spec: set -x exec etcd \ --initial-advertise-peer-urls=https://${NODE_NODE_ENVVAR_NAME_IP}:2380 \ - --cert-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-serving/etcd-serving-NODE_NAME.crt \ - --key-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-serving/etcd-serving-NODE_NAME.key \ - --trusted-ca-file=/etc/kubernetes/static-pod-resources/configmaps/etcd-serving-ca/ca-bundle.crt \ + --cert-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-serving/etcd-serving-NODE_NAME.crt \ + --key-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-serving/etcd-serving-NODE_NAME.key \ + --trusted-ca-file=/etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt \ --client-cert-auth=true \ - --peer-cert-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ - --peer-key-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ - --peer-trusted-ca-file=/etc/kubernetes/static-pod-resources/configmaps/etcd-peer-client-ca/ca-bundle.crt \ + --peer-cert-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ + --peer-key-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ + --peer-trusted-ca-file=/etc/kubernetes/static-pod-certs/configmaps/etcd-peer-client-ca/ca-bundle.crt \ --peer-client-cert-auth=true \ --advertise-client-urls=https://${NODE_NODE_ENVVAR_NAME_IP}:2379 \ --listen-client-urls=https://${LISTEN_ON_ALL_IPS}:2379 \ @@ -1012,12 +1035,12 @@ ${COMPUTED_ENV_VARS} --endpoints https://${NODE_NODE_ENVVAR_NAME_ETCD_DNS_NAME}:9978 \ --metrics-addr https://${LISTEN_ON_ALL_IPS}:9979 \ --listen-addr ${LOCALHOST_IP}:9977 \ - --key /etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ - --key-file /etc/kubernetes/static-pod-resources/secrets/etcd-all-serving-metrics/etcd-serving-metrics-NODE_NAME.key \ - --cert /etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ - --cert-file /etc/kubernetes/static-pod-resources/secrets/etcd-all-serving-metrics/etcd-serving-metrics-NODE_NAME.crt \ - --cacert /etc/kubernetes/static-pod-resources/configmaps/etcd-peer-client-ca/ca-bundle.crt \ - --trusted-ca-file /etc/kubernetes/static-pod-resources/configmaps/etcd-metrics-proxy-serving-ca/ca-bundle.crt + --key /etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ + --key-file /etc/kubernetes/static-pod-certs/secrets/etcd-all-serving-metrics/etcd-serving-metrics-NODE_NAME.key \ + --cert /etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ + --cert-file /etc/kubernetes/static-pod-certs/secrets/etcd-all-serving-metrics/etcd-serving-metrics-NODE_NAME.crt \ + --cacert /etc/kubernetes/static-pod-certs/configmaps/etcd-peer-client-ca/ca-bundle.crt \ + --trusted-ca-file /etc/kubernetes/static-pod-certs/configmaps/etcd-metrics-proxy-serving-ca/ca-bundle.crt env: ${COMPUTED_ENV_VARS} resources: @@ -1159,13 +1182,13 @@ spec: set -x exec etcd \ --initial-advertise-peer-urls=https://${NODE_NODE_ENVVAR_NAME_IP}:2380 \ - --cert-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-serving/etcd-serving-NODE_NAME.crt \ - --key-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-serving/etcd-serving-NODE_NAME.key \ - --trusted-ca-file=/etc/kubernetes/static-pod-resources/configmaps/etcd-serving-ca/ca-bundle.crt \ + --cert-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-serving/etcd-serving-NODE_NAME.crt \ + --key-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-serving/etcd-serving-NODE_NAME.key \ + --trusted-ca-file=/etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt \ --client-cert-auth=true \ - --peer-cert-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ - --peer-key-file=/etc/kubernetes/static-pod-resources/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ - --peer-trusted-ca-file=/etc/kubernetes/static-pod-resources/configmaps/etcd-peer-client-ca/ca-bundle.crt \ + --peer-cert-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt \ + --peer-key-file=/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key \ + --peer-trusted-ca-file=/etc/kubernetes/static-pod-certs/configmaps/etcd-peer-client-ca/ca-bundle.crt \ --peer-client-cert-auth=true \ --advertise-client-urls=https://${NODE_NODE_ENVVAR_NAME_IP}:2379 \ --listen-client-urls=https://${LISTEN_ON_ALL_IPS}:2379 \ @@ -1195,8 +1218,6 @@ ${COMPUTED_ENV_VARS} name: static-pod-dir - mountPath: /etc/kubernetes/etcd-backup-dir name: etcd-backup-dir - - mountPath: /etc/kubernetes/static-pod-resources - name: resource-dir - mountPath: /etc/kubernetes/static-pod-certs name: cert-dir - mountPath: /var/lib/etcd/ @@ -1214,9 +1235,6 @@ ${COMPUTED_ENV_VARS} - hostPath: path: /etc/kubernetes/static-pod-resources/etcd-member name: etcd-backup-dir - - hostPath: - path: /etc/kubernetes/static-pod-resources/etcd-pod-REVISION - name: resource-dir - hostPath: path: /etc/kubernetes/static-pod-resources/etcd-certs name: cert-dir diff --git a/pkg/operator/starter.go b/pkg/operator/starter.go index 114eff788..dd2c669af 100644 --- a/pkg/operator/starter.go +++ b/pkg/operator/starter.go @@ -253,6 +253,10 @@ var RevisionSecrets = []revision.RevisionResource{ var CertConfigMaps = []revision.RevisionResource{ {Name: "restore-etcd-pod"}, {Name: "etcd-scripts"}, + {Name: "etcd-serving-ca"}, + {Name: "etcd-peer-client-ca"}, + {Name: "etcd-metrics-proxy-serving-ca"}, + {Name: "etcd-metrics-proxy-client-ca"}, } var CertSecrets = []revision.RevisionResource{ diff --git a/pkg/operator/targetconfigcontroller/etcd_env.go b/pkg/operator/targetconfigcontroller/etcd_env.go index 9f1d3d3d4..13cbc5c78 100644 --- a/pkg/operator/targetconfigcontroller/etcd_env.go +++ b/pkg/operator/targetconfigcontroller/etcd_env.go @@ -30,7 +30,10 @@ var envVarFns = []envVarFunc{ getDNSName, getFixedEtcdEnvVars, getEtcdName, - getAllClusterMembers, + getAllEtcdEndpoints, + getEtcdctlEnvVars, + getHeartbeatInterval, + getElectionTimeout, } // getEtcdEnvVars returns the env vars that need to be set on the etcd static pods that will be rendered. @@ -38,6 +41,8 @@ var envVarFns = []envVarFunc{ // ETCD_DATA_DIR // ETCDCTL_API // ETCD_QUOTA_BACKEND_BYTES +// ETCD_HEARTBEAT_INTERVAL +// ETCD_ELECTION_TIMEOUT // ETCD_INITIAL_CLUSTER_STATE // NODE_%s_IP // NODE_%s_ETCD_DNS_NAME @@ -72,48 +77,67 @@ func getFixedEtcdEnvVars(envVarContext envVarContext) (map[string]string, error) return map[string]string{ "ETCD_DATA_DIR": "/var/lib/etcd", "ETCD_QUOTA_BACKEND_BYTES": "7516192768", // 7 gig - "ETCDCTL_API": "3", "ETCD_INITIAL_CLUSTER_STATE": "existing", }, nil } -func getAllClusterMembers(envVarContext envVarContext) (map[string]string, error) { - network, err := envVarContext.networkLister.Get("cluster") +func getEtcdctlEnvVars(envVarContext envVarContext) (map[string]string, error) { + endpoints, err := getEtcdGrpcEndpoints(envVarContext) if err != nil { return nil, err } + return map[string]string{ + "ETCDCTL_API": "3", + "ETCDCTL_CACERT": "/etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt", + "ETCDCTL_CERT": "/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.crt", + "ETCDCTL_KEY": "/etc/kubernetes/static-pod-certs/secrets/etcd-all-peer/etcd-peer-NODE_NAME.key", + "ETCDCTL_ENDPOINTS": endpoints, + }, nil +} - ret := map[string]string{} +func getEtcdGrpcEndpoints(envVarContext envVarContext) (string, error) { + network, err := envVarContext.networkLister.Get("cluster") + if err != nil { + return "", err + } endpoints := []string{} for _, nodeInfo := range envVarContext.status.NodeStatuses { node, err := envVarContext.nodeLister.Get(nodeInfo.NodeName) if err != nil { - return nil, err + return "", err } endpointIP, err := dnshelpers.GetEscapedPreferredInternalIPAddressForNodeName(network, node) if err != nil { - return nil, err + return "", err } endpoints = append(endpoints, fmt.Sprintf("https://%s:2379", endpointIP)) } hostEtcdEndpoints, err := envVarContext.endpointLister.Endpoints(operatorclient.TargetNamespace).Get("host-etcd-2") if err != nil { - return nil, err + return "", err } if bootstrapIP := hostEtcdEndpoints.Annotations["alpha.installer.openshift.io/etcd-bootstrap"]; len(bootstrapIP) > 0 { urlHost, err := dnshelpers.GetURLHostForIP(bootstrapIP) if err != nil { - return nil, err + return "", err } endpoints = append(endpoints, "https://"+urlHost+":2379") } - ret["ALL_ETCD_ENDPOINTS"] = strings.Join(endpoints, ",") + return strings.Join(endpoints, ","), nil +} - return ret, nil +func getAllEtcdEndpoints(envVarContext envVarContext) (map[string]string, error) { + endpoints, err := getEtcdGrpcEndpoints(envVarContext) + if err != nil { + return nil, err + } + return map[string]string{ + "ALL_ETCD_ENDPOINTS": endpoints, + }, nil } func getEtcdName(envVarContext envVarContext) (map[string]string, error) { @@ -183,6 +207,46 @@ func getDNSName(envVarContext envVarContext) (map[string]string, error) { return ret, nil } +func getHeartbeatInterval(envVarContext envVarContext) (map[string]string, error) { + heartbeat := "100" // etcd default + + infrastructure, err := envVarContext.infrastructureLister.Get("cluster") + if err != nil { + return nil, err + } + + if status := infrastructure.Status.PlatformStatus; status != nil { + switch { + case status.Azure != nil: + heartbeat = "500" + } + } + + return map[string]string{ + "ETCD_HEARTBEAT_INTERVAL": heartbeat, + }, nil +} + +func getElectionTimeout(envVarContext envVarContext) (map[string]string, error) { + timeout := "1000" // etcd default + + infrastructure, err := envVarContext.infrastructureLister.Get("cluster") + if err != nil { + return nil, err + } + + if status := infrastructure.Status.PlatformStatus; status != nil { + switch { + case status.Azure != nil: + timeout = "2500" + } + } + + return map[string]string{ + "ETCD_ELECTION_TIMEOUT": timeout, + }, nil +} + func envVarSafe(nodeName string) string { return strings.ReplaceAll(strings.ReplaceAll(nodeName, "-", "_"), ".", "_") } diff --git a/vendor/github.com/openshift/library-go/pkg/operator/staticpod/installerpod/cmd.go b/vendor/github.com/openshift/library-go/pkg/operator/staticpod/installerpod/cmd.go index d3ad76e62..1ccde0ac9 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/staticpod/installerpod/cmd.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/staticpod/installerpod/cmd.go @@ -221,9 +221,12 @@ func (o *InstallOptions) copySecretsAndConfigMaps(ctx context.Context, resourceD return err } for filename, content := range secret.Data { - // TODO fix permissions klog.Infof("Writing secret manifest %q ...", path.Join(contentDir, filename)) - if err := ioutil.WriteFile(path.Join(contentDir, filename), content, 0600); err != nil { + filePerms := os.FileMode(0600) + if strings.HasSuffix(filename, ".sh") { + filePerms = 0700 + } + if err := ioutil.WriteFile(path.Join(contentDir, filename), content, filePerms); err != nil { return err } } @@ -240,9 +243,14 @@ func (o *InstallOptions) copySecretsAndConfigMaps(ctx context.Context, resourceD } for filename, content := range configmap.Data { klog.Infof("Writing config file %q ...", path.Join(contentDir, filename)) - if err := ioutil.WriteFile(path.Join(contentDir, filename), []byte(content), 0644); err != nil { + filePerms := os.FileMode(0644) + if strings.HasSuffix(filename, ".sh") { + filePerms = 0755 + } + if err := ioutil.WriteFile(path.Join(contentDir, filename), []byte(content), filePerms); err != nil { return err } + } } diff --git a/vendor/modules.txt b/vendor/modules.txt index 8df3630c8..3a9eb16c5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -190,7 +190,7 @@ github.com/openshift/client-go/operator/informers/externalversions/operator/v1 github.com/openshift/client-go/operator/informers/externalversions/operator/v1alpha1 github.com/openshift/client-go/operator/listers/operator/v1 github.com/openshift/client-go/operator/listers/operator/v1alpha1 -# github.com/openshift/library-go v0.0.0-20200226171210-caa110959f91 +# github.com/openshift/library-go v0.0.0-20200227110433-19fff9ed3c27 github.com/openshift/library-go/pkg/assets github.com/openshift/library-go/pkg/config/client github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers