k8s-infra: routing, TLS (rebased) (#9329)

Introduce an External Server Exposer Strategy, 
responsible for exposing service ports associated with external servers,
making them accessible from outside the cluster.
Move server exposure to shared k8s infra level:
- multi-host: unique hostname for each component, like Che Openshift infrastructure.
- single-host: single hostname for all components. Can be used in conjunction with TLS.
- default-host: default ingress hostname. Can be used for local development without dynamic DNS (based on ingress IP).
Add basic TLS support.
Signed-off-by: Guy Daich <guy.daich@sap.com>

assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties

@@ -454,6 +454,14 @@ che.infra.kubernetes.client.http.connection_pool.max_idle=5
 # in minutes
 che.infra.kubernetes.client.http.connection_pool.keep_alive_min=5
 
+# Creates Ingresses with Transport Layer Security (TLS) enabled
+# In OpenShift infrastructure, Routes will be TLS-enabled
+che.infra.kubernetes.tls_enabled=false
+
+# Name of a secret that should be used when creating workspace ingresses with TLS
+# Ignored by OpenShift infrastructure
+che.infra.kubernetes.tls_secret=
+
 ### OpenShift Infra parameters
 #
 # Since OpenShift infrastructure reuse Kubernetes infrastructure components
@@ -464,10 +472,6 @@ che.infra.kubernetes.client.http.connection_pool.keep_alive_min=5
 # If not set, every workspace will be created in a new project, where project name = workspace id
 che.infra.openshift.project=
 
-# Create routes with Transport Layer Security (TLS) enabled
-che.infra.openshift.tls_enabled=false
-
-
 # Single port mode wildcard domain host & port. nip.io is used by default
 che.singleport.wildcard_domain.host=NULL
 che.singleport.wildcard_domain.port=NULL

assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che_aliases.properties

@@ -29,6 +29,7 @@ che.infra.kubernetes.pvc.jobs.memorylimit=che.infra.openshift.pvc.jobs.memorylim
 che.infra.kubernetes.pvc.access_mode=che.infra.openshift.pvc.access_mode
 che.infra.kubernetes.installer_server_min_port=che.infra.openshift.installer_server_min_port
 che.infra.kubernetes.installer_server_max_port=che.infra.openshift.installer_server_max_port
+che.infra.kubernetes.tls_enabled=che.infra.openshift.tls_enabled
 
 che.infra.docker.daemon_url=docker.client.daemon_url
 che.infra.docker.certificates_folder=docker.client.certificates_folder

deploy/kubernetes/helm/che/charts/che-keycloak/templates/deployment.yaml

@@ -79,6 +79,7 @@ spec:
         - mountPath: /opt/jboss/keycloak/standalone/log
           name: keycloak-log
       restartPolicy: Always
+      serviceAccountName: che-keycloak
       volumes:
       - name: keycloak-data
         persistentVolumeClaim:

deploy/kubernetes/helm/che/charts/che-keycloak/templates/endpoints-monitor-role.yaml

@@ -0,0 +1,16 @@
+# Copyright (c) 2012-2017 Red Hat, Inc
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+#
+
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: che-endpoints-monitor
+rules:
+- apiGroups: [""]
+  resources: ["services", "endpoints", "pods"]
+  verbs: ["get", "list"]

deploy/kubernetes/helm/che/charts/che-keycloak/templates/endpoints-monitor-rolebinding.yaml

@@ -0,0 +1,19 @@
+# Copyright (c) 2012-2017 Red Hat, Inc
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+#
+
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: {{ .Release.Namespace -}} -keycloak-role-binding
+roleRef:
+  kind: Role
+  name: che-endpoints-monitor
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  name: che-keycloak
+  namespace: {{ .Release.Namespace }}

deploy/kubernetes/helm/che/charts/che-keycloak/templates/ingress.yaml

@@ -13,28 +13,34 @@ metadata:
     kubernetes.io/ingress.class: "nginx"
     {{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/proxy-read-timeout: "3600"
     {{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/proxy-connect-timeout: "3600"
-{{- if .Values.global.tlsEnabled }}
+{{- if .Values.global.tls.enabled }}
+    {{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/ssl-redirect: "true"
     kubernetes.io/tls-acme: "true"
 {{- else }}
     {{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/ssl-redirect: "false"
 {{- end }}
 spec:
-{{- if .Values.global.tlsEnabled }}
+{{- if .Values.global.tls.enabled }}
   tls:
   - hosts:
     - {{ template "keycloakHost" . }}
-    secretName: keycloak-tls
+    secretName: che-tls
 {{- end }}
   rules:
-{{- if .Values.global.isHostBased }}
+{{- if eq .Values.global.serverStrategy "default-host" }}
+  - http:
+      paths:
+      - path: /auth/
+{{- else if eq .Values.global.serverStrategy "single-host" }}
   - host: {{ template "keycloakHost" . }}
     http:
       paths:
-      - path: /
+      - path: /auth/
 {{- else }}
-  - http:
+  - host: {{ template "keycloakHost" . }}
+    http:
       paths:
-      - path: /auth/
+      - path: /
 {{- end }}
         backend:
           serviceName: keycloak

deploy/kubernetes/helm/che/charts/che-keycloak/templates/keycloak-configure-job.yaml

@@ -27,7 +27,7 @@ spec:
         command: ["/scripts/keycloak_config.sh"]
         env:
           - name: HTTP_PROTOCOL
-{{- if .Values.global.tlsEnabled }}
+{{- if .Values.global.tls.enabled }}
             value: "https"
 {{- else }}
             value: "http"
@@ -38,5 +38,6 @@ spec:
             value: {{ .Values.requireAdminPasswordChange | quote }}
           - name: CHE_HOST
             value: {{ template "cheHost" . }}
+      serviceAccountName: che-keycloak
       restartPolicy: Never
   backoffLimit: 4

deploy/kubernetes/helm/che/charts/che-keycloak/templates/keycloak-serviceaccount.yaml

@@ -0,0 +1,13 @@
+# Copyright (c) 2012-2017 Red Hat, Inc
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+#
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: che-keycloak
+  name: che-keycloak

deploy/kubernetes/helm/che/charts/che-postgres/templates/deployment.yaml

@@ -19,6 +19,9 @@ spec:
       labels:
         io.kompose.service: postgres
     spec:
+      securityContext:
+        runAsUser: 26
+        fsGroup: 26
       containers:
       - env:
         - name: POSTGRESQL_USER

deploy/kubernetes/helm/che/readme.md

@@ -1,23 +1,28 @@
 # Deploy single user Che to Kubernetes using Helm
 
 ## Prerequisites
+- Start minikube with sufficient resources and RBAC 
+  - `minikube start --cpus 2 --memory 4096 --extra-config=apiserver.Authorization.Mode=RBAC`
+  - `kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default`
 - Install the [Helm](https://github.com/kubernetes/helm/blob/master/docs/install.md) CLI
 - Set your default Kubernetes context (this is required to use helm):
   - In Minikube this is set for you automatically
   - Otherwise, you may have to modify the KUBECONFIG environment variable and then type `kubectl config use-context <my-context>`
 - Install tiller on your cluster:
-  - Create a [tiller serviceAccount](https://github.com/kubernetes/helm/blob/master/docs/rbac.md) and bind it to the almighty cluster-admin role: `kubectl apply -f ./tiller-rbac.yaml`
+  - Create a [tiller serviceAccount](https://github.com/kubernetes/helm/blob/master/docs/rbac.md): `kubectl create serviceaccount tiller --namespace kube-system`
+   - Bind it to the almighty cluster-admin role: `kubectl apply -f ./tiller-rbac.yaml`
   - Install tiller itself: `helm init --service-account tiller`
 - Ensure that you have an NGINX-based ingress controller. Note: This is the default ingress controller on Minikube. You can start it with `minikube addons enable ingress`
 - DNS discovery should be enabled. Note: It is enabled by default in minikube.
+
 ## Deployment Process
 ### Obtain the Address of your Kubernetes Cluser
 - If your cluster is running on Minikube, simply type `minikube ip` at your terminal
 - If yourr cluster is in the cloud, obtain the hostname or ip address from your cloud provider
 
 In production, you should specify a hostname (see [here](https://github.com/eclipse/che/issues/8694) why). In case you don't have a hostname (e.g. during development), and would still want to use a host-based configuration, you can use services such as nip.io or xip.io.
 
-In case you're specifying a hostname, simply pass it as the value of the `cheDomain` parameter below.
+In case you're specifying a hostname, simply pass it as the value of the `ingressDomain` parameter below.
 
 If you must use an ip address (e.g. your corporate policy prevents you from using nip.io), you would also have to set `isHostBased` to `false`.
 
@@ -32,57 +37,62 @@ The context of the commands below is the directory in which this readme file res
 - Or, you can override default values during installation, using the `--set` flag:
 
   ```bash
-  helm upgrade --install <my-che-installation> --namespace <my-che-namespace> --set global.cheDomain=<my-hostname> --set cheImage=<my-image> ./
+  helm upgrade --install <my-che-installation> --namespace <my-che-namespace> --set global.ingressDomain=<my-hostname> --set cheImage=<my-image> ./
   ```
 
-#### Deployment types
-Currenty, only minikube deployment is supported.
+#### Deployment Options
 
-##### Single User
+##### Single User 
 Only Che will be deployed.
 
   ```bash
-  helm upgrade --install <che-release> --namespace <che-namespace> --set global.cheDomain=<domain> ./
+  helm upgrade --install <che-release-name> --namespace <che-namespace> --set global.ingressDomain=<domain> ./
   ```
-
-##### Multi User
+* Master: `https://che-<che-namespace>.domain`
+* Workspaces servers: `https://server-host.domain`
+
+##### Multi User 
 Che, KeyCloak and Postgres will be deployed.
 
   ```bash
-  helm upgrade --install <che-release> --namespace <che-namespace> --set global.multiuser=true --set global.cheDomain=<domain> ./
+  helm upgrade --install <che-release-name> --namespace <che-namespace> -f ./values/multi-user.yaml --set global.ingressDomain=<domain> ./
   ```
 
-##### No Host:
- Ingress will serve requests on minikube-ip.
- Path based routing to Che, Secondary servers (KeyCloak) and Workspace servers.
+* Master: `https://che-<che-namespace>.domain`
+* Keycloak:  `https://keycloak-<che-namespace>.domain`
+* Workspaces servers: `https://server-host.domain`
 
+##### Default Host
+All Ingress specs are created without a host attribute (defaults to *).
+Path based routing to all components.
+Multi User configuration is enabled. 
+
   ```bash
-  helm upgrade --install <che-release> --namespace <che-namespace> --set global.isHostbased=false --set global.cheDomain=<minikube-ip> ./
-   Master: http://<minikube-ip>/
-   Workspaces: http://<minikube-ip>/<path-to-server>
-   Keycloak (if multiuser) : http://<minikube-ip>/auth/
+  helm upgrade --install <che-release-name> --namespace <che-namespace> -f ./values/default-host.yaml --set global.ingressDomain=<domain> ./
   ```
+
+* Master: `http://<domain>/`
+* Keycloak:  `http://<domain>/auth/`
+* Workspaces servers: `http://<domain>/<path-to-server>`
 
-##### Host (partial):
- WS Master Ingress will serve requests on provided domain
- Workspaces: Ingress will serve requests on minikube-ip, Path Based routing to workspaces.
- KeyCloak : dedicated hostname
-
-   ```bash
-   helm upgrade --install <che-release> --namespace <che-namespace> --set global.cheDomain=<minikube-ip>.xip.io ./
-   Master: http://master.<minikube-ip>.xip.io
-   Workspaces: http://<minikube-ip>/<path-to-server>
-   Keycloak (if multiuser): http://keycloak.<minikube-ip>.xip.io/
-   ```
+##### TLS-enabled
+Cert-Manager is used to issue LetsEncrypt certificates.
+To avoid rate-limit issues, we use a single hostname for all ingresses.
+Path based routing to all components.
+Multi User configuration is enabled. 
 
+  ```bash
+  helm install --name <cert-manager-release-name> stable/cert-manager
+  helm upgrade --install <che-release-name> --namespace <che-namespace> -f ./values/tls.yaml --set global.ingressDomain=<domain> ./
+  ```
 
-##### Future options:
-- Path Based: single hostname for all components (che, keycloak, WS servers)
-- Host Based: unique host for each component
-- TLS
+* Master: `https://che-<che-namespace>.domain/`
+* Keycloak:  `https://che-<che-namespace>.domain/auth/`
+* Workspaces servers: `https://che-<che-namespace>.domain/<path-to-server>`
 
 ## Deleting a Deployment
 You can delete a deployment using the following command:
+
 ``` bash
-helm delete <my-che-installation>
+helm delete <che-release-name>
 ```

deploy/kubernetes/helm/che/templates/_hostHelper.tpl

@@ -1,7 +1,9 @@
 {{- define "cheHost" }}
-{{- if .Values.global.isHostBased }}
-{{- printf "master.%s" .Values.global.cheDomain }}
+{{- if eq .Values.global.serverStrategy "default-host" }}
+{{- printf "%s" .Values.global.ingressDomain }}
+{{- else if eq .Values.global.serverStrategy "single-host" }}
+{{- printf "che-%s.%s" .Release.Namespace .Values.global.ingressDomain }}
 {{- else }}
-{{- printf "%s" .Values.global.cheDomain }}
+{{- printf "che-%s.%s" .Release.Namespace .Values.global.ingressDomain }}
 {{- end }}
 {{- end }}

deploy/kubernetes/helm/che/templates/_keycloakAuthUrlHelper.tpl

@@ -1,15 +1,21 @@
 {{- define "keycloakAuthUrl" }}
-{{- if .Values.global.isHostBased }}
-{{- if .Values.global.tlsEnabled }}
-{{- printf "https://keycloak.%s/auth" .Values.global.cheDomain }}
-{{- else }}
-{{- printf "http://keycloak.%s/auth" .Values.global.cheDomain }}
-{{- end }}
-{{- else }}
-{{- if .Values.global.tlsEnabled }}
-{{- printf "https://%s/auth" .Values.global.cheDomain }}
-{{- else }}
-{{- printf "http://%s/auth" .Values.global.cheDomain }}
-{{- end }}
-{{- end }}
+  {{- if eq .Values.global.serverStrategy "default-host" }}
+    {{- if .Values.global.tls.enabled }}
+      {{- printf "https://%s/auth" .Values.global.ingressDomain }}
+    {{- else }}
+      {{- printf "http://%s/auth" .Values.global.ingressDomain }}
+    {{- end }}
+  {{- else if eq .Values.global.serverStrategy "single-host" }}
+    {{- if .Values.global.tls.enabled }}
+      {{- printf "https://che-%s.%s/auth" .Release.Namespace .Values.global.ingressDomain }}
+    {{- else }}
+      {{- printf "http://che-%s.%s/auth" .Release.Namespace .Values.global.ingressDomain }}
+    {{- end }}
+  {{- else }}
+    {{- if .Values.global.tls.enabled }}
+      {{- printf "https://keycloak-%s.%s/auth" .Release.Namespace .Values.global.ingressDomain }}
+    {{- else }}
+      {{- printf "http://keycloak-%s.%s/auth" .Release.Namespace .Values.global.ingressDomain }}
+    {{- end }}
+  {{- end }}
 {{- end }}

deploy/kubernetes/helm/che/templates/_keycloakHostHelper.tpl

@@ -1,7 +1,9 @@
 {{- define "keycloakHost" }}
-{{- if .Values.global.isHostBased }}
-{{- printf "keycloak.%s" .Values.global.cheDomain }}
+{{- if eq .Values.global.serverStrategy "default-host" }}
+{{- printf "%s" .Values.global.ingressDomain }}
+{{- else if eq .Values.global.serverStrategy "single-host" }}
+{{- printf "che-%s.%s" .Release.Namespace .Values.global.ingressDomain }}
 {{- else }}
-{{- printf "%s" .Values.global.cheDomain }}
-{{- end }}
+{{- printf "keycloak-%s.%s" .Release.Namespace .Values.global.ingressDomain }}
 {{- end }}
+{{- end }}

deploy/kubernetes/helm/che/templates/cert-issuer.yaml

@@ -5,15 +5,20 @@
 # http://www.eclipse.org/legal/epl-v10.html
 #
 
-{{- if .Values.global.tlsEnabled }}
+{{- if and .Values.global.tls }}
+{{- if and .Values.global.tls.enabled .Values.global.tls.useCertManager }}
 apiVersion: certmanager.k8s.io/v1alpha1
-kind: Issuer
+kind: ClusterIssuer
 metadata:
   name: letsencrypt
 spec:
   acme:
     # The ACME server URL
+{{- if .Values.global.tls.useStaging }}
+    server: https://acme-staging.api.letsencrypt.org/directory
+{{- else }}
     server: https://acme-v01.api.letsencrypt.org/directory
+{{- end }}
     # Email address used for ACME registration
     email: user@example.com
     # Name of a secret used to store the ACME account private key
@@ -22,3 +27,4 @@ spec:
     # Enable the HTTP-01 challenge provider
     http01: {}
 {{- end }}
+{{- end }}