contrib: update k8s liveness and readiness probes (#4405)

* Fix liveness probe for dgraph alpha.
* Add liveness probe to dgraph zero.
* Update k8s manifests to include both liveness and readiness probes.


(cherry picked from commit ed072d0)

contrib/config/kubernetes/dgraph-ha/dgraph-ha.yaml

@@ -169,6 +169,24 @@ spec:
             else
               exec dgraph zero --my=$(hostname -f):5080 --peer dgraph-zero-0.dgraph-zero.${POD_NAMESPACE}.svc.cluster.local:5080 --idx $idx --replicas 3
             fi
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 6080
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 6
+          successThreshold: 1
+        readinessProbe:
+          httpGet:
+            path: /state
+            port: 6080
+          initialDelaySeconds: 5
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 6
+          successThreshold: 1
       terminationGracePeriodSeconds: 60
       volumes:
       - name: datadir
@@ -286,6 +304,24 @@ spec:
           - |
             set -ex
             dgraph alpha --my=$(hostname -f):7080 --lru_mb 2048 --zero dgraph-zero-0.dgraph-zero.${POD_NAMESPACE}.svc.cluster.local:5080
+        livenessProbe:
+          httpGet:
+            path: /health?live=1
+            port: 8080
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 6
+          successThreshold: 1
+        readinessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+          initialDelaySeconds: 5
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 6
+          successThreshold: 1
       terminationGracePeriodSeconds: 600
       volumes:
       - name: datadir

contrib/config/kubernetes/helm/values.yaml

@@ -111,7 +111,7 @@ zero:
   livenessProbe:
     enabled: false
     port: 6080
-    path: /state
+    path: /health
     initialDelaySeconds: 30
     periodSeconds: 10
     timeoutSeconds: 5
@@ -215,7 +215,7 @@ alpha:
   livenessProbe:
     enabled: false
     port: 8080
-    path: /health
+    path: /health?live=1
     initialDelaySeconds: 30
     periodSeconds: 10
     timeoutSeconds: 5

dgraph/cmd/alpha/run.go

@@ -277,13 +277,17 @@ func grpcPort() int {
 
 func healthCheck(w http.ResponseWriter, r *http.Request) {
 	x.AddCorsHeaders(w)
-	if err := x.HealthCheck(); err != nil {
-		w.WriteHeader(http.StatusServiceUnavailable)
-		_, err = w.Write([]byte(err.Error()))
-		if err != nil {
-			glog.V(2).Infof("Error while writing health check response: %v", err)
+
+	_, ok := r.URL.Query()["live"]
+	if !ok {
+		if err := x.HealthCheck(); err != nil {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			_, err = w.Write([]byte(err.Error()))
+			if err != nil {
+				glog.V(2).Infof("Error while writing health check response: %v", err)
+			}
+			return
 		}
-		return
 	}
 
 	info := struct {

dgraph/cmd/zero/http.go

@@ -232,6 +232,13 @@ func (st *state) getState(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func (st *state) pingResponse(w http.ResponseWriter, r *http.Request) {
+	x.AddCorsHeaders(w)
+
+	w.WriteHeader(http.StatusOK)
+	_, _ = w.Write([]byte("OK"))
+}
+
 func (st *state) serveHTTP(l net.Listener) {
 	srv := &http.Server{
 		ReadTimeout:  10 * time.Second,

dgraph/cmd/zero/run.go

@@ -219,6 +219,7 @@ func run() {
 	st.serveGRPC(grpcListener, store)
 	st.serveHTTP(httpListener)
 
+	http.HandleFunc("/health", st.pingResponse)
 	http.HandleFunc("/state", st.getState)
 	http.HandleFunc("/removeNode", st.removeNode)
 	http.HandleFunc("/moveTablet", st.moveTablet)