feat: etcd cluster single node failure APISIX startup failure

apisix/cli/etcd.lua

@@ -32,6 +32,8 @@ local tonumber = tonumber
 local str_format = string.format
 local str_sub = string.sub
 local table_concat = table.concat
+local table_insert = table.insert
+local io_stderr = io.stderr
 
 local _M = {}
 
@@ -187,6 +189,7 @@ function _M.init(env, args)
     end
 
     -- check the etcd cluster version
+    local etcd_healthy_hosts = {}
     for index, host in ipairs(yaml_conf.etcd.host) do
         local version_url = host .. "/version"
         local errmsg
@@ -206,29 +209,38 @@ function _M.init(env, args)
                              version_url, err, retry_time))
         end
 
-        if not res then
-            errmsg = str_format("request etcd endpoint \'%s\' error, %s\n", version_url, err)
-            util.die(errmsg)
-        end
+        if res then
+            local body, _, err = dkjson.decode(res)
+            if err or (body and not body["etcdcluster"]) then
+                errmsg = str_format("got malformed version message: \"%s\" from etcd \"%s\"\n", res,
+                        version_url)
+                util.die(errmsg)
+            end
 
-        local body, _, err = dkjson.decode(res)
-        if err or (body and not body["etcdcluster"]) then
-            errmsg = str_format("got malformed version message: \"%s\" from etcd \"%s\"\n", res,
-                                version_url)
-            util.die(errmsg)
-        end
+            local cluster_version = body["etcdcluster"]
+            if compare_semantic_version(cluster_version, env.min_etcd_version) then
+                util.die("etcd cluster version ", cluster_version,
+                        " is less than the required version ",
+                        env.min_etcd_version, ", please upgrade your etcd cluster\n")
+            end
 
-        local cluster_version = body["etcdcluster"]
-        if compare_semantic_version(cluster_version, env.min_etcd_version) then
-            util.die("etcd cluster version ", cluster_version,
-                     " is less than the required version ",
-                     env.min_etcd_version,
-                     ", please upgrade your etcd cluster\n")
+            table_insert(etcd_healthy_hosts, host)
+        else
+            io_stderr:write(str_format("request etcd endpoint \'%s\' error, %s\n", version_url,
+                    err))
         end
     end
 
+    if #etcd_healthy_hosts <= 0 then
+        util.die("all etcd nodes are unavailable\n")
+    end
+
+    if (#etcd_healthy_hosts / host_count * 100) <= 50 then
+        util.die("the etcd cluster needs at least 50% and above healthy nodes\n")
+    end
+
     local etcd_ok = false
-    for index, host in ipairs(yaml_conf.etcd.host) do
+    for index, host in ipairs(etcd_healthy_hosts) do
         local is_success = true
 
         local errmsg
@@ -358,7 +370,7 @@ function _M.init(env, args)
     end
 
     if not etcd_ok then
-        util.die("none of the configured etcd works well")
+        util.die("none of the configured etcd works well\n")
     end
 end
 

t/cli/test_etcd_ha.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ETCD_NAME_0=etcd0
+ETCD_NAME_1=etcd1
+ETCD_NAME_2=etcd2
+
+echo '
+etcd:
+  host:
+    - "http://127.0.0.1:23790"
+    - "http://127.0.0.1:23791"
+    - "http://127.0.0.1:23792"
+' > conf/config.yaml
+
+docker-compose -f ./t/cli/docker-compose-etcd-cluster.yaml up -d
+
+# case 1: stop one etcd node (result: start successful)
+docker stop ${ETCD_NAME_0}
+
+out=$(make init 2>&1)
+if echo "$out" | grep "23790" | grep "connection refused"; then
+    echo "passed: APISIX successfully to start, stop only one etcd node"
+else
+    echo "failed: stop only one etcd node APISIX should start normally"
+    exit 1
+fi
+
+# case 2: stop two etcd nodes (result: start failure)
+docker stop ${ETCD_NAME_1}
+
+out=$(make init 2>&1)
+if echo "$out" | grep "23791" | grep "connection refused"; then
+    echo "passed: APISIX failed to start, etcd cluster must have two or more healthy nodes"
+else
+    echo "failed: two etcd nodes have been stopped, APISIX should fail to start"
+    exit 1
+fi
+
+# case 3: stop all etcd nodes (result: start failure)
+docker stop ${ETCD_NAME_2}
+
+out=$(make init 2>&1)
+if echo "$out" | grep "23792" | grep "connection refused"; then
+    echo "passed: APISIX failed to start, all etcd nodes have stopped"
+else
+    echo "failed: all etcd nodes have stopped, APISIX should not be able to start"
+    exit 1
+fi
+
+# stop etcd docker container
+docker-compose -f ./t/cli/docker-compose-etcd-cluster.yaml down