seed routing test with loop pub IP

KiraCore · Apr 7, 2021 · 9f7dea9 · 9f7dea9
1 parent 09109a2
commit 9f7dea9
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 34 deletions.
diff --git a/docker/kira/container/healthcheck.sh b/docker/kira/container/healthcheck.sh
@@ -8,6 +8,12 @@ sleep 30 # rate limit not to overextend the log files
 
 HALT_CHECK="${COMMON_DIR}/halt"
 EXIT_CHECK="${COMMON_DIR}/exit"
+EXCEPTION_COUNTER_FILE="$COMMON_DIR/exception_counter"
+
+touch "$EXCEPTION_COUNTER_FILE"
+
+EXCEPTION_COUNTER=$(cat $EXCEPTION_COUNTER_FILE || echo "")
+(! $(isNaturalNumber "$EXCEPTION_COUNTER")) && EXCEPTION_COUNTER=0
 
 if [ -f "$EXIT_CHECK" ]; then
   echo "INFO: Ensuring sekaid process is killed"
@@ -18,6 +24,7 @@ fi
 
 if [ -f "$HALT_CHECK" ]; then
   echo "INFO: health heck => STOP (halted)"
+  echo "0" > $EXCEPTION_COUNTER_FILE
   exit 0
 fi
 
@@ -34,13 +41,23 @@ elif [ "${NODE_TYPE,,}" == "validator" ]; then
   $SELF_CONTAINER/validator/healthcheck.sh || FAILED="true"
 else
   echo "ERROR: Unknown node type '$NODE_TYPE'"
-  exit 1
+  FAILED="true"
 fi
 
-if [ "$FAILED" == "true" ] ; then
+if [ "${FAILED,,}" == "true" ] ; then
     echo "ERROR: $NODE_TYPE healthcheck failed"
+    EXCEPTION_COUNTER=$(($EXCEPTION_COUNTER + 1))
+
+    if [ $EXCEPTION_COUNTER -ge 2 ] ; then
+        echo "WARNINIG: Unhealthy status, node will reboot"
+        echo "0" > $EXCEPTION_COUNTER_FILE
+        pkill -15 sekaid || echo "WARNING: Failed to kill sekaid"
+        sleep 5
+    else
+        echo "$EXCEPTION_COUNTER" > $EXCEPTION_COUNTER_FILE
+    fi
     exit 1
 else
+    echo "0" > $EXCEPTION_COUNTER_FILE
     exit 0
 fi
-
diff --git a/docker/kira/container/sentry/healthcheck.sh b/docker/kira/container/sentry/healthcheck.sh
@@ -5,21 +5,19 @@ set -x
 
 LIP_FILE="$COMMON_READ/local_ip"
 PIP_FILE="$COMMON_READ/public_ip"
-EXCEPTION_COUNTER_FILE="$COMMON_DIR/exception_counter"
+
 COMMON_CONSENSUS="$COMMON_READ/consensus"
 COMMON_LATEST_BLOCK_HEIGHT="$COMMON_READ/latest_block_height"
 BLOCK_HEIGHT_FILE="$SELF_LOGS/latest_block_height"
 EXECUTED_CHECK="$COMMON_DIR/executed"
 
-touch "$BLOCK_HEIGHT_FILE" "$EXCEPTION_COUNTER_FILE"
+touch "$BLOCK_HEIGHT_FILE"
 
-EXCEPTION_COUNTER=$(cat $EXCEPTION_COUNTER_FILE || echo "")
 LATEST_BLOCK_HEIGHT=$(cat $COMMON_LATEST_BLOCK_HEIGHT || echo "")
 CONSENSUS=$(cat $COMMON_CONSENSUS | jq -rc || echo "")
 CONSENSUS_STOPPED=$(echo "$CONSENSUS" | jq -rc '.consensus_stopped' || echo "")
 HEIGHT=$(sekaid status 2>&1 | jq -rc '.SyncInfo.latest_block_height' || echo "")
 
-(! $(isNaturalNumber "$EXCEPTION_COUNTER")) && EXCEPTION_COUNTER=0
 (! $(isNaturalNumber "$HEIGHT")) && HEIGHT=$(sekaid status 2>&1 | jq -rc '.sync_info.latest_block_height' || echo "")
 (! $(isNaturalNumber "$HEIGHT")) && HEIGHT=0
 (! $(isNaturalNumber "$LATEST_BLOCK_HEIGHT")) && LATEST_BLOCK_HEIGHT=0
@@ -38,14 +36,6 @@ if [ $PREVIOUS_HEIGHT -ge $HEIGHT ]; then
   if [ $LATEST_BLOCK_HEIGHT -ge 1 ] && [ $LATEST_BLOCK_HEIGHT -le $HEIGHT ] && [ "$CONSENSUS_STOPPED" == "true" ] ; then
       echoWarn "WARNINIG: Cosnensus halted, lack of block production is not result of the issue with the node"
   else
-      EXCEPTION_COUNTER=$(($i + 1))
-      if [ $EXCEPTION_COUNTER -ge 4 ] ; then
-          echoWarn "WARNINIG: Node will be setup from scratch during future restart"
-          echo "0" > $EXCEPTION_COUNTER_FILE
-          rm -fv $EXECUTED_CHECK
-      else
-          echo "$EXCEPTION_COUNTER" > $EXCEPTION_COUNTER_FILE
-      fi
       exit 1
   fi
 else
@@ -77,5 +67,4 @@ else
     echo "OFFLINE" > "$COMMON_DIR/external_address_status"
 fi
 
-echo "0" > $EXCEPTION_COUNTER_FILE
 exit 0
diff --git a/workstation/containers/start-seed.sh b/workstation/containers/start-seed.sh
@@ -6,6 +6,7 @@ CONTAINER_NAME="seed"
 COMMON_PATH="$DOCKER_COMMON/$CONTAINER_NAME"
 COMMON_LOGS="$COMMON_PATH/logs"
 HALT_FILE="$COMMON_PATH/halt"
+EXIT_FILE="$COMMON_PATH/exit"
 
 CPU_CORES=$(cat /proc/cpuinfo | grep processor | wc -l || echo "0")
 RAM_MEMORY=$(grep MemTotal /proc/meminfo | awk '{print $2}' || echo "0")
@@ -32,14 +33,24 @@ set -e
 echo "INFO: Setting up $CONTAINER_NAME config vars..."
 # * Config sentry/configs/config.toml
 
-SENTRY_SEED=$(echo "${SENTRY_NODE_ID}@sentry:$DEFAULT_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r')
+SENTRY_SEED=$(echo "${SENTRY_NODE_ID}@sentry:$KIRA_SENTRY_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r')
 PRIV_SENTRY_SEED=$(echo "${PRIV_SENTRY_NODE_ID}@priv_sentry:$KIRA_PRIV_SENTRY_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r')
 
 mkdir -p "$COMMON_LOGS"
 cp -a -v -f $KIRA_SECRETS/seed_node_key.json $COMMON_PATH/node_key.json
 
 # cleanup
-rm -f -v "$COMMON_LOGS/start.log" "$COMMON_PATH/executed" "$HALT_FILE"
+rm -f -v "$COMMON_LOGS/start.log" "$COMMON_PATH/executed" "$HALT_FILE" "$EXIT_FILE"
+
+PUBLIC_IP=$(cat "$DOCKER_COMMON_RO/public_ip" | xargs || echo "")
+if ($(isPublicIp $PUBLIC_IP)) && timeout 3 nc -z $PUBLIC_IP $KIRA_SENTRY_P2P_PORT ; then
+    PUB_SENTRY_SEED=$(echo "${SENTRY_NODE_ID}@$PUBLIC_IP:$KIRA_SENTRY_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r')
+    CFG_seeds="tcp://$PUB_SENTRY_SEED"
+else
+    CFG_seeds=""
+fi
+
+CFG_persistent_peers="tcp://$SENTRY_SEED"
 
 echo "INFO: Starting seed node..."
 
@@ -62,8 +73,8 @@ docker run -d \
     -e CFG_rpc_laddr="tcp://0.0.0.0:$DEFAULT_RPC_PORT" \
     -e CFG_p2p_laddr="tcp://0.0.0.0:$DEFAULT_P2P_PORT" \
     -e CFG_external_address="" \
-    -e CFG_seeds="" \
-    -e CFG_persistent_peers="tcp://$SENTRY_SEED" \
+    -e CFG_seeds="$CFG_seeds" \
+    -e CFG_persistent_peers="$CFG_persistent_peers" \
     -e CFG_private_peer_ids="$PRIV_SENTRY_NODE_ID,$VALIDATOR_NODE_ID,$SNAPSHOT_NODE_ID,$PRIV_SENTRY_NODE_ID" \
     -e CFG_unconditional_peer_ids="$PRIV_SENTRY_NODE_ID,$SENTRY_NODE_ID" \
     -e CFG_addr_book_strict="true" \

diff --git a/workstation/containers/start-sentry.sh b/workstation/containers/start-sentry.sh
@@ -35,6 +35,7 @@ set -e
 echo "INFO: Setting up $CONTAINER_NAME config vars..."
 # * Config sentry/configs/config.toml
 
+SEED_SEED=$(echo "${SEED_NODE_ID}@seed:$KIRA_SEED_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r')
 VALIDATOR_SEED=$(echo "${VALIDATOR_NODE_ID}@validator:$DEFAULT_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r')
 PRIV_SENTRY_SEED=$(echo "${PRIV_SENTRY_NODE_ID}@priv_sentry:$KIRA_PRIV_SENTRY_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r')
 
@@ -58,6 +59,7 @@ if [ "${EXTERNAL_SYNC,,}" == "true" ] ; then
         echo "INFO: Node will sync from the private sentry..."
         CFG_persistent_peers="tcp://$PRIV_SENTRY_SEED"
     else
+        echo "INFO: Node will sync blocks from its own seed list..."
         CFG_persistent_peers=""
     fi
 else

diff --git a/workstation/kira/seeds-edit.sh b/workstation/kira/seeds-edit.sh
@@ -131,28 +131,28 @@ while : ; do
             ( ! $(isNodeId "$sentry_node_id")) && sentry_node_id=$(timeout 1 curl ${dnsStandalone}:$DEFAULT_RPC_PORT/status 2>/dev/null | jq -r '.node_info.id' 2>/dev/null || echo "")
             priv_sentry_node_id=$(timeout 1 curl -f "$dns:$DEFAULT_INTERX_PORT/download/priv_sentry_node_id" || echo "")
 
-            if ($(isNodeId "$seed_node_id")) && timeout 1 nc -z $dns 16656 ; then 
-                 tmp_addr="${seed_node_id}@${dns}:16656"
-                 [ -z "$DETECTED_NODES" ] && DETECTED_NODES="$tmp_addr" || DETECTED_NODES="${DETECTED_NODES},$tmp_addr"
-                 echoInfo "INFO: Port 16656 is exposed by '$dns'" ; 
+            if ($(isNodeId "$seed_node_id")) && timeout 1 nc -z $dns $KIRA_SEED_P2P_PORT ; then 
+                tmp_addr="${seed_node_id}@${dns}:$KIRA_SEED_P2P_PORT"
+                [ -z "$DETECTED_NODES" ] && DETECTED_NODES="$tmp_addr" || DETECTED_NODES="${DETECTED_NODES},$tmp_addr"
+                echoInfo "INFO: Port $KIRA_SEED_P2P_PORT is exposed by '$dns'" ; 
             else 
-                echoInfo "INFO: Port 16656 is not exposed as '$dns'" ; 
+                echoInfo "INFO: Port $KIRA_SEED_P2P_PORT is not exposed as '$dns'" ; 
             fi
 
-            if ($(isNodeId "$sentry_node_id")) && timeout 1 nc -z $dns 26656 ; then 
-                 tmp_addr="${sentry_node_id}@${dns}:26656"
-                 [ -z "$DETECTED_NODES" ] && DETECTED_NODES="$tmp_addr" || DETECTED_NODES="${DETECTED_NODES},$tmp_addr"
-                 echoInfo "INFO: Port 26656 is exposed as '$dns'" ; 
+            if ($(isNodeId "$sentry_node_id")) && timeout 1 nc -z $dns $KIRA_SENTRY_P2P_PORT ; then 
+                tmp_addr="${sentry_node_id}@${dns}:$KIRA_SENTRY_P2P_PORT"
+                [ -z "$DETECTED_NODES" ] && DETECTED_NODES="$tmp_addr" || DETECTED_NODES="${DETECTED_NODES},$tmp_addr"
+                echoInfo "INFO: Port $KIRA_SENTRY_P2P_PORT is exposed as '$dns'" ; 
             else 
-                echoInfo "INFO: Port 26656 is not exposed by '$dns'" ; 
+                echoInfo "INFO: Port $KIRA_SENTRY_P2P_PORT is not exposed by '$dns'" ; 
             fi
 
-            if ($(isNodeId "$priv_sentry_node_id")) && timeout 1 nc -z $dns 36656 ; then 
-                tmp_addr="${priv_sentry_node_id}@${dns}:36656"
+            if ($(isNodeId "$priv_sentry_node_id")) && timeout 1 nc -z $dns $KIRA_PRIV_SENTRY_P2P_PORT ; then 
+                tmp_addr="${priv_sentry_node_id}@${dns}:$KIRA_PRIV_SENTRY_P2P_PORT"
                 [ -z "$DETECTED_NODES" ] && DETECTED_NODES="$tmp_addr" || DETECTED_NODES="${DETECTED_NODES},$tmp_addr"
-                echoInfo "INFO: Port 36656 is exposed as '$dns'" ; 
+                echoInfo "INFO: Port $KIRA_PRIV_SENTRY_P2P_PORT is exposed as '$dns'" ; 
             else 
-                echoInfo "INFO: Port 36656 is not exposed by '$dns'" ; 
+                echoInfo "INFO: Port $KIRA_PRIV_SENTRY_P2P_PORT is not exposed by '$dns'" ; 
             fi
         else
             DETECTED_NODES="${nodeId}@${dns}:${port}"