diff --git a/docker/kira/container/healthcheck.sh b/docker/kira/container/healthcheck.sh index cb116d2ad..1db17696c 100644 --- a/docker/kira/container/healthcheck.sh +++ b/docker/kira/container/healthcheck.sh @@ -8,6 +8,12 @@ sleep 30 # rate limit not to overextend the log files HALT_CHECK="${COMMON_DIR}/halt" EXIT_CHECK="${COMMON_DIR}/exit" +EXCEPTION_COUNTER_FILE="$COMMON_DIR/exception_counter" + +touch "$EXCEPTION_COUNTER_FILE" + +EXCEPTION_COUNTER=$(cat $EXCEPTION_COUNTER_FILE || echo "") +(! $(isNaturalNumber "$EXCEPTION_COUNTER")) && EXCEPTION_COUNTER=0 if [ -f "$EXIT_CHECK" ]; then echo "INFO: Ensuring sekaid process is killed" @@ -18,6 +24,7 @@ fi if [ -f "$HALT_CHECK" ]; then echo "INFO: health heck => STOP (halted)" + echo "0" > $EXCEPTION_COUNTER_FILE exit 0 fi @@ -34,13 +41,23 @@ elif [ "${NODE_TYPE,,}" == "validator" ]; then $SELF_CONTAINER/validator/healthcheck.sh || FAILED="true" else echo "ERROR: Unknown node type '$NODE_TYPE'" - exit 1 + FAILED="true" fi -if [ "$FAILED" == "true" ] ; then +if [ "${FAILED,,}" == "true" ] ; then echo "ERROR: $NODE_TYPE healthcheck failed" + EXCEPTION_COUNTER=$(($EXCEPTION_COUNTER + 1)) + + if [ $EXCEPTION_COUNTER -ge 2 ] ; then + echo "WARNINIG: Unhealthy status, node will reboot" + echo "0" > $EXCEPTION_COUNTER_FILE + pkill -15 sekaid || echo "WARNING: Failed to kill sekaid" + sleep 5 + else + echo "$EXCEPTION_COUNTER" > $EXCEPTION_COUNTER_FILE + fi exit 1 else + echo "0" > $EXCEPTION_COUNTER_FILE exit 0 fi - diff --git a/docker/kira/container/sentry/healthcheck.sh b/docker/kira/container/sentry/healthcheck.sh index ea5930af9..a534aecfa 100644 --- a/docker/kira/container/sentry/healthcheck.sh +++ b/docker/kira/container/sentry/healthcheck.sh @@ -5,21 +5,19 @@ set -x LIP_FILE="$COMMON_READ/local_ip" PIP_FILE="$COMMON_READ/public_ip" -EXCEPTION_COUNTER_FILE="$COMMON_DIR/exception_counter" + COMMON_CONSENSUS="$COMMON_READ/consensus" COMMON_LATEST_BLOCK_HEIGHT="$COMMON_READ/latest_block_height" BLOCK_HEIGHT_FILE="$SELF_LOGS/latest_block_height" EXECUTED_CHECK="$COMMON_DIR/executed" -touch "$BLOCK_HEIGHT_FILE" "$EXCEPTION_COUNTER_FILE" +touch "$BLOCK_HEIGHT_FILE" -EXCEPTION_COUNTER=$(cat $EXCEPTION_COUNTER_FILE || echo "") LATEST_BLOCK_HEIGHT=$(cat $COMMON_LATEST_BLOCK_HEIGHT || echo "") CONSENSUS=$(cat $COMMON_CONSENSUS | jq -rc || echo "") CONSENSUS_STOPPED=$(echo "$CONSENSUS" | jq -rc '.consensus_stopped' || echo "") HEIGHT=$(sekaid status 2>&1 | jq -rc '.SyncInfo.latest_block_height' || echo "") -(! $(isNaturalNumber "$EXCEPTION_COUNTER")) && EXCEPTION_COUNTER=0 (! $(isNaturalNumber "$HEIGHT")) && HEIGHT=$(sekaid status 2>&1 | jq -rc '.sync_info.latest_block_height' || echo "") (! $(isNaturalNumber "$HEIGHT")) && HEIGHT=0 (! $(isNaturalNumber "$LATEST_BLOCK_HEIGHT")) && LATEST_BLOCK_HEIGHT=0 @@ -38,14 +36,6 @@ if [ $PREVIOUS_HEIGHT -ge $HEIGHT ]; then if [ $LATEST_BLOCK_HEIGHT -ge 1 ] && [ $LATEST_BLOCK_HEIGHT -le $HEIGHT ] && [ "$CONSENSUS_STOPPED" == "true" ] ; then echoWarn "WARNINIG: Cosnensus halted, lack of block production is not result of the issue with the node" else - EXCEPTION_COUNTER=$(($i + 1)) - if [ $EXCEPTION_COUNTER -ge 4 ] ; then - echoWarn "WARNINIG: Node will be setup from scratch during future restart" - echo "0" > $EXCEPTION_COUNTER_FILE - rm -fv $EXECUTED_CHECK - else - echo "$EXCEPTION_COUNTER" > $EXCEPTION_COUNTER_FILE - fi exit 1 fi else @@ -77,5 +67,4 @@ else echo "OFFLINE" > "$COMMON_DIR/external_address_status" fi -echo "0" > $EXCEPTION_COUNTER_FILE exit 0 \ No newline at end of file diff --git a/workstation/containers/start-seed.sh b/workstation/containers/start-seed.sh index 4a18aa258..f4b674552 100644 --- a/workstation/containers/start-seed.sh +++ b/workstation/containers/start-seed.sh @@ -6,6 +6,7 @@ CONTAINER_NAME="seed" COMMON_PATH="$DOCKER_COMMON/$CONTAINER_NAME" COMMON_LOGS="$COMMON_PATH/logs" HALT_FILE="$COMMON_PATH/halt" +EXIT_FILE="$COMMON_PATH/exit" CPU_CORES=$(cat /proc/cpuinfo | grep processor | wc -l || echo "0") RAM_MEMORY=$(grep MemTotal /proc/meminfo | awk '{print $2}' || echo "0") @@ -32,14 +33,24 @@ set -e echo "INFO: Setting up $CONTAINER_NAME config vars..." # * Config sentry/configs/config.toml -SENTRY_SEED=$(echo "${SENTRY_NODE_ID}@sentry:$DEFAULT_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r') +SENTRY_SEED=$(echo "${SENTRY_NODE_ID}@sentry:$KIRA_SENTRY_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r') PRIV_SENTRY_SEED=$(echo "${PRIV_SENTRY_NODE_ID}@priv_sentry:$KIRA_PRIV_SENTRY_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r') mkdir -p "$COMMON_LOGS" cp -a -v -f $KIRA_SECRETS/seed_node_key.json $COMMON_PATH/node_key.json # cleanup -rm -f -v "$COMMON_LOGS/start.log" "$COMMON_PATH/executed" "$HALT_FILE" +rm -f -v "$COMMON_LOGS/start.log" "$COMMON_PATH/executed" "$HALT_FILE" "$EXIT_FILE" + +PUBLIC_IP=$(cat "$DOCKER_COMMON_RO/public_ip" | xargs || echo "") +if ($(isPublicIp $PUBLIC_IP)) && timeout 3 nc -z $PUBLIC_IP $KIRA_SENTRY_P2P_PORT ; then + PUB_SENTRY_SEED=$(echo "${SENTRY_NODE_ID}@$PUBLIC_IP:$KIRA_SENTRY_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r') + CFG_seeds="tcp://$PUB_SENTRY_SEED" +else + CFG_seeds="" +fi + +CFG_persistent_peers="tcp://$SENTRY_SEED" echo "INFO: Starting seed node..." @@ -62,8 +73,8 @@ docker run -d \ -e CFG_rpc_laddr="tcp://0.0.0.0:$DEFAULT_RPC_PORT" \ -e CFG_p2p_laddr="tcp://0.0.0.0:$DEFAULT_P2P_PORT" \ -e CFG_external_address="" \ - -e CFG_seeds="" \ - -e CFG_persistent_peers="tcp://$SENTRY_SEED" \ + -e CFG_seeds="$CFG_seeds" \ + -e CFG_persistent_peers="$CFG_persistent_peers" \ -e CFG_private_peer_ids="$PRIV_SENTRY_NODE_ID,$VALIDATOR_NODE_ID,$SNAPSHOT_NODE_ID,$PRIV_SENTRY_NODE_ID" \ -e CFG_unconditional_peer_ids="$PRIV_SENTRY_NODE_ID,$SENTRY_NODE_ID" \ -e CFG_addr_book_strict="true" \ diff --git a/workstation/containers/start-sentry.sh b/workstation/containers/start-sentry.sh index 7764aeb9e..877e459b4 100644 --- a/workstation/containers/start-sentry.sh +++ b/workstation/containers/start-sentry.sh @@ -35,6 +35,7 @@ set -e echo "INFO: Setting up $CONTAINER_NAME config vars..." # * Config sentry/configs/config.toml +SEED_SEED=$(echo "${SEED_NODE_ID}@seed:$KIRA_SEED_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r') VALIDATOR_SEED=$(echo "${VALIDATOR_NODE_ID}@validator:$DEFAULT_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r') PRIV_SENTRY_SEED=$(echo "${PRIV_SENTRY_NODE_ID}@priv_sentry:$KIRA_PRIV_SENTRY_P2P_PORT" | xargs | tr -d '\n' | tr -d '\r') @@ -58,6 +59,7 @@ if [ "${EXTERNAL_SYNC,,}" == "true" ] ; then echo "INFO: Node will sync from the private sentry..." CFG_persistent_peers="tcp://$PRIV_SENTRY_SEED" else + echo "INFO: Node will sync blocks from its own seed list..." CFG_persistent_peers="" fi else diff --git a/workstation/kira/seeds-edit.sh b/workstation/kira/seeds-edit.sh index e81d2608c..1cb2b6c7a 100644 --- a/workstation/kira/seeds-edit.sh +++ b/workstation/kira/seeds-edit.sh @@ -131,28 +131,28 @@ while : ; do ( ! $(isNodeId "$sentry_node_id")) && sentry_node_id=$(timeout 1 curl ${dnsStandalone}:$DEFAULT_RPC_PORT/status 2>/dev/null | jq -r '.node_info.id' 2>/dev/null || echo "") priv_sentry_node_id=$(timeout 1 curl -f "$dns:$DEFAULT_INTERX_PORT/download/priv_sentry_node_id" || echo "") - if ($(isNodeId "$seed_node_id")) && timeout 1 nc -z $dns 16656 ; then - tmp_addr="${seed_node_id}@${dns}:16656" - [ -z "$DETECTED_NODES" ] && DETECTED_NODES="$tmp_addr" || DETECTED_NODES="${DETECTED_NODES},$tmp_addr" - echoInfo "INFO: Port 16656 is exposed by '$dns'" ; + if ($(isNodeId "$seed_node_id")) && timeout 1 nc -z $dns $KIRA_SEED_P2P_PORT ; then + tmp_addr="${seed_node_id}@${dns}:$KIRA_SEED_P2P_PORT" + [ -z "$DETECTED_NODES" ] && DETECTED_NODES="$tmp_addr" || DETECTED_NODES="${DETECTED_NODES},$tmp_addr" + echoInfo "INFO: Port $KIRA_SEED_P2P_PORT is exposed by '$dns'" ; else - echoInfo "INFO: Port 16656 is not exposed as '$dns'" ; + echoInfo "INFO: Port $KIRA_SEED_P2P_PORT is not exposed as '$dns'" ; fi - if ($(isNodeId "$sentry_node_id")) && timeout 1 nc -z $dns 26656 ; then - tmp_addr="${sentry_node_id}@${dns}:26656" - [ -z "$DETECTED_NODES" ] && DETECTED_NODES="$tmp_addr" || DETECTED_NODES="${DETECTED_NODES},$tmp_addr" - echoInfo "INFO: Port 26656 is exposed as '$dns'" ; + if ($(isNodeId "$sentry_node_id")) && timeout 1 nc -z $dns $KIRA_SENTRY_P2P_PORT ; then + tmp_addr="${sentry_node_id}@${dns}:$KIRA_SENTRY_P2P_PORT" + [ -z "$DETECTED_NODES" ] && DETECTED_NODES="$tmp_addr" || DETECTED_NODES="${DETECTED_NODES},$tmp_addr" + echoInfo "INFO: Port $KIRA_SENTRY_P2P_PORT is exposed as '$dns'" ; else - echoInfo "INFO: Port 26656 is not exposed by '$dns'" ; + echoInfo "INFO: Port $KIRA_SENTRY_P2P_PORT is not exposed by '$dns'" ; fi - if ($(isNodeId "$priv_sentry_node_id")) && timeout 1 nc -z $dns 36656 ; then - tmp_addr="${priv_sentry_node_id}@${dns}:36656" + if ($(isNodeId "$priv_sentry_node_id")) && timeout 1 nc -z $dns $KIRA_PRIV_SENTRY_P2P_PORT ; then + tmp_addr="${priv_sentry_node_id}@${dns}:$KIRA_PRIV_SENTRY_P2P_PORT" [ -z "$DETECTED_NODES" ] && DETECTED_NODES="$tmp_addr" || DETECTED_NODES="${DETECTED_NODES},$tmp_addr" - echoInfo "INFO: Port 36656 is exposed as '$dns'" ; + echoInfo "INFO: Port $KIRA_PRIV_SENTRY_P2P_PORT is exposed as '$dns'" ; else - echoInfo "INFO: Port 36656 is not exposed by '$dns'" ; + echoInfo "INFO: Port $KIRA_PRIV_SENTRY_P2P_PORT is not exposed by '$dns'" ; fi else DETECTED_NODES="${nodeId}@${dns}:${port}"