fast-reboot

#!/bin/bash -e

REBOOT_USER=$(logname)
REBOOT_TIME=$(date)
REBOOT_CAUSE_FILE="/host/reboot-cause/reboot-cause.txt"
WARM_DIR=/host/warmboot
REDIS_FILE=dump.rdb
REBOOT_SCRIPT_NAME=$(basename $0)
REBOOT_TYPE="${REBOOT_SCRIPT_NAME}"
VERBOSE=no
FORCE=no
STRICT=no
REBOOT_METHOD="/sbin/kexec -e"
ASSISTANT_IP_LIST=""
ASSISTANT_SCRIPT="/usr/local/bin/neighbor_advertiser"
WATCHDOG_UTIL="/usr/local/bin/watchdogutil"
DEVPATH="/usr/share/sonic/device"
PLATFORM=$(sonic-cfggen -H -v DEVICE_METADATA.localhost.platform)
PLATFORM_PLUGIN="${REBOOT_TYPE}_plugin"
LOG_SSD_HEALTH="/usr/local/bin/log_ssd_health"
SSD_FW_UPDATE="ssd-fw-upgrade"
# Require 100M available on the hard drive for warm reboot temp files,
# Size is in 1K blocks:
MIN_HD_SPACE_NEEDED=100000

EXIT_SUCCESS=0
EXIT_FAILURE=1
EXIT_NOT_SUPPORTED=2
EXIT_FILE_SYSTEM_FULL=3
EXIT_NEXT_IMAGE_NOT_EXISTS=4
EXIT_ORCHAGENT_SHUTDOWN=10
EXIT_SYNCD_SHUTDOWN=11
EXIT_FAST_REBOOT_DUMP_FAILURE=12
EXIT_FILTER_FDB_ENTRIES_FAILURE=13
EXIT_COUNTERPOLL_DISABLE_FAILURE=14
EXIT_NO_CONTROL_PLANE_ASSISTANT=20
EXIT_SONIC_INSTALLER_VERIFY_REBOOT=21

function error()
{
    echo $@ >&2
}

function debug()
{
    if [[ x"${VERBOSE}" == x"yes" ]]; then
        echo `date` $@
    fi
    logger "$@"
}

function showHelpAndExit()
{
    echo "Usage: ${REBOOT_SCRIPT_NAME} [options]"
    echo "    -h,-? : get this help"
    echo "    -v    : turn on verbose"
    echo "    -f    : force execution"
    echo "    -r    : reboot with /sbin/reboot"
    echo "    -k    : reboot with /sbin/kexec -e [default]"
    echo "    -x    : execute script with -x flag"
    echo "    -c    : specify control plane assistant IP list"
    echo "    -s    : strict mode: do not proceed without:"
    echo "            - control plane assistant IP list."

    exit "${EXIT_SUCCESS}"
}

function parseOptions()
{
    while getopts "vfh?rkxc:s" opt; do
        case ${opt} in
            h|\? )
                showHelpAndExit
                ;;
            v )
                VERBOSE=yes
                ;;
            f )
                FORCE=yes
                ;;
            r )
                REBOOT_METHOD="/sbin/reboot"
                ;;
            k )
                REBOOT_METHOD="/sbin/kexec -e"
                ;;
            x )
                set -x
                ;;
            c )
                ASSISTANT_IP_LIST=${OPTARG}
                ;;
            s )
                STRICT=yes
                ;;
        esac
    done
}

function common_clear()
{
    debug "${REBOOT_TYPE} failure ($?) cleanup ..."

    /sbin/kexec -u || /bin/true

    teardown_control_plane_assistant
}

function clear_fast_boot()
{
    common_clear

    sonic-db-cli STATE_DB DEL "FAST_REBOOT|system" &>/dev/null || /bin/true
}

function clear_warm_boot()
{
    common_clear

    result=`timeout 10s config warm_restart disable; if [[ $? == 124 ]]; then echo timeout; else echo "code ($?)"; fi` || /bin/true
    debug "Cancel warm-reboot: ${result}"

    TIMESTAMP=`date +%Y%m%d-%H%M%S`
    if [[ -f ${WARM_DIR}/${REDIS_FILE} ]]; then
        mv -f ${WARM_DIR}/${REDIS_FILE} ${WARM_DIR}/${REDIS_FILE}.${TIMESTAMP} || /bin/true
    fi
}

function init_warm_reboot_states()
{
    # If the current running instanace was booted up with warm reboot. Then
    # the current DB contents will likely mark warm reboot is done.
    # Clear these states so that the next boot up image won't get confused.
    if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
        sonic-db-cli STATE_DB eval "
            for _, key in ipairs(redis.call('keys', 'WARM_RESTART_TABLE|*')) do
                redis.call('hdel', key, 'state')
            end
        " 0 >/dev/null
    fi
}

function initialize_pre_shutdown()
{
    debug "Initialize pre-shutdown ..."
    TABLE="WARM_RESTART_TABLE|warm-shutdown"
    RESTORE_COUNT=`sonic-db-cli STATE_DB hget "${TABLE}" restore_count`
    if [[ -z "$RESTORE_COUNT" ]]; then
        sonic-db-cli STATE_DB hset "${TABLE}" "restore_count" "0" > /dev/null
    fi
    sonic-db-cli STATE_DB hset "${TABLE}" "state" "requesting" > /dev/null
}

function request_pre_shutdown()
{
    debug "Requesting pre-shutdown ..."
    /usr/bin/docker exec -i syncd /usr/bin/syncd_request_shutdown --pre &> /dev/null || {
        error "Failed to request pre-shutdown"
    }
}

function recover_issu_bank_file_instruction()
{
    debug "To recover (${ISSU_BANK_FILE}) file, do the following:"
    debug "$ docker exec -it syncd sx_api_dbg_generate_dump.py"
    debug "$ docker exec -it syncd cat /tmp/sdkdump | grep 'ISSU Bank'"
    debug "Command above will print the VALUE of ISSU BANK - 0 or 1, use this VALUE in the next command"
    debug "$ printf VALUE > /host/warmboot/issu_bank.txt"
}

function check_issu_bank_file()
{
    ISSU_BANK_FILE=/host/warmboot/issu_bank.txt
    MLNX_ISSU_BANK_BROKEN=102

    if [[ ! -s "$ISSU_BANK_FILE" ]]; then
        error "(${ISSU_BANK_FILE}) does NOT exist or empty ..."
        recover_issu_bank_file_instruction
        if [[ "$1" = true ]]; then
            exit "${MLNX_ISSU_BANK_BROKEN}"
        fi
        return
    fi

    issu_file_chars_count=`stat -c %s ${ISSU_BANK_FILE}`;
    issu_file_content=`awk '{print $0}' ${ISSU_BANK_FILE}`

    if [[ $issu_file_chars_count != 1 ]] ||
        [[ "$issu_file_content" != "0" && "$issu_file_content" != "1" ]]; then
        error "(${ISSU_BANK_FILE}) is broken ..."
        recover_issu_bank_file_instruction
        if [[ "$1" = true ]]; then
            exit "${MLNX_ISSU_BANK_BROKEN}"
        fi
    fi
}

function wait_for_pre_shutdown_complete_or_fail()
{
    debug "Waiting for pre-shutdown ..."
    TABLE="WARM_RESTART_TABLE|warm-shutdown"
    STATE="requesting"
    declare -i waitcount
    declare -i retrycount
    waitcount=0
    retrycount=0
    # Wait up to 60 seconds for pre-shutdown to complete
    while [[ ${waitcount} -lt 600 ]]; do
        # timeout doesn't work with -i option of "docker exec". Therefore we have
        # to invoke docker exec directly below.
        STATE=`timeout 5s sonic-db-cli STATE_DB hget "${TABLE}" state; if [[ $? == 124 ]]; then echo "timed out"; fi`

        if [[ x"${STATE}" == x"timed out" ]]; then
            waitcount+=50
            retrycount+=1
            debug "Timed out getting pre-shutdown state (${waitcount}) retry count ${retrycount} ..."
            if [[ retrycount -gt 2 ]]; then
                break
            fi
        elif [[ x"${STATE}" != x"requesting" ]]; then
            break
        else
            sleep 0.1
            waitcount+=1
        fi
    done

    if [[ x"${STATE}" != x"pre-shutdown-succeeded" ]]; then
        debug "Syncd pre-shutdown failed: ${STATE} ..."
    else
        debug "Pre-shutdown succeeded ..."
    fi
}

function backup_database()
{
    debug "Backing up database ..."
    # Dump redis content to a file 'dump.rdb' in warmboot directory
    mkdir -p $WARM_DIR
    # Delete keys in stateDB except FDB_TABLE|*, MIRROR_SESSION_TABLE|*, WARM_RESTART_ENABLE_TABLE|*
    sonic-db-cli STATE_DB eval "
        for _, k in ipairs(redis.call('keys', '*')) do
            if not string.match(k, 'FDB_TABLE|') and not string.match(k, 'WARM_RESTART_TABLE|') \
                                          and not string.match(k, 'MIRROR_SESSION_TABLE|') \
                                          and not string.match(k, 'WARM_RESTART_ENABLE_TABLE|') then
                redis.call('del', k)
            end
        end
    " 0 > /dev/null
    # Delete COPP_TABLE keys in APPL_DB
    sonic-db-cli APPL_DB eval "
        for _, k in ipairs(redis.call('keys', '*')) do
            if string.match(k, 'COPP_TABLE:') then
                redis.call('del', k)
            end
        end
    " 0 > /dev/null
    sonic-db-cli SAVE > /dev/null
    #TODO : need a script to copy all rdb files if there is multiple db instances config
    docker cp database:/var/lib/redis/$REDIS_FILE $WARM_DIR
    docker exec -i database rm /var/lib/redis/$REDIS_FILE
}

function setup_control_plane_assistant()
{
    if [[ -n "${ASSISTANT_IP_LIST}" && -x ${ASSISTANT_SCRIPT} ]]; then
        debug "Setting up control plane assistant: ${ASSISTANT_IP_LIST} ..."
        ${ASSISTANT_SCRIPT} -s ${ASSISTANT_IP_LIST} -m set
    elif [[ X"${STRICT}" == X"yes" ]]; then
        debug "Strict mode: fail due to lack of control plane assistant ..."
        exit ${EXIT_NO_CONTROL_PLANE_ASSISTANT}
    fi
}

function teardown_control_plane_assistant()
{
    if [[ -n "${ASSISTANT_IP_LIST}" && -x ${ASSISTANT_SCRIPT} ]]; then
        debug "Tearing down control plane assistant: ${ASSISTANT_IP_LIST} ..."
        ${ASSISTANT_SCRIPT} -s ${ASSISTANT_IP_LIST} -m reset
    fi
}

function is_secureboot() {
    grep -Eq 'secure_boot_enable=[1y]' /proc/cmdline
}

function setup_reboot_variables()
{
    # Kernel and initrd image
    NEXT_SONIC_IMAGE=$(sonic-installer list | grep "Next: " | cut -d ' ' -f 2)
    IMAGE_PATH="/host/image-${NEXT_SONIC_IMAGE#SONiC-OS-}"
    if grep -q aboot_platform= /host/machine.conf; then
        if is_secureboot; then
            KERNEL_IMAGE=""
            BOOT_OPTIONS="SONIC_BOOT_TYPE=${BOOT_TYPE_ARG} secure_boot_enable=1"
        else
            KERNEL_IMAGE="$(ls $IMAGE_PATH/boot/vmlinuz-*)"
            BOOT_OPTIONS="$(cat "$IMAGE_PATH/kernel-cmdline" | tr '\n' ' ') SONIC_BOOT_TYPE=${BOOT_TYPE_ARG}"
        fi
    elif grep -q onie_platform= /host/machine.conf; then
        KERNEL_OPTIONS=$(cat /host/grub/grub.cfg | sed "/$NEXT_SONIC_IMAGE'/,/}/"'!'"g" | grep linux)
        KERNEL_IMAGE="/host$(echo $KERNEL_OPTIONS | cut -d ' ' -f 2)"
        BOOT_OPTIONS="$(echo $KERNEL_OPTIONS | sed -e 's/\s*linux\s*/BOOT_IMAGE=/') SONIC_BOOT_TYPE=${BOOT_TYPE_ARG}"
    else
        error "Unknown bootloader. ${REBOOT_TYPE} is not supported."
        exit "${EXIT_NOT_SUPPORTED}"
    fi
    INITRD=$(echo $KERNEL_IMAGE | sed 's/vmlinuz/initrd.img/g')
}

function reboot_pre_check()
{
    # Make sure that the file system is normal: read-write able
    filename="/host/test-`date +%Y%m%d-%H%M%S`"
    if [[ ! -f ${filename} ]]; then
        touch ${filename}
    fi
    rm ${filename}

    # Make sure /host has enough space for warm reboot temp files
    avail=$(df -k /host | tail -1 | awk '{ print $4 }')
    if [[ ${avail} -lt ${MIN_HD_SPACE_NEEDED} ]]; then
        debug "/host has ${avail}K bytes available, not enough for warm reboot."
        exit ${EXIT_FILE_SYSTEM_FULL}
    fi

    # Verify the next image by sonic-installer
    INSTALLER_VERIFY_RC=0
    sonic-installer verify-next-image > /dev/null || INSTALLER_VERIFY_RC=$?
    if [[ INSTALLER_VERIFY_RC -ne 0 ]]; then
        error "Failed to verify next image. Exit code: $INSTALLER_VERIFY_RC"
        exit ${EXIT_SONIC_INSTALLER_VERIFY_REBOOT}
    fi
    
    # Make sure ASIC configuration has not changed between images
    ASIC_CONFIG_CHECK_SCRIPT="/usr/local/bin/asic_config_check"
    ASIC_CONFIG_CHECK_SUCCESS=0
    if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
        ASIC_CONFIG_CHECK_EXIT_CODE=0
        ${ASIC_CONFIG_CHECK_SCRIPT} || ASIC_CONFIG_CHECK_EXIT_CODE=$?

        if [[ "${ASIC_CONFIG_CHECK_EXIT_CODE}" != "${ASIC_CONFIG_CHECK_SUCCESS}" ]]; then
            if [[ x"${FORCE}" == x"yes" ]]; then
                debug "Ignoring ASIC config checksum failure..."
            else
                error "ASIC config may have changed: errno=${ASIC_CONFIG_CHECK_EXIT_CODE}"
                exit "${EXIT_FAILURE}"
            fi
        fi
    fi
}

function load_aboot_secureboot_kernel() {
    local next_image="$IMAGE_PATH/sonic.swi"
    echo "Loading next image from $next_image"
    unzip -qp "$next_image" boot0 | \
        swipath=$next_image kexec=true loadonly=true ENV_EXTRA_CMDLINE="$BOOT_OPTIONS" bash -
}

function load_kernel() {
    # Load kernel into the memory
    /sbin/kexec -l "$KERNEL_IMAGE" --initrd="$INITRD" --append="$BOOT_OPTIONS"
}

function unload_kernel()
{
    # Unload the previously loaded kernel if any loaded
    if [[ "$(cat /sys/kernel/kexec_loaded)" -eq 1 ]]; then
        /sbin/kexec -u
    fi
}

# main starts here
parseOptions $@

# Check root privileges
if [[ "$EUID" -ne 0 ]]
then
    echo "This command must be run as root" >&2
    exit "${EXIT_FAILURE}"
fi

sonic_asic_type=$(sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type)

# Check reboot type supported
BOOT_TYPE_ARG="cold"
case "$REBOOT_TYPE" in
    "fast-reboot")
        BOOT_TYPE_ARG=$REBOOT_TYPE
        trap clear_fast_boot EXIT HUP INT QUIT TERM KILL ABRT ALRM
        sonic-db-cli STATE_DB SET "FAST_REBOOT|system" "1" "EX" "180" &>/dev/null
        ;;
    "warm-reboot")
        if [[ "$sonic_asic_type" == "mellanox" ]]; then
            REBOOT_TYPE="fastfast-reboot"
            BOOT_TYPE_ARG="fastfast"
            # source mlnx-ffb.sh file with
            # functions to check ISSU upgrade possibility
            source mlnx-ffb.sh
        else
            BOOT_TYPE_ARG="warm"
        fi
        trap clear_warm_boot EXIT HUP INT QUIT TERM KILL ABRT ALRM
        config warm_restart enable system
        ;;
    *)
        error "Not supported reboot type: $REBOOT_TYPE"
        exit "${EXIT_NOT_SUPPORTED}"
        ;;
esac

unload_kernel

setup_reboot_variables

reboot_pre_check

# Install new FW for mellanox platforms before control plane goes down
# So on boot switch will not spend time to upgrade FW increasing the CP downtime
if [[ "$sonic_asic_type" == "mellanox" ]]; then
    MLNX_EXIT_SUCCESS=0
    MLNX_EXIT_FW_ERROR=100
    MLNX_EXIT_FFB_FAILURE=101

    MLNX_FW_UPGRADE_SCRIPT="/usr/bin/mlnx-fw-upgrade.sh"


    if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
        check_ffb || {
            error "Warm reboot is not supported"
            exit "${MLNX_EXIT_FFB_FAILURE}"
        }
    fi

    debug "Prepare MLNX ASIC to ${REBOOT_TYPE}: install new FW if required"

    ${MLNX_FW_UPGRADE_SCRIPT} --upgrade
    MLNX_EXIT_CODE="$?"
    if [[ "${MLNX_EXIT_CODE}" != "${MLNX_EXIT_SUCCESS}" ]]; then
        error "Failed to burn MLNX FW: errno=${MLNX_EXIT_CODE}"
        exit "${MLNX_EXIT_FW_ERROR}"
    fi
fi


if is_secureboot && grep -q aboot_machine= /host/machine.conf; then
    load_aboot_secureboot_kernel
else
    load_kernel
fi

if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
    # Dump the ARP and FDB tables to files also as default routes for both IPv4 and IPv6
    # into /host/fast-reboot
    DUMP_DIR=/host/fast-reboot
    CONFIG_DB_FILE=/etc/sonic/config_db.json
    mkdir -p $DUMP_DIR
    FAST_REBOOT_DUMP_RC=0
    /usr/local/bin/fast-reboot-dump.py -t $DUMP_DIR || FAST_REBOOT_DUMP_RC=$?
    if [[ FAST_REBOOT_DUMP_RC -ne 0 ]]; then
        error "Failed to run fast-reboot-dump.py. Exit code: $FAST_REBOOT_DUMP_RC"
        unload_kernel
        exit "${EXIT_FAST_REBOOT_DUMP_FAILURE}"
    fi

    FILTER_FDB_ENTRIES_RC=0
    # Filter FDB entries using MAC addresses from ARP table
    /usr/local/bin/filter_fdb_entries -f $DUMP_DIR/fdb.json -a $DUMP_DIR/arp.json -c $CONFIG_DB_FILE || FILTER_FDB_ENTRIES_RC=$?
    if [[ FILTER_FDB_ENTRIES_RC -ne 0 ]]; then
        error "Failed to filter FDb entries. Exit code: $FILTER_FDB_ENTRIES_RC"
        unload_kernel
        exit "${EXIT_FILTER_FDB_ENTRIES_FAILURE}"
    fi
fi

init_warm_reboot_states

setup_control_plane_assistant

if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
    # Freeze orchagent for warm restart
    # Ask orchagent_restart_check to try freeze 5 times with interval of 2 seconds,
    # it is possible that the orchagent is in transient state and no opportunity to be freezed
    # Note: assume that 2*5 seconds is enough for orchagent to process the request and respone freeze or not
    debug "Pausing orchagent ..."
    docker exec -i swss /usr/bin/orchagent_restart_check -w 2000 -r 5 > /dev/null || RESTARTCHECK_RC=$?
    if [[ RESTARTCHECK_RC -ne 0 ]]; then
        error "RESTARTCHECK failed"
        if [[ x"${FORCE}" == x"yes" ]]; then
            debug "Ignoring orchagent pausing failure ..."
        else
            exit "${EXIT_ORCHAGENT_SHUTDOWN}"
        fi
    fi
fi

# We are fully committed to reboot from this point on becasue critical
# service will go down and we cannot recover from it.
set +e

if [ -x ${LOG_SSD_HEALTH} ]; then
    debug "Collecting logs to check ssd health before fast-reboot..."
    ${LOG_SSD_HEALTH}
fi


# Kill nat docker after saving the conntrack table
debug "Stopping nat ..."
/usr/local/bin/dump_nat_entries.py
docker kill nat > /dev/null || true
systemctl stop nat
debug "Stopped nat ..."

# Kill radv before stopping BGP service to prevent annoucing our departure.
debug "Stopping radv service..."
systemctl stop radv
debug "Stopped radv service..."

# Kill bgpd to start the bgp graceful restart procedure
debug "Stopping bgp ..."
systemctl stop bgp
debug "Stopped  bgp ..."

# Kill lldp, otherwise it sends informotion about reboot.
# We call `docker kill lldp` to ensure the container stops as quickly as possible,
# then immediately call `systemctl stop lldp` to prevent the service from
# restarting the container automatically.
docker kill lldp &> /dev/null || debug "Docker lldp is not running ($?) ..."
systemctl stop lldp

if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
    # Kill teamd processes inside of teamd container with SIGUSR2 to allow them to send last LACP frames
    # We call `docker kill teamd` to ensure the container stops as quickly as possible,
    # then immediately call `systemctl stop teamd` to prevent the service from
    # restarting the container automatically.
    # Note: teamd must be killed before syncd, because it will send the last packet through CPU port
    debug "Stopping teamd ..."
    docker exec -i teamd pkill -USR2 teamd || [ $? == 1 ]
    while docker exec -i teamd pgrep teamd > /dev/null; do
      sleep 0.05
    done
    docker kill teamd &> /dev/null || debug "Docker teamd is not running ($?) ..."
    systemctl stop teamd
    debug "Stopped teamd ..."
fi

debug "Stopping swss service ..."
systemctl stop swss
debug "Stopped swss service ..."

# Pre-shutdown syncd
if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
    initialize_pre_shutdown

    BEFORE_PRE_SHUTDOWN=true

    if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
        check_issu_bank_file "$BEFORE_PRE_SHUTDOWN"
    fi

    request_pre_shutdown

    wait_for_pre_shutdown_complete_or_fail

    if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
        check_issu_bank_file
    fi

    # Warm reboot: dump state to host disk
    if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
        sonic-db-cli ASIC_DB FLUSHDB > /dev/null
        sonic-db-cli COUNTERS_DB FLUSHDB > /dev/null
        sonic-db-cli FLEX_COUNTER_DB FLUSHDB > /dev/null
    fi

    # TODO: backup_database preserves FDB_TABLE
    # need to cleanup as well for fastfast boot case
    backup_database
fi

# Stop teamd gracefully
if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
    debug "Stopping teamd ..."
    # Send USR1 signal to all teamd instances to stop them
    # It will prepare teamd for warm-reboot
    # Note: We must send USR1 signal before syncd, because it will send the last packet through CPU port
    docker exec -i teamd pkill -USR1 teamd > /dev/null || [ $? == 1 ]
    debug "Stopped  teamd ..."
fi

debug "Stopping syncd ..."
systemctl stop syncd || debug "Ignore stopping syncd service error $?"
debug "Stopped  syncd ..."

# Kill other containers to make the reboot faster
# We call `docker kill ...` to ensure the container stops as quickly as possible,
# then immediately call `systemctl stop ...` to prevent the service from
# restarting the container automatically.
debug "Stopping all remaining containers ..."
for CONTAINER_NAME in $(docker ps --format '{{.Names}}'); do
    CONTAINER_STOP_RC=0
    docker kill $CONTAINER_NAME &> /dev/null || CONTAINER_STOP_RC=$?
    systemctl stop $CONTAINER_NAME || debug "Ignore stopping $CONTAINER_NAME error $?"
    if [[ CONTAINER_STOP_RC -ne 0 ]]; then
        debug "Failed killing container $CONTAINER_NAME RC $CONTAINER_STOP_RC ."
    fi
done
debug "Stopped all remaining containers ..."

# Stop the docker container engine. Otherwise we will have a broken docker storage
systemctl stop docker.service || debug "Ignore stopping docker service error $?"

# Stop kernel modules for Nephos platform
if [[ "$sonic_asic_type" = 'nephos' ]];
then
  systemctl stop nps-modules-`uname -r`.service || debug "Ignore stopping nps service error $?"
fi

# Stop opennsl modules for Broadcom platform
if [[ "$sonic_asic_type" = 'broadcom' ]];
then
  service_name=$(systemctl list-units --plain --no-pager --no-legend --type=service | grep opennsl | cut -f 1 -d' ')
  systemctl stop "$service_name"
fi

if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
    CONFIG_DB_FILE=/etc/sonic/config_db.json
    COUNTERPOLL_DISABLE_RC=0
    # Disable counters in config_db.json
    /usr/local/bin/counterpoll config-db disable $CONFIG_DB_FILE || COUNTERPOLL_DISABLE_RC=$?
    if [[ COUNTERPOLL_DISABLE_RC -ne 0 ]]; then
        error "Failed to disable counterpoll. Exit code: $COUNTERPOLL_DISABLE_RC"
        /usr/local/bin/counterpoll config-db enable $CONFIG_DB_FILE || COUNTERPOLL_DISABLE_RC=$?
        unload_kernel
        exit "${EXIT_COUNTERPOLL_DISABLE_FAILURE}"
    fi
fi

# Update the reboot cause file to reflect that user issued this script
# Upon next boot, the contents of this file will be used to determine the
# cause of the previous reboot
echo "User issued '${REBOOT_SCRIPT_NAME}' command [User: ${REBOOT_USER}, Time: ${REBOOT_TIME}]" > ${REBOOT_CAUSE_FILE}

# Wait until all buffers synced with disk
sync
sleep 1
sync

# sync the current system time to CMOS
if [ -x /sbin/hwclock ]; then
    /sbin/hwclock -w || /bin/true
fi

if [ -x ${DEVPATH}/${PLATFORM}/${SSD_FW_UPDATE} ]; then
    debug "updating ssd fw for${REBOOT_TYPE}"
    ${DEVPATH}/${PLATFORM}/${SSD_FW_UPDATE} ${REBOOT_TYPE}
fi

# Enable Watchdog Timer
if [ -x ${WATCHDOG_UTIL} ]; then
    debug "Enabling Watchdog before ${REBOOT_TYPE}"
    ${WATCHDOG_UTIL} arm
fi

# Run platform specific reboot plugin
if [ -x ${DEVPATH}/${PLATFORM}/${PLATFORM_PLUGIN} ]; then
    debug "Running ${PLATFORM} specific plugin..."
    ${DEVPATH}/${PLATFORM}/${PLATFORM_PLUGIN}
fi

# Reboot: explicity call Linux native reboot under sbin
debug "Rebooting with ${REBOOT_METHOD} to ${NEXT_SONIC_IMAGE} ..."
exec ${REBOOT_METHOD}

# Should never reach here
error "${REBOOT_TYPE} failed!"
exit "${EXIT_FAILURE}"