[sonic-package-manager] support warm/fast reboot for extension packages

config/main.py

@@ -1993,20 +1993,28 @@ def warm_restart(ctx, redis_unix_socket_path):
     ctx.obj = {'db': config_db, 'state_db': state_db, 'prefix': prefix}
 
 @warm_restart.command('enable')
-@click.argument('module', metavar='<module>', default='system', required=False, type=click.Choice(["system", "swss", "bgp", "teamd"]))
+@click.argument('module', metavar='<module>', default='system', required=False)
 @click.pass_context
 def warm_restart_enable(ctx, module):
     state_db = ctx.obj['state_db']
+    config_db = ctx.obj['db']
+    feature_table = config_db.get_table('FEATURE')
+    if module != 'system' and module not in feature_table:
+        exit('Feature {} is unknown'.format(module))
     prefix = ctx.obj['prefix']
     _hash = '{}{}'.format(prefix, module)
     state_db.set(state_db.STATE_DB, _hash, 'enable', 'true')
     state_db.close(state_db.STATE_DB)
 
 @warm_restart.command('disable')
-@click.argument('module', metavar='<module>', default='system', required=False, type=click.Choice(["system", "swss", "bgp", "teamd"]))
+@click.argument('module', metavar='<module>', default='system', required=False)
 @click.pass_context
 def warm_restart_enable(ctx, module):
     state_db = ctx.obj['state_db']
+    config_db = ctx.obj['db']
+    feature_table = config_db.get_table('FEATURE')
+    if module != 'system' and module not in feature_table:
+        exit('Feature {} is unknown'.format(module))
     prefix = ctx.obj['prefix']
     _hash = '{}{}'.format(prefix, module)
     state_db.set(state_db.STATE_DB, _hash, 'enable', 'false')

scripts/fast-reboot

@@ -7,6 +7,7 @@ WARM_DIR=/host/warmboot
 REDIS_FILE=dump.rdb
 REBOOT_SCRIPT_NAME=$(basename $0)
 REBOOT_TYPE="${REBOOT_SCRIPT_NAME}"
+SHUTDOWN_ORDER_FILE="/etc/sonic/${REBOOT_TYPE}_order"
 VERBOSE=no
 FORCE=no
 IGNORE_ASIC=no
@@ -567,82 +568,72 @@ if [ -x ${LOG_SSD_HEALTH} ]; then
 fi
 
 
-# Kill nat docker after saving the conntrack table
-debug "Stopping nat ..."
-/usr/local/bin/dump_nat_entries.py
-docker kill nat > /dev/null || true
-systemctl stop nat
-debug "Stopped nat ..."
-
-# Kill radv before stopping BGP service to prevent announcing our departure.
-debug "Stopping radv service..."
-systemctl stop radv
-debug "Stopped radv service..."
-
-# Kill bgpd to start the bgp graceful restart procedure
-debug "Stopping bgp ..."
-systemctl stop bgp
-debug "Stopped  bgp ..."
-
-# Kill sflow docker	
-debug "Stopping sflow ..."	
-container kill sflow &> /dev/null || debug "Docker sflow is not running ($?) ..."
-systemctl stop sflow
-debug "Stopped sflow ..."
-
-# Kill lldp, otherwise it sends informotion about reboot.
-# We call `docker kill lldp` to ensure the container stops as quickly as possible,
-# then immediately call `systemctl stop lldp` to prevent the service from
-# restarting the container automatically.
-container kill lldp &> /dev/null || debug "Docker lldp is not running ($?) ..."
-systemctl stop lldp
-
-if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
-    debug "Stopping teamd ..."
-    systemctl stop teamd
-    debug "Stopped teamd ..."
+if [[ -f ${SHUTDOWN_ORDER_FILE} ]]; then
+    SERVICES_TO_STOP="$(cat ${SHUTDOWN_ORDER_FILE})"
+else
+    # TODO: to be removed once sonic-buildimage change is in
+    if [[ "${REBOOT_TYPE}" == "fast-reboot" ]]; then
+        SERVICES_TO_STOP="nat radv bgp sflow lldp swss teamd syncd"
+    elif [[ "${REBOOT_TYPE}" == "fastfast-reboot" || "${REBOOT_TYPE}" == "warm-reboot" ]]; then
+        SERVICES_TO_STOP="nat radv bgp sflow lldp teamd swss syncd"
+    else
+        error "Unexpected reboot type ${REBOOT_TYPE}"
+        exit $EXIT_FAILURE
+    fi
 fi
 
-debug "Stopping swss service ..."
-systemctl stop swss
-debug "Stopped swss service ..."
+for service in ${SERVICES_TO_STOP}; do
+    debug "Stopping ${service} ..."
 
-if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
-    # Pre-shutdown syncd
-    initialize_pre_shutdown
+    # TODO: These exceptions for nat, sflow, lldp
+    # have to be coded in corresponding service scripts
 
-    if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
-        check_issu_bank_file
+    if [[ "${service}" = "nat" ]]; then
+        /usr/local/bin/dump_nat_entries.py
     fi
 
-    request_pre_shutdown
-
-    wait_for_pre_shutdown_complete_or_fail
-
-    if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
-        check_issu_bank_file
+    if [[ "${service}" = "nat" || "${service}" = "sflow" || "${service}" = "lldp" ]]; then
+        container kill "${service}"  &> /dev/null || debug "Docker ${service} is not running ($?) ..."
     fi
 
-    # Warm reboot: dump state to host disk
-    if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
-        sonic-db-cli ASIC_DB FLUSHDB > /dev/null
-        sonic-db-cli COUNTERS_DB FLUSHDB > /dev/null
-        sonic-db-cli FLEX_COUNTER_DB FLUSHDB > /dev/null
+    if [[ "${service}" = "syncd" ]]; then
+        systemctl stop ${service} || debug "Ignore stopping ${service} service error $?"
+    else
+        systemctl stop ${service}
     fi
 
-    # TODO: backup_database preserves FDB_TABLE
-    # need to cleanup as well for fastfast boot case
-    backup_database
+    debug "Stopped ${service}"
 
-    # Stop teamd gracefully
-    debug "Stopping teamd ..."
-    systemctl stop teamd
-    debug "Stopped  teamd ..."
-fi
+    if [[ "${service}" = "swss" ]]; then
+        if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
+            # Pre-shutdown syncd
+            initialize_pre_shutdown
+
+            if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
+                check_issu_bank_file
+            fi
 
-debug "Stopping syncd ..."
-systemctl stop syncd || debug "Ignore stopping syncd service error $?"
-debug "Stopped  syncd ..."
+            request_pre_shutdown
+
+            wait_for_pre_shutdown_complete_or_fail
+
+            if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
+                check_issu_bank_file
+            fi
+
+            # Warm reboot: dump state to host disk
+            if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
+                sonic-db-cli ASIC_DB FLUSHDB > /dev/null
+                sonic-db-cli COUNTERS_DB FLUSHDB > /dev/null
+                sonic-db-cli FLEX_COUNTER_DB FLUSHDB > /dev/null
+            fi
+
+            # TODO: backup_database preserves FDB_TABLE
+            # need to cleanup as well for fastfast boot case
+            backup_database
+        fi
+    fi
+done
 
 # Kill other containers to make the reboot faster
 # We call `docker kill ...` to ensure the container stops as quickly as possible,

scripts/generate_shutdown_order.py

@@ -0,0 +1,15 @@
+#!/usr/bin/python3
+
+''' This script is used to generate initial warm/fast shutdown order file '''
+
+from sonic_package_manager import PackageManager
+
+def main():
+    manager = PackageManager.get_manager()
+    installed_packages = manager.get_installed_packages()
+    print('installed packages {}'.format(installed_packages))
+    manager.service_creator.generate_shutdown_sequence_files(installed_packages)
+    print('Done.')
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -95,6 +95,7 @@
         'scripts/fdbshow',
         'scripts/gearboxutil',
         'scripts/generate_dump',
+        'scripts/generate_shutdown_order.py',
         'scripts/intfutil',
         'scripts/intfstat',
         'scripts/ipintutil',
@@ -186,9 +187,10 @@
         'sonic-py-common',
         'sonic-yang-mgmt',
         'swsssdk>=2.0.1',
-        'tabulate>=0.8.2',
-        'www-authenticate>=0.9.2',
-        'xmltodict>=0.12.0',
+        'tabulate==0.8.2',
+        'toposort==1.6',
+        'www-authenticate==0.9.2',
+        'xmltodict==0.12.0',
     ],
     setup_requires= [
         'pytest-runner',

sonic_package_manager/manager.py

@@ -6,7 +6,7 @@
 import pkgutil
 import tempfile
 from inspect import signature
-from typing import Any, Iterable, Callable, Dict, Optional
+from typing import Any, Iterable, List, Callable, Dict, Optional
 
 import docker
 import filelock
@@ -375,6 +375,14 @@ def install_from_source(self,
                 self.service_creator.create(package, state=feature_state, owner=default_owner)
                 exits.callback(rollback(self.service_creator.remove, package))
 
+                self.service_creator.generate_shutdown_sequence_files(
+                    self._get_installed_packages_and(package)
+                )
+                exits.callback(rollback(
+                    self.service_creator.generate_shutdown_sequence_files,
+                    self.get_installed_packages())
+                )
+
                 if not skip_host_plugins:
                     self._install_cli_plugins(package)
                     exits.callback(rollback(self._uninstall_cli_plugins, package))
@@ -429,6 +437,9 @@ def uninstall(self, name: str, force=False):
         try:
             self._uninstall_cli_plugins(package)
             self.service_creator.remove(package)
+            self.service_creator.generate_shutdown_sequence_files(
+                self._get_installed_packages_except(package)
+            )
 
             # Clean containers based on this image
             containers = self.docker.ps(filters={'ancestor': package.image_id},
@@ -525,8 +536,8 @@ def upgrade_from_source(self,
                                             old_package, 'start'))
 
                 self.service_creator.remove(old_package, deregister_feature=False)
-                exits.callback(rollback(self.service_creator.create,
-                                        old_package, register_feature=False))
+                exits.callback(rollback(self.service_creator.create, old_package,
+                                        register_feature=False))
 
                 # Clean containers based on the old image
                 containers = self.docker.ps(filters={'ancestor': old_package.image_id},
@@ -538,6 +549,14 @@ def upgrade_from_source(self,
                 exits.callback(rollback(self.service_creator.remove, new_package,
                                         register_feature=False))
 
+                self.service_creator.generate_shutdown_sequence_files(
+                    self._get_installed_packages_and(new_package)
+                )
+                exits.callback(rollback(
+                    self.service_creator.generate_shutdown_sequence_files,
+                    self._get_installed_packages_and(old_package))
+                )
+
                 if self.feature_registry.is_feature_enabled(new_feature):
                     self._systemctl_action(new_package, 'start')
                     exits.callback(rollback(self._systemctl_action,
@@ -817,10 +836,19 @@ def get_installed_packages(self) -> Dict[str, Package]:
         """
 
         return {
-            entry.name: self.get_installed_package(entry.name)
-            for entry in self.database if entry.installed
+            entry.name: entry for entry in self.get_installed_packages_list()
         }
 
+    def get_installed_packages_list(self) -> List[Package]:
+        """ Returns a list of installed packages.
+
+        Returns:
+            Installed packages dictionary.
+        """
+
+        return [self.get_installed_package(entry.name) 
+                for entry in self.database if entry.installed]
+
     def _migrate_package_database(self, old_package_database: PackageDatabase):
         """ Performs part of package migration process.
         For every package in  old_package_database that is not listed in current

sonic_package_manager/manifest.py

@@ -92,8 +92,10 @@ class ManifestRoot(ManifestNode):
 
         def marshal(self, value: Optional[dict]):
             result = {}
-            if value is None:
-                value = {}
+            value = value or {}
+
+            if not isinstance(value, dict):
+                raise ManifestError(f'"{self.key}" field has to be a dictionary')
 
             for item in self.items:
                 next_value = value.get(item.key)
@@ -115,7 +117,7 @@ def marshal(self, value):
             if value is None:
                 if self.default is not None:
                     return self.default
-                raise ManifestError(f'{self.key} is a required field but it is missing')
+                raise ManifestError(f'"{self.key}" is a required field but it is missing')
             try:
                 return_value = self.type.marshal(value)
             except Exception as err:
@@ -130,10 +132,12 @@ class ManifestArray(ManifestNode):
         type: Any
 
         def marshal(self, value):
-            if value is None:
-                return []
-
             return_value = []
+            value = value or []
+
+            if not isinstance(value, list):
+                raise ManifestError(f'"{self.key}" has to be of type list')
+
             try:
                 for item in value:
                     return_value.append(self.type.marshal(item))
@@ -173,6 +177,14 @@ def unmarshal(self, value):
             ManifestField('asic-service', DefaultMarshaller(bool), False),
             ManifestField('host-service', DefaultMarshaller(bool), True),
             ManifestField('delayed', DefaultMarshaller(bool), False),
+            ManifestRoot('warm-shutdown', [
+                ManifestArray('after', DefaultMarshaller(str)),
+                ManifestArray('before', DefaultMarshaller(str)),
+            ]),
+            ManifestRoot('fast-shutdown', [
+                ManifestArray('after', DefaultMarshaller(str)),
+                ManifestArray('before', DefaultMarshaller(str)),
+            ]),
         ]),
         ManifestRoot('container', [
             ManifestField('privileged', DefaultMarshaller(bool), False),
@@ -187,6 +199,7 @@ def unmarshal(self, value):
         ]),
         ManifestArray('processes', ManifestRoot('processes', [
             ManifestField('name', DefaultMarshaller(str)),
+            ManifestField('reconciles', DefaultMarshaller(bool), False),
         ])),
         ManifestRoot('cli', [
             ManifestField('mandatory', DefaultMarshaller(bool), False),