From d3f51dd7801a493fe766d0960b1b6f5befe000c9 Mon Sep 17 00:00:00 2001 From: dgsudharsan Date: Fri, 24 Jul 2020 09:56:58 -0700 Subject: [PATCH 1/8] Copp Changes --- scripts/db_migrator.py | 23 +++++++++++++++++++++-- scripts/fast-reboot | 8 ++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index 17ec3ea6e6..1a0d642b97 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -37,7 +37,7 @@ def __init__(self, namespace, socket=None): none-zero values. build: sequentially increase within a minor version domain. """ - self.CURRENT_VERSION = 'version_1_0_3' + self.CURRENT_VERSION = 'version_1_0_4' self.TABLE_NAME = 'VERSIONS' self.TABLE_KEY = 'DATABASE' @@ -238,6 +238,17 @@ def mlnx_migrate_buffer_pool_size(self): log_info("Successfully migrate mlnx buffer pool size to the latest.") return True + def migrate_copp_table(self): + ''' + Delete the existing COPP table + ''' + if self.appDB is None: + return + + keys = self.appDB.keys(self.appDB.APPL_DB, "COPP_TABLE:*") + for copp_key in keys: + self.appDB.delete(self.appDB.APPL_DB, copp_key) + def version_unknown(self): """ version_unknown tracks all SONiC versions that doesn't have a version @@ -288,9 +299,17 @@ def version_1_0_2(self): def version_1_0_3(self): """ - Current latest version. Nothing to do here. + version_1_0_3 """ log_info('Handling version_1_0_3') + self.migrate_copp_table() + self.set_version('version_1_0_4') + + def version_1_0_4(self): + """ + Current latest version. Nothing to do here. + """ + log_info('Handling version_1_0_4') return None diff --git a/scripts/fast-reboot b/scripts/fast-reboot index e18af200c3..9a7ed3781c 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -246,6 +246,14 @@ function backup_database() end end " 0 > /dev/null + # Delete COPP_TABLE keys in APPL_DB + sonic-db-cli APPL_DB eval " + for _, k in ipairs(redis.call('keys', '*')) do + if string.match(k, 'COPP_TABLE:') then + redis.call('del', k) + end + end + " 0 > /dev/null sonic-db-cli SAVE > /dev/null #TODO : need a script to copy all rdb files if there is multiple db instances config docker cp database:/var/lib/redis/$REDIS_FILE $WARM_DIR From 439d8033bb583801e29539ab1b1fa769ddec7924 Mon Sep 17 00:00:00 2001 From: dgsudharsan Date: Mon, 14 Sep 2020 13:52:44 -0700 Subject: [PATCH 2/8] resolving merge --- scripts/db_migrator.py | 95 ------------------------------------------ 1 file changed, 95 deletions(-) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index 36949738d6..85662e7563 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -142,101 +142,6 @@ def migrate_intf_table(self): self.appDB.set(self.appDB.APPL_DB, table, 'NULL', 'NULL') if_db.append(if_name) - def mlnx_migrate_buffer_pool_size(self): - """ - On Mellanox platform the buffer pool size changed since - version with new SDK 4.3.3052, SONiC to SONiC update - from version with old SDK will be broken without migration. - This migration is specifically for Mellanox platform. - """ - # Buffer pools defined in version 1_0_2 - buffer_pools = ['ingress_lossless_pool', 'egress_lossless_pool', 'ingress_lossy_pool', 'egress_lossy_pool'] - - # Old default buffer pool values on Mellanox platform - spc1_t0_default_value = [{'ingress_lossless_pool': '4194304'}, {'egress_lossless_pool': '16777152'}, {'ingress_lossy_pool': '7340032'}, {'egress_lossy_pool': '7340032'}] - spc1_t1_default_value = [{'ingress_lossless_pool': '2097152'}, {'egress_lossless_pool': '16777152'}, {'ingress_lossy_pool': '5242880'}, {'egress_lossy_pool': '5242880'}] - spc2_t0_default_value = [{'ingress_lossless_pool': '8224768'}, {'egress_lossless_pool': '35966016'}, {'ingress_lossy_pool': '8224768'}, {'egress_lossy_pool': '8224768'}] - spc2_t1_default_value = [{'ingress_lossless_pool': '12042240'}, {'egress_lossless_pool': '35966016'}, {'ingress_lossy_pool': '12042240'}, {'egress_lossy_pool': '12042240'}] - - # New default buffer pool configuration on Mellanox platform - spc1_t0_default_config = {"ingress_lossless_pool": { "size": "5029836", "type": "ingress", "mode": "dynamic" }, - "ingress_lossy_pool": { "size": "5029836", "type": "ingress", "mode": "dynamic" }, - "egress_lossless_pool": { "size": "14024599", "type": "egress", "mode": "dynamic" }, - "egress_lossy_pool": {"size": "5029836", "type": "egress", "mode": "dynamic" } } - spc1_t1_default_config = {"ingress_lossless_pool": { "size": "2097100", "type": "ingress", "mode": "dynamic" }, - "ingress_lossy_pool": { "size": "2097100", "type": "ingress", "mode": "dynamic" }, - "egress_lossless_pool": { "size": "14024599", "type": "egress", "mode": "dynamic" }, - "egress_lossy_pool": {"size": "2097100", "type": "egress", "mode": "dynamic" } } - spc2_t0_default_config = {"ingress_lossless_pool": { "size": "14983147", "type": "ingress", "mode": "dynamic" }, - "ingress_lossy_pool": { "size": "14983147", "type": "ingress", "mode": "dynamic" }, - "egress_lossless_pool": { "size": "34340822", "type": "egress", "mode": "dynamic" }, - "egress_lossy_pool": {"size": "14983147", "type": "egress", "mode": "dynamic" } } - spc2_t1_default_config = {"ingress_lossless_pool": { "size": "9158635", "type": "ingress", "mode": "dynamic" }, - "ingress_lossy_pool": { "size": "9158635", "type": "ingress", "mode": "dynamic" }, - "egress_lossless_pool": { "size": "34340822", "type": "egress", "mode": "dynamic" }, - "egress_lossy_pool": {"size": "9158635", "type": "egress", "mode": "dynamic" } } - # 3800 platform has gearbox installed so the buffer pool size is different with other Spectrum2 platform - spc2_3800_t0_default_config = {"ingress_lossless_pool": { "size": "28196784", "type": "ingress", "mode": "dynamic" }, - "ingress_lossy_pool": { "size": "28196784", "type": "ingress", "mode": "dynamic" }, - "egress_lossless_pool": { "size": "34340832", "type": "egress", "mode": "dynamic" }, - "egress_lossy_pool": {"size": "28196784", "type": "egress", "mode": "dynamic" } } - spc2_3800_t1_default_config = {"ingress_lossless_pool": { "size": "17891280", "type": "ingress", "mode": "dynamic" }, - "ingress_lossy_pool": { "size": "17891280", "type": "ingress", "mode": "dynamic" }, - "egress_lossless_pool": { "size": "34340832", "type": "egress", "mode": "dynamic" }, - "egress_lossy_pool": {"size": "17891280", "type": "egress", "mode": "dynamic" } } - - # Try to get related info from DB - buffer_pool_conf = {} - device_data = self.configDB.get_table('DEVICE_METADATA') - if 'localhost' in device_data.keys(): - hwsku = device_data['localhost']['hwsku'] - platform = device_data['localhost']['platform'] - else: - log_error("Trying to get DEVICE_METADATA from DB but doesn't exist, skip migration") - return False - buffer_pool_conf = self.configDB.get_table('BUFFER_POOL') - - # Get current buffer pool configuration, only migrate configuration which - # with default values, if it's not default, leave it as is. - pool_size_in_db_list = [] - pools_in_db = buffer_pool_conf.keys() - - # Buffer pool numbers is different with default, don't need migrate - if len(pools_in_db) != len(buffer_pools): - return True - - # If some buffer pool is not default ones, don't need migrate - for buffer_pool in buffer_pools: - if buffer_pool not in pools_in_db: - return True - pool_size_in_db_list.append({buffer_pool: buffer_pool_conf[buffer_pool]['size']}) - - # To check if the buffer pool size is equal to old default values - new_buffer_pool_conf = None - if pool_size_in_db_list == spc1_t0_default_value: - new_buffer_pool_conf = spc1_t0_default_config - elif pool_size_in_db_list == spc1_t1_default_value: - new_buffer_pool_conf = spc1_t1_default_config - elif pool_size_in_db_list == spc2_t0_default_value: - if platform == 'x86_64-mlnx_msn3800-r0': - new_buffer_pool_conf = spc2_3800_t0_default_config - else: - new_buffer_pool_conf = spc2_t0_default_config - elif pool_size_in_db_list == spc2_t1_default_value: - if platform == 'x86_64-mlnx_msn3800-r0': - new_buffer_pool_conf = spc2_3800_t1_default_config - else: - new_buffer_pool_conf = spc2_t1_default_config - else: - # It's not using default buffer pool configuration, no migration needed. - log_info("buffer pool size is not old default value, no need to migrate") - return True - # Migrate old buffer conf to latest. - for pool in buffer_pools: - self.configDB.set_entry('BUFFER_POOL', pool, new_buffer_pool_conf[pool]) - log_info("Successfully migrate mlnx buffer pool size to the latest.") - return True - def migrate_copp_table(self): ''' Delete the existing COPP table From fbf7b2289677da78c6399d40dd0e04d2cd5f2aa2 Mon Sep 17 00:00:00 2001 From: dgsudharsan Date: Mon, 19 Oct 2020 19:52:36 -0700 Subject: [PATCH 3/8] Addressing code review comments --- scripts/db_migrator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index 85662e7563..59a37e4cc5 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -211,15 +211,15 @@ def version_1_0_3(self): if self.mellanox_buffer_migrator.mlnx_migrate_buffer_pool_size('version_1_0_3', 'version_1_0_4') and self.mellanox_buffer_migrator.mlnx_migrate_buffer_profile('version_1_0_3', 'version_1_0_4'): self.set_version('version_1_0_4') - self.migrate_copp_table() self.set_version('version_1_0_4') return 'version_1_0_4' def version_1_0_4(self): """ - Current latest version. Nothing to do here. + Current latest version. Always perform copp table migration at current latest version """ log.log_info('Handling version_1_0_4') + self.migrate_copp_table() return None def get_version(self): From 6954f704d2c59a29d9c343bc33ff4ceeeb536b8f Mon Sep 17 00:00:00 2001 From: dgsudharsan Date: Tue, 20 Oct 2020 07:54:40 -0700 Subject: [PATCH 4/8] UT Fixes --- scripts/db_migrator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index 59a37e4cc5..1dafc0a15e 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -150,6 +150,8 @@ def migrate_copp_table(self): return keys = self.appDB.keys(self.appDB.APPL_DB, "COPP_TABLE:*") + if keys is None: + return for copp_key in keys: self.appDB.delete(self.appDB.APPL_DB, copp_key) From b781463b2ac90780a5a9eb7ef10247952fb0d0f3 Mon Sep 17 00:00:00 2001 From: dgsudharsan Date: Wed, 11 Nov 2020 09:08:50 -0800 Subject: [PATCH 5/8] Moving migrate copp table to common migration --- scripts/db_migrator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index 52e886120f..2c4e13973d 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -221,10 +221,9 @@ def version_1_0_3(self): def version_1_0_4(self): """ - Current latest version. Always perform copp table migration at current latest version + Current latest version. Nothing to do here. """ log.log_info('Handling version_1_0_4') - self.migrate_copp_table() return None def get_version(self): @@ -259,6 +258,7 @@ def common_migration_ops(self): # Update all tables that do not exist in configDB but are present in INIT_CFG for init_table_key, init_table_val in table_val.items(): self.configDB.set_entry(init_cfg_table, init_table_key, init_table_val) + self.migrate_copp_table() def migrate(self): version = self.get_version() From 5310d3269b3adc25a2bc504b0622a36b13842d76 Mon Sep 17 00:00:00 2001 From: dgsudharsan Date: Wed, 11 Nov 2020 09:11:18 -0800 Subject: [PATCH 6/8] Cleanup --- scripts/db_migrator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index 2c4e13973d..d9a6630570 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -215,8 +215,8 @@ def version_1_0_3(self): if self.asic_type == "mellanox": if self.mellanox_buffer_migrator.mlnx_migrate_buffer_pool_size('version_1_0_3', 'version_1_0_4') and self.mellanox_buffer_migrator.mlnx_migrate_buffer_profile('version_1_0_3', 'version_1_0_4'): self.set_version('version_1_0_4') - - self.set_version('version_1_0_4') + else: + self.set_version('version_1_0_4') return 'version_1_0_4' def version_1_0_4(self): @@ -224,6 +224,7 @@ def version_1_0_4(self): Current latest version. Nothing to do here. """ log.log_info('Handling version_1_0_4') + return None def get_version(self): From eb43280152bfb31c855d46f18f3c8ca7e4df5c87 Mon Sep 17 00:00:00 2001 From: dgsudharsan Date: Wed, 11 Nov 2020 09:12:21 -0800 Subject: [PATCH 7/8] Cleanup --- scripts/db_migrator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index d9a6630570..8db6738442 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -217,6 +217,7 @@ def version_1_0_3(self): self.set_version('version_1_0_4') else: self.set_version('version_1_0_4') + return 'version_1_0_4' def version_1_0_4(self): From 3aebe246c4d8c5caa40bd4980ef648eecaabbc8c Mon Sep 17 00:00:00 2001 From: dgsudharsan Date: Wed, 11 Nov 2020 11:44:45 -0800 Subject: [PATCH 8/8] Addressing review comments --- scripts/db_migrator.py | 1 + scripts/fast-reboot | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index 8db6738442..326eed41f7 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -260,6 +260,7 @@ def common_migration_ops(self): # Update all tables that do not exist in configDB but are present in INIT_CFG for init_table_key, init_table_val in table_val.items(): self.configDB.set_entry(init_cfg_table, init_table_key, init_table_val) + self.migrate_copp_table() def migrate(self): diff --git a/scripts/fast-reboot b/scripts/fast-reboot index 01796aaf75..48c643ae6a 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -247,14 +247,6 @@ function backup_database() end end " 0 > /dev/null - # Delete COPP_TABLE keys in APPL_DB - sonic-db-cli APPL_DB eval " - for _, k in ipairs(redis.call('keys', '*')) do - if string.match(k, 'COPP_TABLE:') then - redis.call('del', k) - end - end - " 0 > /dev/null sonic-db-cli SAVE > /dev/null #TODO : need a script to copy all rdb files if there is multiple db instances config docker cp database:/var/lib/redis/$REDIS_FILE $WARM_DIR