From 7dfd4f0647bb9518a84bb574bce86d69d0af5cf5 Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Wed, 12 Aug 2020 21:21:31 -0700 Subject: [PATCH 1/8] [filter-fdb] Call Filter FDB Main From Within Test Code Code coverage requires that python code be run with the same process. Current test code was invoking filter fdb via shell which launches new process and so coverage is not available. This PR calles the main method from within test code. signed-off-by: Tamer Ahmed --- scripts/__init__.py | 0 scripts/filter_fdb_entries.py | 6 +++--- tests/filter_fdb_entries_test.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 scripts/__init__.py diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/filter_fdb_entries.py b/scripts/filter_fdb_entries.py index 31d4204ec9..9c64628f09 100755 --- a/scripts/filter_fdb_entries.py +++ b/scripts/filter_fdb_entries.py @@ -124,13 +124,13 @@ def file_exists_or_raise(filename): if not os.path.exists(filename): raise Exception("file '{0}' does not exist".format(filename)) -def main(): +def main(argv): parser = argparse.ArgumentParser() parser.add_argument('-f', '--fdb', type=str, default='/tmp/fdb.json', help='fdb file name') parser.add_argument('-a', '--arp', type=str, default='/tmp/arp.json', help='arp file name') parser.add_argument('-c', '--config_db', type=str, default='/tmp/config_db.json', help='config db file name') parser.add_argument('-b', '--backup_file', type=bool, default=True, help='Back up old fdb entries file') - args = parser.parse_args() + args = parser.parse_args(argv) fdb_filename = args.fdb arp_filename = args.arp @@ -152,7 +152,7 @@ def main(): res = 0 try: syslog.openlog('filter_fdb_entries') - res = main() + res = main(sys.argv[1:]) except KeyboardInterrupt: syslog.syslog(syslog.LOG_NOTICE, "SIGINT received. Quitting") res = 1 diff --git a/tests/filter_fdb_entries_test.py b/tests/filter_fdb_entries_test.py index af1f7712c3..30ae82291b 100644 --- a/tests/filter_fdb_entries_test.py +++ b/tests/filter_fdb_entries_test.py @@ -7,6 +7,7 @@ from collections import defaultdict from filter_fdb_input.test_vectors import filterFdbEntriesTestVector +from scripts.filter_fdb_entries import main as filterFdbMain class TestFilterFdbEntries(object): """ @@ -162,16 +163,15 @@ def testFilterFdbEntries(self, testData): """ try: self.__setUp(testData) - - stdout, stderr, rc = self.__runCommand([ - "scripts/filter_fdb_entries.py", + argv = [ "-a", self.ARP_FILENAME, "-f", self.FDB_FILENAME, "-c", self.CONFIG_DB_FILENAME, - ]) + ] + rc = filterFdbMain(argv) assert rc == 0, "Filter_fdb_entries.py failed with '{0}'".format(stderr) assert self.__verifyOutput(), "Test failed for test data: {0}".format(testData) finally: From 8c9701ccb7a7645d80573b97d408560d3f2e1e36 Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Thu, 13 Aug 2020 12:51:56 -0700 Subject: [PATCH 2/8] TESTING.. NO MERGE --- scripts/__init__.py | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/scripts/__init__.py b/scripts/__init__.py index e69de29bb2..4265cc3e6c 100644 --- a/scripts/__init__.py +++ b/scripts/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python diff --git a/setup.py b/setup.py index 88f3fbe848..8e163e50c3 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ 'tests': ['acl_input/*', 'mock_tables/*.py', 'mock_tables/*.json', 'filter_fdb_input/*'] }, scripts=[ + 'scripts/__init__.py', 'scripts/aclshow', 'scripts/asic_config_check', 'scripts/boot_part', From 40d3e527ad0a225d55226829a891e83f6000942b Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Fri, 14 Aug 2020 14:35:32 -0700 Subject: [PATCH 3/8] convert filter_fdb_entries into console scripts --- fdb_utils/__init__.py | 0 {scripts => fdb_utils}/filter_fdb_entries.py | 0 pytest.ini | 2 +- scripts/__init__.py | 1 - scripts/fast-reboot | 2 +- setup.py | 3 +-- tests/filter_fdb_entries_test.py | 2 +- 7 files changed, 4 insertions(+), 6 deletions(-) create mode 100644 fdb_utils/__init__.py rename {scripts => fdb_utils}/filter_fdb_entries.py (100%) delete mode 100644 scripts/__init__.py diff --git a/fdb_utils/__init__.py b/fdb_utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/filter_fdb_entries.py b/fdb_utils/filter_fdb_entries.py similarity index 100% rename from scripts/filter_fdb_entries.py rename to fdb_utils/filter_fdb_entries.py diff --git a/pytest.ini b/pytest.ini index d1975890b9..b20e0d15b4 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] filterwarnings = ignore::DeprecationWarning -addopts = --cov=acl_loader --cov=clear --cov=config --cov=connect --cov=consutil --cov=counterpoll --cov=crm --cov=debug --cov=fwutil --cov=pcieutil --cov=pfcwd --cov=psuutil --cov=pddf_fanutil --cov=pddf_ledutil --cov=pddf_psuutil --cov=pddf_thermalutil --cov=scripts --cov=sfputil --cov=show --cov=sonic_installer --cov=ssdutil --cov=utilities_common --cov=watchdogutil --cov-report html --cov-report term --cov-report xml +addopts = --cov=acl_loader --cov=clear --cov=config --cov=connect --cov=consutil --cov=counterpoll --cov=crm --cov=debug --cov=fdb_utils --cov=fwutil --cov=pcieutil --cov=pfcwd --cov=psuutil --cov=pddf_fanutil --cov=pddf_ledutil --cov=pddf_psuutil --cov=pddf_thermalutil --cov=scripts --cov=sfputil --cov=show --cov=sonic_installer --cov=ssdutil --cov=utilities_common --cov=watchdogutil --cov-report html --cov-report term --cov-report xml diff --git a/scripts/__init__.py b/scripts/__init__.py deleted file mode 100644 index 4265cc3e6c..0000000000 --- a/scripts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -#!/usr/bin/env python diff --git a/scripts/fast-reboot b/scripts/fast-reboot index 302cdf7f1f..8a5b5fe71c 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -456,7 +456,7 @@ if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then FILTER_FDB_ENTRIES_RC=0 # Filter FDB entries using MAC addresses from ARP table - /usr/bin/filter_fdb_entries.py -f $DUMP_DIR/fdb.json -a $DUMP_DIR/arp.json -c $CONFIG_DB_FILE || FILTER_FDB_ENTRIES_RC=$? + /usr/bin/filter_fdb_entries -f $DUMP_DIR/fdb.json -a $DUMP_DIR/arp.json -c $CONFIG_DB_FILE || FILTER_FDB_ENTRIES_RC=$? if [[ FILTER_FDB_ENTRIES_RC -ne 0 ]]; then error "Failed to filter FDb entries. Exit code: $FILTER_FDB_ENTRIES_RC" unload_kernel diff --git a/setup.py b/setup.py index 8e163e50c3..e274a9a56d 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,6 @@ 'tests': ['acl_input/*', 'mock_tables/*.py', 'mock_tables/*.json', 'filter_fdb_input/*'] }, scripts=[ - 'scripts/__init__.py', 'scripts/aclshow', 'scripts/asic_config_check', 'scripts/boot_part', @@ -79,7 +78,6 @@ 'scripts/fast-reboot-dump.py', 'scripts/fdbclear', 'scripts/fdbshow', - 'scripts/filter_fdb_entries.py', 'scripts/gearboxutil', 'scripts/generate_dump', 'scripts/intfutil', @@ -141,6 +139,7 @@ 'sonic_installer = sonic_installer.main:sonic_installer', # Deprecated 'undebug = undebug.main:cli', 'watchdogutil = watchdogutil.main:watchdogutil', + 'filter_fdb_entries = fdb_utils.filter_fdb_entries:main', ] }, # NOTE: sonic-utilities also depends on other packages that are either only diff --git a/tests/filter_fdb_entries_test.py b/tests/filter_fdb_entries_test.py index 30ae82291b..a0cb2e318c 100644 --- a/tests/filter_fdb_entries_test.py +++ b/tests/filter_fdb_entries_test.py @@ -7,7 +7,7 @@ from collections import defaultdict from filter_fdb_input.test_vectors import filterFdbEntriesTestVector -from scripts.filter_fdb_entries import main as filterFdbMain +from fdb_utils.filter_fdb_entries import main as filterFdbMain class TestFilterFdbEntries(object): """ From 6d507f52795bd7151300c31ccce6f6548201df48 Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Fri, 14 Aug 2020 14:43:14 -0700 Subject: [PATCH 4/8] remove __main__ construct --- fdb_utils/filter_fdb_entries.py | 23 ++++++----------------- tests/filter_fdb_entries_test.py | 1 + 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/fdb_utils/filter_fdb_entries.py b/fdb_utils/filter_fdb_entries.py index 9c64628f09..6299f6aa38 100755 --- a/fdb_utils/filter_fdb_entries.py +++ b/fdb_utils/filter_fdb_entries.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import json import sys import os @@ -130,7 +128,7 @@ def main(argv): parser.add_argument('-a', '--arp', type=str, default='/tmp/arp.json', help='arp file name') parser.add_argument('-c', '--config_db', type=str, default='/tmp/config_db.json', help='config db file name') parser.add_argument('-b', '--backup_file', type=bool, default=True, help='Back up old fdb entries file') - args = parser.parse_args(argv) + args = parser.parse_args(argv[1:]) fdb_filename = args.fdb arp_filename = args.arp @@ -138,30 +136,21 @@ def main(argv): backup_file = args.backup_file try: + syslog.openlog('filter_fdb_entries') file_exists_or_raise(fdb_filename) file_exists_or_raise(arp_filename) file_exists_or_raise(config_db_filename) except Exception as e: syslog.syslog(syslog.LOG_ERR, "Got an exception %s: Traceback: %s" % (str(e), traceback.format_exc())) - else: - filter_fdb_entries(fdb_filename, arp_filename, config_db_filename, backup_file) - - return 0 - -if __name__ == '__main__': - res = 0 - try: - syslog.openlog('filter_fdb_entries') - res = main(sys.argv[1:]) except KeyboardInterrupt: syslog.syslog(syslog.LOG_NOTICE, "SIGINT received. Quitting") res = 1 except Exception as e: syslog.syslog(syslog.LOG_ERR, "Got an exception %s: Traceback: %s" % (str(e), traceback.format_exc())) res = 2 + else: + filter_fdb_entries(fdb_filename, arp_filename, config_db_filename, backup_file) finally: syslog.closelog() - try: - sys.exit(res) - except SystemExit: - os._exit(res) + + return 0 diff --git a/tests/filter_fdb_entries_test.py b/tests/filter_fdb_entries_test.py index a0cb2e318c..1c6a0a0001 100644 --- a/tests/filter_fdb_entries_test.py +++ b/tests/filter_fdb_entries_test.py @@ -164,6 +164,7 @@ def testFilterFdbEntries(self, testData): try: self.__setUp(testData) argv = [ + "filter_fdb_entries", "-a", self.ARP_FILENAME, "-f", From ede5e983bd38dd820838869d52f667a9994eac7f Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Fri, 14 Aug 2020 14:50:47 -0700 Subject: [PATCH 5/8] nitpicking... --- {fdb_utils => fdbutils}/__init__.py | 0 {fdb_utils => fdbutils}/filter_fdb_entries.py | 0 setup.py | 2 +- tests/filter_fdb_entries_test.py | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename {fdb_utils => fdbutils}/__init__.py (100%) rename {fdb_utils => fdbutils}/filter_fdb_entries.py (100%) diff --git a/fdb_utils/__init__.py b/fdbutils/__init__.py similarity index 100% rename from fdb_utils/__init__.py rename to fdbutils/__init__.py diff --git a/fdb_utils/filter_fdb_entries.py b/fdbutils/filter_fdb_entries.py similarity index 100% rename from fdb_utils/filter_fdb_entries.py rename to fdbutils/filter_fdb_entries.py diff --git a/setup.py b/setup.py index e274a9a56d..eea2586d41 100644 --- a/setup.py +++ b/setup.py @@ -122,6 +122,7 @@ 'counterpoll = counterpoll.main:cli', 'crm = crm.main:cli', 'debug = debug.main:cli', + 'filter_fdb_entries = fdbutils.filter_fdb_entries:main', 'pfcwd = pfcwd.main:cli', 'sfputil = sfputil.main:cli', 'ssdutil = ssdutil.main:ssdutil', @@ -139,7 +140,6 @@ 'sonic_installer = sonic_installer.main:sonic_installer', # Deprecated 'undebug = undebug.main:cli', 'watchdogutil = watchdogutil.main:watchdogutil', - 'filter_fdb_entries = fdb_utils.filter_fdb_entries:main', ] }, # NOTE: sonic-utilities also depends on other packages that are either only diff --git a/tests/filter_fdb_entries_test.py b/tests/filter_fdb_entries_test.py index 1c6a0a0001..8b17c8abf4 100644 --- a/tests/filter_fdb_entries_test.py +++ b/tests/filter_fdb_entries_test.py @@ -7,7 +7,7 @@ from collections import defaultdict from filter_fdb_input.test_vectors import filterFdbEntriesTestVector -from fdb_utils.filter_fdb_entries import main as filterFdbMain +from fdbutils.filter_fdb_entries import main as filterFdbMain class TestFilterFdbEntries(object): """ From 3a00245c7ead7ca7ad2787d60902efe5a02ff4d3 Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Fri, 14 Aug 2020 18:01:45 -0700 Subject: [PATCH 6/8] fix build failure and make lgtm happy --- fdbutils/filter_fdb_entries.py | 7 ++----- setup.py | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/fdbutils/filter_fdb_entries.py b/fdbutils/filter_fdb_entries.py index 6299f6aa38..684e0020e6 100755 --- a/fdbutils/filter_fdb_entries.py +++ b/fdbutils/filter_fdb_entries.py @@ -1,5 +1,4 @@ import json -import sys import os import argparse import syslog @@ -135,6 +134,7 @@ def main(argv): config_db_filename = args.config_db backup_file = args.backup_file + res = 0 try: syslog.openlog('filter_fdb_entries') file_exists_or_raise(fdb_filename) @@ -145,12 +145,9 @@ def main(argv): except KeyboardInterrupt: syslog.syslog(syslog.LOG_NOTICE, "SIGINT received. Quitting") res = 1 - except Exception as e: - syslog.syslog(syslog.LOG_ERR, "Got an exception %s: Traceback: %s" % (str(e), traceback.format_exc())) - res = 2 else: filter_fdb_entries(fdb_filename, arp_filename, config_db_filename, backup_file) finally: syslog.closelog() - return 0 + return res diff --git a/setup.py b/setup.py index eea2586d41..30fa218a16 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ 'ssdutil', 'pfc', 'psuutil', + 'fdbutils', 'fwutil', 'pcieutil', 'pddf_fanutil', From ca0826bc7962c9ccd4406d03c476a6ba7bdb188c Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Fri, 14 Aug 2020 18:30:25 -0700 Subject: [PATCH 7/8] rename fdbutils -> fdbutil --- {fdbutils => fdbutil}/__init__.py | 0 {fdbutils => fdbutil}/filter_fdb_entries.py | 0 pytest.ini | 2 +- setup.py | 4 ++-- tests/filter_fdb_entries_test.py | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename {fdbutils => fdbutil}/__init__.py (100%) rename {fdbutils => fdbutil}/filter_fdb_entries.py (100%) diff --git a/fdbutils/__init__.py b/fdbutil/__init__.py similarity index 100% rename from fdbutils/__init__.py rename to fdbutil/__init__.py diff --git a/fdbutils/filter_fdb_entries.py b/fdbutil/filter_fdb_entries.py similarity index 100% rename from fdbutils/filter_fdb_entries.py rename to fdbutil/filter_fdb_entries.py diff --git a/pytest.ini b/pytest.ini index b20e0d15b4..40d110e3b0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] filterwarnings = ignore::DeprecationWarning -addopts = --cov=acl_loader --cov=clear --cov=config --cov=connect --cov=consutil --cov=counterpoll --cov=crm --cov=debug --cov=fdb_utils --cov=fwutil --cov=pcieutil --cov=pfcwd --cov=psuutil --cov=pddf_fanutil --cov=pddf_ledutil --cov=pddf_psuutil --cov=pddf_thermalutil --cov=scripts --cov=sfputil --cov=show --cov=sonic_installer --cov=ssdutil --cov=utilities_common --cov=watchdogutil --cov-report html --cov-report term --cov-report xml +addopts = --cov=acl_loader --cov=clear --cov=config --cov=connect --cov=consutil --cov=counterpoll --cov=crm --cov=debug --cov=fdbutil --cov=fwutil --cov=pcieutil --cov=pfcwd --cov=psuutil --cov=pddf_fanutil --cov=pddf_ledutil --cov=pddf_psuutil --cov=pddf_thermalutil --cov=scripts --cov=sfputil --cov=show --cov=sonic_installer --cov=ssdutil --cov=utilities_common --cov=watchdogutil --cov-report html --cov-report term --cov-report xml diff --git a/setup.py b/setup.py index 30fa218a16..7d20a0b11f 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ 'ssdutil', 'pfc', 'psuutil', - 'fdbutils', + 'fdbutil', 'fwutil', 'pcieutil', 'pddf_fanutil', @@ -123,7 +123,7 @@ 'counterpoll = counterpoll.main:cli', 'crm = crm.main:cli', 'debug = debug.main:cli', - 'filter_fdb_entries = fdbutils.filter_fdb_entries:main', + 'filter_fdb_entries = fdbutil.filter_fdb_entries:main', 'pfcwd = pfcwd.main:cli', 'sfputil = sfputil.main:cli', 'ssdutil = ssdutil.main:ssdutil', diff --git a/tests/filter_fdb_entries_test.py b/tests/filter_fdb_entries_test.py index 8b17c8abf4..fbbff67ea9 100644 --- a/tests/filter_fdb_entries_test.py +++ b/tests/filter_fdb_entries_test.py @@ -7,7 +7,7 @@ from collections import defaultdict from filter_fdb_input.test_vectors import filterFdbEntriesTestVector -from fdbutils.filter_fdb_entries import main as filterFdbMain +from fdbutil.filter_fdb_entries import main as filterFdbMain class TestFilterFdbEntries(object): """ From b989f74dc1f16c4b2cad2f75ae776b609820f946 Mon Sep 17 00:00:00 2001 From: Tamer Ahmed Date: Fri, 14 Aug 2020 21:29:24 -0700 Subject: [PATCH 8/8] fix lack of argv when called from Python generated script. --- fdbutil/filter_fdb_entries.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fdbutil/filter_fdb_entries.py b/fdbutil/filter_fdb_entries.py index 684e0020e6..23380ed0ef 100755 --- a/fdbutil/filter_fdb_entries.py +++ b/fdbutil/filter_fdb_entries.py @@ -1,4 +1,5 @@ import json +import sys import os import argparse import syslog @@ -121,7 +122,7 @@ def file_exists_or_raise(filename): if not os.path.exists(filename): raise Exception("file '{0}' does not exist".format(filename)) -def main(argv): +def main(argv=sys.argv): parser = argparse.ArgumentParser() parser.add_argument('-f', '--fdb', type=str, default='/tmp/fdb.json', help='fdb file name') parser.add_argument('-a', '--arp', type=str, default='/tmp/arp.json', help='arp file name')