From 5f748566924c5625f4305c8845c320ffea08aec3 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sat, 18 Jul 2020 23:44:41 +0000 Subject: [PATCH 01/25] Introduce sonic-py-common package --- .gitignore | 6 + .../build_templates/sonic_debian_extension.j2 | 6 + rules/sonic-py-common.dep | 8 + rules/sonic-py-common.mk | 7 + slave.mk | 2 + src/sonic-py-common/setup.py | 37 ++ .../sonic_py_common/__init__.py | 0 .../sonic_py_common/daemon_base.py | 82 +++++ .../sonic_py_common/device_info.py | 346 ++++++++++++++++++ src/sonic-py-common/sonic_py_common/logger.py | 56 +++ 10 files changed, 550 insertions(+) create mode 100644 rules/sonic-py-common.dep create mode 100644 rules/sonic-py-common.mk create mode 100644 src/sonic-py-common/setup.py create mode 100644 src/sonic-py-common/sonic_py_common/__init__.py create mode 100644 src/sonic-py-common/sonic_py_common/daemon_base.py create mode 100644 src/sonic-py-common/sonic_py_common/device_info.py create mode 100644 src/sonic-py-common/sonic_py_common/logger.py diff --git a/.gitignore b/.gitignore index 948a266ba451..38f7f9742f32 100644 --- a/.gitignore +++ b/.gitignore @@ -32,10 +32,16 @@ installer/x86_64/platforms/ src/sonic-config-engine/**/*.pyc src/sonic-config-engine/build src/sonic-config-engine/sonic_config_engine.egg-info + src/sonic-daemon-base/**/*.pyc src/sonic-daemon-base/build src/sonic-daemon-base/sonic_daemon_base.egg-info +src/sonic-py-common/**/*.pyc +src/sonic-py-common/build +src/sonic-py-common/dist +src/sonic-py-common/sonic_py_common.egg-info + # Misc. files asic_config_checksum files/Aboot/boot0 diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 1732580b5ac8..050d2567e677 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -163,6 +163,12 @@ sudo cp {{daemon_base_py2_wheel_path}} $FILESYSTEM_ROOT/$DAEMON_BASE_PY2_WHEEL_N sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $DAEMON_BASE_PY2_WHEEL_NAME sudo rm -rf $FILESYSTEM_ROOT/$DAEMON_BASE_PY2_WHEEL_NAME +# Install sonic-py-common Python 2 package +SONIC_PY_COMMON_PY2_WHEEL_NAME=$(basename {{sonic_py_common_py2_wheel_path}}) +sudo cp {{sonic_py_common_py2_wheel_path}} $FILESYSTEM_ROOT/$SONIC_PY_COMMON_PY2_WHEEL_NAME +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SONIC_PY_COMMON_PY2_WHEEL_NAME +sudo rm -rf $FILESYSTEM_ROOT/$SONIC_PY_COMMON_PY2_WHEEL_NAME + # Install built Python Click package (and its dependencies via 'apt-get -y install -f') # Do this before installing sonic-utilities so that it doesn't attempt to install # an older version as part of its dependencies diff --git a/rules/sonic-py-common.dep b/rules/sonic-py-common.dep new file mode 100644 index 000000000000..e26529d69c95 --- /dev/null +++ b/rules/sonic-py-common.dep @@ -0,0 +1,8 @@ +SPATH := $($(SONIC_PY_COMMON_PY2)_SRC_PATH) +DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-py-common.mk rules/sonic-py-common.dep +DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) +DEP_FILES += $(shell git ls-files $(SPATH)) + +$(SONIC_PY_COMMON_PY2)_CACHE_MODE := GIT_CONTENT_SHA +$(SONIC_PY_COMMON_PY2)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(SONIC_PY_COMMON_PY2)_DEP_FILES := $(DEP_FILES) diff --git a/rules/sonic-py-common.mk b/rules/sonic-py-common.mk new file mode 100644 index 000000000000..e7afc184fd8d --- /dev/null +++ b/rules/sonic-py-common.mk @@ -0,0 +1,7 @@ +# SONIC_PY_COMMON_PY2 package + +SONIC_PY_COMMON_PY2 = sonic_py_common-1.0-py2-none-any.whl +$(SONIC_PY_COMMON_PY2)_SRC_PATH = $(SRC_PATH)/sonic-py-common +$(SONIC_PY_COMMON_PY2)_DEPENDS += $(SWSSSDK_PY2) +$(SONIC_PY_COMMON_PY2)_PYTHON_VERSION = 2 +SONIC_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2) diff --git a/slave.mk b/slave.mk index cd22d665cfb8..9b145e9688ad 100644 --- a/slave.mk +++ b/slave.mk @@ -805,6 +805,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ $$(addprefix $(FILES_PATH)/,$$($$*_FILES)) \ $(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_ZTP))) \ $(addprefix $(PYTHON_DEBS_PATH)/,$(SONIC_UTILS)) \ + $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY2)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2)) \ @@ -835,6 +836,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ export installer_debs="$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$($*_INSTALLS))" export lazy_installer_debs="$(foreach deb, $($*_LAZY_INSTALLS),$(foreach device, $($(deb)_PLATFORM),$(addprefix $(device)@, $(IMAGE_DISTRO_DEBS_PATH)/$(deb))))" export installer_images="$(addprefix $(TARGET_PATH)/,$($*_DOCKERS))" + export sonic_py_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY2))" export config_engine_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE))" export swsssdk_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY2))" export swsssdk_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY3))" diff --git a/src/sonic-py-common/setup.py b/src/sonic-py-common/setup.py new file mode 100644 index 000000000000..3f984940d539 --- /dev/null +++ b/src/sonic-py-common/setup.py @@ -0,0 +1,37 @@ +from setuptools import setup + +dependencies = [ + 'swsssdk>=2.0.1', +] + +high_performance_deps = [ + 'swsssdk[high_perf]>=2.0.1', +] + +setup( + name='sonic-py-common', + version='1.0', + description='Common Python libraries for SONiC', + license='Apache 2.0', + author='SONiC Team', + author_email='linuxnetdev@microsoft.com', + url='https://github.com/Azure/SONiC', + maintainer='Joe LeVeque', + maintainer_email='jolevequ@microsoft.com', + install_requires=dependencies, + extras_require={ + 'high_perf': high_performance_deps, + }, + packages=[ + 'sonic_py_common', + ], + classifiers=[ + 'Intended Audience :: Developers', + 'Operating System :: Linux', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python', + ], + keywords='SONiC sonic PYTHON python COMMON common', +) + diff --git a/src/sonic-py-common/sonic_py_common/__init__.py b/src/sonic-py-common/sonic_py_common/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/sonic-py-common/sonic_py_common/daemon_base.py b/src/sonic-py-common/sonic_py_common/daemon_base.py new file mode 100644 index 000000000000..1eadef4e66c7 --- /dev/null +++ b/src/sonic-py-common/sonic_py_common/daemon_base.py @@ -0,0 +1,82 @@ +import imp +import os +import signal +import subprocess +import sys + +from . import device_info, logger + +# +# Constants ==================================================================== +# +REDIS_TIMEOUT_MSECS = 0 + +EEPROM_MODULE_NAME = 'eeprom' +EEPROM_CLASS_NAME = 'board' + +# +# Helper functions ============================================================= +# + +def db_connect(db_name): + from swsscommon import swsscommon + return swsscommon.DBConnector(db_name, REDIS_TIMEOUT_MSECS, True) + +# +# DaemonBase =================================================================== +# + +class DaemonBase(logger.Logger): + def __init__(self, log_identifier): + super(DaemonBase, self).__init__(log_identifier, logger.FACILITY_DAEMON) + + # Register our default signal handlers, unless the signal already has a + # handler registered, most likely from a subclass implementation + if not signal.getsignal(signal.SIGHUP): + signal.signal(signal.SIGHUP, self.signal_handler) + if not signal.getsignal(signal.SIGINT): + signal.signal(signal.SIGINT, self.signal_handler) + if not signal.getsignal(signal.SIGTERM): + signal.signal(signal.SIGTERM, self.signal_handler) + + # Default signal handler; can be overridden by subclass + def signal_handler(self, sig, frame): + if sig == signal.SIGHUP: + log_info("DaemonBase: Caught SIGHUP - ignoring...") + elif sig == signal.SIGINT: + log_info("DaemonBase: Caught SIGINT - exiting...") + sys.exit(128 + sig) + elif sig == signal.SIGTERM: + log_info("DaemonBase: Caught SIGTERM - exiting...") + sys.exit(128 + sig) + else: + log_warning("DaemonBase: Caught unhandled signal '{}'".format(sig)) + + # Loads platform specific psuutil module from source + def load_platform_util(self, module_name, class_name): + platform_util = None + + # Get path to platform and hwsku + (platform_path, hwsku_path) = device_info.get_path_to_platform_and_hwsku() + + try: + module_file = "/".join([platform_path, "plugins", module_name + ".py"]) + module = imp.load_source(module_name, module_file) + except IOError, e: + raise IOError("Failed to load platform module '%s': %s" % (module_name, str(e))) + + try: + platform_util_class = getattr(module, class_name) + # board class of eeprom requires 4 paramerters, need special treatment here. + if module_name == EEPROM_MODULE_NAME and class_name == EEPROM_CLASS_NAME: + platform_util = platform_util_class('','','','') + else: + platform_util = platform_util_class() + except AttributeError, e: + raise AttributeError("Failed to instantiate '%s' class: %s" % (class_name, str(e))) + + return platform_util + + # Runs daemon + def run(self): + raise NotImplementedError() diff --git a/src/sonic-py-common/sonic_py_common/device_info.py b/src/sonic-py-common/sonic_py_common/device_info.py new file mode 100644 index 000000000000..5522112634c3 --- /dev/null +++ b/src/sonic-py-common/sonic_py_common/device_info.py @@ -0,0 +1,346 @@ +import glob +import os +import subprocess +import yaml +from natsort import natsorted + +# TODD: Replace with swsscommon +from swsssdk import ConfigDBConnector, SonicDBConfig + +USR_SHARE_SONIC_PATH = "/usr/share/sonic" +HOST_DEVICE_PATH = USR_SHARE_SONIC_PATH + "/device" +CONTAINER_PLATFORM_PATH = USR_SHARE_SONIC_PATH + "/platform" + +MACHINE_CONF_PATH = "/host/machine.conf" +SONIC_VERSION_YAML_PATH = "/etc/sonic/sonic_version.yml" + +# Port configuration file names +PORT_CONFIG_FILE = "port_config.ini" +PLATFORM_JSON_FILE = "platform.json" + +# Multi-NPU constants +NPU_NAME_PREFIX = "asic" +NAMESPACE_PATH_GLOB = "/run/netns/*" +ASIC_CONF_FILENAME = "asic.conf" +FRONTEND_ASIC_SUB_ROLE = "FrontEnd" +BACKEND_ASIC_SUB_ROLE = "BackEnd" + + +def get_machine_info(): + """ + Retreives data from the machine configuration file + + Returns: + A dictionary containing the key/value pairs as found in the machine + configuration file + """ + if not os.path.isfile(MACHINE_CONF_PATH): + return None + + machine_vars = {} + with open(MACHINE_CONF_PATH) as machine_conf_file: + for line in machine_conf_file: + tokens = line.split('=') + if len(tokens) < 2: + continue + machine_vars[tokens[0]] = tokens[1].strip() + + return machine_vars + + +def get_platform(): + """ + Retrieve the device's platform identifier + + Returns: + A string containing the device's platform identifier + """ + # First, attempt to retrieve the platform string from Config DB + config_db = ConfigDBConnector() + config_db.connect() + + metadata = config_db.get_table('DEVICE_METADATA') + + if 'localhost' in metadata and 'platform' in metadata['localhost']: + return metadata['localhost']['platform'] + + # If we were unable to retrieve the platform string from Config DB, attempt + # to retrieve it from the machine configuration file + machine_info = get_machine_info() + if machine_info: + if machine_info.has_key('onie_platform'): + return machine_info['onie_platform'] + elif machine_info.has_key('aboot_platform'): + return machine_info['aboot_platform'] + + return None + + +def get_hwsku(): + """ + Retrieve the device's hardware SKU identifier + + Returns: + A string containing the device's hardware SKU identifier + """ + config_db = ConfigDBConnector() + config_db.connect() + + metadata = config_db.get_table('DEVICE_METADATA') + + if 'localhost' in metadata and 'hwsku' in metadata['localhost']: + return metadata['localhost']['hwsku'] + + return "" + + +def get_platform_and_hwsku(): + """ + Convenience function which retrieves both the device's platform identifier + and hardware SKU identifier + + Returns: + A tuple of two strings, the first containing the device's + platform identifier, the second containing the device's + hardware SKU identifier + """ + platform = get_platform() + hwsku = get_hwsku() + + return (platform, hwsku) + + +def get_asic_conf_file_path(): + """ + Retrieves the path to the ASIC conguration file on the device + + Args: + platform: Platform string of the device + + Returns: + A string containing the path to the ASIC conguration file on success, + None on failure + """ + asic_conf_path_candidates = [] + + asic_conf_path_candidates.append(os.path.join(CONTAINER_PLATFORM_PATH, ASIC_CONF_FILENAME)) + + platform = get_platform() + if platform: + asic_conf_path_candidates.append(os.path.join(HOST_DEVICE_PATH, platform, ASIC_CONF_FILENAME)) + + for asic_conf_file_path in asic_conf_path_candidates: + if os.path.isfile(asic_conf_file_path): + return asic_conf_file_path + + return None + + +def get_paths_to_platform_and_hwsku_dirs(): + """ + Retreives the paths to the device's platform and hardware SKU data + directories + + Returns: + A tuple of two strings, the first containing the path to the platform + directory of the device, the second containing the path to the hardware + SKU directory of the device + """ + # Get platform and hwsku + (platform, hwsku) = get_platform_and_hwsku() + + # Determine whether we're running in a container or on the host + platform_path_host = os.path.join(HOST_DEVICE_PATH, platform) + + if os.path.isdir(CONTAINER_PLATFORM_PATH): + platform_path = CONTAINER_PLATFORM_PATH + elif os.path.isdir(platform_path_host): + platform_path = platform_path_host + else: + raise OSError("Failed to locate platform directory") + + hwsku_path = os.path.join(platform_path, hwsku) + + return (platform_path, hwsku_path) + + +def get_path_to_port_config_file(): + """ + Retrieves the path to the device's port configuration file + + Returns: + A string containing the path the the device's port configuration file + """ + # Get platform and hwsku path + (platform_path, hwsku_path) = get_paths_to_platform_and_hwsku_dirs() + + # First check for the presence of the new 'platform.json' file + port_config_file_path = os.path.join(platform_path, PLATFORM_JSON_FILE) + if not os.path.isfile(port_config_file_path): + # platform.json doesn't exist. Try loading the legacy 'port_config.ini' file + port_config_file_path = os.path.join(hwsku_path, PORT_CONFIG_FILE) + if not os.path.isfile(port_config_file_path): + raise OSError("Failed to detect port config file: {}".format(port_config_file_path)) + + return port_config_file_path + + +def get_sonic_version_info(): + if not os.path.isfile(SONIC_VERSION_YAML_PATH): + return None + + data = {} + with open(SONIC_VERSION_YAML_PATH) as stream: + if yaml.__version__ >= "5.1": + data = yaml.full_load(stream) + else: + data = yaml.load(stream) + + return data + + +# +# Multi-NPU functionality +# + +def get_num_npus(): + platform = get_platform() + asic_conf_file_path = get_asic_conf_file_path() + if asic_conf_file_path is None: + return 1 + with open(asic_conf_file_path) as asic_conf_file: + for line in asic_conf_file: + tokens = line.split('=') + if len(tokens) < 2: + continue + if tokens[0].lower() == 'num_asic': + num_npus = tokens[1].strip() + return int(num_npus) + + +def is_multi_npu(): + num_npus = get_num_npus() + return (num_npus > 1) + + +def get_npu_id_from_name(npu_name): + if npu_name.startswith(NPU_NAME_PREFIX): + return npu_name[len(NPU_NAME_PREFIX):] + else: + return None + + +def get_namespaces(): + """ + In a multi NPU platform, each NPU is in a Linux Namespace. + This method returns list of all the Namespace present on the device + """ + ns_list = [] + for path in glob.glob(NAMESPACE_PATH_GLOB): + ns = os.path.basename(path) + ns_list.append(ns) + return natsorted(ns_list) + + +def get_all_namespaces(): + """ + In case of Multi-Asic platform, Each ASIC will have a linux network namespace created. + So we loop through the databases in different namespaces and depending on the sub_role + decide whether this is a front end ASIC/namespace or a back end one. + """ + front_ns = [] + back_ns = [] + num_npus = get_num_npus() + SonicDBConfig.load_sonic_global_db_config() + + if is_multi_npu(): + for npu in range(num_npus): + namespace = "{}{}".format(NPU_NAME_PREFIX, npu) + config_db = ConfigDBConnector(use_unix_socket_path=True, namespace=namespace) + config_db.connect() + + metadata = config_db.get_table('DEVICE_METADATA') + if metadata['localhost']['sub_role'] == FRONTEND_ASIC_SUB_ROLE: + front_ns.append(namespace) + elif metadata['localhost']['sub_role'] == BACKEND_ASIC_SUB_ROLE: + back_ns.append(namespace) + + return {'front_ns':front_ns, 'back_ns':back_ns} + + +def _valid_mac_address(mac): + return bool(re.match("^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$", mac)) + + +def get_system_mac(namespace=None): + version_info = get_sonic_version_info() + + if (version_info['asic_type'] == 'mellanox'): + # With Mellanox ONIE release(2019.05-5.2.0012) and above + # "onie_base_mac" was added to /host/machine.conf: + # onie_base_mac=e4:1d:2d:44:5e:80 + # So we have another way to get the mac address besides decode syseeprom + # By this can mitigate the dependency on the hw-management service + base_mac_key = "onie_base_mac" + machine_vars = get_machine_info() + if machine_vars is not None and base_mac_key in machine_vars: + mac = machine_vars[base_mac_key] + mac = mac.strip() + if _valid_mac_address(mac): + return mac + + hw_mac_entry_cmds = [ "sudo decode-syseeprom -m" ] + elif (version_info['asic_type'] == 'marvell'): + # Try valid mac in eeprom, else fetch it from eth0 + platform = get_platform() + hwsku = get_hwsku() + profile_cmd = 'cat' + HOST_DEVICE_PATH + '/' + platform +'/'+ hwsku +'/profile.ini | grep switchMacAddress | cut -f2 -d=' + hw_mac_entry_cmds = [ profile_cmd, "sudo decode-syseeprom -m", "ip link show eth0 | grep ether | awk '{print $2}'" ] + else: + mac_address_cmd = "cat /sys/class/net/eth0/address" + if namespace is not None: + mac_address_cmd = "sudo ip netns exec {} {}".format(namespace, mac_address_cmd) + + hw_mac_entry_cmds = [mac_address_cmd] + + for get_mac_cmd in hw_mac_entry_cmds: + proc = subprocess.Popen(get_mac_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (mac, err) = proc.communicate() + if err: + continue + mac = mac.strip() + if _valid_mac_address(mac): + break + + if not _valid_mac_address(mac): + return None + + # Align last byte of MAC if necessary + if version_info and version_info['asic_type'] == 'centec': + last_byte = mac[-2:] + aligned_last_byte = format(int(int(last_byte, 16) + 1), '02x') + mac = mac[:-2] + aligned_last_byte + return mac + + +def get_system_routing_stack(): + """ + Retrieves the routing stack being utilized on this device + + Returns: + A string containing the name of the routing stack in use on the device + """ + command = "sudo docker ps | grep bgp | awk '{print$2}' | cut -d'-' -f3 | cut -d':' -f1" + + try: + proc = subprocess.Popen(command, + stdout=subprocess.PIPE, + shell=True, + stderr=subprocess.STDOUT) + stdout = proc.communicate()[0] + proc.wait() + result = stdout.rstrip('\n') + except OSError, e: + raise OSError("Cannot detect routing stack") + + return result diff --git a/src/sonic-py-common/sonic_py_common/logger.py b/src/sonic-py-common/sonic_py_common/logger.py new file mode 100644 index 000000000000..dc5a685d99fe --- /dev/null +++ b/src/sonic-py-common/sonic_py_common/logger.py @@ -0,0 +1,56 @@ +import os +import syslog + +""" + Logging functionality for SONiC Python applications +""" + +FACILITY_USER = syslog.LOG_USER +FACILITY_DAEMON = syslog.LOG_DAEMON + +class Logger(object): + """ + Logger class for SONiC Python applications + """ + def __init__(self, log_identifier=None, log_facility=FACILITY_USER): + self.syslog = syslog + + if not log_identifier: + log_identifier = os.path.basename(sys.argv[0]) + + self.syslog.openlog(ident=log_identifier, + logoption=(syslog.LOG_PID | syslog.LOG_NDELAY), + facility=log_facility) + + def __del__(self): + self.syslog.closelog() + + def log_error(self, msg, also_print_to_console=False): + self.syslog.syslog(self.syslog.LOG_ERR, msg) + + if also_print_to_console: + print(msg) + + def log_warning(self, msg, also_print_to_console=False): + self.syslog.syslog(self.syslog.LOG_WARNING, msg) + + if also_print_to_console: + print(msg) + + def log_notice(self, msg, also_print_to_console=False): + self.syslog.syslog(self.syslog.LOG_NOTICE, msg) + + if also_print_to_console: + print(msg) + + def log_info(self, msg, also_print_to_console=False): + self.syslog.syslog(self.syslog.LOG_INFO, msg) + + if also_print_to_console: + print(msg) + + def log_debug(self, msg, also_print_to_console=False): + self.syslog.syslog(self.syslog.LOG_DEBUG, msg) + + if also_print_to_console: + print(msg) From ac83ce3b49163484beaca411e5796f0cba8646b2 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sun, 19 Jul 2020 00:06:06 +0000 Subject: [PATCH 02/25] Install sonic_py_common in PMon container --- rules/docker-platform-monitor.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/docker-platform-monitor.mk b/rules/docker-platform-monitor.mk index 11a6f1c94c86..4fd9015272e4 100644 --- a/rules/docker-platform-monitor.mk +++ b/rules/docker-platform-monitor.mk @@ -13,6 +13,7 @@ endif $(DOCKER_PLATFORM_MONITOR)_PYTHON_DEBS += $(SONIC_LEDD) $(SONIC_XCVRD) $(SONIC_PSUD) $(SONIC_SYSEEPROMD) $(SONIC_THERMALCTLD) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PLATFORM_COMMON_PY2) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SWSSSDK_PY2) +$(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PLATFORM_API_PY2) $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_DAEMON_BASE_PY2) From 55a1d79925be895baf1544ab8a222a6a9e8ae53f Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sun, 19 Jul 2020 00:09:21 +0000 Subject: [PATCH 03/25] Add sonic-py-common as dependency of mlnx-platform-api --- platform/mellanox/mlnx-platform-api.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/mellanox/mlnx-platform-api.mk b/platform/mellanox/mlnx-platform-api.mk index 7bbbc3c70b0e..c579b4539283 100644 --- a/platform/mellanox/mlnx-platform-api.mk +++ b/platform/mellanox/mlnx-platform-api.mk @@ -3,7 +3,7 @@ SONIC_PLATFORM_API_PY2 = mlnx_platform_api-1.0-py2-none-any.whl $(SONIC_PLATFORM_API_PY2)_SRC_PATH = $(PLATFORM_PATH)/mlnx-platform-api $(SONIC_PLATFORM_API_PY2)_PYTHON_VERSION = 2 -$(SONIC_PLATFORM_API_PY2)_DEPENDS = $(SONIC_PLATFORM_COMMON_PY2) $(SONIC_DAEMON_BASE_PY2) $(SONIC_CONFIG_ENGINE) +$(SONIC_PLATFORM_API_PY2)_DEPENDS = $(SONIC_PY_COMMON_PY2) $(SONIC_PLATFORM_COMMON_PY2) $(SONIC_DAEMON_BASE_PY2) $(SONIC_CONFIG_ENGINE) SONIC_PYTHON_WHEELS += $(SONIC_PLATFORM_API_PY2) export mlnx_platform_api_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_API_PY2))" From 862e786126d681715aa4bc6ca5d0b3efb203667d Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sun, 19 Jul 2020 00:10:12 +0000 Subject: [PATCH 04/25] Add sonic-py-common as dependency of sonic-thermalctld --- rules/sonic-thermalctld.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/sonic-thermalctld.mk b/rules/sonic-thermalctld.mk index 775082e7bbce..cf42c2544af9 100644 --- a/rules/sonic-thermalctld.mk +++ b/rules/sonic-thermalctld.mk @@ -2,5 +2,5 @@ SONIC_THERMALCTLD = python-sonic-thermalctld_1.0-1_all.deb $(SONIC_THERMALCTLD)_SRC_PATH = $(SRC_PATH)/sonic-platform-daemons/sonic-thermalctld -$(SONIC_THERMALCTLD)_WHEEL_DEPENDS = $(SONIC_DAEMON_BASE_PY2) +$(SONIC_THERMALCTLD)_WHEEL_DEPENDS = $(SONIC_PY_COMMON_PY2) $(SONIC_DAEMON_BASE_PY2) SONIC_PYTHON_STDEB_DEBS += $(SONIC_THERMALCTLD) From 284c0db40b2c4007eb9b9c14e67aa16337bfd0e8 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sun, 19 Jul 2020 00:12:18 +0000 Subject: [PATCH 05/25] Fix LGTM alerts --- src/sonic-py-common/sonic_py_common/daemon_base.py | 2 -- src/sonic-py-common/sonic_py_common/device_info.py | 1 - 2 files changed, 3 deletions(-) diff --git a/src/sonic-py-common/sonic_py_common/daemon_base.py b/src/sonic-py-common/sonic_py_common/daemon_base.py index 1eadef4e66c7..eec2c9233320 100644 --- a/src/sonic-py-common/sonic_py_common/daemon_base.py +++ b/src/sonic-py-common/sonic_py_common/daemon_base.py @@ -1,7 +1,5 @@ import imp -import os import signal -import subprocess import sys from . import device_info, logger diff --git a/src/sonic-py-common/sonic_py_common/device_info.py b/src/sonic-py-common/sonic_py_common/device_info.py index 5522112634c3..f9d1177fbf3b 100644 --- a/src/sonic-py-common/sonic_py_common/device_info.py +++ b/src/sonic-py-common/sonic_py_common/device_info.py @@ -204,7 +204,6 @@ def get_sonic_version_info(): # def get_num_npus(): - platform = get_platform() asic_conf_file_path = get_asic_conf_file_path() if asic_conf_file_path is None: return 1 From 1de13859c8b87b13c4a1002ff997d976bd069ab4 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sun, 19 Jul 2020 01:05:07 +0000 Subject: [PATCH 06/25] Update docstring for get_asic_conf_file_path() --- src/sonic-py-common/sonic_py_common/device_info.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/sonic-py-common/sonic_py_common/device_info.py b/src/sonic-py-common/sonic_py_common/device_info.py index f9d1177fbf3b..6135db11a91d 100644 --- a/src/sonic-py-common/sonic_py_common/device_info.py +++ b/src/sonic-py-common/sonic_py_common/device_info.py @@ -114,9 +114,6 @@ def get_asic_conf_file_path(): """ Retrieves the path to the ASIC conguration file on the device - Args: - platform: Platform string of the device - Returns: A string containing the path to the ASIC conguration file on success, None on failure From 97fef087456644073af60e173724d8191a6520fc Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sun, 19 Jul 2020 01:06:33 +0000 Subject: [PATCH 07/25] Update comment --- src/sonic-py-common/sonic_py_common/daemon_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-py-common/sonic_py_common/daemon_base.py b/src/sonic-py-common/sonic_py_common/daemon_base.py index eec2c9233320..e6c6a9b1775d 100644 --- a/src/sonic-py-common/sonic_py_common/daemon_base.py +++ b/src/sonic-py-common/sonic_py_common/daemon_base.py @@ -50,7 +50,7 @@ def signal_handler(self, sig, frame): else: log_warning("DaemonBase: Caught unhandled signal '{}'".format(sig)) - # Loads platform specific psuutil module from source + # Loads platform specific platform module from source def load_platform_util(self, module_name, class_name): platform_util = None From bedbc1a21d806a0f08f1d9eed58f2c65f183a7da Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sun, 19 Jul 2020 01:08:47 +0000 Subject: [PATCH 08/25] Docstring style --- .../celestica/x86_64-cel_seastone-r0/sonic_platform/helper.py | 4 ++-- src/sonic-bgpcfgd/.cache/v/cache/lastfailed | 1 + src/sonic-py-common/sonic_py_common/logger.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 src/sonic-bgpcfgd/.cache/v/cache/lastfailed diff --git a/device/celestica/x86_64-cel_seastone-r0/sonic_platform/helper.py b/device/celestica/x86_64-cel_seastone-r0/sonic_platform/helper.py index 86fc5ea726ef..79d22fc377d6 100644 --- a/device/celestica/x86_64-cel_seastone-r0/sonic_platform/helper.py +++ b/device/celestica/x86_64-cel_seastone-r0/sonic_platform/helper.py @@ -3,7 +3,7 @@ import os import struct import subprocess -from sonic_daemon_base.daemon_base import DaemonBase +from sonic_py_common import device_info from mmap import * HOST_CHK_CMD = "docker > /dev/null 2>&1" @@ -13,7 +13,7 @@ class APIHelper(): def __init__(self): - (self.platform, self.hwsku) = DaemonBase().get_platform_and_hwsku() + (self.platform, self.hwsku) = device_info.get_platform_and_hwsku() def is_host(self): return os.system(HOST_CHK_CMD) == 0 diff --git a/src/sonic-bgpcfgd/.cache/v/cache/lastfailed b/src/sonic-bgpcfgd/.cache/v/cache/lastfailed new file mode 100644 index 000000000000..9e26dfeeb6e6 --- /dev/null +++ b/src/sonic-bgpcfgd/.cache/v/cache/lastfailed @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/sonic-py-common/sonic_py_common/logger.py b/src/sonic-py-common/sonic_py_common/logger.py index dc5a685d99fe..8c0ee87b83f6 100644 --- a/src/sonic-py-common/sonic_py_common/logger.py +++ b/src/sonic-py-common/sonic_py_common/logger.py @@ -2,7 +2,7 @@ import syslog """ - Logging functionality for SONiC Python applications +Logging functionality for SONiC Python applications """ FACILITY_USER = syslog.LOG_USER @@ -10,7 +10,7 @@ class Logger(object): """ - Logger class for SONiC Python applications + Logger class for SONiC Python applications """ def __init__(self, log_identifier=None, log_facility=FACILITY_USER): self.syslog = syslog From f49c4da16588da72687d5f2f03d3bae12633bb85 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Mon, 20 Jul 2020 22:37:22 +0000 Subject: [PATCH 09/25] Remove BGP cache file that snuck in to a commit --- src/sonic-bgpcfgd/.cache/v/cache/lastfailed | 1 - 1 file changed, 1 deletion(-) delete mode 100644 src/sonic-bgpcfgd/.cache/v/cache/lastfailed diff --git a/src/sonic-bgpcfgd/.cache/v/cache/lastfailed b/src/sonic-bgpcfgd/.cache/v/cache/lastfailed deleted file mode 100644 index 9e26dfeeb6e6..000000000000 --- a/src/sonic-bgpcfgd/.cache/v/cache/lastfailed +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file From d11f1a631a5f150785aa3920a8c2bdaf2ffeb4b0 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Mon, 20 Jul 2020 22:48:31 +0000 Subject: [PATCH 10/25] Add TODO comment above moving multi-ASIC functionality to its own module --- src/sonic-py-common/sonic_py_common/device_info.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sonic-py-common/sonic_py_common/device_info.py b/src/sonic-py-common/sonic_py_common/device_info.py index 6135db11a91d..97121680abf9 100644 --- a/src/sonic-py-common/sonic_py_common/device_info.py +++ b/src/sonic-py-common/sonic_py_common/device_info.py @@ -19,6 +19,7 @@ PLATFORM_JSON_FILE = "platform.json" # Multi-NPU constants +# TODO: Move Multi-ASIC-related functions and constants to a "multi_asic.py" module NPU_NAME_PREFIX = "asic" NAMESPACE_PATH_GLOB = "/run/netns/*" ASIC_CONF_FILENAME = "asic.conf" From 40a450b4390b5ff852cc261234bda301ed10ca6c Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Mon, 20 Jul 2020 22:52:04 +0000 Subject: [PATCH 11/25] Revert changes to celestica helper.py file; Will save for a separate PR --- .../celestica/x86_64-cel_seastone-r0/sonic_platform/helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/device/celestica/x86_64-cel_seastone-r0/sonic_platform/helper.py b/device/celestica/x86_64-cel_seastone-r0/sonic_platform/helper.py index 79d22fc377d6..86fc5ea726ef 100644 --- a/device/celestica/x86_64-cel_seastone-r0/sonic_platform/helper.py +++ b/device/celestica/x86_64-cel_seastone-r0/sonic_platform/helper.py @@ -3,7 +3,7 @@ import os import struct import subprocess -from sonic_py_common import device_info +from sonic_daemon_base.daemon_base import DaemonBase from mmap import * HOST_CHK_CMD = "docker > /dev/null 2>&1" @@ -13,7 +13,7 @@ class APIHelper(): def __init__(self): - (self.platform, self.hwsku) = device_info.get_platform_and_hwsku() + (self.platform, self.hwsku) = DaemonBase().get_platform_and_hwsku() def is_host(self): return os.system(HOST_CHK_CMD) == 0 From d65f6839035938e6d344fbcfe716f6ebb384285e Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Wed, 22 Jul 2020 03:45:01 +0000 Subject: [PATCH 12/25] Add ability to control minimum log priority, set default to NOTICE --- src/sonic-py-common/sonic_py_common/logger.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/sonic-py-common/sonic_py_common/logger.py b/src/sonic-py-common/sonic_py_common/logger.py index 8c0ee87b83f6..7963d65017d8 100644 --- a/src/sonic-py-common/sonic_py_common/logger.py +++ b/src/sonic-py-common/sonic_py_common/logger.py @@ -8,6 +8,13 @@ FACILITY_USER = syslog.LOG_USER FACILITY_DAEMON = syslog.LOG_DAEMON +PRIORITY_ERROR = syslog.LOG_ERR +PRIORITY_WARNING = syslog.LOG_WARNING +PRIORITY_NOTICE = syslog.LOG_NOTICE +PRIORITY_INFO = syslog.LOG_INFO +PRIORITY_DEBUG = syslog.LOG_DEBUG + + class Logger(object): """ Logger class for SONiC Python applications @@ -22,9 +29,15 @@ def __init__(self, log_identifier=None, log_facility=FACILITY_USER): logoption=(syslog.LOG_PID | syslog.LOG_NDELAY), facility=log_facility) + # Set the default minimum log priority to PRIORITY_NOTICE + self.set_min_log_priority(PRIORITY_NOTICE) + def __del__(self): self.syslog.closelog() + def set_min_log_priority(priority): + self.syslog.setlogmask(self.syslog.LOG_UPTO(priority)) + def log_error(self, msg, also_print_to_console=False): self.syslog.syslog(self.syslog.LOG_ERR, msg) From 4288a8b1310d6dd07c87654a21ccd8a298e9c981 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Wed, 22 Jul 2020 04:06:16 +0000 Subject: [PATCH 13/25] Fix LGTM alert --- src/sonic-py-common/sonic_py_common/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-py-common/sonic_py_common/logger.py b/src/sonic-py-common/sonic_py_common/logger.py index 7963d65017d8..f2517bb33880 100644 --- a/src/sonic-py-common/sonic_py_common/logger.py +++ b/src/sonic-py-common/sonic_py_common/logger.py @@ -35,7 +35,7 @@ def __init__(self, log_identifier=None, log_facility=FACILITY_USER): def __del__(self): self.syslog.closelog() - def set_min_log_priority(priority): + def set_min_log_priority(self, priority): self.syslog.setlogmask(self.syslog.LOG_UPTO(priority)) def log_error(self, msg, also_print_to_console=False): From 0b488e23239343331aa307c9b3e3fe834cf24f21 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Wed, 22 Jul 2020 18:05:56 +0000 Subject: [PATCH 14/25] Add convenience functions for setting logger minimum priority --- .../sonic_py_common/daemon_base.py | 2 +- src/sonic-py-common/sonic_py_common/logger.py | 87 +++++++++++++------ 2 files changed, 60 insertions(+), 29 deletions(-) diff --git a/src/sonic-py-common/sonic_py_common/daemon_base.py b/src/sonic-py-common/sonic_py_common/daemon_base.py index e6c6a9b1775d..b7ba772e3f74 100644 --- a/src/sonic-py-common/sonic_py_common/daemon_base.py +++ b/src/sonic-py-common/sonic_py_common/daemon_base.py @@ -26,7 +26,7 @@ def db_connect(db_name): class DaemonBase(logger.Logger): def __init__(self, log_identifier): - super(DaemonBase, self).__init__(log_identifier, logger.FACILITY_DAEMON) + super(DaemonBase, self).__init__(log_identifier, logger.Logger.FACILITY_DAEMON) # Register our default signal handlers, unless the signal already has a # handler registered, most likely from a subclass implementation diff --git a/src/sonic-py-common/sonic_py_common/logger.py b/src/sonic-py-common/sonic_py_common/logger.py index f2517bb33880..40e6ad9785b8 100644 --- a/src/sonic-py-common/sonic_py_common/logger.py +++ b/src/sonic-py-common/sonic_py_common/logger.py @@ -5,20 +5,21 @@ Logging functionality for SONiC Python applications """ -FACILITY_USER = syslog.LOG_USER -FACILITY_DAEMON = syslog.LOG_DAEMON - -PRIORITY_ERROR = syslog.LOG_ERR -PRIORITY_WARNING = syslog.LOG_WARNING -PRIORITY_NOTICE = syslog.LOG_NOTICE -PRIORITY_INFO = syslog.LOG_INFO -PRIORITY_DEBUG = syslog.LOG_DEBUG - class Logger(object): """ Logger class for SONiC Python applications """ + FACILITY_USER = syslog.LOG_USER + FACILITY_DAEMON = syslog.LOG_DAEMON + + PRIORITY_ERROR = syslog.LOG_ERR + PRIORITY_WARNING = syslog.LOG_WARNING + PRIORITY_NOTICE = syslog.LOG_NOTICE + PRIORITY_INFO = syslog.LOG_INFO + PRIORITY_DEBUG = syslog.LOG_DEBUG + + def __init__(self, log_identifier=None, log_facility=FACILITY_USER): self.syslog = syslog @@ -30,40 +31,70 @@ def __init__(self, log_identifier=None, log_facility=FACILITY_USER): facility=log_facility) # Set the default minimum log priority to PRIORITY_NOTICE - self.set_min_log_priority(PRIORITY_NOTICE) + self.set_min_log_priority(self.PRIORITY_NOTICE) def __del__(self): self.syslog.closelog() + def set_min_log_priority(self, priority): + """ + Sets the minimum log priority level to . All log messages + with a priority lower than will not be logged + + Args: + priority: The minimum priority at which to log messages + """ self.syslog.setlogmask(self.syslog.LOG_UPTO(priority)) - def log_error(self, msg, also_print_to_console=False): - self.syslog.syslog(self.syslog.LOG_ERR, msg) + def set_min_log_priority_error(self): + """ + Convenience function to set minimum log priority to PRIORITY_ERROR + """ + self.set_min_log_priority(self.PRIORITY_ERROR) - if also_print_to_console: - print(msg) + def set_min_log_priority_warning(self): + """ + Convenience function to set minimum log priority to PRIORITY_WARNING + """ + self.set_min_log_priority(self.PRIORITY_WARNING) - def log_warning(self, msg, also_print_to_console=False): - self.syslog.syslog(self.syslog.LOG_WARNING, msg) + def set_min_log_priority_notice(self): + """ + Convenience function to set minimum log priority to PRIORITY_NOTICE + """ + self.set_min_log_priority(self.PRIORITY_NOTICE) - if also_print_to_console: - print(msg) + def set_min_log_priority_info(self): + """ + Convenience function to set minimum log priority to PRIORITY_INFO + """ + self.set_min_log_priority(self.PRIORITY_INFO) - def log_notice(self, msg, also_print_to_console=False): - self.syslog.syslog(self.syslog.LOG_NOTICE, msg) + def set_min_log_priority_debug(self): + """ + Convenience function to set minimum log priority to PRIORITY_DEBUG + """ + self.set_min_log_priority(self.PRIORITY_DEBUG) - if also_print_to_console: - print(msg) - def log_info(self, msg, also_print_to_console=False): - self.syslog.syslog(self.syslog.LOG_INFO, msg) + def log(self, priority, msg, also_print_to_console=False): + self.syslog.syslog(priority, msg) if also_print_to_console: print(msg) - def log_debug(self, msg, also_print_to_console=False): - self.syslog.syslog(self.syslog.LOG_DEBUG, msg) + def log_error(self, msg, also_print_to_console=False): + self.log(self.PRIORITY_ERROR, msg, also_print_to_console) - if also_print_to_console: - print(msg) + def log_warning(self, msg, also_print_to_console=False): + self.log(self.PRIORITY_WARNING, msg, also_print_to_console) + + def log_notice(self, msg, also_print_to_console=False): + self.log(self.PRIORITY_NOTICE, msg, also_print_to_console) + + def log_info(self, msg, also_print_to_console=False): + self.log(self.PRIORITY_INFO, msg, also_print_to_console) + + def log_debug(self, msg, also_print_to_console=False): + self.log(self.PRIORITY_DEBUG, msg, also_print_to_console) From 1dec66a35c328e4a6fe8ce2b0192e76d3e2efe96 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Wed, 22 Jul 2020 18:16:50 +0000 Subject: [PATCH 15/25] Modify imports in daemon_base.py to simplifys usage of Logger --- src/sonic-py-common/sonic_py_common/daemon_base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/sonic-py-common/sonic_py_common/daemon_base.py b/src/sonic-py-common/sonic_py_common/daemon_base.py index b7ba772e3f74..12014fb20dae 100644 --- a/src/sonic-py-common/sonic_py_common/daemon_base.py +++ b/src/sonic-py-common/sonic_py_common/daemon_base.py @@ -2,7 +2,8 @@ import signal import sys -from . import device_info, logger +from . import device_info +from .logger import Logger # # Constants ==================================================================== @@ -24,9 +25,9 @@ def db_connect(db_name): # DaemonBase =================================================================== # -class DaemonBase(logger.Logger): +class DaemonBase(Logger): def __init__(self, log_identifier): - super(DaemonBase, self).__init__(log_identifier, logger.Logger.FACILITY_DAEMON) + super(DaemonBase, self).__init__(log_identifier, Logger.FACILITY_DAEMON) # Register our default signal handlers, unless the signal already has a # handler registered, most likely from a subclass implementation From f8c19a2a15023f3c1d525570ef1c1ea19f189e4f Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Wed, 22 Jul 2020 20:10:13 +0000 Subject: [PATCH 16/25] Remove extra blank lines pre PEP8, add comments as visual separators instead --- src/sonic-py-common/sonic_py_common/logger.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sonic-py-common/sonic_py_common/logger.py b/src/sonic-py-common/sonic_py_common/logger.py index 40e6ad9785b8..96ae7eb2bd58 100644 --- a/src/sonic-py-common/sonic_py_common/logger.py +++ b/src/sonic-py-common/sonic_py_common/logger.py @@ -19,7 +19,6 @@ class Logger(object): PRIORITY_INFO = syslog.LOG_INFO PRIORITY_DEBUG = syslog.LOG_DEBUG - def __init__(self, log_identifier=None, log_facility=FACILITY_USER): self.syslog = syslog @@ -36,6 +35,9 @@ def __init__(self, log_identifier=None, log_facility=FACILITY_USER): def __del__(self): self.syslog.closelog() + # + # Methods for setting minimum log priority + # def set_min_log_priority(self, priority): """ @@ -77,6 +79,9 @@ def set_min_log_priority_debug(self): """ self.set_min_log_priority(self.PRIORITY_DEBUG) + # + # Methods for logging messages + # def log(self, priority, msg, also_print_to_console=False): self.syslog.syslog(priority, msg) From 50e7b4984a73a2d8bec724c91569eefbaa15f2b5 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Wed, 22 Jul 2020 21:14:03 +0000 Subject: [PATCH 17/25] Build Python 3 verison of package, install in host OS and SNMP container --- files/build_templates/sonic_debian_extension.j2 | 6 ++++++ rules/docker-snmp.mk | 2 +- rules/sonic-py-common.dep | 6 +++++- rules/sonic-py-common.mk | 8 ++++++++ slave.mk | 2 ++ 5 files changed, 22 insertions(+), 2 deletions(-) diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 050d2567e677..eb5aad4eedf6 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -169,6 +169,12 @@ sudo cp {{sonic_py_common_py2_wheel_path}} $FILESYSTEM_ROOT/$SONIC_PY_COMMON_PY2 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SONIC_PY_COMMON_PY2_WHEEL_NAME sudo rm -rf $FILESYSTEM_ROOT/$SONIC_PY_COMMON_PY2_WHEEL_NAME +# Install sonic-py-common Python 3 package +SONIC_PY_COMMON_PY3_WHEEL_NAME=$(basename {{sonic_py_common_py3_wheel_path}}) +sudo cp {{sonic_py_common_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_PY_COMMON_PY3_WHEEL_NAME +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SONIC_PY_COMMON_PY3_WHEEL_NAME +sudo rm -rf $FILESYSTEM_ROOT/$SONIC_PY_COMMON_PY3_WHEEL_NAME + # Install built Python Click package (and its dependencies via 'apt-get -y install -f') # Do this before installing sonic-utilities so that it doesn't attempt to install # an older version as part of its dependencies diff --git a/rules/docker-snmp.mk b/rules/docker-snmp.mk index 39d5f2584b36..0b2f2e5da3cd 100644 --- a/rules/docker-snmp.mk +++ b/rules/docker-snmp.mk @@ -14,7 +14,7 @@ $(DOCKER_SNMP)_DBG_DEPENDS += $(SNMP_DBG) $(SNMPD_DBG) $(LIBSNMP_DBG) $(DOCKER_SNMP)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_IMAGE_PACKAGES) -$(DOCKER_SNMP)_PYTHON_WHEELS += $(SONIC_PLATFORM_COMMON_PY3) $(SWSSSDK_PY3) $(ASYNCSNMP_PY3) +$(DOCKER_SNMP)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) $(SONIC_PLATFORM_COMMON_PY3) $(SWSSSDK_PY3) $(ASYNCSNMP_PY3) $(DOCKER_SNMP)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_BUSTER) SONIC_DOCKER_IMAGES += $(DOCKER_SNMP) diff --git a/rules/sonic-py-common.dep b/rules/sonic-py-common.dep index e26529d69c95..9ecedb068a4f 100644 --- a/rules/sonic-py-common.dep +++ b/rules/sonic-py-common.dep @@ -3,6 +3,10 @@ DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-py-common.mk rules/sonic-p DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) DEP_FILES += $(shell git ls-files $(SPATH)) -$(SONIC_PY_COMMON_PY2)_CACHE_MODE := GIT_CONTENT_SHA +$(SONIC_PY_COMMON_PY2)_CACHE_MODE := GIT_CONTENT_SHA $(SONIC_PY_COMMON_PY2)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) $(SONIC_PY_COMMON_PY2)_DEP_FILES := $(DEP_FILES) + +$(SONIC_PY_COMMON_PY3)_CACHE_MODE := GIT_CONTENT_SHA +$(SONIC_PY_COMMON_PY3)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(SONIC_PY_COMMON_PY3)_DEP_FILES := $(DEP_FILES) diff --git a/rules/sonic-py-common.mk b/rules/sonic-py-common.mk index e7afc184fd8d..b730ef55857b 100644 --- a/rules/sonic-py-common.mk +++ b/rules/sonic-py-common.mk @@ -5,3 +5,11 @@ $(SONIC_PY_COMMON_PY2)_SRC_PATH = $(SRC_PATH)/sonic-py-common $(SONIC_PY_COMMON_PY2)_DEPENDS += $(SWSSSDK_PY2) $(SONIC_PY_COMMON_PY2)_PYTHON_VERSION = 2 SONIC_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2) + +# SONIC_PY_COMMON_PY3 package + +SONIC_PY_COMMON_PY3 = sonic_py_common-1.0-py3-none-any.whl +$(SONIC_PY_COMMON_PY3)_SRC_PATH = $(SRC_PATH)/sonic-py-common +$(SONIC_PY_COMMON_PY3)_DEPENDS += $(SWSSSDK_PY3) +$(SONIC_PY_COMMON_PY3)_PYTHON_VERSION = 3 +SONIC_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) diff --git a/slave.mk b/slave.mk index 9b145e9688ad..7cab2c28bcb4 100644 --- a/slave.mk +++ b/slave.mk @@ -806,6 +806,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ $(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_ZTP))) \ $(addprefix $(PYTHON_DEBS_PATH)/,$(SONIC_UTILS)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY2)) \ + $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY3)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2)) \ @@ -837,6 +838,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ export lazy_installer_debs="$(foreach deb, $($*_LAZY_INSTALLS),$(foreach device, $($(deb)_PLATFORM),$(addprefix $(device)@, $(IMAGE_DISTRO_DEBS_PATH)/$(deb))))" export installer_images="$(addprefix $(TARGET_PATH)/,$($*_DOCKERS))" export sonic_py_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY2))" + export sonic_py_common_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY3))" export config_engine_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE))" export swsssdk_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY2))" export swsssdk_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY3))" From c1e84de4fb944ecc75c0b79c49f9b98b4488b9c0 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Wed, 22 Jul 2020 21:15:48 +0000 Subject: [PATCH 18/25] Install Python 3 package using pip3 --- files/build_templates/sonic_debian_extension.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index eb5aad4eedf6..37b264304fb3 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -172,7 +172,7 @@ sudo rm -rf $FILESYSTEM_ROOT/$SONIC_PY_COMMON_PY2_WHEEL_NAME # Install sonic-py-common Python 3 package SONIC_PY_COMMON_PY3_WHEEL_NAME=$(basename {{sonic_py_common_py3_wheel_path}}) sudo cp {{sonic_py_common_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_PY_COMMON_PY3_WHEEL_NAME -sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SONIC_PY_COMMON_PY3_WHEEL_NAME +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_PY_COMMON_PY3_WHEEL_NAME sudo rm -rf $FILESYSTEM_ROOT/$SONIC_PY_COMMON_PY3_WHEEL_NAME # Install built Python Click package (and its dependencies via 'apt-get -y install -f') From a3d280e5de0f4515bd1522364616d6fa3ac26af5 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Thu, 23 Jul 2020 00:50:16 +0000 Subject: [PATCH 19/25] Make exception statements Python3-compliant --- src/sonic-py-common/sonic_py_common/daemon_base.py | 4 ++-- src/sonic-py-common/sonic_py_common/device_info.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sonic-py-common/sonic_py_common/daemon_base.py b/src/sonic-py-common/sonic_py_common/daemon_base.py index 12014fb20dae..c5cfb975456c 100644 --- a/src/sonic-py-common/sonic_py_common/daemon_base.py +++ b/src/sonic-py-common/sonic_py_common/daemon_base.py @@ -61,7 +61,7 @@ def load_platform_util(self, module_name, class_name): try: module_file = "/".join([platform_path, "plugins", module_name + ".py"]) module = imp.load_source(module_name, module_file) - except IOError, e: + except IOError as e: raise IOError("Failed to load platform module '%s': %s" % (module_name, str(e))) try: @@ -71,7 +71,7 @@ def load_platform_util(self, module_name, class_name): platform_util = platform_util_class('','','','') else: platform_util = platform_util_class() - except AttributeError, e: + except AttributeError as e: raise AttributeError("Failed to instantiate '%s' class: %s" % (class_name, str(e))) return platform_util diff --git a/src/sonic-py-common/sonic_py_common/device_info.py b/src/sonic-py-common/sonic_py_common/device_info.py index 97121680abf9..432965b0851a 100644 --- a/src/sonic-py-common/sonic_py_common/device_info.py +++ b/src/sonic-py-common/sonic_py_common/device_info.py @@ -337,7 +337,7 @@ def get_system_routing_stack(): stdout = proc.communicate()[0] proc.wait() result = stdout.rstrip('\n') - except OSError, e: + except OSError as e: raise OSError("Cannot detect routing stack") return result From f2d0fb0dd32c6fb4419051d65f6a2d6186826846 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Thu, 23 Jul 2020 07:45:00 +0000 Subject: [PATCH 20/25] Add natsort as dependency --- src/sonic-py-common/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sonic-py-common/setup.py b/src/sonic-py-common/setup.py index 3f984940d539..f08e6b6bc113 100644 --- a/src/sonic-py-common/setup.py +++ b/src/sonic-py-common/setup.py @@ -1,6 +1,7 @@ from setuptools import setup dependencies = [ + 'natsort', 'swsssdk>=2.0.1', ] From f1a541f87919f0aa8cdf006ef746d9ffd99ba7da Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Thu, 23 Jul 2020 21:24:40 +0000 Subject: [PATCH 21/25] Separate standard imports from third-party imports with a blank line --- src/sonic-py-common/sonic_py_common/device_info.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sonic-py-common/sonic_py_common/device_info.py b/src/sonic-py-common/sonic_py_common/device_info.py index 432965b0851a..72b9e5a28644 100644 --- a/src/sonic-py-common/sonic_py_common/device_info.py +++ b/src/sonic-py-common/sonic_py_common/device_info.py @@ -1,6 +1,7 @@ import glob import os import subprocess + import yaml from natsort import natsorted From 59f56d134ebcc84c6b9b321fa75f27095f6e2db1 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Thu, 23 Jul 2020 21:26:33 +0000 Subject: [PATCH 22/25] Add pyyaml as dependency --- src/sonic-py-common/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sonic-py-common/setup.py b/src/sonic-py-common/setup.py index f08e6b6bc113..b1294472624e 100644 --- a/src/sonic-py-common/setup.py +++ b/src/sonic-py-common/setup.py @@ -2,6 +2,7 @@ dependencies = [ 'natsort', + 'pyyaml', 'swsssdk>=2.0.1', ] From 461a56cb75374d971a142b79b042a5884d67eea4 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Thu, 23 Jul 2020 21:32:49 +0000 Subject: [PATCH 23/25] Add 'LOG_' prefix to facilities and priorities --- src/sonic-py-common/sonic_py_common/logger.py | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/sonic-py-common/sonic_py_common/logger.py b/src/sonic-py-common/sonic_py_common/logger.py index 96ae7eb2bd58..ee5e9f35fea0 100644 --- a/src/sonic-py-common/sonic_py_common/logger.py +++ b/src/sonic-py-common/sonic_py_common/logger.py @@ -10,16 +10,16 @@ class Logger(object): """ Logger class for SONiC Python applications """ - FACILITY_USER = syslog.LOG_USER - FACILITY_DAEMON = syslog.LOG_DAEMON + LOG_FACILITY_USER = syslog.LOG_USER + LOG_FACILITY_DAEMON = syslog.LOG_DAEMON - PRIORITY_ERROR = syslog.LOG_ERR - PRIORITY_WARNING = syslog.LOG_WARNING - PRIORITY_NOTICE = syslog.LOG_NOTICE - PRIORITY_INFO = syslog.LOG_INFO - PRIORITY_DEBUG = syslog.LOG_DEBUG + LOG_PRIORITY_ERROR = syslog.LOG_ERR + LOG_PRIORITY_WARNING = syslog.LOG_WARNING + LOG_PRIORITY_NOTICE = syslog.LOG_NOTICE + LOG_PRIORITY_INFO = syslog.LOG_INFO + LOG_PRIORITY_DEBUG = syslog.LOG_DEBUG - def __init__(self, log_identifier=None, log_facility=FACILITY_USER): + def __init__(self, log_identifier=None, log_facility=LOG_FACILITY_USER): self.syslog = syslog if not log_identifier: @@ -29,8 +29,8 @@ def __init__(self, log_identifier=None, log_facility=FACILITY_USER): logoption=(syslog.LOG_PID | syslog.LOG_NDELAY), facility=log_facility) - # Set the default minimum log priority to PRIORITY_NOTICE - self.set_min_log_priority(self.PRIORITY_NOTICE) + # Set the default minimum log priority to LOG_PRIORITY_NOTICE + self.set_min_log_priority(self.LOG_PRIORITY_NOTICE) def __del__(self): self.syslog.closelog() @@ -51,33 +51,33 @@ def set_min_log_priority(self, priority): def set_min_log_priority_error(self): """ - Convenience function to set minimum log priority to PRIORITY_ERROR + Convenience function to set minimum log priority to LOG_PRIORITY_ERROR """ - self.set_min_log_priority(self.PRIORITY_ERROR) + self.set_min_log_priority(self.LOG_PRIORITY_ERROR) def set_min_log_priority_warning(self): """ - Convenience function to set minimum log priority to PRIORITY_WARNING + Convenience function to set minimum log priority to LOG_PRIORITY_WARNING """ - self.set_min_log_priority(self.PRIORITY_WARNING) + self.set_min_log_priority(self.LOG_PRIORITY_WARNING) def set_min_log_priority_notice(self): """ - Convenience function to set minimum log priority to PRIORITY_NOTICE + Convenience function to set minimum log priority to LOG_PRIORITY_NOTICE """ - self.set_min_log_priority(self.PRIORITY_NOTICE) + self.set_min_log_priority(self.LOG_PRIORITY_NOTICE) def set_min_log_priority_info(self): """ - Convenience function to set minimum log priority to PRIORITY_INFO + Convenience function to set minimum log priority to LOG_PRIORITY_INFO """ - self.set_min_log_priority(self.PRIORITY_INFO) + self.set_min_log_priority(self.LOG_PRIORITY_INFO) def set_min_log_priority_debug(self): """ - Convenience function to set minimum log priority to PRIORITY_DEBUG + Convenience function to set minimum log priority to LOG_PRIORITY_DEBUG """ - self.set_min_log_priority(self.PRIORITY_DEBUG) + self.set_min_log_priority(self.LOG_PRIORITY_DEBUG) # # Methods for logging messages @@ -90,16 +90,16 @@ def log(self, priority, msg, also_print_to_console=False): print(msg) def log_error(self, msg, also_print_to_console=False): - self.log(self.PRIORITY_ERROR, msg, also_print_to_console) + self.log(self.LOG_PRIORITY_ERROR, msg, also_print_to_console) def log_warning(self, msg, also_print_to_console=False): - self.log(self.PRIORITY_WARNING, msg, also_print_to_console) + self.log(self.LOG_PRIORITY_WARNING, msg, also_print_to_console) def log_notice(self, msg, also_print_to_console=False): - self.log(self.PRIORITY_NOTICE, msg, also_print_to_console) + self.log(self.LOG_PRIORITY_NOTICE, msg, also_print_to_console) def log_info(self, msg, also_print_to_console=False): - self.log(self.PRIORITY_INFO, msg, also_print_to_console) + self.log(self.LOG_PRIORITY_INFO, msg, also_print_to_console) def log_debug(self, msg, also_print_to_console=False): - self.log(self.PRIORITY_DEBUG, msg, also_print_to_console) + self.log(self.LOG_PRIORITY_DEBUG, msg, also_print_to_console) From 95419cd47333f1bf9cd37f2e2e2f3f5fc00e050d Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Fri, 24 Jul 2020 18:02:34 +0000 Subject: [PATCH 24/25] Install python3-setuptools in build_debian.sh --- build_debian.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build_debian.sh b/build_debian.sh index 4a6751fa40ca..86d77a265a1d 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -275,6 +275,7 @@ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y in openssh-server \ python \ python-setuptools \ + python3-setuptools \ python-apt \ traceroute \ iputils-ping \ From 3365c7bbb69be59e8bd515aa96b452b3362eb080 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sat, 25 Jul 2020 04:55:47 +0000 Subject: [PATCH 25/25] Add missing 'LOG_' prefix --- src/sonic-py-common/sonic_py_common/daemon_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-py-common/sonic_py_common/daemon_base.py b/src/sonic-py-common/sonic_py_common/daemon_base.py index c5cfb975456c..d88b07edcace 100644 --- a/src/sonic-py-common/sonic_py_common/daemon_base.py +++ b/src/sonic-py-common/sonic_py_common/daemon_base.py @@ -27,7 +27,7 @@ def db_connect(db_name): class DaemonBase(Logger): def __init__(self, log_identifier): - super(DaemonBase, self).__init__(log_identifier, Logger.FACILITY_DAEMON) + super(DaemonBase, self).__init__(log_identifier, Logger.LOG_FACILITY_DAEMON) # Register our default signal handlers, unless the signal already has a # handler registered, most likely from a subclass implementation