Skip to content

Commit

Permalink
[201911] Introduce sonic-py-common package (#5063)
Browse files Browse the repository at this point in the history
Consolidate common SONiC Python-language functionality into one shared package (sonic-py-common) and eliminate duplicate code.

The package currently includes four modules:
- daemon_base
- device_info
- logger
- task_base

NOTE: This is a combination of all changes from #5003, #5049 and some changes from #5043 backported to align with the 201911 branch. As part of the 201911 port, I am not installing the Python 3 package in the base image or in the VS container, because we do not have pip3 installed, and we do not intend to migrate to Python 3 in 201911.
  • Loading branch information
jleveque authored Aug 3, 2020
1 parent 4e558bc commit 6556c40
Show file tree
Hide file tree
Showing 17 changed files with 656 additions and 9 deletions.
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -141,10 +141,16 @@ installer/x86_64/platforms/
src/sonic-config-engine/**/*.pyc
src/sonic-config-engine/build
src/sonic-config-engine/sonic_config_engine.egg-info

src/sonic-daemon-base/**/*.pyc
src/sonic-daemon-base/build
src/sonic-daemon-base/sonic_daemon_base.egg-info

src/sonic-py-common/**/*.pyc
src/sonic-py-common/build
src/sonic-py-common/dist
src/sonic-py-common/sonic_py_common.egg-info

# Misc. files
files/initramfs-tools/arista-convertfs
files/initramfs-tools/union-mount
Expand Down
18 changes: 12 additions & 6 deletions files/build_templates/sonic_debian_extension.j2
Original file line number Diff line number Diff line change
Expand Up @@ -97,12 +97,6 @@ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y in
python-yaml \
python-bitarray

# Install SONiC config engine Python package
CONFIG_ENGINE_WHEEL_NAME=$(basename {{config_engine_wheel_path}})
sudo cp {{config_engine_wheel_path}} $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $CONFIG_ENGINE_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME

# Install Python client for Redis
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==2.10.6"

Expand All @@ -121,6 +115,18 @@ sudo cp {{swsssdk_py2_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SWSSSDK_PY2_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME

# Install sonic-py-common Python 2 package
SONIC_PY_COMMON_PY2_WHEEL_NAME=$(basename {{sonic_py_common_py2_wheel_path}})
sudo cp {{sonic_py_common_py2_wheel_path}} $FILESYSTEM_ROOT/$SONIC_PY_COMMON_PY2_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SONIC_PY_COMMON_PY2_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SONIC_PY_COMMON_PY2_WHEEL_NAME

# Install SONiC config engine Python package
CONFIG_ENGINE_WHEEL_NAME=$(basename {{config_engine_wheel_path}})
sudo cp {{config_engine_wheel_path}} $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $CONFIG_ENGINE_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME

# Install sonic-platform-common Python 2 package
PLATFORM_COMMON_PY2_WHEEL_NAME=$(basename {{platform_common_py2_wheel_path}})
sudo cp {{platform_common_py2_wheel_path}} $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY2_WHEEL_NAME
Expand Down
2 changes: 1 addition & 1 deletion platform/mellanox/mlnx-platform-api.mk
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
SONIC_PLATFORM_API_PY2 = mlnx_platform_api-1.0-py2-none-any.whl
$(SONIC_PLATFORM_API_PY2)_SRC_PATH = $(PLATFORM_PATH)/mlnx-platform-api
$(SONIC_PLATFORM_API_PY2)_PYTHON_VERSION = 2
$(SONIC_PLATFORM_API_PY2)_DEPENDS = $(SONIC_PLATFORM_COMMON_PY2) $(SONIC_DAEMON_BASE_PY2) $(SONIC_CONFIG_ENGINE)
$(SONIC_PLATFORM_API_PY2)_DEPENDS = $(SONIC_PY_COMMON_PY2) $(SONIC_PLATFORM_COMMON_PY2) $(SONIC_DAEMON_BASE_PY2) $(SONIC_CONFIG_ENGINE)
SONIC_PYTHON_WHEELS += $(SONIC_PLATFORM_API_PY2)

export mlnx_platform_api_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_API_PY2))"
3 changes: 3 additions & 0 deletions platform/vs/docker-sonic-vs.mk
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ $(DOCKER_SONIC_VS)_DEPENDS += $(SWSS) \

$(DOCKER_SONIC_VS)_PYTHON_DEBS += $(SONIC_UTILS)

$(DOCKER_SONIC_VS)_PYTHON_WHEELS += $(SWSSSDK_PY2) \
$(SONIC_PY_COMMON_PY2)

ifeq ($(INSTALL_DEBUG_TOOLS), y)
$(DOCKER_SONIC_VS)_DEPENDS += $(SWSS_DBG) \
$(LIBSWSSCOMMON_DBG) \
Expand Down
1 change: 1 addition & 0 deletions rules/docker-config-engine-stretch.mk
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
DOCKER_CONFIG_ENGINE_STRETCH = docker-config-engine-stretch.gz
$(DOCKER_CONFIG_ENGINE_STRETCH)_PATH = $(DOCKERS_PATH)/docker-config-engine-stretch
$(DOCKER_CONFIG_ENGINE_STRETCH)_PYTHON_WHEELS += $(SWSSSDK_PY2)
$(DOCKER_CONFIG_ENGINE_STRETCH)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2)
$(DOCKER_CONFIG_ENGINE_STRETCH)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE)
$(DOCKER_CONFIG_ENGINE_STRETCH)_LOAD_DOCKERS += $(DOCKER_BASE_STRETCH)

Expand Down
1 change: 1 addition & 0 deletions rules/docker-config-engine.mk
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
DOCKER_CONFIG_ENGINE = docker-config-engine.gz
$(DOCKER_CONFIG_ENGINE)_PATH = $(DOCKERS_PATH)/docker-config-engine
$(DOCKER_CONFIG_ENGINE)_PYTHON_WHEELS += $(SWSSSDK_PY2)
$(DOCKER_CONFIG_ENGINE)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2)
$(DOCKER_CONFIG_ENGINE)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE)
$(DOCKER_CONFIG_ENGINE)_LOAD_DOCKERS += $(DOCKER_BASE)
SONIC_DOCKER_IMAGES += $(DOCKER_CONFIG_ENGINE)
1 change: 1 addition & 0 deletions rules/docker-platform-monitor.mk
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ endif
$(DOCKER_PLATFORM_MONITOR)_PYTHON_DEBS += $(SONIC_LEDD) $(SONIC_XCVRD) $(SONIC_PSUD) $(SONIC_SYSEEPROMD) $(SONIC_THERMALCTLD)
$(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PLATFORM_COMMON_PY2)
$(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SWSSSDK_PY2)
$(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2)
$(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PLATFORM_API_PY2)
$(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_DAEMON_BASE_PY2)

Expand Down
2 changes: 1 addition & 1 deletion rules/docker-snmp-sv2.mk
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ $(DOCKER_SNMP_SV2)_DBG_DEPENDS += $(SNMP_DBG) $(SNMPD_DBG) $(LIBSNMP_DBG)

$(DOCKER_SNMP_SV2)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_IMAGE_PACKAGES)

$(DOCKER_SNMP_SV2)_PYTHON_WHEELS += $(SONIC_PLATFORM_COMMON_PY3) $(SWSSSDK_PY3) $(ASYNCSNMP_PY3)
$(DOCKER_SNMP_SV2)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) $(SONIC_PLATFORM_COMMON_PY3) $(SWSSSDK_PY3) $(ASYNCSNMP_PY3)
$(DOCKER_SNMP_SV2)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_STRETCH)

SONIC_DOCKER_IMAGES += $(DOCKER_SNMP_SV2)
Expand Down
15 changes: 15 additions & 0 deletions rules/sonic-py-common.mk
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# SONIC_PY_COMMON_PY2 package

SONIC_PY_COMMON_PY2 = sonic_py_common-1.0-py2-none-any.whl
$(SONIC_PY_COMMON_PY2)_SRC_PATH = $(SRC_PATH)/sonic-py-common
$(SONIC_PY_COMMON_PY2)_DEPENDS += $(SWSSSDK_PY2)
$(SONIC_PY_COMMON_PY2)_PYTHON_VERSION = 2
SONIC_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2)

# SONIC_PY_COMMON_PY3 package

SONIC_PY_COMMON_PY3 = sonic_py_common-1.0-py3-none-any.whl
$(SONIC_PY_COMMON_PY3)_SRC_PATH = $(SRC_PATH)/sonic-py-common
$(SONIC_PY_COMMON_PY3)_DEPENDS += $(SWSSSDK_PY3)
$(SONIC_PY_COMMON_PY3)_PYTHON_VERSION = 3
SONIC_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3)
2 changes: 1 addition & 1 deletion rules/sonic-thermalctld.mk
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@

SONIC_THERMALCTLD = python-sonic-thermalctld_1.0-1_all.deb
$(SONIC_THERMALCTLD)_SRC_PATH = $(SRC_PATH)/sonic-platform-daemons/sonic-thermalctld
$(SONIC_THERMALCTLD)_WHEEL_DEPENDS = $(SONIC_DAEMON_BASE_PY2)
$(SONIC_THERMALCTLD)_WHEEL_DEPENDS = $(SONIC_PY_COMMON_PY2) $(SONIC_DAEMON_BASE_PY2)
SONIC_PYTHON_STDEB_DEBS += $(SONIC_THERMALCTLD)
4 changes: 4 additions & 0 deletions slave.mk
Original file line number Diff line number Diff line change
Expand Up @@ -636,6 +636,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
$(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(DEBS_PATH)/,$(SONIC_ZTP))) \
$(addprefix $(STRETCH_FILES_PATH)/, $(if $(filter $(CONFIGURED_ARCH),amd64), $(IXGBE_DRIVER))) \
$(addprefix $(PYTHON_DEBS_PATH)/,$(SONIC_UTILS)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY2)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY3)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2)) \
$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2)) \
Expand Down Expand Up @@ -664,6 +666,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
export installer_debs="$(addprefix $(STRETCH_DEBS_PATH)/,$($*_INSTALLS))"
export lazy_installer_debs="$(foreach deb, $($*_LAZY_INSTALLS),$(foreach device, $($(deb)_PLATFORM),$(addprefix $(device)@, $(STRETCH_DEBS_PATH)/$(deb))))"
export installer_images="$(addprefix $(TARGET_PATH)/,$($*_DOCKERS))"
export sonic_py_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY2))"
export sonic_py_common_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY3))"
export config_engine_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE))"
export swsssdk_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY2))"
export platform_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2))"
Expand Down
39 changes: 39 additions & 0 deletions src/sonic-py-common/setup.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
from setuptools import setup

dependencies = [
'natsort',
'pyyaml',
'swsssdk>=2.0.1',
]

high_performance_deps = [
'swsssdk[high_perf]>=2.0.1',
]

setup(
name='sonic-py-common',
version='1.0',
description='Common Python libraries for SONiC',
license='Apache 2.0',
author='SONiC Team',
author_email='linuxnetdev@microsoft.com',
url='https://github.com/Azure/SONiC',
maintainer='Joe LeVeque',
maintainer_email='jolevequ@microsoft.com',
install_requires=dependencies,
extras_require={
'high_perf': high_performance_deps,
},
packages=[
'sonic_py_common',
],
classifiers=[
'Intended Audience :: Developers',
'Operating System :: Linux',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python',
],
keywords='SONiC sonic PYTHON python COMMON common',
)

Empty file.
81 changes: 81 additions & 0 deletions src/sonic-py-common/sonic_py_common/daemon_base.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
import imp
import signal
import sys

from . import device_info
from .logger import Logger

#
# Constants ====================================================================
#
REDIS_TIMEOUT_MSECS = 0

EEPROM_MODULE_NAME = 'eeprom'
EEPROM_CLASS_NAME = 'board'

#
# Helper functions =============================================================
#

def db_connect(db_name):
from swsscommon import swsscommon
return swsscommon.DBConnector(db_name, REDIS_TIMEOUT_MSECS, True)

#
# DaemonBase ===================================================================
#

class DaemonBase(Logger):
def __init__(self, log_identifier):
super(DaemonBase, self).__init__(log_identifier, Logger.LOG_FACILITY_DAEMON)

# Register our default signal handlers, unless the signal already has a
# handler registered, most likely from a subclass implementation
if not signal.getsignal(signal.SIGHUP):
signal.signal(signal.SIGHUP, self.signal_handler)
if not signal.getsignal(signal.SIGINT):
signal.signal(signal.SIGINT, self.signal_handler)
if not signal.getsignal(signal.SIGTERM):
signal.signal(signal.SIGTERM, self.signal_handler)

# Default signal handler; can be overridden by subclass
def signal_handler(self, sig, frame):
if sig == signal.SIGHUP:
log_info("DaemonBase: Caught SIGHUP - ignoring...")
elif sig == signal.SIGINT:
log_info("DaemonBase: Caught SIGINT - exiting...")
sys.exit(128 + sig)
elif sig == signal.SIGTERM:
log_info("DaemonBase: Caught SIGTERM - exiting...")
sys.exit(128 + sig)
else:
log_warning("DaemonBase: Caught unhandled signal '{}'".format(sig))

# Loads platform specific platform module from source
def load_platform_util(self, module_name, class_name):
platform_util = None

# Get path to platform and hwsku
(platform_path, hwsku_path) = device_info.get_paths_to_platform_and_hwsku()

try:
module_file = "/".join([platform_path, "plugins", module_name + ".py"])
module = imp.load_source(module_name, module_file)
except IOError as e:
raise IOError("Failed to load platform module '%s': %s" % (module_name, str(e)))

try:
platform_util_class = getattr(module, class_name)
# board class of eeprom requires 4 paramerters, need special treatment here.
if module_name == EEPROM_MODULE_NAME and class_name == EEPROM_CLASS_NAME:
platform_util = platform_util_class('','','','')
else:
platform_util = platform_util_class()
except AttributeError as e:
raise AttributeError("Failed to instantiate '%s' class: %s" % (class_name, str(e)))

return platform_util

# Runs daemon
def run(self):
raise NotImplementedError()
Loading

0 comments on commit 6556c40

Please sign in to comment.