Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[system-health] Led color shall be controlled by configuration when system is booting #12487

Merged
merged 2 commits into from
Dec 1, 2022
Merged

[system-health] Led color shall be controlled by configuration when system is booting #12487

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2e5f655
[system-health] Led color shall be controlled by configuration when s…
Junchao-Mellanox Oct 21, 2022
efde7fb
Fix unit test issue
Junchao-Mellanox Oct 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/system-health/health_checker/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class Config(object):
DEFAULT_LED_CONFIG = {
'fault': 'red',
'normal': 'green',
'booting': 'orange_blink'
'booting': 'red'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why changed from 'orange' blinking to 'red'? How to distinguish between fault state or system hung stat during boot?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is for backward compatible. Before this change, booting stage is not used, so "orange_blink" is not used too, system led color is always red during system booting time because not all services are ready. In the new change, I set "booting stage" to red to make sure backward compatible: if user does not provide system health configuration file, system led color will be red which is the same behavior as before; otherwise, system led will be set to the color specified in system health configuration file.

}

# System health configuration file name
Expand Down
23 changes: 16 additions & 7 deletions src/system-health/health_checker/manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,13 @@
from .service_checker import ServiceChecker
from .hardware_checker import HardwareChecker
from .user_defined_checker import UserDefinedChecker
from . import utils


class HealthCheckerManager(object):
"""
Manage all system health checkers and system health configuration.
"""
boot_timeout = None

def __init__(self):
self._checkers = []
self.config = Config()
Expand Down Expand Up @@ -42,9 +41,7 @@ def check(self, chassis):
checker = UserDefinedChecker(udc)
self._do_check(checker, stats)

led_status = 'normal' if HealthChecker.summary == HealthChecker.STATUS_OK else 'fault'
self._set_system_led(chassis, self.config, led_status)

self._set_system_led(chassis)
return stats

def _do_check(self, checker, stats):
Expand Down Expand Up @@ -75,10 +72,22 @@ def _do_check(self, checker, stats):
else:
stats['Internal'].update(entry)

def _set_system_led(self, chassis, config, status):
def _set_system_led(self, chassis):
try:
chassis.set_status_led(config.get_led_color(status))
chassis.set_status_led(self._get_led_target_color())
except NotImplementedError:
print('chassis.set_status_led is not implemented')
except Exception as e:
print('Failed to set system led due to - {}'.format(repr(e)))

def _get_led_target_color(self):
"""Get target LED color according to health status and system uptime

Returns:
str: LED color
"""
if HealthChecker.summary == HealthChecker.STATUS_OK:
return self.config.get_led_color('normal')
else:
uptime = utils.get_uptime()
return self.config.get_led_color('booting') if uptime < self.config.get_bootup_timeout() else self.config.get_led_color('fault')
16 changes: 8 additions & 8 deletions src/system-health/tests/test_system_health.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,7 @@ def test_config():

assert config.get_led_color('fault') == 'red'
assert config.get_led_color('normal') == 'green'
assert config.get_led_color('booting') == 'orange_blink'
assert config.get_led_color('booting') == 'red'

config._last_mtime = 1
config._config_file = 'notExistFile'
Expand Down Expand Up @@ -498,10 +498,10 @@ def test_manager(mock_hw_info, mock_service_info, mock_udc_info):
assert stat['Internal']['UserDefinedChecker - some check']['status'] == 'Not OK'

chassis.set_status_led.side_effect = NotImplementedError()
manager._set_system_led(chassis, manager.config, 'normal')
manager._set_system_led(chassis)

chassis.set_status_led.side_effect = RuntimeError()
manager._set_system_led(chassis, manager.config, 'normal')
manager._set_system_led(chassis)

def test_utils():
output = utils.run_command('some invalid command')
Expand Down Expand Up @@ -568,7 +568,7 @@ def test_get_app_ready_status(mock_config_db, mock_run, mock_docker_client):
'has_global_scope': 'True',
'has_per_asic_scope': 'False',
'check_up_status': 'True'
},
},
'snmp': {
'state': 'enabled',
'has_global_scope': 'True',
Expand Down Expand Up @@ -659,10 +659,10 @@ def test_get_all_system_status_not_ok():
result = sysmon.get_all_system_status()
print("result:{}".format(result))
assert result == 'DOWN'

def test_post_unit_status():
sysmon = Sysmonitor()
sysmon.post_unit_status("mock_bgp", 'OK', 'Down', 'mock reason', '-')
sysmon.post_unit_status("mock_bgp", 'OK', 'Down', 'mock reason', '-')
result = swsscommon.SonicV2Connector.get_all(MockConnector, 0, 'ALL_SERVICE_STATUS|mock_bgp')
print(result)
assert result['service_status'] == 'OK'
Expand All @@ -671,7 +671,7 @@ def test_post_unit_status():

def test_post_system_status():
sysmon = Sysmonitor()
sysmon.post_system_status("UP")
sysmon.post_system_status("UP")
result = swsscommon.SonicV2Connector.get(MockConnector, 0, "SYSTEM_READY|SYSTEM_STATE", 'Status')
print("post system status result:{}".format(result))
assert result == "UP"
Expand All @@ -689,7 +689,7 @@ def test_publish_system_status():
@patch('health_checker.sysmonitor.Sysmonitor.publish_system_status', test_publish_system_status())
def test_update_system_status():
sysmon = Sysmonitor()
sysmon.update_system_status()
sysmon.update_system_status()
result = swsscommon.SonicV2Connector.get(MockConnector, 0, "SYSTEM_READY|SYSTEM_STATE", 'Status')
assert result == "UP"

Expand Down