From 071c9b3d1ee5e793694065da8bf0ba61c394f363 Mon Sep 17 00:00:00 2001
From: yozhao101 <56170650+yozhao101@users.noreply.github.com>
Date: Wed, 6 Nov 2019 11:02:57 -0800
Subject: [PATCH] [Services] Restart LLDP service upon unexpected critical
 process exit. (#3713)

Signed-off-by: Yong Zhao <yozhao@microsoft.com>

    RB=2124908
    G=lnos-reviewers
    R=pchaudha,pmao,vapatil,zxu
    A=zxu
---
 dockers/docker-lldp-sv2/Dockerfile.j2      | 2 ++
 dockers/docker-lldp-sv2/critical_processes | 3 +++
 dockers/docker-lldp-sv2/supervisord.conf   | 6 ++++++
 files/build_templates/lldp.service.j2      | 4 ++++
 rules/docker-lldp-sv2.mk                   | 1 +
 5 files changed, 16 insertions(+)
 create mode 100644 dockers/docker-lldp-sv2/critical_processes

diff --git a/dockers/docker-lldp-sv2/Dockerfile.j2 b/dockers/docker-lldp-sv2/Dockerfile.j2
index ec5de2a25a31..9ca40f57bf46 100644
--- a/dockers/docker-lldp-sv2/Dockerfile.j2
+++ b/dockers/docker-lldp-sv2/Dockerfile.j2
@@ -48,5 +48,7 @@ COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 COPY ["lldpd.conf.j2", "/usr/share/sonic/templates/"]
 COPY ["lldpd", "/etc/default/"]
 COPY ["lldpmgrd", "/usr/bin/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor"]
 
 ENTRYPOINT ["/usr/bin/supervisord"]
diff --git a/dockers/docker-lldp-sv2/critical_processes b/dockers/docker-lldp-sv2/critical_processes
new file mode 100644
index 000000000000..b845b70bb3f7
--- /dev/null
+++ b/dockers/docker-lldp-sv2/critical_processes
@@ -0,0 +1,3 @@
+lldpd
+lldp-syncd
+lldpmgrd
diff --git a/dockers/docker-lldp-sv2/supervisord.conf b/dockers/docker-lldp-sv2/supervisord.conf
index 5feb3543f1bc..3f3f5beabc8d 100644
--- a/dockers/docker-lldp-sv2/supervisord.conf
+++ b/dockers/docker-lldp-sv2/supervisord.conf
@@ -3,6 +3,12 @@ logfile_maxbytes=1MB
 logfile_backups=2
 nodaemon=true
 
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener
+events=PROCESS_STATE_EXITED
+autostart=true
+autorestart=unexpected
+
 [program:start.sh]
 command=/usr/bin/start.sh
 priority=1
diff --git a/files/build_templates/lldp.service.j2 b/files/build_templates/lldp.service.j2
index c317e18efc5c..6da216a92450 100644
--- a/files/build_templates/lldp.service.j2
+++ b/files/build_templates/lldp.service.j2
@@ -3,12 +3,16 @@ Description=LLDP container
 Requires=updategraph.service
 After=updategraph.service
 Before=ntp-config.service
+StartLimitIntervalSec=1200
+StartLimitBurst=3
 
 [Service]
 User={{ sonicadmin_user }}
 ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh wait
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+Restart=always
+RestartSec=30
 
 [Install]
 WantedBy=multi-user.target
diff --git a/rules/docker-lldp-sv2.mk b/rules/docker-lldp-sv2.mk
index f00a5257b90a..0eb4c2381677 100644
--- a/rules/docker-lldp-sv2.mk
+++ b/rules/docker-lldp-sv2.mk
@@ -14,3 +14,4 @@ $(DOCKER_LLDP_SV2)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
 
 $(DOCKER_LLDP_SV2)_BASE_IMAGE_FILES += lldpctl:/usr/bin/lldpctl
 $(DOCKER_LLDP_SV2)_BASE_IMAGE_FILES += monit_lldp:/etc/monit/conf.d
+$(DOCKER_LLDP_SV2)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)