Fault tolerance

README.md

@@ -227,6 +227,8 @@ All the configurations are grouped in *sections* and some of them can vary depen
   - `DashboardURL`: Sets the url where the frontend is running. Unless you are using a custom domain, you should keep this value as https://beta.lanthorn.ai/.
   - `EnableSlackNotifications`: A boolean parameter to enable/disable the Slack integration for notifications and daily reports. We recommend not editing this parameter directly and manage it from the [UI](https://beta.lanthorn.ai) to configure your workspace correctly.
   - `SlackChannel`: Configures the slack channel used by the notifications. The chosen slack channel must exist in the configured workspace.
+  - `OccupancyAlertsMinInterval`:  Sets the desired interval (in seconds) between occupancy alerts.
+  - `MaxThreadRestarts`: Defines the number of restarts allowed per thread.
 
 - `[Api]`
   - `Host`: Configures the host IP of the processor's API (inside docker). We recommend don't change that value and keep it as *0.0.0.0*.

amd64-usbtpu.Dockerfile

@@ -95,12 +95,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 ENV DEV_ALLOW_ALL_ORIGINS=true
 ENV AWS_SHARED_CREDENTIALS_FILE=/repo/.aws/credentials
 ENV AWS_CONFIG_FILE=/repo/.aws/config
-ENV CONFIG_FILE=config-coral.ini
 
 RUN cd / && apt-get update && apt-get install -y git python3-edgetpu && git clone \
     https://github.com/google-coral/project-posenet.git && sed -i 's/sudo / /g' \
     /project-posenet/install_requirements.sh && sh /project-posenet/install_requirements.sh
 ENV PYTHONPATH=$PYTHONPATH:/project-posenet
+ENV CONFIG_FILE=config-coral.ini
 
 COPY . /repo
 WORKDIR /repo

config-coral.ini

@@ -11,6 +11,8 @@ EnableSlackNotifications = no
 SlackChannel = lanthorn-notifications
 ; OccupancyAlertsMinInterval time is measured in seconds (if interval < 0 then no occupancy alerts are triggered)
 OccupancyAlertsMinInterval = 180
+MaxThreadRestarts = 5
+
 
 [API]
 Host = 0.0.0.0

config-jetson.ini

@@ -14,6 +14,7 @@ EnableSlackNotifications = no
 SlackChannel = lanthorn-notifications
 ; OccupancyAlertsMinInterval time is measured in seconds (if interval < 0 then no occupancy alerts are triggered)
 OccupancyAlertsMinInterval = 180
+MaxThreadRestarts = 5
 
 [API]
 Host = 0.0.0.0

config-x86-gpu.ini

@@ -23,6 +23,8 @@ DashboardURL = http://0.0.0.0:8000
 ScreenshotsDirectory = /repo/data/processor/static/screenshots
 EnableSlackNotifications = no
 SlackChannel = lanthorn-notifications
+OccupancyAlertsMinInterval = 180
+MaxThreadRestarts = 5
 
 [Area_0]
 Id = area0

config-x86-openvino.ini

@@ -25,6 +25,7 @@ EnableSlackNotifications = no
 SlackChannel = lanthorn-notifications
 ; OccupancyAlertsMinInterval time is measured in seconds (if interval < 0 then no occupancy alerts are triggered)
 OccupancyAlertsMinInterval = 180
+MaxThreadRestarts = 5
 
 [Area_0]
 Id = area0

config-x86.ini

@@ -25,6 +25,7 @@ EnableSlackNotifications = no
 SlackChannel = lanthorn-notifications
 ; OccupancyAlertsMinInterval time is measured in seconds (if interval < 0 then no occupancy alerts are triggered)
 OccupancyAlertsMinInterval = 180
+MaxThreadRestarts = 5
 
 [Area_0]
 Id = area0

coral-dev-board.Dockerfile

@@ -78,15 +78,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 ENV DEV_ALLOW_ALL_ORIGINS=true
 ENV AWS_SHARED_CREDENTIALS_FILE=/repo/.aws/credentials
 ENV AWS_CONFIG_FILE=/repo/.aws/config
-ENV CONFIG_FILE=config-coral.ini
-
 
 RUN cd / && apt-get update && apt-get install -y git python3-edgetpu && git clone \
     https://github.com/google-coral/project-posenet.git && sed -i 's/sudo / /g' \
     /project-posenet/install_requirements.sh && sh /project-posenet/install_requirements.sh
 ENV PYTHONPATH=$PYTHONPATH:/project-posenet
-
+ENV CONFIG_FILE=config-coral.ini
 # Also if you use opencv: LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libgomp.so.1.0.0"
+
 COPY . /repo
 WORKDIR /repo
 HEALTHCHECK --interval=30s --retries=2 --start-period=15s CMD bash healthcheck.bash

libs/area_threading.py

@@ -34,17 +34,20 @@ def __init__(self, config, area):
 
     def run(self):
         self.engine = AreaEngine(self.config, self.area)
+        restarts = 0
+        max_restarts = int(self.config.get_section_dict("App")["MaxThreadRestarts"])
         while True:
             try:
                 self.engine.process_area()
             except Exception as e:
                 logging.error(e, exc_info=True)
+                logging.info(f"Exception processing area {self.area['name']}")
+                if restarts == max_restarts:
+                    raise e
                 # Sleep the thread for 5 seconds and try to process the area again
                 time.sleep(5)
-                logging.info(f"Exception processing area {self.area['name']}")
                 logging.info("Restarting the area processing")
-
-        self.engine.process_area()
+                restarts += 1
 
     def stop(self):
         self.engine.stop_process_area()

libs/engine_threading.py

@@ -44,15 +44,20 @@ def __init__(self, config, source, live_feed_enabled=True):
 
     def run(self):
         self.engine = CvEngine(self.config, self.source['section'], self.live_feed_enabled)
+        restarts = 0
+        max_restarts = int(self.config.get_section_dict("App")["MaxThreadRestarts"])
         while True:
             try:
                 self.engine.process_video(self.source['url'])
             except Exception as e:
                 logging.error(e, exc_info=True)
+                logging.info(f"Exception processing video for source {self.source['name']}")
+                if restarts == max_restarts:
+                    raise e
                 # Sleep the thread for 5 seconds and try to process the video again
                 time.sleep(5)
-                logging.info(f"Exception processing video for source {self.source['name']}")
                 logging.info("Restarting the video processing")
+                restarts += 1
 
     def stop(self):
         self.engine.stop_process_video()

x86.Dockerfile

@@ -82,8 +82,8 @@ ENV AWS_SHARED_CREDENTIALS_FILE=/repo/.aws/credentials
 ENV AWS_CONFIG_FILE=/repo/.aws/config
 ENV CONFIG_FILE=config-x86.ini
 
-
 COPY . /repo
 WORKDIR /repo
+
 HEALTHCHECK --interval=30s --retries=2 --start-period=15s CMD bash healthcheck.bash
 CMD supervisord -c supervisord.conf -n