Fault tolerance

amd64-usbtpu.Dockerfile

@@ -83,6 +83,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 	python3-requests \
         build-essential \
         libedgetpu1-std \
+        supervisor \
     && rm -rf /var/lib/apt/lists/* \
     && python3 -m pip install --upgrade pip setuptools==41.0.0 wheel && pip install -r /requirements.txt \
 	https://dl.google.com/coral/python/tflite_runtime-2.1.0.post1-cp37-cp37m-linux_x86_64.whl \
@@ -94,6 +95,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 ENV DEV_ALLOW_ALL_ORIGINS=true
 ENV AWS_SHARED_CREDENTIALS_FILE=/repo/.aws/credentials
 ENV AWS_CONFIG_FILE=/repo/.aws/config
+ENV CONFIG_FILE=config-coral.ini
 
 RUN cd / && apt-get update && apt-get install -y git python3-edgetpu && git clone \
     https://github.com/google-coral/project-posenet.git && sed -i 's/sudo / /g' \
@@ -102,5 +104,5 @@ ENV PYTHONPATH=$PYTHONPATH:/project-posenet
 
 COPY . /repo
 WORKDIR /repo
-ENTRYPOINT ["bash", "start_services.bash"]
-CMD ["config-coral.ini"]
+HEALTHCHECK --interval=30s --retries=2 --start-period=15s CMD bash healthcheck.bash
+CMD supervisord -c supervisord.conf -n

coral-dev-board.Dockerfile

@@ -67,6 +67,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         python3-pip \
         python3-scipy \
         python3-wget \
+        supervisor \
     && rm -rf /var/lib/apt/lists/* \
     && python3 -m pip install --upgrade pip setuptools==41.0.0 && pip install -r /requirements.txt \
         https://dl.google.com/coral/python/tflite_runtime-2.1.0.post1-cp37-cp37m-linux_aarch64.whl \
@@ -77,6 +78,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 ENV DEV_ALLOW_ALL_ORIGINS=true
 ENV AWS_SHARED_CREDENTIALS_FILE=/repo/.aws/credentials
 ENV AWS_CONFIG_FILE=/repo/.aws/config
+ENV CONFIG_FILE=config-coral.ini
+
 
 RUN cd / && apt-get update && apt-get install -y git python3-edgetpu && git clone \
     https://github.com/google-coral/project-posenet.git && sed -i 's/sudo / /g' \
@@ -86,5 +89,5 @@ ENV PYTHONPATH=$PYTHONPATH:/project-posenet
 # Also if you use opencv: LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libgomp.so.1.0.0"
 COPY . /repo
 WORKDIR /repo
-ENTRYPOINT ["bash", "start_services.bash"]
-CMD ["config-coral.ini"]
+HEALTHCHECK --interval=30s --retries=2 --start-period=15s CMD bash healthcheck.bash
+CMD supervisord -c supervisord.conf -n

healthcheck.bash

@@ -0,0 +1,6 @@
+#!/bin/bash
+if fatalError=$(supervisorctl -c supervisord.conf status all | grep -i "FATAL\|UNKNOWN"); then
+    exit 1;
+else
+    exit 0;
+fi

jetson-nano.Dockerfile

@@ -82,6 +82,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         python3-pip \
         python3-scipy \
         python3-wget \
+        supervisor \
     && rm -rf /var/lib/apt/lists/* \
     && ln -sf $(which gcc) /usr/local/bin/gcc-aarch64-linux-gnu \
     && ln -sf $(which g++) /usr/local/bin/g++-aarch64-linux-gnu \
@@ -96,8 +97,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 ENV DEV_ALLOW_ALL_ORIGINS=true
 ENV AWS_SHARED_CREDENTIALS_FILE=/repo/.aws/credentials
 ENV AWS_CONFIG_FILE=/repo/.aws/config
+ENV CONFIG_FILE=config-jetson.ini
 
 COPY . /repo/
 WORKDIR /repo
-ENTRYPOINT ["bash", "start_services.bash"]
-CMD ["config-jetson.ini"]
+HEALTHCHECK --interval=30s --retries=2 --start-period=15s CMD bash healthcheck.bash
+CMD supervisord -c supervisord.conf -n

jetson-tx2.Dockerfile

@@ -80,6 +80,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         python3-pip \
         python3-scipy \
         python3-wget \
+        supervisor \
     && rm -rf /var/lib/apt/lists/* \
     && ln -sf $(which gcc) /usr/local/bin/gcc-aarch64-linux-gnu \
     && ln -sf $(which g++) /usr/local/bin/g++-aarch64-linux-gnu \
@@ -92,8 +93,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 ENV DEV_ALLOW_ALL_ORIGINS=true
 ENV AWS_SHARED_CREDENTIALS_FILE=/repo/.aws/credentials
 ENV AWS_CONFIG_FILE=/repo/.aws/config
+ENV CONFIG_FILE=config-jetson.ini
 
 COPY . /repo/
 WORKDIR /repo
-ENTRYPOINT ["bash", "start_services.bash"]
-CMD ["config-jetson.ini"]
+HEALTHCHECK --interval=30s --retries=2 --start-period=15s CMD bash healthcheck.bash
+CMD supervisord -c supervisord.conf -n

libs/area_reporting.py

@@ -30,8 +30,10 @@ def __init__(self, config, area):
             camera['file_path'] = os.path.join(self.log_dir, camera['id'], "objects_log")
             camera['last_processed_time'] = time.time()
 
-        self.mail_service = MailService(config)
-        self.slack_service = SlackService(config)
+        if self.should_send_email_notifications:
+            self.mail_service = MailService(config)
+        if self.should_send_slack_notifications:
+            self.slack_service = SlackService(config)
 
     def process_area(self):
         # Sleep for a while so cameras start processing

libs/area_threading.py

@@ -1,4 +1,5 @@
 import os
+import time
 from threading import Thread
 from libs.area_reporting import AreaReporting as AreaEngine
 import logging
@@ -33,6 +34,16 @@ def __init__(self, config, area):
 
     def run(self):
         self.engine = AreaEngine(self.config, self.area)
+        while True:
+            try:
+                self.engine.process_area()
+            except Exception as e:
+                logging.error(e, exc_info=True)
+                # Sleep the thread for 5 seconds and try to process the area again
+                time.sleep(5)
+                logging.info(f"Exception processing area {self.area['name']}")
+                logging.info("Restarting the area processing")
+
         self.engine.process_area()
 
     def stop(self):

libs/engine_threading.py

@@ -1,4 +1,5 @@
 import os
+import time
 from shutil import rmtree
 from threading import Thread
 from libs.distancing import Distancing as CvEngine
@@ -43,7 +44,15 @@ def __init__(self, config, source, live_feed_enabled=True):
 
     def run(self):
         self.engine = CvEngine(self.config, self.source['section'], self.live_feed_enabled)
-        self.engine.process_video(self.source['url'])
+        while True:
+            try:
+                self.engine.process_video(self.source['url'])
+            except Exception as e:
+                logging.error(e, exc_info=True)
+                # Sleep the thread for 5 seconds and try to process the video again
+                time.sleep(5)
+                logging.info(f"Exception processing video for source {self.source['name']}")
+                logging.info("Restarting the video processing")
 
     def stop(self):
         self.engine.stop_process_video()

sample_startup.bash

@@ -3,7 +3,7 @@ config="$1"
 
 # check if video file exists, if not download it
 line=$(cat $config | grep VideoPath)
-videoPath="$(cut -d'=' -f2 <<<"$line")"
+videoPath="$(cut -d'=' -f2 <<<"$line" | xargs)"
 if [ ! -f "$videoPath" ]; then
     echo "video file at $videoPath not exists, downloading..."
     sh '/repo/download_sample_video.sh'

supervisord.conf

@@ -0,0 +1,42 @@
+[supervisord]
+
+[unix_http_server]
+file=/var/run/supervisor.sock
+chmod=0700
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix:///var/run/supervisor.sock
+
+[program:api]
+command=python3 run_processor_api.py --config %(ENV_CONFIG_FILE)s
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=true
+
+[program:core]
+command=python3 run_processor_core.py --config %(ENV_CONFIG_FILE)s
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=true
+
+[program:startup]
+command=/repo/sample_startup.bash %(ENV_CONFIG_FILE)s
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:reports]
+command=python3 create_reports.py --config %(ENV_CONFIG_FILE)s
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=true

x86-gpu.Dockerfile

@@ -68,6 +68,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         python3-pip \
         python3-scipy \
         python3-wget \
+        supervisor \
     && rm -rf /var/lib/apt/lists/* \
     && python3 -m pip install --upgrade pip setuptools==41.0.0 && pip install -r /requirements.txt \
     && apt-get purge -y \
@@ -80,7 +81,10 @@ ENV DEV_ALLOW_ALL_ORIGINS=true
 ENV AWS_SHARED_CREDENTIALS_FILE=/repo/.aws/credentials
 ENV AWS_CONFIG_FILE=/repo/.aws/config
 ENV TF_FORCE_GPU_ALLOW_GROWTH=true
+ENV CONFIG_FILE=config-x86-gpu.ini
+
 COPY . /repo
 WORKDIR /repo
-ENTRYPOINT ["bash", "start_services.bash"]
-CMD ["config-x86-gpu.ini"]
+
+HEALTHCHECK --interval=30s --retries=2 --start-period=15s CMD bash healthcheck.bash
+CMD supervisord -c supervisord.conf -n

x86-openvino.Dockerfile

@@ -69,6 +69,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         python3-pip \
         python3-scipy \
         python3-wget \
+        supervisor \
     && rm -rf /var/lib/apt/lists/* \
     && python3 -m pip install --upgrade pip setuptools==41.0.0 wheel && pip install -r /requirements.txt \
     && apt-get purge -y \
@@ -82,7 +83,10 @@ ADD docker/x86-openvino/openvino_setupvars.py /opt/openvino_setupvars.py
 ENV DEV_ALLOW_ALL_ORIGINS=true
 ENV AWS_SHARED_CREDENTIALS_FILE=/repo/.aws/credentials
 ENV AWS_CONFIG_FILE=/repo/.aws/config
+ENV CONFIG_FILE=config-x86-openvino.ini
 
 COPY . /repo
 WORKDIR /repo
-CMD env `python3 /opt/openvino_setupvars.py` bash start_services.bash config-x86-openvino.ini
+
+HEALTHCHECK --interval=30s --retries=2 --start-period=15s CMD bash healthcheck.bash
+CMD env `python3 /opt/openvino_setupvars.py` supervisord -c supervisord.conf -n

x86.Dockerfile

@@ -68,6 +68,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         python3-pip \
         python3-scipy \
         python3-wget \
+        supervisor \
     && rm -rf /var/lib/apt/lists/* \
     && python3 -m pip install --upgrade pip setuptools==41.0.0 && pip install -r /requirements.txt \
     && apt-get purge -y \
@@ -79,8 +80,10 @@ RUN apt-get update && apt-get install -y python3-dev && pip3 install torch==1.5
 ENV DEV_ALLOW_ALL_ORIGINS=true
 ENV AWS_SHARED_CREDENTIALS_FILE=/repo/.aws/credentials
 ENV AWS_CONFIG_FILE=/repo/.aws/config
+ENV CONFIG_FILE=config-x86.ini
+
 
 COPY . /repo
 WORKDIR /repo
-ENTRYPOINT ["bash", "start_services.bash"]
-CMD ["config-x86.ini"]
+HEALTHCHECK --interval=30s --retries=2 --start-period=15s CMD bash healthcheck.bash
+CMD supervisord -c supervisord.conf -n