From 9e7d56fe625c8060bd686dc2afad34231758b491 Mon Sep 17 00:00:00 2001
From: xwu2git <3871873+xwu2git@users.noreply.github.com>
Date: Mon, 22 Jul 2019 22:00:45 -0700
Subject: [PATCH 1/2] gracefully exit by handling SIGTERM and keep database
 consistent

---
 analytics/feeder/main.py              | 18 +++++++++++++-----
 maintenance/cleanup/Dockerfile        |  2 +-
 maintenance/cleanup/main.py           | 12 +++++++++---
 maintenance/where_indexing/Dockerfile |  2 +-
 maintenance/where_indexing/main.py    |  7 +++++++
 sensor/onvif/Dockerfile               |  2 +-
 sensor/onvif/main.py                  |  5 +++++
 sensor/simulation/Dockerfile          |  2 +-
 sensor/simulation/main.py             |  9 +++++++++
 trigger/health/Dockerfile             |  2 +-
 trigger/health/main.py                |  9 +++++++++
 web/cloud/Dockerfile                  |  2 +-
 web/cloud/main.py                     | 15 ++++++++++++++-
 web/local/Dockerfile                  |  2 +-
 web/local/main.py                     | 14 +++++++++++++-
 15 files changed, 86 insertions(+), 17 deletions(-)

diff --git a/analytics/feeder/main.py b/analytics/feeder/main.py
index 208ec4805..56e3cd526 100644
--- a/analytics/feeder/main.py
+++ b/analytics/feeder/main.py
@@ -2,6 +2,7 @@
 
 from db_ingest import DBIngest
 from db_query import DBQuery
+from signal import signal, SIGTERM
 
 from watchdog.observers import Observer
 from watchdog.events import FileSystemEventHandler
@@ -14,10 +15,7 @@
 from threading import Timer
 from probe import probe, run
 import time
-import datetime
 import os
-import sys
-import signal
 import tempfile
 import shutil
 import re
@@ -114,12 +112,13 @@ def stop(self):
         logger.info(" ### Stopping Feeder ### ")
 
         self._threadflag = False
-        self.mqttclient.loop_stop()
-        self.observer.stop()
 
         logger.debug("Unregistering algorithm from DB")
         self.db_alg.delete(self.alg_id)
 
+        self.mqttclient.loop_stop()
+        self.observer.stop()
+
     def startmqtt(self):
         self.mqttclient = mqtt.Client("feeder_" + self.alg_id)
         self.mqttclient.connect(self.mqtthost)
@@ -345,6 +344,15 @@ def ingest(self):
 
 if __name__ == '__main__':
     smtc_feeder = Feeder()
+
+    def quit_nicely(signum, sigframe):
+        try:
+            smtc_feeder.stop()
+        except Exception as e:
+            print("quit exception"+str(e))
+        exit(143)
+    signal(SIGTERM, quit_nicely)
+
     try:
         smtc_feeder.start()
     except KeyboardInterrupt:
diff --git a/maintenance/cleanup/Dockerfile b/maintenance/cleanup/Dockerfile
index 436f05ca8..705f917e6 100644
--- a/maintenance/cleanup/Dockerfile
+++ b/maintenance/cleanup/Dockerfile
@@ -4,7 +4,7 @@ FROM centos:7.6.1810
 RUN yum -y install epel-release && yum -y install python36-requests python36-ply && rm -rf /var/cache/yum/*
 
 COPY    *.py /home/
-CMD     ["/bin/bash","-c","/home/main.py"]
+CMD     ["/home/main.py"]
 
 ####
 ARG  USER
diff --git a/maintenance/cleanup/main.py b/maintenance/cleanup/main.py
index 4f2f57bc4..421012105 100755
--- a/maintenance/cleanup/main.py
+++ b/maintenance/cleanup/main.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 from db_query import DBQuery
+from signal import signal, SIGTERM
 import time
 import os
 
@@ -10,13 +11,19 @@
 office=list(map(float, os.environ["OFFICE"].split(",")))
 dbhost=os.environ["DBHOST"]
 
-while True:
+def quit_nicely(signum, sigframe):
+    exit(143)
+
+# set signal to quit nicely
+signal(SIGTERM, quit_nicely)
+
+while True: 
     time.sleep(service_interval)
 
     print("Searching...",flush=True)
     for index in indexes:
         db=DBQuery(index=index,office=office,host=dbhost)
-        for r in db.search("time<now-"+str(retention_time*1000)):
+        for r in db.search("time<now-"+str(retention_time*1000), size=500):
             # delete the record
             db.delete(r["_id"])
 
@@ -26,4 +33,3 @@
                     os.remove(r["_source"]["path"])
                 except Exception as e:
                     print(str(e))
-
diff --git a/maintenance/where_indexing/Dockerfile b/maintenance/where_indexing/Dockerfile
index 436f05ca8..705f917e6 100644
--- a/maintenance/where_indexing/Dockerfile
+++ b/maintenance/where_indexing/Dockerfile
@@ -4,7 +4,7 @@ FROM centos:7.6.1810
 RUN yum -y install epel-release && yum -y install python36-requests python36-ply && rm -rf /var/cache/yum/*
 
 COPY    *.py /home/
-CMD     ["/bin/bash","-c","/home/main.py"]
+CMD     ["/home/main.py"]
 
 ####
 ARG  USER
diff --git a/maintenance/where_indexing/main.py b/maintenance/where_indexing/main.py
index 4315e2914..04b0836d1 100755
--- a/maintenance/where_indexing/main.py
+++ b/maintenance/where_indexing/main.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 from db_query import DBQuery
+from signal import signal, SIGTERM
 import os
 import time
 
@@ -12,6 +13,12 @@
 office=list(map(float, os.environ["OFFICE"].split(",")))
 dbhost=os.environ["DBHOST"]
 
+def quit_nicely(signum, sigframe):
+    exit(143)
+
+# set signal to quit nicely
+signal(SIGTERM, quit_nicely)
+
 dbq=DBQuery(index=indexes[0],office=office,host=dbhost)
 dba=DBQuery(index=indexes[1],office=office,host=dbhost)
 while True:
diff --git a/sensor/onvif/Dockerfile b/sensor/onvif/Dockerfile
index fefc79e7f..346b1ce87 100644
--- a/sensor/onvif/Dockerfile
+++ b/sensor/onvif/Dockerfile
@@ -26,7 +26,7 @@ RUN  wget ${ZEEP_REPO} && \
      rm v${ONVIF_ZEEP_VER}.tar.gz
 
 COPY    *.py /home/
-CMD     ["/bin/bash","-c","/home/main.py"]
+CMD     ["/home/main.py"]
 WORKDIR /home
 EXPOSE  80
 
diff --git a/sensor/onvif/main.py b/sensor/onvif/main.py
index e82afa373..985c34002 100755
--- a/sensor/onvif/main.py
+++ b/sensor/onvif/main.py
@@ -8,6 +8,7 @@
 import subprocess
 from onvif import ONVIFCamera, ONVIFError
 import xml.etree.ElementTree as ET
+from signal import SIGTERM, signal
 
 from db_query import DBQuery
 from db_ingest import DBIngest
@@ -317,4 +318,8 @@ def discover_all_onvif_cameras():
         time.sleep(60)
 
 if __name__ == "__main__":
+    def quit_nicely(signum, sigframe):
+        exit(143)
+    signal(SIGTERM, quit_nicely)
+
     discover_all_onvif_cameras()
diff --git a/sensor/simulation/Dockerfile b/sensor/simulation/Dockerfile
index 41e167879..cefcbd0f7 100644
--- a/sensor/simulation/Dockerfile
+++ b/sensor/simulation/Dockerfile
@@ -2,7 +2,7 @@
 FROM ubuntu:18.04
 RUN apt-get update && apt-get install -y -q python3-requests vlc && rm -rf /var/lib/apt/lists/*
 COPY *.py /home/
-ENTRYPOINT ["/bin/bash","-c","/home/main.py"]
+CMD ["/home/main.py"]
 
 ####
 ARG  USER
diff --git a/sensor/simulation/main.py b/sensor/simulation/main.py
index 5b3d33923..4b03f2666 100755
--- a/sensor/simulation/main.py
+++ b/sensor/simulation/main.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 from db_ingest import DBIngest
+from signal import signal, SIGTERM
 import subprocess
 import socket
 import math
@@ -61,6 +62,14 @@ def geo_point(origin, distance, tc):
     "status": "idle",
 })
 
+def quit_nicely(signum, sigframe):
+    try:
+        db.delete(r["_id"])
+    except Exception as e:
+        print("quit exception: "+str(e))
+    exit(143)
+signal(SIGTERM, quit_nicely)
+
 # run rtspatt
 while True:
     simulated_root="/mnt/simulated"
diff --git a/trigger/health/Dockerfile b/trigger/health/Dockerfile
index 5a8bdcda3..dcc655268 100644
--- a/trigger/health/Dockerfile
+++ b/trigger/health/Dockerfile
@@ -3,7 +3,7 @@ FROM centos:7.6.1810
 
 RUN yum install -y epel-release && yum install -y python36-requests python36-ply && rm -rf /var/cache/yum/*
 COPY *.py /home/
-CMD  ["/bin/bash","-c","/home/main.py"]
+CMD  ["/home/main.py"]
 
 ####
 ARG  USER
diff --git a/trigger/health/main.py b/trigger/health/main.py
index 2d3c5840b..4c7761be9 100755
--- a/trigger/health/main.py
+++ b/trigger/health/main.py
@@ -2,6 +2,7 @@
 
 from db_ingest import DBIngest
 from db_query import DBQuery
+from signal import SIGTERM, signal
 import datetime
 import time
 import os
@@ -18,6 +19,14 @@
 })
 print(rt,flush=True)
 
+def quit_nicely(signum, sigframe):
+    try:
+        dbt.delete(rt["_id"])
+    except Exception as e:
+        print("quit exception: "+str(e))
+    exit(143)
+signal(SIGTERM, quit_nicely)
+
 dbs=DBQuery(index="sensors",office=office,host=dbhost)
 dba=DBQuery(index="algorithms",office=office,host=dbhost)
 dbat=DBIngest(index="alerts",office=office,host=dbhost)
diff --git a/web/cloud/Dockerfile b/web/cloud/Dockerfile
index e152392f1..519eae1f5 100644
--- a/web/cloud/Dockerfile
+++ b/web/cloud/Dockerfile
@@ -5,7 +5,7 @@ RUN yum -y install epel-release && yum -y install nginx python36-tornado python3
 
 COPY    *.conf /etc/nginx/
 COPY    *.py /home/
-CMD     ["/bin/bash","-c","/home/main.py&/usr/sbin/nginx"]
+CMD     ["/home/main.py"]
 EXPOSE  8080
 
 ####
diff --git a/web/cloud/main.py b/web/cloud/main.py
index f6f8a685e..db56cb5e9 100755
--- a/web/cloud/main.py
+++ b/web/cloud/main.py
@@ -6,6 +6,8 @@
 from hint import HintHandler
 from redirect import RedirectHandler
 from stats import StatsHandler
+from subprocess import Popen
+from signal import signal, SIGTERM, SIGQUIT
 
 app = web.Application([
     (r'/api/search',SearchHandler),
@@ -21,4 +23,15 @@
     parse_command_line()
     print("Listening to " + options.ip + ":" + str(options.port))
     app.listen(options.port, address=options.ip)
-    ioloop.IOLoop.instance().start()
+
+    tornado1=ioloop.IOLoop.instance();
+    nginx1=Popen(["/usr/sbin/nginx"])
+
+    # set SIGTERM/SIGQUIT handler
+    def quit_nicely(signum, frame):
+        nginx1.send_signal(SIGQUIT)
+        tornado1.add_callback(tornado1.stop)
+        
+    signal(SIGTERM, quit_nicely)
+    tornado1.start()
+    nginx1.wait()
diff --git a/web/local/Dockerfile b/web/local/Dockerfile
index c4d5ffe38..519eae1f5 100644
--- a/web/local/Dockerfile
+++ b/web/local/Dockerfile
@@ -5,7 +5,7 @@ RUN yum -y install epel-release && yum -y install nginx python36-tornado python3
 
 COPY    *.conf /etc/nginx/
 COPY    *.py /home/
-CMD     ["/bin/bash","-c","/usr/sbin/nginx&/home/main.py"]
+CMD     ["/home/main.py"]
 EXPOSE  8080
 
 ####
diff --git a/web/local/main.py b/web/local/main.py
index 6c38bfcba..48d87861f 100755
--- a/web/local/main.py
+++ b/web/local/main.py
@@ -4,6 +4,8 @@
 from tornado.options import define, options, parse_command_line
 from workload import WorkloadHandler
 from db_ingest import DBIngest
+from subprocess import Popen
+from signal import signal, SIGTERM, SIGQUIT
 import socket
 import os
 
@@ -31,4 +33,14 @@
     print("Listening to " + options.ip + ":" + str(options.port))
 
     app.listen(options.port, address=options.ip)
-    ioloop.IOLoop.instance().start()
+    tornado1=ioloop.IOLoop.instance();
+    nginx1=Popen(["/usr/sbin/nginx"])
+    
+    def quit_nicely(signum, frame):
+        tornado1.add_callback(tornado1.stop)
+        nginx1.send_signal(SIGQUIT)
+        db.delete(r["_id"])
+
+    signal(SIGTERM, quit_nicely)
+    tornado1.start()
+    nginx1.wait()

From 9601de1744b87ac9cc0e1315be3b8aa470a7fc07 Mon Sep 17 00:00:00 2001
From: xwu2git <3871873+xwu2git@users.noreply.github.com>
Date: Mon, 22 Jul 2019 22:33:08 -0700
Subject: [PATCH 2/2] office -> recording fix

---
 volume/html/js/office.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/volume/html/js/office.js b/volume/html/js/office.js
index e533c9997..6dea288fd 100644
--- a/volume/html/js/office.js
+++ b/volume/html/js/office.js
@@ -29,7 +29,7 @@ $("#pg-office").on(":initpage", function(e, queries, office) {
                 var line=$("<tr><td>"+v._source.name+"</td><td>"+v._id+"</td><td>"+source+"</td><td>"+v._source.status+"</td><td>"+latency+"</td><td>"+performance+"</td><td>"+v._source.skip+" frame(s)</td></tr>");
                 tbody.append(line);
                 line.find("a").click(function () {
-                    selectPage("recording",['sensor="'+v._source.source+'"']);
+                    selectPage("recording",['sensor="'+v._source.source+'"',office]);
                 });
             });