Watchdog for bot process

freqtrade/watchdog.py

@@ -0,0 +1,79 @@
+import os
+import signal
+import time
+import logging
+from multiprocessing import Value
+
+logger = logging.getLogger('freqtrade.watchdog')
+
+WATCHDOG_TIMEOUT = 300
+
+
+class Watchdog:
+
+    shared_heartbeat = Value('d', 0.0)
+    kill_signal = None
+
+    def heartbeat(self) -> None:
+        logger.debug("Heartbeat")
+        self.shared_heartbeat.value = time.time()
+
+    def exit_gracefully(self, signum, frame):
+        logger.warning("Kill signal: {}".format(signum))
+        self.kill_signal = signum
+
+    def kill(self, pid):
+        logger.info("Stopping pid {}".format(pid))
+        os.kill(pid, signal.SIGTERM)  # Better use sigint and then sigterm?
+        os.wait()
+
+    def start(self) -> bool:
+        self.heartbeat()
+        pid = os.fork()
+        if pid != 0:
+            # In watchdog proces, run it
+            if not self.run(pid):
+                # Got exit signal
+                return False
+            else:
+                # Forked new children, continue to main
+                self.heartbeat()
+                return True
+        else:
+            # In children process, continue to main
+            return True
+
+    def run(self, pid) -> bool:
+        logger.info("Watchdog started")
+        self.orig_SIGINT = signal.signal(signal.SIGINT, self.exit_gracefully)
+        self.orig_SIGTERM = signal.signal(signal.SIGTERM, self.exit_gracefully)
+        try:
+            while True:
+                if self.kill_signal:
+                    raise KeyboardInterrupt()
+
+                timeout = time.time() - self.shared_heartbeat.value
+
+                if timeout > WATCHDOG_TIMEOUT:
+                    logger.warning("Kill process due to timeout: {}".format(timeout))
+                    self.kill(pid)
+                    new_pid = os.fork()
+                    if new_pid == 0:
+                        logger.info("New children forked")
+                        signal.signal(signal.SIGINT, self.orig_SIGINT)
+                        signal.signal(signal.SIGTERM, self.orig_SIGTERM)
+                        return True
+                    else:
+                        pid = new_pid
+
+                time.sleep(1)
+
+        except Exception as ex:
+            logger.exception(ex)
+            self.kill(pid)
+            return False
+
+        except KeyboardInterrupt:
+            logger.info("Watchdog stopped")
+            self.kill(pid)
+            return False