80 lines
2.3 KiB
Python
80 lines
2.3 KiB
Python
import os
|
|
import signal
|
|
import time
|
|
import logging
|
|
from multiprocessing import Value
|
|
|
|
logger = logging.getLogger('freqtrade.watchdog')
|
|
|
|
WATCHDOG_TIMEOUT = 300
|
|
|
|
|
|
class Watchdog:
|
|
|
|
shared_heartbeat = Value('d', 0.0)
|
|
kill_signal = None
|
|
|
|
def heartbeat(self) -> None:
|
|
logger.debug("Heartbeat")
|
|
self.shared_heartbeat.value = time.time()
|
|
|
|
def exit_gracefully(self, signum, frame):
|
|
logger.warning("Kill signal: {}".format(signum))
|
|
self.kill_signal = signum
|
|
|
|
def kill(self, pid):
|
|
logger.info("Stopping pid {}".format(pid))
|
|
os.kill(pid, signal.SIGTERM) # Better use sigint and then sigterm?
|
|
os.wait()
|
|
|
|
def start(self) -> bool:
|
|
self.heartbeat()
|
|
pid = os.fork()
|
|
if pid != 0:
|
|
# In watchdog proces, run it
|
|
if not self.run(pid):
|
|
# Got exit signal
|
|
return False
|
|
else:
|
|
# Forked new children, continue to main
|
|
self.heartbeat()
|
|
return True
|
|
else:
|
|
# In children process, continue to main
|
|
return True
|
|
|
|
def run(self, pid) -> bool:
|
|
logger.info("Watchdog started")
|
|
self.orig_SIGINT = signal.signal(signal.SIGINT, self.exit_gracefully)
|
|
self.orig_SIGTERM = signal.signal(signal.SIGTERM, self.exit_gracefully)
|
|
try:
|
|
while True:
|
|
if self.kill_signal:
|
|
raise KeyboardInterrupt()
|
|
|
|
timeout = time.time() - self.shared_heartbeat.value
|
|
|
|
if timeout > WATCHDOG_TIMEOUT:
|
|
logger.warning("Kill process due to timeout: {}".format(timeout))
|
|
self.kill(pid)
|
|
new_pid = os.fork()
|
|
if new_pid == 0:
|
|
logger.info("New children forked")
|
|
signal.signal(signal.SIGINT, self.orig_SIGINT)
|
|
signal.signal(signal.SIGTERM, self.orig_SIGTERM)
|
|
return True
|
|
else:
|
|
pid = new_pid
|
|
|
|
time.sleep(1)
|
|
|
|
except Exception as ex:
|
|
logger.exception(ex)
|
|
self.kill(pid)
|
|
return False
|
|
|
|
except KeyboardInterrupt:
|
|
logger.info("Watchdog stopped")
|
|
self.kill(pid)
|
|
return False
|