Merge pull request #1648 from hroff-1902/sd-watchdog

Support for systemd watchdog

docs/bot-usage.md

@@ -38,7 +38,7 @@ optional arguments:
   --db-url PATH         Override trades database URL, this is useful if
                         dry_run is enabled or in custom deployments (default:
                         None).
-
+  --sd-notify           Notify systemd service manager.
 ```
 
 ### How to use a different configuration file?

docs/configuration.md

@@ -69,6 +69,7 @@ Mandatory Parameters are marked as **Required**.
 | `strategy` | DefaultStrategy | Defines Strategy class to use.
 | `strategy_path` | null | Adds an additional strategy lookup path (must be a folder).
 | `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second.
+| `internals.sd_notify` | false | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
 
 ### Parameters in the strategy
 

docs/installation.md

@@ -428,6 +428,19 @@ For this to be persistent (run when user is logged out) you'll need to enable `l
 sudo loginctl enable-linger "$USER"
 ```
 
+If you run the bot as a service, you can use systemd service manager as a software watchdog monitoring freqtrade bot 
+state and restarting it in the case of failures. If the `internals.sd_notify` parameter is set to true in the 
+configuration or the `--sd-notify` command line option is used, the bot will send keep-alive ping messages to systemd 
+using the sd_notify (systemd notifications) protocol and will also tell systemd its current state (Running or Stopped) 
+when it changes. 
+
+The `freqtrade.service.watchdog` file contains an example of the service unit configuration file which uses systemd 
+as the watchdog.
+
+!!! Note: 
+The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a 
+Docker container.
+
 ------
 
 ## Windows

freqtrade.service.watchdog

@@ -0,0 +1,30 @@
+[Unit]
+Description=Freqtrade Daemon
+After=network.target
+
+[Service]
+# Set WorkingDirectory and ExecStart to your file paths accordingly
+# NOTE: %h will be resolved to /home/<username>
+WorkingDirectory=%h/freqtrade
+ExecStart=/usr/bin/freqtrade --sd-notify
+
+Restart=always
+#Restart=on-failure
+
+# Note that we use Type=notify here
+Type=notify
+
+# Currently required if Type=notify
+NotifyAccess=all
+
+StartLimitInterval=1min
+StartLimitBurst=5
+
+TimeoutStartSec=1min
+
+# Use here (process_throttle_secs * 2) or longer time interval
+WatchdogSec=20
+
+[Install]
+WantedBy=default.target
+

freqtrade/arguments.py

@@ -127,6 +127,12 @@ class Arguments(object):
             type=str,
             metavar='PATH',
         )
+        self.parser.add_argument(
+            '--sd-notify',
+            help='Notify systemd service manager.',
+            action='store_true',
+            dest='sd_notify',
+        )
 
     @staticmethod
     def backtesting_options(parser: argparse.ArgumentParser) -> None:
@@ -140,7 +146,6 @@ class Arguments(object):
             dest='position_stacking',
             default=False
         )
-
         parser.add_argument(
             '--dmmp', '--disable-max-market-positions',
             help='Disable applying `max_open_trades` during backtest '

freqtrade/configuration.py

@@ -123,6 +123,10 @@ class Configuration(object):
         set_loggers(config['verbosity'])
         logger.info('Verbosity set to %s', config['verbosity'])
 
+        # Support for sd_notify
+        if self.args.sd_notify:
+            config['internals'].update({'sd_notify': True})
+
         # Add dynamic_whitelist if found
         if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist:
             # Update to volumePairList (the previous default)

freqtrade/constants.py

@@ -172,7 +172,8 @@ CONF_SCHEMA = {
             'type': 'object',
             'properties': {
                 'process_throttle_secs': {'type': 'number'},
-                'interval': {'type': 'integer'}
+                'interval': {'type': 'integer'},
+                'sd_notify': {'type': 'boolean'},
             }
         }
     },

freqtrade/freqtradebot.py

@@ -11,6 +11,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import arrow
 from requests.exceptions import RequestException
+import sdnotify
 
 from freqtrade import (DependencyException, OperationalException,
                        TemporaryError, __version__, constants, persistence)
@@ -51,6 +52,10 @@ class FreqtradeBot(object):
 
         # Init objects
         self.config = config
+
+        self._sd_notify = sdnotify.SystemdNotifier() if \
+            self.config.get('internals', {}).get('sd_notify', False) else None
+
         self.strategy: IStrategy = StrategyResolver(self.config).strategy
 
         self.rpc: RPCManager = RPCManager(self)
@@ -76,6 +81,11 @@ class FreqtradeBot(object):
         self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist']
         self._init_modules()
 
+        # Tell the systemd that we completed initialization phase
+        if self._sd_notify:
+            logger.debug("sd_notify: READY=1")
+            self._sd_notify.notify("READY=1")
+
     def _init_modules(self) -> None:
         """
         Initializes all modules and updates the config
@@ -99,9 +109,22 @@ class FreqtradeBot(object):
         :return: None
         """
         logger.info('Cleaning up modules ...')
+
         self.rpc.cleanup()
         persistence.cleanup()
 
+    def stopping(self) -> None:
+        # Tell systemd that we are exiting now
+        if self._sd_notify:
+            logger.debug("sd_notify: STOPPING=1")
+            self._sd_notify.notify("STOPPING=1")
+
+    def reconfigure(self) -> None:
+        # Tell systemd that we initiated reconfiguring
+        if self._sd_notify:
+            logger.debug("sd_notify: RELOADING=1")
+            self._sd_notify.notify("RELOADING=1")
+
     def worker(self, old_state: State = None) -> State:
         """
         Trading routine that must be run at each loop
@@ -119,16 +142,27 @@ class FreqtradeBot(object):
             if state == State.RUNNING:
                 self.rpc.startup_messages(self.config, self.pairlists)
 
-        if state == State.STOPPED:
+        throttle_secs = self.config.get('internals', {}).get(
-            time.sleep(1)
-        elif state == State.RUNNING:
-            min_secs = self.config.get('internals', {}).get(
             'process_throttle_secs',
             constants.PROCESS_THROTTLE_SECS
         )
 
-            self._throttle(func=self._process,
+        if state == State.STOPPED:
-                           min_secs=min_secs)
+            # Ping systemd watchdog before sleeping in the stopped state
+            if self._sd_notify:
+                logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: STOPPED.")
+                self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: STOPPED.")
+
+            time.sleep(throttle_secs)
+
+        elif state == State.RUNNING:
+            # Ping systemd watchdog before throttling
+            if self._sd_notify:
+                logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: RUNNING.")
+                self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: RUNNING.")
+
+            self._throttle(func=self._process, min_secs=throttle_secs)
+
         return state
 
     def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any:

freqtrade/main.py

@@ -60,6 +60,7 @@ def main(sysargv: List[str]) -> None:
         logger.exception('Fatal exception!')
     finally:
         if freqtrade:
+            freqtrade.stopping()
             freqtrade.rpc.send_msg({
                 'type': RPCMessageType.STATUS_NOTIFICATION,
                 'status': 'process died'
@@ -72,6 +73,8 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot:
     """
     Cleans up current instance, reloads the configuration and returns the new instance
     """
+    freqtrade.reconfigure()
+
     # Clean up current modules
     freqtrade.cleanup()
 

requirements.txt

@@ -24,3 +24,6 @@ py_find_1st==1.1.3
 
 # Load ticker files 30% faster
 python-rapidjson==0.7.0
+
+# Notify systemd
+sdnotify==0.3.2