From 8730852d6ede9fb4c3fe6ffe07c0305ab8068cd4 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 10 Mar 2019 20:05:33 +0300 Subject: [PATCH] Support for systemd watchdog via sd_notify --- docs/bot-usage.md | 2 +- docs/configuration.md | 1 + docs/installation.md | 13 +++++++++++ freqtrade.service.watchdog | 30 +++++++++++++++++++++++++ freqtrade/arguments.py | 7 +++++- freqtrade/configuration.py | 4 ++++ freqtrade/constants.py | 3 ++- freqtrade/freqtradebot.py | 45 ++++++++++++++++++++++++++++++-------- requirements.txt | 5 ++++- 9 files changed, 97 insertions(+), 13 deletions(-) create mode 100644 freqtrade.service.watchdog diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 8b10fb6e9..739a89c07 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -38,7 +38,7 @@ optional arguments: --db-url PATH Override trades database URL, this is useful if dry_run is enabled or in custom deployments (default: None). - + --sd-notify Notify systemd service manager. ``` ### How to use a different configuration file? diff --git a/docs/configuration.md b/docs/configuration.md index d7e774595..1874bef4b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -67,6 +67,7 @@ Mandatory Parameters are marked as **Required**. | `strategy` | DefaultStrategy | Defines Strategy class to use. | `strategy_path` | null | Adds an additional strategy lookup path (must be a folder). | `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second. +| `internals.sd_notify` | false | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details. ### Parameters in the strategy diff --git a/docs/installation.md b/docs/installation.md index 80223f954..bd6c50c5a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -428,6 +428,19 @@ For this to be persistent (run when user is logged out) you'll need to enable `l sudo loginctl enable-linger "$USER" ``` +If you run the bot as a service, you can use systemd service manager as a software watchdog monitoring freqtrade bot +state and restarting it in the case of failures. If the `internals.sd_notify` parameter is set to true in the +configuration or the `--sd-notify` command line option is used, the bot will send keep-alive ping messages to systemd +using the sd_notify (systemd notifications) protocol and will also tell systemd its current state (Running or Stopped) +when it changes. + +The `freqtrade.service.watchdog` file contains an example of the service unit configuration file which uses systemd +as the watchdog. + +!!! Note: +The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a +Docker container. + ------ ## Windows diff --git a/freqtrade.service.watchdog b/freqtrade.service.watchdog new file mode 100644 index 000000000..ba491fa53 --- /dev/null +++ b/freqtrade.service.watchdog @@ -0,0 +1,30 @@ +[Unit] +Description=Freqtrade Daemon +After=network.target + +[Service] +# Set WorkingDirectory and ExecStart to your file paths accordingly +# NOTE: %h will be resolved to /home/ +WorkingDirectory=%h/freqtrade +ExecStart=/usr/bin/freqtrade --sd-notify + +Restart=always +#Restart=on-failure + +# Note that we use Type=notify here +Type=notify + +# Currently required if Type=notify +NotifyAccess=all + +StartLimitInterval=1min +StartLimitBurst=5 + +TimeoutStartSec=1min + +# Use here (process_throttle_secs * 2) or longer time interval +WatchdogSec=20 + +[Install] +WantedBy=default.target + diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index ee19f6fe1..604386426 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -127,6 +127,12 @@ class Arguments(object): type=str, metavar='PATH', ) + self.parser.add_argument( + '--sd-notify', + help='Notify systemd service manager.', + action='store_true', + dest='sd_notify', + ) @staticmethod def backtesting_options(parser: argparse.ArgumentParser) -> None: @@ -140,7 +146,6 @@ class Arguments(object): dest='position_stacking', default=False ) - parser.add_argument( '--dmmp', '--disable-max-market-positions', help='Disable applying `max_open_trades` during backtest ' diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index e96305993..6ca0299f0 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -122,6 +122,10 @@ class Configuration(object): set_loggers(config['verbosity']) logger.info('Verbosity set to %s', config['verbosity']) + # Support for sd_notify + if self.args.sd_notify: + config['internals'].update({'sd_notify': True}) + # Add dynamic_whitelist if found if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: # Update to volumePairList (the previous default) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 4d0907d78..ff623df14 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -171,7 +171,8 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'process_throttle_secs': {'type': 'number'}, - 'interval': {'type': 'integer'} + 'interval': {'type': 'integer'}, + 'sd_notify': {'type': 'boolean'}, } } }, diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index dce3136df..4f3edd600 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -11,6 +11,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple import arrow from requests.exceptions import RequestException +import sdnotify from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) @@ -51,6 +52,10 @@ class FreqtradeBot(object): # Init objects self.config = config + + self._sd_notify = sdnotify.SystemdNotifier() if \ + self.config.get('internals', {}).get('sd_notify', False) else None + self.strategy: IStrategy = StrategyResolver(self.config).strategy self.rpc: RPCManager = RPCManager(self) @@ -76,6 +81,11 @@ class FreqtradeBot(object): self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] self._init_modules() + # Tell the systemd that we completed initialization phase + if self._sd_notify: + logger.debug("sd_notify: READY=1") + self._sd_notify.notify('READY=1') + def _init_modules(self) -> None: """ Initializes all modules and updates the config @@ -99,6 +109,12 @@ class FreqtradeBot(object): :return: None """ logger.info('Cleaning up modules ...') + + # Tell systemd that we are stopping now + if self._sd_notify: + logger.debug("sd_notify: STOPPING=1") + self._sd_notify.notify('STOPPING=1') + self.rpc.cleanup() persistence.cleanup() @@ -119,16 +135,27 @@ class FreqtradeBot(object): if state == State.RUNNING: self.rpc.startup_messages(self.config, self.pairlists) - if state == State.STOPPED: - time.sleep(1) - elif state == State.RUNNING: - min_secs = self.config.get('internals', {}).get( - 'process_throttle_secs', - constants.PROCESS_THROTTLE_SECS - ) + throttle_secs = self.config.get('internals', {}).get( + 'process_throttle_secs', + constants.PROCESS_THROTTLE_SECS + ) + + if state == State.STOPPED: + # Ping systemd watchdog before sleeping in the stopped state + if self._sd_notify: + logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: STOPPED.") + self._sd_notify.notify('WATCHDOG=1\nSTATUS=State: STOPPED.') + + time.sleep(throttle_secs) + + elif state == State.RUNNING: + # Ping systemd watchdog before throttling + if self._sd_notify: + logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: RUNNING.") + self._sd_notify.notify('WATCHDOG=1\nSTATUS=State: RUNNING.') + + self._throttle(func=self._process, min_secs=throttle_secs) - self._throttle(func=self._process, - min_secs=min_secs) return state def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: diff --git a/requirements.txt b/requirements.txt index f1ab77b0a..4c223eed4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,5 +22,8 @@ scikit-optimize==0.5.2 # find first, C search in arrays py_find_1st==1.1.3 -#Load ticker files 30% faster +# Load ticker files 30% faster python-rapidjson==0.7.0 + +# Notify systemd +sdnotify==0.3.2