Merge remote-tracking branch 'origin/develop' into logging-syslog

config_full.json.example

@@ -119,7 +119,8 @@
     "initial_state": "running",
     "forcebuy_enable": false,
     "internals": {
-        "process_throttle_secs": 5
+        "process_throttle_secs": 5,
+        "heartbeat_interval": 60
     },
     "strategy": "DefaultStrategy",
     "strategy_path": "user_data/strategies/"

docs/advanced-setup.md

@@ -0,0 +1,33 @@
+# Advanced Post-installation Tasks
+
+This page explains some advanced tasks and configuration options that can be performed after the bot installation and may be uselful in some environments.
+
+If you do not know what things mentioned here mean, you probably do not need it.
+
+## Configure the bot running as a systemd service
+
+Copy the `freqtrade.service` file to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup.
+
+After that you can start the daemon with:
+
+```bash
+systemctl --user start freqtrade
+```
+
+For this to be persistent (run when user is logged out) you'll need to enable `linger` for your freqtrade user.
+
+```bash
+sudo loginctl enable-linger "$USER"
+```
+
+If you run the bot as a service, you can use systemd service manager as a software watchdog monitoring freqtrade bot 
+state and restarting it in the case of failures. If the `internals.sd_notify` parameter is set to true in the 
+configuration or the `--sd-notify` command line option is used, the bot will send keep-alive ping messages to systemd 
+using the sd_notify (systemd notifications) protocol and will also tell systemd its current state (Running or Stopped) 
+when it changes. 
+
+The `freqtrade.service.watchdog` file contains an example of the service unit configuration file which uses systemd 
+as the watchdog.
+
+!!! Note
+    The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a Docker container.

docs/configuration.md

@@ -98,6 +98,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `strategy` | DefaultStrategy | Defines Strategy class to use.
 | `strategy_path` | null | Adds an additional strategy lookup path (must be a directory).
 | `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second.
+| `internals.heartbeat_interval` | 60 | Print heartbeat message every X seconds. Set to 0 to disable heartbeat messages.
 | `internals.sd_notify` | false | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
 | `logfile` | | Specify Logfile. Uses a rolling strategy of 10 files, with 1Mb per file.
 | `user_data_dir` | cwd()/user_data | Directory containing user data. Defaults to `./user_data/`.
@@ -330,7 +331,7 @@ This configuration enables binance, as well as rate limiting to avoid bans from
     Optimal settings for rate limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings.
     We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step.
 
-#### Advanced FreqTrade Exchange configuration
+#### Advanced Freqtrade Exchange configuration
 
 Advanced options can be configured using the `_ft_has_params` setting, which will override Defaults and exchange-specific behaviours.
 
@@ -350,6 +351,13 @@ For example, to test the order type `FOK` with Kraken, and modify candle_limit t
 !!! Warning
     Please make sure to fully understand the impacts of these settings before modifying them.
 
+#### Random notes for other exchanges
+
+* The Ocean (ccxt id: 'theocean') exchange uses Web3 functionality and requires web3 package to be installed:
+```shell
+$ pip3 install web3
+```
+
 ### What values can be used for fiat_display_currency?
 
 The `fiat_display_currency` configuration parameter sets the base currency to use for the

docs/installation.md

@@ -151,7 +151,7 @@ python3 -m venv .env
 source .env/bin/activate
 ```
 
-#### 3. Install FreqTrade
+#### 3. Install Freqtrade
 
 Clone the git repository:
 
@@ -192,33 +192,9 @@ freqtrade -c config.json
 
 *Note*: If you run the bot on a server, you should consider using [Docker](docker.md) or a terminal multiplexer like `screen` or [`tmux`](https://en.wikipedia.org/wiki/Tmux) to avoid that the bot is stopped on logout.
 
-#### 7. [Optional] Configure `freqtrade` as a `systemd` service
+#### 7. (Optional) Post-installation Tasks
 
-From the freqtrade repo... copy `freqtrade.service` to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup.
-
-After that you can start the daemon with:
-
-```bash
-systemctl --user start freqtrade
-```
-
-For this to be persistent (run when user is logged out) you'll need to enable `linger` for your freqtrade user.
-
-```bash
-sudo loginctl enable-linger "$USER"
-```
-
-If you run the bot as a service, you can use systemd service manager as a software watchdog monitoring freqtrade bot 
-state and restarting it in the case of failures. If the `internals.sd_notify` parameter is set to true in the 
-configuration or the `--sd-notify` command line option is used, the bot will send keep-alive ping messages to systemd 
-using the sd_notify (systemd notifications) protocol and will also tell systemd its current state (Running or Stopped) 
-when it changes. 
-
-The `freqtrade.service.watchdog` file contains an example of the service unit configuration file which uses systemd 
-as the watchdog.
-
-!!! Note
-    The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a Docker container.
+On Linux, as an optional post-installation task, you can setup the bot to run as a `systemd` service. See [Advanced Post-installation Tasks](advanced-setup.md) for details.
 
 ------
 

freqtrade/freqtradebot.py

@@ -6,26 +6,27 @@ import logging
 import traceback
 from datetime import datetime
 from math import isclose
+from os import getpid
 from typing import Any, Dict, List, Optional, Tuple
 
 import arrow
 from requests.exceptions import RequestException
 
-from freqtrade import (DependencyException, InvalidOrderException,
-                       __version__, constants, persistence)
+from freqtrade import (DependencyException, InvalidOrderException, __version__,
+                       constants, persistence)
+from freqtrade.configuration import validate_config_consistency
 from freqtrade.data.converter import order_book_to_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.edge import Edge
-from freqtrade.configuration import validate_config_consistency
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
 from freqtrade.persistence import Trade
+from freqtrade.resolvers import (ExchangeResolver, PairListResolver,
+                                 StrategyResolver)
 from freqtrade.rpc import RPCManager, RPCMessageType
-from freqtrade.resolvers import ExchangeResolver, StrategyResolver, PairListResolver
 from freqtrade.state import State
-from freqtrade.strategy.interface import SellType, IStrategy
+from freqtrade.strategy.interface import IStrategy, SellType
 from freqtrade.wallets import Wallets
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -50,13 +51,15 @@ class FreqtradeBot:
         # Init objects
         self.config = config
 
+        self._heartbeat_msg = 0
+
+        self.heartbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60)
+
         self.strategy: IStrategy = StrategyResolver(self.config).strategy
 
         # Check config consistency here since strategies can set certain options
         validate_config_consistency(config)
 
-        self.rpc: RPCManager = RPCManager(self)
-
         self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange
 
         self.wallets = Wallets(self.config, self.exchange)
@@ -83,6 +86,13 @@ class FreqtradeBot:
         initial_state = self.config.get('initial_state')
         self.state = State[initial_state.upper()] if initial_state else State.STOPPED
 
+        # RPC runs in separate threads, can start handling external commands just after
+        # initialization, even before Freqtradebot has a chance to start its throttling,
+        # so anything in the Freqtradebot instance should be ready (initialized), including
+        # the initial state of the bot.
+        # Keep this at the end of this initialization method.
+        self.rpc: RPCManager = RPCManager(self)
+
     def cleanup(self) -> None:
         """
         Cleanup pending resources on an already stopped bot
@@ -145,6 +155,11 @@ class FreqtradeBot:
             self.check_handle_timedout()
             Trade.session.flush()
 
+        if (self.heartbeat_interval
+           and (arrow.utcnow().timestamp - self._heartbeat_msg > self.heartbeat_interval)):
+            logger.info(f"Bot heartbeat. PID={getpid()}")
+            self._heartbeat_msg = arrow.utcnow().timestamp
+
     def _extend_whitelist_with_trades(self, whitelist: List[str], trades: List[Any]):
         """
         Extend whitelist with pairs from open trades
@@ -438,7 +453,7 @@ class FreqtradeBot:
         try:
             # Create entity and execute trade
             if not self.create_trades():
-                logger.info('Found no buy signals for whitelisted currencies. Trying again...')
+                logger.debug('Found no buy signals for whitelisted currencies. Trying again...')
         except DependencyException as exception:
             logger.warning('Unable to create trade: %s', exception)
 

mkdocs.yml

@@ -22,6 +22,7 @@ nav:
         - Strategy analysis: strategy_analysis_example.md
         - Plotting: plotting.md
     - SQL Cheatsheet: sql_cheatsheet.md
+    - Advanced Post-installation Tasks: advanced-setup.md
     - Sandbox Testing: sandbox-testing.md
     - Deprecated Features: deprecated.md
     - Contributors Guide: developer.md

scripts/download_backtest_data.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-
-
-print("This script has been integrated into freqtrade "
-      "and its functionality is available by calling `freqtrade download-data`.")
-print("Please check the documentation on https://www.freqtrade.io/en/latest/backtesting/ "
-      "for details.")
-
-sys.exit(1)

scripts/get_market_pairs.py

@@ -1,103 +0,0 @@
-"""
-This script was adapted from ccxt here:
-https://github.com/ccxt/ccxt/blob/master/examples/py/arbitrage-pairs.py
-"""
-import os
-import sys
-import traceback
-
-root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-sys.path.append(root + '/python')
-
-import ccxt  # noqa: E402
-
-
-def style(s, style):
-    return style + s + '\033[0m'
-
-
-def green(s):
-    return style(s, '\033[92m')
-
-
-def blue(s):
-    return style(s, '\033[94m')
-
-
-def yellow(s):
-    return style(s, '\033[93m')
-
-
-def red(s):
-    return style(s, '\033[91m')
-
-
-def pink(s):
-    return style(s, '\033[95m')
-
-
-def bold(s):
-    return style(s, '\033[1m')
-
-
-def underline(s):
-    return style(s, '\033[4m')
-
-
-def dump(*args):
-    print(' '.join([str(arg) for arg in args]))
-
-
-def print_supported_exchanges():
-    dump('Supported exchanges:', green(', '.join(ccxt.exchanges)))
-
-
-try:
-
-    if len(sys.argv) < 2:
-        dump("Usage: python " + sys.argv[0], green('id'))
-        print_supported_exchanges()
-        sys.exit(1)
-
-    id = sys.argv[1]  # get exchange id from command line arguments
-
-    # check if the exchange is supported by ccxt
-    exchange_found = id in ccxt.exchanges
-
-    if exchange_found:
-        dump('Instantiating', green(id), 'exchange')
-
-        # instantiate the exchange by id
-        exchange = getattr(ccxt, id)({
-            # 'proxy':'https://cors-anywhere.herokuapp.com/',
-        })
-
-        # load all markets from the exchange
-        markets = exchange.load_markets()
-
-        # output a list of all market symbols
-        dump(green(id), 'has', len(exchange.symbols), 'symbols:', exchange.symbols)
-
-        tuples = list(ccxt.Exchange.keysort(markets).items())
-
-        # debug
-        for (k, v) in tuples:
-            print(v)
-
-        # output a table of all markets
-        dump(pink('{:<15} {:<15} {:<15} {:<15}'.format('id', 'symbol', 'base', 'quote')))
-
-        for (k, v) in tuples:
-            dump('{:<15} {:<15} {:<15} {:<15}'.format(v['id'], v['symbol'], v['base'], v['quote']))
-
-    else:
-
-        dump('Exchange ' + red(id) + ' not found')
-        print_supported_exchanges()
-
-except Exception as e:
-    dump('[' + type(e).__name__ + ']', str(e))
-    dump(traceback.format_exc())
-    dump("Usage: python " + sys.argv[0], green('id'))
-    print_supported_exchanges()
-    sys.exit(1)

scripts/plot_dataframe.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-
-
-print("This script has been integrated into freqtrade "
-      "and its functionality is available by calling `freqtrade plot-dataframe`.")
-print("Please check the documentation on https://www.freqtrade.io/en/latest/plotting/ "
-      "for details.")
-
-sys.exit(1)

scripts/plot_profit.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-
-
-print("This script has been integrated into freqtrade "
-      "and its functionality is available by calling `freqtrade plot-profit`.")
-print("Please check the documentation on https://www.freqtrade.io/en/latest/plotting/ "
-      "for details.")
-
-sys.exit(1)

tests/test_freqtradebot.py

@@ -1521,6 +1521,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog,
 
 
 def test_process_maybe_execute_buys(mocker, default_conf, caplog) -> None:
+    caplog.set_level(logging.DEBUG)
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trades', MagicMock(return_value=False))
@@ -3647,3 +3648,27 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker):
     ftbot = get_patched_freqtradebot(mocker, edge_conf)
     ftbot.startup()
     assert reinit_mock.call_count == 0
+
+
+def test_process_i_am_alive(default_conf, mocker, caplog):
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+
+    ftbot = get_patched_freqtradebot(mocker, default_conf)
+    message = r"Bot heartbeat\. PID=.*"
+    ftbot.process()
+    assert log_has_re(message, caplog)
+    assert ftbot._heartbeat_msg != 0
+
+    caplog.clear()
+    # Message is not shown before interval is up
+    ftbot.process()
+    assert not log_has_re(message, caplog)
+
+    caplog.clear()
+    # Set clock - 70 seconds
+    ftbot._heartbeat_msg -= 70
+
+    ftbot.process()
+    assert log_has_re(message, caplog)