From 87ff7be5503de08ec6b3fb2e579a9fab30b2632e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 23 Oct 2019 07:06:02 +0200 Subject: [PATCH 01/50] Use exchange.name instead of config['exchange']['name'] --- freqtrade/utils.py | 8 ++++---- tests/test_utils.py | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 0e7134235..b001f0316 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -86,9 +86,9 @@ def start_download_data(args: Dict[str, Any]) -> None: pairs_not_available: List[str] = [] + # Init exchange + exchange = ExchangeResolver(config['exchange']['name'], config).exchange try: - # Init exchange - exchange = ExchangeResolver(config['exchange']['name'], config).exchange if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( @@ -110,7 +110,7 @@ def start_download_data(args: Dict[str, Any]) -> None: finally: if pairs_not_available: logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {config['exchange']['name']}.") + f"on exchange {exchange.name}.") def start_list_timeframes(args: Dict[str, Any]) -> None: @@ -127,5 +127,5 @@ def start_list_timeframes(args: Dict[str, Any]) -> None: if args['print_one_column']: print('\n'.join(exchange.timeframes)) else: - print(f"Timeframes available for the exchange `{config['exchange']['name']}`: " + print(f"Timeframes available for the exchange `{exchange.name}`: " f"{', '.join(exchange.timeframes)}") diff --git a/tests/test_utils.py b/tests/test_utils.py index 586a6891b..558f5642c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -100,7 +100,7 @@ def test_list_timeframes(mocker, capsys): ] start_list_timeframes(get_args(args)) captured = capsys.readouterr() - assert re.match("Timeframes available for the exchange `bittrex`: " + assert re.match("Timeframes available for the exchange `Bittrex`: " "1m, 5m, 30m, 1h, 1d", captured.out) @@ -111,7 +111,7 @@ def test_list_timeframes(mocker, capsys): ] start_list_timeframes(get_args(args)) captured = capsys.readouterr() - assert re.match("Timeframes available for the exchange `bittrex`: " + assert re.match("Timeframes available for the exchange `Bittrex`: " "1m, 5m, 30m, 1h, 1d", captured.out) @@ -125,7 +125,7 @@ def test_list_timeframes(mocker, capsys): '1d': '1d', '3d': '3d', } - patch_exchange(mocker, api_mock=api_mock) + patch_exchange(mocker, api_mock=api_mock, id='binance') # Test with --exchange binance args = [ "list-timeframes", @@ -133,7 +133,7 @@ def test_list_timeframes(mocker, capsys): ] start_list_timeframes(get_args(args)) captured = capsys.readouterr() - assert re.match("Timeframes available for the exchange `binance`: " + assert re.match("Timeframes available for the exchange `Binance`: " "1m, 5m, 15m, 30m, 1h, 6h, 12h, 1d, 3d", captured.out) @@ -208,7 +208,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): def test_download_data_no_markets(mocker, caplog): dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) - patch_exchange(mocker) + patch_exchange(mocker, id='binance') mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) ) @@ -220,7 +220,7 @@ def test_download_data_no_markets(mocker, caplog): ] start_download_data(get_args(args)) assert dl_mock.call_args[1]['timerange'].starttype == "date" - assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog) + assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange Binance.", caplog) def test_download_data_no_exchange(mocker, caplog): From 6640f4a1b296b77e738cf37a083a5478c42187e3 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 23 Oct 2019 23:57:17 +0300 Subject: [PATCH 02/50] Make flake happy --- freqtrade/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index f9c4aeed7..25e883c76 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -218,4 +218,4 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: elif not (args.get('print_one_column', False) or args.get('list_pairs_print_json', False) or args.get('print_csv', False)): - print(f"{summary_str}.") \ No newline at end of file + print(f"{summary_str}.") From 2e1e080022756ff07eecb468e05279d5655397fe Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 24 Oct 2019 22:33:44 +0300 Subject: [PATCH 03/50] Fix potential race conditions between RPC and Freqtradebot during initialization --- freqtrade/freqtradebot.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ed5116b08..6a1be16a1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -55,8 +55,6 @@ class FreqtradeBot: # Check config consistency here since strategies can set certain options validate_config_consistency(config) - self.rpc: RPCManager = RPCManager(self) - self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange self.wallets = Wallets(self.config, self.exchange) @@ -83,6 +81,13 @@ class FreqtradeBot: initial_state = self.config.get('initial_state') self.state = State[initial_state.upper()] if initial_state else State.STOPPED + # RPC runs in separate threads, can start handling external commands just after + # initialization, even before Freqtradebot has a chance to start its throttling, + # so anything in the Freqtradebot instance should be ready (initialized), including + # the initial state of the bot. + # Keep this at the end of this initialization method. + self.rpc: RPCManager = RPCManager(self) + def cleanup(self) -> None: """ Cleanup pending resources on an already stopped bot From 59e881c59ddbd83e66926bad6ad2d80887ac48e7 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 24 Oct 2019 23:11:07 +0300 Subject: [PATCH 04/50] Remove obsolete scripts --- scripts/download_backtest_data.py | 11 ---- scripts/get_market_pairs.py | 103 ------------------------------ scripts/plot_dataframe.py | 11 ---- scripts/plot_profit.py | 11 ---- {scripts => tests}/pytest.sh | 0 5 files changed, 136 deletions(-) delete mode 100755 scripts/download_backtest_data.py delete mode 100644 scripts/get_market_pairs.py delete mode 100755 scripts/plot_dataframe.py delete mode 100755 scripts/plot_profit.py rename {scripts => tests}/pytest.sh (100%) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py deleted file mode 100755 index a8f919a10..000000000 --- a/scripts/download_backtest_data.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -import sys - - -print("This script has been integrated into freqtrade " - "and its functionality is available by calling `freqtrade download-data`.") -print("Please check the documentation on https://www.freqtrade.io/en/latest/backtesting/ " - "for details.") - -sys.exit(1) diff --git a/scripts/get_market_pairs.py b/scripts/get_market_pairs.py deleted file mode 100644 index cd38bf2fa..000000000 --- a/scripts/get_market_pairs.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -This script was adapted from ccxt here: -https://github.com/ccxt/ccxt/blob/master/examples/py/arbitrage-pairs.py -""" -import os -import sys -import traceback - -root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -sys.path.append(root + '/python') - -import ccxt # noqa: E402 - - -def style(s, style): - return style + s + '\033[0m' - - -def green(s): - return style(s, '\033[92m') - - -def blue(s): - return style(s, '\033[94m') - - -def yellow(s): - return style(s, '\033[93m') - - -def red(s): - return style(s, '\033[91m') - - -def pink(s): - return style(s, '\033[95m') - - -def bold(s): - return style(s, '\033[1m') - - -def underline(s): - return style(s, '\033[4m') - - -def dump(*args): - print(' '.join([str(arg) for arg in args])) - - -def print_supported_exchanges(): - dump('Supported exchanges:', green(', '.join(ccxt.exchanges))) - - -try: - - if len(sys.argv) < 2: - dump("Usage: python " + sys.argv[0], green('id')) - print_supported_exchanges() - sys.exit(1) - - id = sys.argv[1] # get exchange id from command line arguments - - # check if the exchange is supported by ccxt - exchange_found = id in ccxt.exchanges - - if exchange_found: - dump('Instantiating', green(id), 'exchange') - - # instantiate the exchange by id - exchange = getattr(ccxt, id)({ - # 'proxy':'https://cors-anywhere.herokuapp.com/', - }) - - # load all markets from the exchange - markets = exchange.load_markets() - - # output a list of all market symbols - dump(green(id), 'has', len(exchange.symbols), 'symbols:', exchange.symbols) - - tuples = list(ccxt.Exchange.keysort(markets).items()) - - # debug - for (k, v) in tuples: - print(v) - - # output a table of all markets - dump(pink('{:<15} {:<15} {:<15} {:<15}'.format('id', 'symbol', 'base', 'quote'))) - - for (k, v) in tuples: - dump('{:<15} {:<15} {:<15} {:<15}'.format(v['id'], v['symbol'], v['base'], v['quote'])) - - else: - - dump('Exchange ' + red(id) + ' not found') - print_supported_exchanges() - -except Exception as e: - dump('[' + type(e).__name__ + ']', str(e)) - dump(traceback.format_exc()) - dump("Usage: python " + sys.argv[0], green('id')) - print_supported_exchanges() - sys.exit(1) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py deleted file mode 100755 index 62c4bc39f..000000000 --- a/scripts/plot_dataframe.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -import sys - - -print("This script has been integrated into freqtrade " - "and its functionality is available by calling `freqtrade plot-dataframe`.") -print("Please check the documentation on https://www.freqtrade.io/en/latest/plotting/ " - "for details.") - -sys.exit(1) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py deleted file mode 100755 index c9a23c1ee..000000000 --- a/scripts/plot_profit.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -import sys - - -print("This script has been integrated into freqtrade " - "and its functionality is available by calling `freqtrade plot-profit`.") -print("Please check the documentation on https://www.freqtrade.io/en/latest/plotting/ " - "for details.") - -sys.exit(1) diff --git a/scripts/pytest.sh b/tests/pytest.sh similarity index 100% rename from scripts/pytest.sh rename to tests/pytest.sh From b3e028e853789f64754a8ee054a0b45b0a65c371 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 25 Oct 2019 06:43:52 +0200 Subject: [PATCH 05/50] Improve dynamic pairlist documentation --- docs/configuration.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 0eff4da88..dd54b6c54 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -417,10 +417,15 @@ section of the configuration. `askVolume`, `bidVolume` and `quoteVolume`, defaults to `quoteVolume`. * There is a possibility to filter low-value coins that would not allow setting a stop loss (set `precision_filter` parameter to `true` for this). + * `VolumePairList` does not consider `pair_whitelist`, but builds this automatically based the pairlist configuration. Example: ```json +"exchange": { + "pair_whitelist": [], + "pair_blacklist": ["BNB/BTC"] +}, "pairlist": { "method": "VolumePairList", "config": { From 45b83cc5441e4bf08fc30ee4379ebe5a71a0a32d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 25 Oct 2019 07:07:01 +0200 Subject: [PATCH 06/50] Don't require pair_whitelist for dynamicPairlist usecases --- freqtrade/configuration/config_validation.py | 15 ++++++++++- freqtrade/configuration/configuration.py | 3 +++ freqtrade/constants.py | 2 +- tests/test_configuration.py | 26 +++++++++++++++++++- 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 6a8374e6d..93d93263f 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -5,7 +5,7 @@ from jsonschema import Draft4Validator, validators from jsonschema.exceptions import ValidationError, best_match from freqtrade import constants, OperationalException - +from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -64,6 +64,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None: # validating trailing stoploss _validate_trailing_stoploss(conf) _validate_edge(conf) + _validate_whitelist(conf) def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None: @@ -111,3 +112,15 @@ def _validate_edge(conf: Dict[str, Any]) -> None: "Edge and VolumePairList are incompatible, " "Edge will override whatever pairs VolumePairlist selects." ) + + +def _validate_whitelist(conf: Dict[str, Any]) -> None: + """ + Dynamic whitelist does not require pair_whitelist to be set - however StaticWhitelist does. + """ + if conf.get('runmode', RunMode.OTHER) in [RunMode.OTHER, RunMode.PLOT]: + return + + if (conf.get('pairlist', {}).get('method', 'StaticPairList') == 'StaticPairList' + and not conf.get('exchange', {}).get('pair_whitelist')): + raise OperationalException("StaticPairList requires pair_whitelist to be set.") diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 034f8d386..be1c7ab4e 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -179,6 +179,9 @@ class Configuration: config['exchange']['name'] = self.args["exchange"] logger.info(f"Using exchange {config['exchange']['name']}") + if 'pair_whitelist' not in config['exchange']: + config['exchange']['pair_whitelist'] = [] + if 'user_data_dir' in self.args and self.args["user_data_dir"]: config.update({'user_data_dir': self.args["user_data_dir"]}) elif 'user_data_dir' not in config: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e8f3f5783..5fdd45916 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -235,7 +235,7 @@ CONF_SCHEMA = { 'ccxt_config': {'type': 'object'}, 'ccxt_async_config': {'type': 'object'} }, - 'required': ['name', 'pair_whitelist'] + 'required': ['name'] }, 'edge': { 'type': 'object', diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 2aa805fe6..cfb7f9a7f 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -729,6 +729,30 @@ def test_validate_edge(edge_conf): validate_config_consistency(edge_conf) +def test_validate_whitelist(default_conf): + default_conf['runmode'] = RunMode.DRY_RUN + # Test regular case - has whitelist and uses StaticPairlist + validate_config_consistency(default_conf) + conf = deepcopy(default_conf) + del conf['exchange']['pair_whitelist'] + # Test error case + with pytest.raises(OperationalException, + match="StaticPairList requires pair_whitelist to be set."): + + validate_config_consistency(conf) + + conf = deepcopy(default_conf) + + conf.update({"pairlist": { + "method": "VolumePairList", + }}) + # Dynamic whitelist should not care about pair_whitelist + validate_config_consistency(conf) + del conf['exchange']['pair_whitelist'] + + validate_config_consistency(conf) + + def test_load_config_test_comments() -> None: """ Load config with comments @@ -895,7 +919,7 @@ def test_pairlist_resolving_fallback(mocker): # Fix flaky tests if config.json exists args["config"] = None - configuration = Configuration(args) + configuration = Configuration(args, RunMode.OTHER) config = configuration.get_config() assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] From 8201f70a80dd2f767516044f5e767b50461d6352 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 25 Oct 2019 14:19:02 +0200 Subject: [PATCH 07/50] Change loglevel of repeated message to debug --- freqtrade/freqtradebot.py | 2 +- tests/test_freqtradebot.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6a1be16a1..d28014608 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -443,7 +443,7 @@ class FreqtradeBot: try: # Create entity and execute trade if not self.create_trades(): - logger.info('Found no buy signals for whitelisted currencies. Trying again...') + logger.debug('Found no buy signals for whitelisted currencies. Trying again...') except DependencyException as exception: logger.warning('Unable to create trade: %s', exception) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index f1533d867..607cb8f32 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1521,6 +1521,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, def test_process_maybe_execute_buys(mocker, default_conf, caplog) -> None: + caplog.set_level(logging.DEBUG) freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trades', MagicMock(return_value=False)) From 0773a653336f06e91bba0f0a70276dcfaf151874 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 25 Oct 2019 15:00:16 +0200 Subject: [PATCH 08/50] Add I Am Alive Message --- config_full.json.example | 3 ++- docs/configuration.md | 1 + freqtrade/freqtradebot.py | 9 +++++++++ tests/test_freqtradebot.py | 24 ++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/config_full.json.example b/config_full.json.example index c6b229ea3..ebf76eaee 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -119,7 +119,8 @@ "initial_state": "running", "forcebuy_enable": false, "internals": { - "process_throttle_secs": 5 + "process_throttle_secs": 5, + "keep_alive_interval": 60 }, "strategy": "DefaultStrategy", "strategy_path": "user_data/strategies/" diff --git a/docs/configuration.md b/docs/configuration.md index 0eff4da88..c6a12d865 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -98,6 +98,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `strategy` | DefaultStrategy | Defines Strategy class to use. | `strategy_path` | null | Adds an additional strategy lookup path (must be a directory). | `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second. +| `internals.keep_alive_interval` | 60 | Print keepalive message every X seconds. Set to 0 to disable keepalive messages. | `internals.sd_notify` | false | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details. | `logfile` | | Specify Logfile. Uses a rolling strategy of 10 files, with 1Mb per file. | `user_data_dir` | cwd()/user_data | Directory containing user data. Defaults to `./user_data/`. diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d28014608..6c79d04de 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -50,6 +50,10 @@ class FreqtradeBot: # Init objects self.config = config + self._last_alive_msg = 0 + + self.keep_alive_interval = self.config.get('internals', {}).get('keep_alive_interval', 60) + self.strategy: IStrategy = StrategyResolver(self.config).strategy # Check config consistency here since strategies can set certain options @@ -150,6 +154,11 @@ class FreqtradeBot: self.check_handle_timedout() Trade.session.flush() + if (self.keep_alive_interval + and (arrow.utcnow().timestamp - self._last_alive_msg > self.keep_alive_interval)): + logger.info("I am alive.") + self._last_alive_msg = arrow.utcnow().timestamp + def _extend_whitelist_with_trades(self, whitelist: List[str], trades: List[Any]): """ Extend whitelist with pairs from open trades diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 607cb8f32..b29bd0843 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3648,3 +3648,27 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker): ftbot = get_patched_freqtradebot(mocker, edge_conf) ftbot.startup() assert reinit_mock.call_count == 0 + + +def test_process_i_am_alive(default_conf, mocker, caplog): + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + ftbot = get_patched_freqtradebot(mocker, default_conf) + message = "I am alive." + ftbot.process() + assert log_has(message, caplog) + assert ftbot._last_alive_msg != 0 + + caplog.clear() + # Message is not shown before interval is up + ftbot.process() + assert not log_has(message, caplog) + + caplog.clear() + # Set clock - 70 seconds + ftbot._last_alive_msg -= 70 + + ftbot.process() + assert log_has(message, caplog) From e63377980e7f0aa6f2937753eb6033518a6aaf53 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 25 Oct 2019 07:12:50 +0200 Subject: [PATCH 09/50] Improve pairlist documentation --- docs/configuration.md | 5 +++-- tests/test_configuration.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index dd54b6c54..33c296a6a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -75,8 +75,8 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.key` | '' | API key to use for the exchange. Only required when you are in production mode. ***Keep it in secrete, do not disclose publicly.*** | `exchange.secret` | '' | API secret to use for the exchange. Only required when you are in production mode. ***Keep it in secrete, do not disclose publicly.*** | `exchange.password` | '' | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests. ***Keep it in secrete, do not disclose publicly.*** -| `exchange.pair_whitelist` | [] | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Can be overriden by dynamic pairlists (see [below](#dynamic-pairlists)). -| `exchange.pair_blacklist` | [] | List of pairs the bot must absolutely avoid for trading and backtesting. Can be overriden by dynamic pairlists (see [below](#dynamic-pairlists)). +| `exchange.pair_whitelist` | [] | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used when using VolumePairList (see [below](#dynamic-pairlists)). +| `exchange.pair_blacklist` | [] | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#dynamic-pairlists)). | `exchange.ccxt_config` | None | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.ccxt_async_config` | None | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.markets_refresh_interval` | 60 | The interval in minutes in which markets are reloaded. @@ -418,6 +418,7 @@ section of the configuration. * There is a possibility to filter low-value coins that would not allow setting a stop loss (set `precision_filter` parameter to `true` for this). * `VolumePairList` does not consider `pair_whitelist`, but builds this automatically based the pairlist configuration. + * Pairs in `pair_blacklist` are not considered for VolumePairList, even if all other filters would match. Example: diff --git a/tests/test_configuration.py b/tests/test_configuration.py index cfb7f9a7f..545dd5df4 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -825,7 +825,7 @@ def test_pairlist_resolving(): args = Arguments(arglist).get_parsed_arg() - configuration = Configuration(args) + configuration = Configuration(args, RunMode.OTHER) config = configuration.get_config() assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] From 2e896462c14b66e226b2a5e4afb054f99c229fbe Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 25 Oct 2019 19:49:23 +0200 Subject: [PATCH 10/50] Fix wrong volumepairlist message --- freqtrade/pairlist/VolumePairList.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index b9b7977ab..5f53cd17b 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -54,7 +54,7 @@ class VolumePairList(IPairList): """ # Generate dynamic whitelist self._whitelist = self._gen_pair_whitelist( - self._config['stake_currency'], self._sort_key)[:self._number_pairs] + self._config['stake_currency'], self._sort_key) @cached(TTLCache(maxsize=1, ttl=1800)) def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: @@ -91,6 +91,6 @@ class VolumePairList(IPairList): valid_tickers.remove(t) pairs = [s['symbol'] for s in valid_tickers] - logger.info(f"Searching pairs: {self._whitelist}") + logger.info(f"Searching pairs: {pairs[:self._number_pairs]}") return pairs From 2f1d9696cd1fd7efca1c619dc4a5b861353777f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 25 Oct 2019 19:59:04 +0200 Subject: [PATCH 11/50] Change keepalive to heartbeat --- config_full.json.example | 2 +- docs/configuration.md | 2 +- freqtrade/freqtradebot.py | 25 +++++++++++++------------ tests/test_freqtradebot.py | 12 ++++++------ 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index ebf76eaee..5789e49ac 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -120,7 +120,7 @@ "forcebuy_enable": false, "internals": { "process_throttle_secs": 5, - "keep_alive_interval": 60 + "heartbeat_interval": 60 }, "strategy": "DefaultStrategy", "strategy_path": "user_data/strategies/" diff --git a/docs/configuration.md b/docs/configuration.md index c6a12d865..e3d16c57b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -98,7 +98,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `strategy` | DefaultStrategy | Defines Strategy class to use. | `strategy_path` | null | Adds an additional strategy lookup path (must be a directory). | `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second. -| `internals.keep_alive_interval` | 60 | Print keepalive message every X seconds. Set to 0 to disable keepalive messages. +| `internals.heartbeat_interval` | 60 | Print heartbeat message every X seconds. Set to 0 to disable heartbeat messages. | `internals.sd_notify` | false | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details. | `logfile` | | Specify Logfile. Uses a rolling strategy of 10 files, with 1Mb per file. | `user_data_dir` | cwd()/user_data | Directory containing user data. Defaults to `./user_data/`. diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6c79d04de..9bc4fb04d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -6,26 +6,27 @@ import logging import traceback from datetime import datetime from math import isclose +from os import getpid from typing import Any, Dict, List, Optional, Tuple import arrow from requests.exceptions import RequestException -from freqtrade import (DependencyException, InvalidOrderException, - __version__, constants, persistence) +from freqtrade import (DependencyException, InvalidOrderException, __version__, + constants, persistence) +from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.configuration import validate_config_consistency from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date from freqtrade.persistence import Trade +from freqtrade.resolvers import (ExchangeResolver, PairListResolver, + StrategyResolver) from freqtrade.rpc import RPCManager, RPCMessageType -from freqtrade.resolvers import ExchangeResolver, StrategyResolver, PairListResolver from freqtrade.state import State -from freqtrade.strategy.interface import SellType, IStrategy +from freqtrade.strategy.interface import IStrategy, SellType from freqtrade.wallets import Wallets - logger = logging.getLogger(__name__) @@ -50,9 +51,9 @@ class FreqtradeBot: # Init objects self.config = config - self._last_alive_msg = 0 + self._heartbeat_msg = 0 - self.keep_alive_interval = self.config.get('internals', {}).get('keep_alive_interval', 60) + self.hearbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60) self.strategy: IStrategy = StrategyResolver(self.config).strategy @@ -154,10 +155,10 @@ class FreqtradeBot: self.check_handle_timedout() Trade.session.flush() - if (self.keep_alive_interval - and (arrow.utcnow().timestamp - self._last_alive_msg > self.keep_alive_interval)): - logger.info("I am alive.") - self._last_alive_msg = arrow.utcnow().timestamp + if (self.hearbeat_interval + and (arrow.utcnow().timestamp - self._heartbeat_msg > self.hearbeat_interval)): + logger.info(f"Freqtrade heartbeat. PID={getpid()}") + self._heartbeat_msg = arrow.utcnow().timestamp def _extend_whitelist_with_trades(self, whitelist: List[str], trades: List[Any]): """ diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index b29bd0843..cf67e644f 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3656,19 +3656,19 @@ def test_process_i_am_alive(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) ftbot = get_patched_freqtradebot(mocker, default_conf) - message = "I am alive." + message = r"Freqtrade heartbeat. PID=.*" ftbot.process() - assert log_has(message, caplog) - assert ftbot._last_alive_msg != 0 + assert log_has_re(message, caplog) + assert ftbot._heartbeat_msg != 0 caplog.clear() # Message is not shown before interval is up ftbot.process() - assert not log_has(message, caplog) + assert not log_has_re(message, caplog) caplog.clear() # Set clock - 70 seconds - ftbot._last_alive_msg -= 70 + ftbot._heartbeat_msg -= 70 ftbot.process() - assert log_has(message, caplog) + assert log_has_re(message, caplog) From 3929ad4e1f84fe7dca8a676d4236b47565af0468 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Oct 2019 09:21:51 +0200 Subject: [PATCH 12/50] Fix typo --- freqtrade/freqtradebot.py | 8 ++++---- tests/test_freqtradebot.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9bc4fb04d..7251715a7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -53,7 +53,7 @@ class FreqtradeBot: self._heartbeat_msg = 0 - self.hearbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60) + self.heartbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60) self.strategy: IStrategy = StrategyResolver(self.config).strategy @@ -155,9 +155,9 @@ class FreqtradeBot: self.check_handle_timedout() Trade.session.flush() - if (self.hearbeat_interval - and (arrow.utcnow().timestamp - self._heartbeat_msg > self.hearbeat_interval)): - logger.info(f"Freqtrade heartbeat. PID={getpid()}") + if (self.heartbeat_interval + and (arrow.utcnow().timestamp - self._heartbeat_msg > self.heartbeat_interval)): + logger.info(f"Bot heartbeat. PID={getpid()}") self._heartbeat_msg = arrow.utcnow().timestamp def _extend_whitelist_with_trades(self, whitelist: List[str], trades: List[Any]): diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index cf67e644f..8aefaba17 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3656,7 +3656,7 @@ def test_process_i_am_alive(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) ftbot = get_patched_freqtradebot(mocker, default_conf) - message = r"Freqtrade heartbeat. PID=.*" + message = r"Bot heartbeat\. PID=.*" ftbot.process() assert log_has_re(message, caplog) assert ftbot._heartbeat_msg != 0 From ea6b94fd0c57ac5000bd744e6602a421029134af Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sat, 26 Oct 2019 11:54:04 +0300 Subject: [PATCH 13/50] docs: add a tip for The Ocean exchange --- docs/configuration.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index e3d16c57b..ff40b1750 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -331,7 +331,7 @@ This configuration enables binance, as well as rate limiting to avoid bans from Optimal settings for rate limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings. We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. -#### Advanced FreqTrade Exchange configuration +#### Advanced Freqtrade Exchange configuration Advanced options can be configured using the `_ft_has_params` setting, which will override Defaults and exchange-specific behaviours. @@ -351,6 +351,13 @@ For example, to test the order type `FOK` with Kraken, and modify candle_limit t !!! Warning Please make sure to fully understand the impacts of these settings before modifying them. +#### Random notes for other exchanges + +* The Ocean (ccxt id: 'theocean') exchange uses Web3 functionality and requires web3 package to be installed: +```shell +$ pip3 install web3 +``` + ### What values can be used for fiat_display_currency? The `fiat_display_currency` configuration parameter sets the base currency to use for the From d0521d33cedd079cff559a75571cb4cf2f9eef1d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Oct 2019 09:38:29 +0200 Subject: [PATCH 14/50] Refactor whitelist handling fixes #2413 --- freqtrade/freqtradebot.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6a1be16a1..f6f12f7f8 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -72,7 +72,7 @@ class FreqtradeBot: self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None - self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] + self.active_pair_whitelist = self._refresh_whitelist() persistence.init(self.config.get('db_url', None), clean_open_orders=self.config.get('dry_run', False)) @@ -118,21 +118,10 @@ class FreqtradeBot: # Check whether markets have to be reloaded self.exchange._reload_markets() - # Refresh whitelist - self.pairlists.refresh_pairlist() - self.active_pair_whitelist = self.pairlists.whitelist - - # Calculating Edge positioning - if self.edge: - self.edge.calculate() - self.active_pair_whitelist = self.edge.adjust(self.active_pair_whitelist) - # Query trades from persistence layer trades = Trade.get_open_trades() - # Extend active-pair whitelist with pairs from open trades - # It ensures that tickers are downloaded for open trades - self._extend_whitelist_with_trades(self.active_pair_whitelist, trades) + self.active_pair_whitelist = self._refresh_whitelist(trades) # Refreshing candles self.dataprovider.refresh(self._create_pair_whitelist(self.active_pair_whitelist), @@ -150,11 +139,24 @@ class FreqtradeBot: self.check_handle_timedout() Trade.session.flush() - def _extend_whitelist_with_trades(self, whitelist: List[str], trades: List[Any]): + def _refresh_whitelist(self, trades: List[Trade] = []) -> List[str]: """ - Extend whitelist with pairs from open trades + Refresh whitelist from pairlist or edge and extend it with trades. """ - whitelist.extend([trade.pair for trade in trades if trade.pair not in whitelist]) + # Refresh whitelist + self.pairlists.refresh_pairlist() + _whitelist = self.pairlists.whitelist + + # Calculating Edge positioning + if self.edge: + self.edge.calculate() + _whitelist = self.edge.adjust(_whitelist) + + if trades: + # Extend active-pair whitelist with pairs from open trades + # It ensures that tickers are downloaded for open trades + _whitelist.extend([trade.pair for trade in trades if trade.pair not in _whitelist]) + return _whitelist def _create_pair_whitelist(self, pairs: List[str]) -> List[Tuple[str, str]]: """ From bfec9d974b5108acec273f894cab5f36ef793e39 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 26 Oct 2019 13:08:36 +0300 Subject: [PATCH 15/50] docs: Create Advanced Post-installation Tasks section; move systemd stuff there --- docs/advanced-setup.md | 33 +++++++++++++++++++++++++++++++++ docs/installation.md | 30 +++--------------------------- mkdocs.yml | 1 + 3 files changed, 37 insertions(+), 27 deletions(-) create mode 100644 docs/advanced-setup.md diff --git a/docs/advanced-setup.md b/docs/advanced-setup.md new file mode 100644 index 000000000..e6334d2c1 --- /dev/null +++ b/docs/advanced-setup.md @@ -0,0 +1,33 @@ +# Advanced Post-installation Tasks + +This page explains some advanced tasks and configuration options that can be performed after the bot installation and may be uselful in some environments. + +If you do not know what things mentioned here mean, you probably do not need it. + +## Configure the bot running as a systemd service + +Copy the `freqtrade.service` file to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. + +After that you can start the daemon with: + +```bash +systemctl --user start freqtrade +``` + +For this to be persistent (run when user is logged out) you'll need to enable `linger` for your freqtrade user. + +```bash +sudo loginctl enable-linger "$USER" +``` + +If you run the bot as a service, you can use systemd service manager as a software watchdog monitoring freqtrade bot +state and restarting it in the case of failures. If the `internals.sd_notify` parameter is set to true in the +configuration or the `--sd-notify` command line option is used, the bot will send keep-alive ping messages to systemd +using the sd_notify (systemd notifications) protocol and will also tell systemd its current state (Running or Stopped) +when it changes. + +The `freqtrade.service.watchdog` file contains an example of the service unit configuration file which uses systemd +as the watchdog. + +!!! Note + The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a Docker container. diff --git a/docs/installation.md b/docs/installation.md index 68348d4b0..e1e101efd 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -151,7 +151,7 @@ python3 -m venv .env source .env/bin/activate ``` -#### 3. Install FreqTrade +#### 3. Install Freqtrade Clone the git repository: @@ -192,33 +192,9 @@ freqtrade -c config.json *Note*: If you run the bot on a server, you should consider using [Docker](docker.md) or a terminal multiplexer like `screen` or [`tmux`](https://en.wikipedia.org/wiki/Tmux) to avoid that the bot is stopped on logout. -#### 7. [Optional] Configure `freqtrade` as a `systemd` service +#### 7. (Optional) Post-installation Tasks -From the freqtrade repo... copy `freqtrade.service` to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. - -After that you can start the daemon with: - -```bash -systemctl --user start freqtrade -``` - -For this to be persistent (run when user is logged out) you'll need to enable `linger` for your freqtrade user. - -```bash -sudo loginctl enable-linger "$USER" -``` - -If you run the bot as a service, you can use systemd service manager as a software watchdog monitoring freqtrade bot -state and restarting it in the case of failures. If the `internals.sd_notify` parameter is set to true in the -configuration or the `--sd-notify` command line option is used, the bot will send keep-alive ping messages to systemd -using the sd_notify (systemd notifications) protocol and will also tell systemd its current state (Running or Stopped) -when it changes. - -The `freqtrade.service.watchdog` file contains an example of the service unit configuration file which uses systemd -as the watchdog. - -!!! Note - The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a Docker container. +You can also optionally setup the bot to run as a `systemd` service and configure it to send the log messages to the `syslog`/`rsyslog` or `journald` daemons. See [Advanced Post-installation Tasks](advanced-setup.md) for details. ------ diff --git a/mkdocs.yml b/mkdocs.yml index 863731873..2c3f70191 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,6 +22,7 @@ nav: - Strategy analysis: strategy_analysis_example.md - Plotting: plotting.md - SQL Cheatsheet: sql_cheatsheet.md + - Advanced Post-installation Tasks: advanced-setup.md - Sandbox Testing: sandbox-testing.md - Deprecated Features: deprecated.md - Contributors Guide: developer.md From f5351e60e7befbc44672f8f33f6d4076bc3dabf2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Oct 2019 10:08:23 +0200 Subject: [PATCH 16/50] Adjust markets mock --- tests/conftest.py | 29 ++++++++--- tests/exchange/test_exchange.py | 31 ++++------- tests/pairlist/test_pairlist.py | 16 +++--- tests/rpc/test_rpc.py | 91 +++++++++++---------------------- tests/test_freqtradebot.py | 49 ++++++++---------- 5 files changed, 93 insertions(+), 123 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 305221d6d..84612175d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,13 +55,15 @@ def patched_configuration_load_config_file(mocker, config) -> None: ) -def patch_exchange(mocker, api_mock=None, id='bittrex') -> None: +def patch_exchange(mocker, api_mock=None, id='bittrex', mock_markets=True) -> None: mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.validate_ordertypes', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.id', PropertyMock(return_value=id)) mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value=id.title())) + if mock_markets: + mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=get_markets())) if api_mock: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) @@ -69,8 +71,8 @@ def patch_exchange(mocker, api_mock=None, id='bittrex') -> None: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock()) -def get_patched_exchange(mocker, config, api_mock=None, id='bittrex') -> Exchange: - patch_exchange(mocker, api_mock, id) +def get_patched_exchange(mocker, config, api_mock=None, id='bittrex', mock_markets=True) -> Exchange: + patch_exchange(mocker, api_mock, id, mock_markets) config["exchange"]["name"] = id try: exchange = ExchangeResolver(id, config).exchange @@ -85,6 +87,11 @@ def patch_wallet(mocker, free=999.9) -> None: )) +def patch_whitelist(mocker, conf) -> None: + mocker.patch('freqtrade.freqtradebot.FreqtradeBot._refresh_whitelist', + MagicMock(return_value=conf['exchange']['pair_whitelist'])) + + def patch_edge(mocker) -> None: # "ETH/BTC", # "LTC/BTC", @@ -120,6 +127,8 @@ def patch_freqtradebot(mocker, config) -> None: patch_exchange(mocker) mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) + mocker.patch('freqtrade.freqtradebot.FreqtradeBot._refresh_whitelist', + MagicMock(return_value=config['exchange']['pair_whitelist'])) def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: @@ -287,6 +296,10 @@ def ticker_sell_down(): @pytest.fixture def markets(): + return get_markets() + + +def get_markets(): return { 'ETH/BTC': { 'id': 'ethbtc', @@ -369,7 +382,7 @@ def markets(): 'symbol': 'LTC/BTC', 'base': 'LTC', 'quote': 'BTC', - 'active': False, + 'active': True, 'precision': { 'price': 8, 'amount': 8, @@ -394,7 +407,7 @@ def markets(): 'symbol': 'XRP/BTC', 'base': 'XRP', 'quote': 'BTC', - 'active': False, + 'active': True, 'precision': { 'price': 8, 'amount': 8, @@ -419,7 +432,7 @@ def markets(): 'symbol': 'NEO/BTC', 'base': 'NEO', 'quote': 'BTC', - 'active': False, + 'active': True, 'precision': { 'price': 8, 'amount': 8, @@ -444,7 +457,7 @@ def markets(): 'symbol': 'BTT/BTC', 'base': 'BTT', 'quote': 'BTC', - 'active': True, + 'active': False, 'precision': { 'base': 8, 'quote': 8, @@ -494,7 +507,7 @@ def markets(): 'symbol': 'LTC/USDT', 'base': 'LTC', 'quote': 'USDT', - 'active': True, + 'active': False, 'precision': { 'amount': 8, 'price': 8 diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 1e0a5fdc3..d3f50c6da 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -177,16 +177,11 @@ def test_symbol_amount_prec(default_conf, mocker): ''' Test rounds down to 4 Decimal places ''' - api_mock = MagicMock() - api_mock.load_markets = MagicMock(return_value={ - 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' - }) - mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='binance')) markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'amount': 4}}}) - type(api_mock).markets = markets - exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange = get_patched_exchange(mocker, default_conf, id="binance") + mocker.patch('freqtrade.exchange.Exchange.markets', markets) amount = 2.34559 pair = 'ETH/BTC' @@ -198,16 +193,10 @@ def test_symbol_price_prec(default_conf, mocker): ''' Test rounds up to 4 decimal places ''' - api_mock = MagicMock() - api_mock.load_markets = MagicMock(return_value={ - 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' - }) - mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='binance')) - markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'price': 4}}}) - type(api_mock).markets = markets - exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange = get_patched_exchange(mocker, default_conf, id="binance") + mocker.patch('freqtrade.exchange.Exchange.markets', markets) price = 2.34559 pair = 'ETH/BTC' @@ -279,7 +268,7 @@ def test__load_markets(default_conf, mocker, caplog): api_mock.load_markets = MagicMock(return_value=expected_return) type(api_mock).markets = expected_return default_conf['exchange']['pair_whitelist'] = ['ETH/BTC'] - ex = get_patched_exchange(mocker, default_conf, api_mock, id="binance") + ex = get_patched_exchange(mocker, default_conf, api_mock, id="binance", mock_markets=False) assert ex.markets == expected_return @@ -294,7 +283,8 @@ def test__reload_markets(default_conf, mocker, caplog): api_mock.load_markets = load_markets type(api_mock).markets = initial_markets default_conf['exchange']['markets_refresh_interval'] = 10 - exchange = get_patched_exchange(mocker, default_conf, api_mock, id="binance") + exchange = get_patched_exchange(mocker, default_conf, api_mock, id="binance", + mock_markets=False) exchange._last_markets_refresh = arrow.utcnow().timestamp updated_markets = {'ETH/BTC': {}, "LTC/BTC": {}} @@ -1715,15 +1705,16 @@ def test_get_valid_pair_combination(default_conf, mocker, markets): 'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC']), # active markets ([], [], False, True, - ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/USD', 'LTC/USDT', - 'TKN/BTC', 'XLTCUSDT']), + ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'NEO/BTC', + 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC']), # all pairs ([], [], True, False, ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XRP/BTC']), # active pairs ([], [], True, True, - ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/USD', 'LTC/USDT', 'TKN/BTC']), + ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'NEO/BTC', + 'TKN/BTC', 'XRP/BTC']), # all markets, base=ETH, LTC (['ETH', 'LTC'], [], False, False, ['ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 411ae60a3..929fc0ba0 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -80,7 +80,7 @@ def test_refresh_pairlist_dynamic(mocker, markets, tickers, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) # argument: use the whitelist dynamically by exchange-volume - whitelist = ['ETH/BTC', 'TKN/BTC', 'BTT/BTC'] + whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'] freqtradebot.pairlists.refresh_pairlist() assert whitelist == freqtradebot.pairlists.whitelist @@ -108,12 +108,12 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): @pytest.mark.parametrize("precision_filter,base_currency,key,whitelist_result", [ - (False, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'BTT/BTC']), - (False, "BTC", "bidVolume", ['BTT/BTC', 'TKN/BTC', 'ETH/BTC']), - (False, "USDT", "quoteVolume", ['ETH/USDT', 'LTC/USDT']), + (False, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), + (False, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC']), + (False, "USDT", "quoteVolume", ['ETH/USDT']), (False, "ETH", "quoteVolume", []), # this replaces tests that were removed from test_exchange - (True, "BTC", "quoteVolume", ["ETH/BTC", "TKN/BTC"]), - (True, "BTC", "bidVolume", ["TKN/BTC", "ETH/BTC"]) + (True, "BTC", "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC"]), + (True, "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC"]) ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers, base_currency, key, whitelist_result, precision_filter) -> None: @@ -127,7 +127,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers, freqtrade.pairlists._precision_filter = precision_filter freqtrade.config['stake_currency'] = base_currency whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency=base_currency, key=key) - assert whitelist == whitelist_result + assert sorted(whitelist) == sorted(whitelist_result) def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: @@ -160,7 +160,7 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): (['ETH/BTC', 'TKN/BTC', 'TRX/ETH'], "is not compatible with exchange"), # TRX/ETH wrong stake (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"), # BCH/BTC not available (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "is not compatible with exchange"), # BLK/BTC in blacklist - (['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], "Market is not active") # LTC/BTC is inactive + (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active") # BTT/BTC is inactive ]) def test_validate_whitelist(mocker, whitelist_conf, markets, pairlist, whitelist, caplog, log_message): diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 66468927f..a5da9b51e 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -14,7 +14,7 @@ from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State -from tests.conftest import patch_exchange, patch_get_signal +from tests.conftest import patch_exchange, patch_get_signal, get_patched_freqtradebot # Functions for recurrent object patching @@ -26,17 +26,15 @@ def prec_satoshi(a, b) -> float: # Unit tests -def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: +def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) @@ -98,17 +96,15 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: } == results[0] -def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: - patch_exchange(mocker) +def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) @@ -134,7 +130,6 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: def test_rpc_daily_profit(default_conf, update, ticker, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -143,7 +138,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -181,22 +176,20 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, - limit_buy_order, limit_sell_order, markets, mocker) -> None: + limit_buy_order, limit_sell_order, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), ) - patch_exchange(mocker) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -267,9 +260,8 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Test that rpc_trade_statistics can handle trades that lacks # trade.open_rate (it is set to None) -def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, +def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, ticker_sell_up, limit_buy_order, limit_sell_order): - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.rpc.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), @@ -281,10 +273,9 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -343,7 +334,6 @@ def test_rpc_balance_handle_error(default_conf, mocker): 'freqtrade.rpc.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), ) - patch_exchange(mocker) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -352,7 +342,7 @@ def test_rpc_balance_handle_error(default_conf, mocker): get_ticker=MagicMock(side_effect=TemporaryError('Could not load ticker due to xxx')) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) rpc._fiat_converter = CryptoToFiatConverter() @@ -394,7 +384,6 @@ def test_rpc_balance_handle(default_conf, mocker): 'freqtrade.rpc.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), ) - patch_exchange(mocker) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -406,7 +395,7 @@ def test_rpc_balance_handle(default_conf, mocker): side_effect=lambda a, b: f"{b}/{a}" if a == "PAX" else f"{a}/{b}") ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) rpc._fiat_converter = CryptoToFiatConverter() @@ -438,14 +427,13 @@ def test_rpc_balance_handle(default_conf, mocker): def test_rpc_start(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=MagicMock() ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -460,14 +448,13 @@ def test_rpc_start(mocker, default_conf) -> None: def test_rpc_stop(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=MagicMock() ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.RUNNING @@ -483,14 +470,13 @@ def test_rpc_stop(mocker, default_conf) -> None: def test_rpc_stopbuy(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=MagicMock() ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.RUNNING @@ -501,8 +487,7 @@ def test_rpc_stopbuy(mocker, default_conf) -> None: assert freqtradebot.config['max_open_trades'] == 0 -def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: - patch_exchange(mocker) +def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) cancel_order_mock = MagicMock() @@ -518,10 +503,9 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: } ), get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) @@ -606,18 +590,16 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: def test_performance_handle(default_conf, ticker, limit_buy_order, fee, - limit_sell_order, markets, mocker) -> None: - patch_exchange(mocker) + limit_sell_order, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) @@ -641,18 +623,16 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, assert prec_satoshi(res[0]['profit'], 6.2) -def test_rpc_count(mocker, default_conf, ticker, fee, markets) -> None: - patch_exchange(mocker) +def test_rpc_count(mocker, default_conf, ticker, fee) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) @@ -665,9 +645,8 @@ def test_rpc_count(mocker, default_conf, ticker, fee, markets) -> None: assert counts["current"] == 1 -def test_rpcforcebuy(mocker, default_conf, ticker, fee, markets, limit_buy_order) -> None: +def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None: default_conf['forcebuy_enable'] = True - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( @@ -675,11 +654,10 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, markets, limit_buy_order get_balances=MagicMock(return_value=ticker), get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets), buy=buy_mm ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) pair = 'ETH/BTC' @@ -704,7 +682,7 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, markets, limit_buy_order # Test not buying default_conf['stake_amount'] = 0.0000001 - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) pair = 'TKN/BTC' @@ -715,10 +693,9 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, markets, limit_buy_order def test_rpcforcebuy_stopped(mocker, default_conf) -> None: default_conf['forcebuy_enable'] = True default_conf['initial_state'] = 'stopped' - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) pair = 'ETH/BTC' @@ -727,10 +704,9 @@ def test_rpcforcebuy_stopped(mocker, default_conf) -> None: def test_rpcforcebuy_disabled(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) pair = 'ETH/BTC' @@ -739,10 +715,9 @@ def test_rpcforcebuy_disabled(mocker, default_conf) -> None: def test_rpc_whitelist(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_whitelist() assert ret['method'] == 'StaticPairList' @@ -750,14 +725,13 @@ def test_rpc_whitelist(mocker, default_conf) -> None: def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: - patch_exchange(mocker) default_conf['pairlist'] = {'method': 'VolumePairList', 'config': {'number_assets': 4} } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_whitelist() assert ret['method'] == 'VolumePairList' @@ -766,10 +740,9 @@ def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: def test_rpc_blacklist(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_blacklist(None) assert ret['method'] == 'StaticPairList' @@ -785,23 +758,21 @@ def test_rpc_blacklist(mocker, default_conf) -> None: def test_rpc_edge_disabled(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) with pytest.raises(RPCException, match=r'Edge is not enabled.'): rpc._rpc_edge() def test_rpc_edge_enabled(mocker, edge_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': PairInfo(-0.02, 0.66, 3.71, 0.50, 1.71, 10, 60), } )) - freqtradebot = FreqtradeBot(edge_conf) + freqtradebot = get_patched_freqtradebot(mocker, edge_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_edge() diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index f1533d867..ff9a34142 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -23,7 +23,7 @@ from freqtrade.strategy.interface import SellCheckTuple, SellType from freqtrade.worker import Worker from tests.conftest import (get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, patch_edge, patch_exchange, - patch_get_signal, patch_wallet) + patch_get_signal, patch_wallet, patch_whitelist) def patch_RPCManager(mocker) -> MagicMock: @@ -1247,11 +1247,10 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, - markets, limit_buy_order, limit_sell_order) -> None: + limit_buy_order, limit_sell_order) -> None: # When trailing stoploss is set stoploss_limit = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=MagicMock(return_value={ @@ -1262,7 +1261,6 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), stoploss_limit=stoploss_limit ) @@ -1272,7 +1270,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, # disabling ROI default_conf['minimal_roi']['0'] = 999999999 - freqtrade = FreqtradeBot(default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # enabling stoploss on exchange freqtrade.strategy.order_types['stoploss_on_exchange'] = True @@ -1824,20 +1822,18 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, def test_handle_trade_roi(default_conf, ticker, limit_buy_order, - fee, mocker, markets, caplog) -> None: + fee, mocker, caplog) -> None: caplog.set_level(logging.DEBUG) patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtrade, value=(True, False)) freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) @@ -1858,20 +1854,18 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, def test_handle_trade_use_sell_signal( - default_conf, ticker, limit_buy_order, fee, mocker, markets, caplog) -> None: + default_conf, ticker, limit_buy_order, fee, mocker, caplog) -> None: # use_sell_signal is True buy default caplog.set_level(logging.DEBUG) patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trades() @@ -2236,6 +2230,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc get_fee=fee, markets=PropertyMock(return_value=markets) ) + patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2282,6 +2277,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, get_fee=fee, markets=PropertyMock(return_value=markets) ) + patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2331,6 +2327,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe get_fee=fee, markets=PropertyMock(return_value=markets) ) + patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2647,6 +2644,7 @@ def test_execute_sell_market_order(default_conf, ticker, fee, get_fee=fee, markets=PropertyMock(return_value=markets) ) + patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2814,14 +2812,13 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke assert trade.sell_reason == SellType.SELL_SIGNAL.value -def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, markets, mocker, caplog) -> None: +def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2851,7 +2848,7 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, markets, mock assert log_has(f"Pair {trade.pair} is currently locked.", caplog) -def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: +def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2863,7 +2860,6 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) default_conf['ask_strategy'] = { 'ignore_roi_if_buy_signal': True @@ -2885,7 +2881,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m assert trade.sell_reason == SellType.ROI.value -def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, mocker) -> None: +def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2897,9 +2893,9 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), ) default_conf['trailing_stop'] = True + patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) @@ -2937,7 +2933,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value -def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets, +def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, mocker) -> None: buy_price = limit_buy_order['price'] patch_RPCManager(mocker) @@ -2951,10 +2947,11 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), ) default_conf['trailing_stop'] = True default_conf['trailing_stop_positive'] = 0.01 + patch_whitelist(mocker, default_conf) + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) @@ -2994,7 +2991,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, - caplog, mocker, markets) -> None: + caplog, mocker) -> None: buy_price = limit_buy_order['price'] patch_RPCManager(mocker) patch_exchange(mocker) @@ -3007,9 +3004,8 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), ) - + patch_whitelist(mocker, default_conf) default_conf['trailing_stop'] = True default_conf['trailing_stop_positive'] = 0.01 default_conf['trailing_stop_positive_offset'] = 0.011 @@ -3054,7 +3050,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, - caplog, mocker, markets) -> None: + caplog, mocker) -> None: buy_price = limit_buy_order['price'] # buy_price: 0.00001099 @@ -3069,9 +3065,8 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), ) - + patch_whitelist(mocker, default_conf) default_conf['trailing_stop'] = True default_conf['trailing_stop_positive'] = 0.05 default_conf['trailing_stop_positive_offset'] = 0.055 From ef1885c38bd9ab89df7418f6b8110e86e81f6398 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Oct 2019 13:24:26 +0200 Subject: [PATCH 17/50] Fix more tests --- tests/data/test_history.py | 11 ++--- tests/rpc/test_rpc_telegram.py | 73 ++++++++++++---------------------- tests/test_utils.py | 42 +++++++++---------- 3 files changed, 53 insertions(+), 73 deletions(-) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 95382768a..48ef2affd 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -533,21 +533,22 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + + ex = get_patched_exchange(mocker, default_conf) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) ) - ex = get_patched_exchange(mocker, default_conf) timerange = TimeRange.parse_timerange("20190101-20190102") - unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], + unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["BTT/BTC", "LTC/USDT"], timeframes=["1m", "5m"], dl_path=testdatadir, timerange=timerange, erase=False ) assert dl_mock.call_count == 0 - assert "ETH/BTC" in unav_pairs - assert "XRP/BTC" in unav_pairs - assert log_has("Skipping pair ETH/BTC...", caplog) + assert "BTT/BTC" in unav_pairs + assert "LTC/USDT" in unav_pairs + assert log_has("Skipping pair BTT/BTC...", caplog) def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir): diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index a776ad5df..766511d2d 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -22,7 +22,7 @@ from freqtrade.rpc.telegram import Telegram, authorized_only from freqtrade.state import State from freqtrade.strategy.interface import SellType from tests.conftest import (get_patched_freqtradebot, log_has, patch_exchange, - patch_get_signal) + patch_get_signal, patch_whitelist) class DummyCls(Telegram): @@ -143,17 +143,15 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: assert log_has('Exception occurred within Telegram module', caplog) -def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: +def test_status(default_conf, update, mocker, fee, ticker,) -> None: update.message.chat.id = 123 default_conf['telegram']['enabled'] = False default_conf['telegram']['chat_id'] = 123 - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(markets) ) msg_mock = MagicMock() status_table = MagicMock() @@ -184,9 +182,8 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: _status_table=status_table, _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -204,13 +201,11 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: assert status_table.call_count == 1 -def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> None: - patch_exchange(mocker) +def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(markets) ) msg_mock = MagicMock() status_table = MagicMock() @@ -220,9 +215,9 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No _status_table=status_table, _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -256,14 +251,12 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No assert 'ETH/BTC' in msg_mock.call_args_list[0][0][0] -def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) -> None: - patch_exchange(mocker) +def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, - markets=PropertyMock(markets) ) msg_mock = MagicMock() mocker.patch.multiple( @@ -271,10 +264,9 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) default_conf['stake_amount'] = 15.0 - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -307,8 +299,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, - limit_sell_order, markets, mocker) -> None: - patch_exchange(mocker) + limit_sell_order, mocker) -> None: default_conf['max_open_trades'] = 1 mocker.patch( 'freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', @@ -318,7 +309,6 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(markets) ) msg_mock = MagicMock() mocker.patch.multiple( @@ -326,9 +316,8 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -382,7 +371,6 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker @@ -393,9 +381,8 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -420,14 +407,12 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, - limit_buy_order, limit_sell_order, markets, mocker) -> None: - patch_exchange(mocker) + limit_buy_order, limit_sell_order, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(markets) ) msg_mock = MagicMock() mocker.patch.multiple( @@ -435,9 +420,8 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -724,16 +708,16 @@ def test_reload_conf_handle(default_conf, update, mocker) -> None: def test_forcesell_handle(default_conf, update, ticker, fee, - ticker_sell_up, markets, mocker) -> None: + ticker_sell_up, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) patch_exchange(mocker) + patch_whitelist(mocker, default_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets), ) freqtradebot = FreqtradeBot(default_conf) @@ -775,17 +759,18 @@ def test_forcesell_handle(default_conf, update, ticker, fee, def test_forcesell_down_handle(default_conf, update, ticker, fee, - ticker_sell_down, markets, mocker) -> None: + ticker_sell_down, mocker) -> None: mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) patch_exchange(mocker) + patch_whitelist(mocker, default_conf) + mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets), ) freqtradebot = FreqtradeBot(default_conf) @@ -830,17 +815,17 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, } == last_msg -def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker) -> None: +def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None: patch_exchange(mocker) mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + patch_whitelist(mocker, default_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets), ) default_conf['max_open_trades'] = 4 freqtradebot = FreqtradeBot(default_conf) @@ -885,9 +870,8 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: _init=MagicMock(), _send_msg=msg_mock ) - patch_exchange(mocker) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -980,8 +964,7 @@ def test_forcebuy_handle_exception(default_conf, update, markets, mocker) -> Non def test_performance_handle(default_conf, update, ticker, fee, - limit_buy_order, limit_sell_order, markets, mocker) -> None: - patch_exchange(mocker) + limit_buy_order, limit_sell_order, mocker) -> None: msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -992,10 +975,8 @@ def test_performance_handle(default_conf, update, ticker, fee, 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(markets), ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -1018,8 +999,7 @@ def test_performance_handle(default_conf, update, ticker, fee, assert 'ETH/BTC\t6.20% (1)' in msg_mock.call_args_list[0][0][0] -def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> None: - patch_exchange(mocker) +def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -1030,10 +1010,9 @@ def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> Non 'freqtrade.exchange.Exchange', get_ticker=ticker, buy=MagicMock(return_value={'id': 'mocked_order_id'}), - markets=PropertyMock(markets) + get_fee=fee, ) - mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) diff --git a/tests/test_utils.py b/tests/test_utils.py index f64a6924a..0833375c8 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -188,8 +188,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 8 active markets: " - "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/USD, LTC/USDT, TKN/BTC, XLTCUSDT.\n" + assert ("Exchange Bittrex has 9 active markets: " + "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, NEO/BTC, TKN/BTC, XLTCUSDT, XRP/BTC.\n" in captured.out) patch_exchange(mocker, api_mock=api_mock, id="binance") @@ -202,7 +202,7 @@ def test_list_markets(mocker, markets, capsys): pargs['config'] = None start_list_markets(pargs, False) captured = capsys.readouterr() - assert re.match("\nExchange Binance has 8 active markets:\n", + assert re.match("\nExchange Binance has 9 active markets:\n", captured.out) patch_exchange(mocker, api_mock=api_mock, id="bittrex") @@ -227,8 +227,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), True) captured = capsys.readouterr() - assert ("Exchange Bittrex has 7 active pairs: " - "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/USD, LTC/USDT, TKN/BTC.\n" + assert ("Exchange Bittrex has 8 active pairs: " + "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, NEO/BTC, TKN/BTC, XRP/BTC.\n" in captured.out) # Test list-pairs subcommand with --all: all pairs @@ -254,7 +254,7 @@ def test_list_markets(mocker, markets, capsys): start_list_markets(get_args(args), False) captured = capsys.readouterr() assert ("Exchange Bittrex has 5 active markets with ETH, LTC as base currencies: " - "ETH/BTC, ETH/USDT, LTC/USD, LTC/USDT, XLTCUSDT.\n" + "ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, base=LTC @@ -267,7 +267,7 @@ def test_list_markets(mocker, markets, capsys): start_list_markets(get_args(args), False) captured = capsys.readouterr() assert ("Exchange Bittrex has 3 active markets with LTC as base currency: " - "LTC/USD, LTC/USDT, XLTCUSDT.\n" + "LTC/BTC, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, quote=USDT, USD @@ -279,8 +279,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 4 active markets with USDT, USD as quote currencies: " - "ETH/USDT, LTC/USD, LTC/USDT, XLTCUSDT.\n" + assert ("Exchange Bittrex has 3 active markets with USDT, USD as quote currencies: " + "ETH/USDT, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, quote=USDT @@ -292,8 +292,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 3 active markets with USDT as quote currency: " - "ETH/USDT, LTC/USDT, XLTCUSDT.\n" + assert ("Exchange Bittrex has 2 active markets with USDT as quote currency: " + "ETH/USDT, XLTCUSDT.\n" in captured.out) # active markets, base=LTC, quote=USDT @@ -305,21 +305,21 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 2 active markets with LTC as base currency and " - "with USDT as quote currency: LTC/USDT, XLTCUSDT.\n" + assert ("Exchange Bittrex has 1 active market with LTC as base currency and " + "with USDT as quote currency: XLTCUSDT.\n" in captured.out) # active pairs, base=LTC, quote=USDT args = [ '--config', 'config.json.example', "list-pairs", - "--base", "LTC", "--quote", "USDT", + "--base", "LTC", "--quote", "USD", "--print-list", ] start_list_markets(get_args(args), True) captured = capsys.readouterr() assert ("Exchange Bittrex has 1 active pair with LTC as base currency and " - "with USDT as quote currency: LTC/USDT.\n" + "with USD as quote currency: LTC/USD.\n" in captured.out) # active markets, base=LTC, quote=USDT, NONEXISTENT @@ -331,8 +331,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 2 active markets with LTC as base currency and " - "with USDT, NONEXISTENT as quote currencies: LTC/USDT, XLTCUSDT.\n" + assert ("Exchange Bittrex has 1 active market with LTC as base currency and " + "with USDT, NONEXISTENT as quote currencies: XLTCUSDT.\n" in captured.out) # active markets, base=LTC, quote=NONEXISTENT @@ -355,7 +355,7 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 8 active markets:\n" + assert ("Exchange Bittrex has 9 active markets:\n" in captured.out) # Test tabular output, no markets found @@ -378,7 +378,7 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ('["BLK/BTC","BTT/BTC","ETH/BTC","ETH/USDT","LTC/USD","LTC/USDT","TKN/BTC","XLTCUSDT"]' + assert ('["BLK/BTC","ETH/BTC","ETH/USDT","LTC/BTC","LTC/USD","NEO/BTC","TKN/BTC","XLTCUSDT","XRP/BTC"]' in captured.out) # Test --print-csv @@ -391,7 +391,7 @@ def test_list_markets(mocker, markets, capsys): captured = capsys.readouterr() assert ("Id,Symbol,Base,Quote,Active,Is pair" in captured.out) assert ("blkbtc,BLK/BTC,BLK,BTC,True,True" in captured.out) - assert ("BTTBTC,BTT/BTC,BTT,BTC,True,True" in captured.out) + assert ("USD-LTC,LTC/USD,LTC,USD,True,True" in captured.out) # Test --one-column args = [ @@ -402,7 +402,7 @@ def test_list_markets(mocker, markets, capsys): start_list_markets(get_args(args), False) captured = capsys.readouterr() assert re.search(r"^BLK/BTC$", captured.out, re.MULTILINE) - assert re.search(r"^BTT/BTC$", captured.out, re.MULTILINE) + assert re.search(r"^LTC/USD$", captured.out, re.MULTILINE) def test_create_datadir_failed(caplog): From 32df73c056763d1305f80dc37ba5edcaf55d0faa Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Oct 2019 13:28:04 +0200 Subject: [PATCH 18/50] flake --- tests/conftest.py | 6 ++++-- tests/rpc/test_rpc.py | 3 +-- tests/test_freqtradebot.py | 2 +- tests/test_utils.py | 3 ++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 84612175d..0d0511751 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -63,7 +63,8 @@ def patch_exchange(mocker, api_mock=None, id='bittrex', mock_markets=True) -> No mocker.patch('freqtrade.exchange.Exchange.id', PropertyMock(return_value=id)) mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value=id.title())) if mock_markets: - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=get_markets())) + mocker.patch('freqtrade.exchange.Exchange.markets', + PropertyMock(return_value=get_markets())) if api_mock: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) @@ -71,7 +72,8 @@ def patch_exchange(mocker, api_mock=None, id='bittrex', mock_markets=True) -> No mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock()) -def get_patched_exchange(mocker, config, api_mock=None, id='bittrex', mock_markets=True) -> Exchange: +def get_patched_exchange(mocker, config, api_mock=None, id='bittrex', + mock_markets=True) -> Exchange: patch_exchange(mocker, api_mock, id, mock_markets) config["exchange"]["name"] = id try: diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index a5da9b51e..df2261c1f 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -9,12 +9,11 @@ from numpy import isnan from freqtrade import DependencyException, TemporaryError from freqtrade.edge import PairInfo -from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State -from tests.conftest import patch_exchange, patch_get_signal, get_patched_freqtradebot +from tests.conftest import patch_get_signal, get_patched_freqtradebot # Functions for recurrent object patching diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index ff9a34142..a58c12ead 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2951,7 +2951,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, default_conf['trailing_stop'] = True default_conf['trailing_stop_positive'] = 0.01 patch_whitelist(mocker, default_conf) - + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0833375c8..7d6b82809 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -378,7 +378,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ('["BLK/BTC","ETH/BTC","ETH/USDT","LTC/BTC","LTC/USD","NEO/BTC","TKN/BTC","XLTCUSDT","XRP/BTC"]' + assert ('["BLK/BTC","ETH/BTC","ETH/USDT","LTC/BTC","LTC/USD","NEO/BTC",' + '"TKN/BTC","XLTCUSDT","XRP/BTC"]' in captured.out) # Test --print-csv From bf20f3b7d8862ad6a57285a0c2c9cce7f00cb901 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 26 Oct 2019 15:41:31 +0300 Subject: [PATCH 19/50] Remove part which is related to #2418 --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index e1e101efd..fcbce571e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -194,7 +194,7 @@ freqtrade -c config.json #### 7. (Optional) Post-installation Tasks -You can also optionally setup the bot to run as a `systemd` service and configure it to send the log messages to the `syslog`/`rsyslog` or `journald` daemons. See [Advanced Post-installation Tasks](advanced-setup.md) for details. +On Linux, as an optional post-installation task, you can setup the bot to run as a `systemd` service. See [Advanced Post-installation Tasks](advanced-setup.md) for details. ------ From 13ae339a2e2c08344d7ec3224d0e0666fe453672 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Oct 2019 16:34:13 +0200 Subject: [PATCH 20/50] Improve windows Install documentation with hints --- docs/installation.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/installation.md b/docs/installation.md index fcbce571e..afc635673 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -218,6 +218,12 @@ If that is not available on your system, feel free to try the instructions below ### Install freqtrade manually +!!! Note + Make sure to use 64bit Windows to avoid problems with backtesting or hyperopt due to the memory constraints 32bit applications have under Windows. + +!!! Hint + Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under windows can greatly help with installation problems. Check out the [Conda section](#using-conda) in this document. + #### Clone the git repository ```bash From 48d83715a5cff5f2ea68338fde92bff0b2e07105 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 27 Oct 2019 03:44:49 +0300 Subject: [PATCH 21/50] Fix typo in docs (thanks to Escaliert@Slack) --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index ff40b1750..5e936065c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -135,7 +135,7 @@ To allow the bot to trade all the available `stake_currency` in your account set In this case a trade amount is calclulated as: ```python -currency_balanse / (max_open_trades - current_open_trades) +currency_balance / (max_open_trades - current_open_trades) ``` ### Understand minimal_roi From e5487441ba29c93cc0658bd3e9a8f876482e6b4c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Oct 2019 06:08:55 +0100 Subject: [PATCH 22/50] Fix typos --- docs/installation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index afc635673..02870a1c1 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -219,10 +219,10 @@ If that is not available on your system, feel free to try the instructions below ### Install freqtrade manually !!! Note - Make sure to use 64bit Windows to avoid problems with backtesting or hyperopt due to the memory constraints 32bit applications have under Windows. + Make sure to use 64bit Windows and 64bit Python to avoid problems with backtesting or hyperopt due to the memory constraints 32bit applications have under Windows. !!! Hint - Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under windows can greatly help with installation problems. Check out the [Conda section](#using-conda) in this document. + Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under Windows can greatly help with installation problems. Check out the [Conda section](#using-conda) in this document. #### Clone the git repository From 141c454187c96bd8d2f0f85b4309a18e2a8b1bfb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Oct 2019 11:17:01 +0200 Subject: [PATCH 23/50] Add startup-candles-argument for strategy --- freqtrade/strategy/interface.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 014ca9968..48a70b0ce 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -103,6 +103,9 @@ class IStrategy(ABC): # run "populate_indicators" only for new candle process_only_new_candles: bool = False + # Count of candles the strategy requires before producing valid signals + startup_candle_count: int = 0 + # Class level variables (intentional) containing # the dataprovider (dp) (access to other candles, historic data, ...) # and wallets - access to the current balance. @@ -421,6 +424,7 @@ class IStrategy(ABC): def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: """ Creates a dataframe and populates indicators for given ticker data + Used by optimize operations only, not during dry / live runs. """ return {pair: self.advise_indicators(pair_data, {'pair': pair}) for pair, pair_data in tickerdata.items()} From 616fe08bcea0dae438906aaff60434749ff4e375 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Oct 2019 11:29:25 +0200 Subject: [PATCH 24/50] Add subtract_start to timerange object --- freqtrade/configuration/timerange.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index fc759ab6e..527402af7 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -27,6 +27,10 @@ class TimeRange: return (self.starttype == other.starttype and self.stoptype == other.stoptype and self.startts == other.startts and self.stopts == other.stopts) + def subtract_start(self, seconds) -> None: + if self.startts: + self.startts = self.startts - seconds + @staticmethod def parse_timerange(text: Optional[str]): """ From 9e7e051eb42d45fffed0b8318587a0f8e7e78d99 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Oct 2019 11:30:20 +0200 Subject: [PATCH 25/50] add trim-dataframe method --- freqtrade/data/history.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index ed5d80b0e..d385a28ed 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -13,6 +13,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple import arrow +import pytz from pandas import DataFrame from freqtrade import OperationalException, misc @@ -49,6 +50,19 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: return tickerlist[start_index:stop_index] +def trim_dataframe(df: DataFrame, timerange: TimeRange) -> DataFrame: + """ + Trim dataframe based on given timerange + """ + if timerange.starttype == 'date': + start = datetime.fromtimestamp(timerange.startts, tz=pytz.utc) + df = df.loc[df['date'] >= start, :] + if timerange.stoptype == 'date': + stop = datetime.fromtimestamp(timerange.stopts, tz=pytz.utc) + df = df.loc[df['date'] <= stop, :] + return df + + def load_tickerdata_file(datadir: Path, pair: str, ticker_interval: str, timerange: Optional[TimeRange] = None) -> Optional[list]: """ From 9c7696a8ce2e4833c4995b7e2ff30368b6292913 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Oct 2019 13:56:01 +0200 Subject: [PATCH 26/50] Add required_startup to backtesting --- freqtrade/optimize/backtesting.py | 33 ++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fb8c182ee..aa8a6a882 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -15,7 +15,7 @@ from freqtrade import OperationalException from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.dataprovider import DataProvider -from freqtrade.exchange import timeframe_to_minutes +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -90,6 +90,9 @@ class Backtesting: self.ticker_interval = str(self.config.get('ticker_interval')) self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval) + # Get maximum required startup period + self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) + self.required_startup_s = self.required_startup * timeframe_to_seconds(self.ticker_interval) # Load one (first) strategy self._set_strategy(self.strategylist[0]) @@ -418,11 +421,19 @@ class Backtesting: timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) + + logger.info('Using indicator startup period: %s ...', self.required_startup) + + # Timerange_startup is timerange - startup-candles + timerange_startup = deepcopy(timerange) + timerange_startup.subtract_start(self.required_startup_s) + data = history.load_data( datadir=Path(self.config['datadir']), pairs=pairs, ticker_interval=self.ticker_interval, timerange=timerange, + startup_candles=self.required_startup ) if not data: @@ -439,11 +450,14 @@ class Backtesting: min_date, max_date = history.get_timeframe(data) logger.info( - 'Backtesting with data from %s up to %s (%s days)..', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days + 'Loading backtest data from %s up to %s (%s days)..', + min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days ) + if not timerange_startup.starttype: + # If no startts was defined, we need to move the backtesting start + logger.info("Moving start-date by %s candles.", self.required_startup) + timerange.startts = min_date.timestamp + self.required_startup_s + timerange.starttype = 'date' for strat in self.strategylist: logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) @@ -452,6 +466,15 @@ class Backtesting: # need to reprocess data every time to populate signals preprocessed = self.strategy.tickerdata_to_dataframe(data) + # Trim startup period from analyzed dataframe + for pair, df in preprocessed.items(): + preprocessed[pair] = history.trim_dataframe(df, timerange) + min_date, max_date = history.get_timeframe(preprocessed) + + logger.info( + 'Backtesting with data from %s up to %s (%s days)..', + min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days + ) # Execute backtest and print results all_results[self.strategy.get_strategy_name()] = self.backtest( { From 704121c197798c2c0ad3829ff464a997d60fa1ff Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Oct 2019 14:02:53 +0200 Subject: [PATCH 27/50] Move most logic to history --- freqtrade/data/history.py | 30 ++++++++++++++++++++++++------ freqtrade/optimize/backtesting.py | 12 ++++-------- tests/optimize/test_backtesting.py | 2 +- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index d385a28ed..71ac5c9a7 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -8,6 +8,7 @@ Includes: import logging import operator +from copy import deepcopy from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -19,7 +20,7 @@ from pandas import DataFrame from freqtrade import OperationalException, misc from freqtrade.configuration import TimeRange from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv -from freqtrade.exchange import Exchange, timeframe_to_minutes +from freqtrade.exchange import Exchange, timeframe_to_minutes, timeframe_to_seconds logger = logging.getLogger(__name__) @@ -127,7 +128,8 @@ def load_pair_history(pair: str, refresh_pairs: bool = False, exchange: Optional[Exchange] = None, fill_up_missing: bool = True, - drop_incomplete: bool = True + drop_incomplete: bool = True, + startup_candles: int = 0, ) -> DataFrame: """ Loads cached ticker history for the given pair. @@ -140,9 +142,15 @@ def load_pair_history(pair: str, :param exchange: Exchange object (needed when using "refresh_pairs") :param fill_up_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. + :param startup_candles: Additional candles to load at the start of the period :return: DataFrame with ohlcv data """ + timerange_startup = deepcopy(timerange) + if startup_candles: + logger.info('Using indicator startup period: %s ...', startup_candles) + timerange_startup.subtract_start(timeframe_to_seconds(ticker_interval) * startup_candles) + # The user forced the refresh of pairs if refresh_pairs: download_pair_history(datadir=datadir, @@ -151,11 +159,11 @@ def load_pair_history(pair: str, ticker_interval=ticker_interval, timerange=timerange) - pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) + pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange_startup) if pairdata: - if timerange: - _validate_pairdata(pair, pairdata, timerange) + if timerange_startup: + _validate_pairdata(pair, pairdata, timerange_startup) return parse_ticker_dataframe(pairdata, ticker_interval, pair=pair, fill_missing=fill_up_missing, drop_incomplete=drop_incomplete) @@ -174,10 +182,20 @@ def load_data(datadir: Path, exchange: Optional[Exchange] = None, timerange: Optional[TimeRange] = None, fill_up_missing: bool = True, + startup_candles: int = 0, ) -> Dict[str, DataFrame]: """ Loads ticker history data for a list of pairs - :return: dict(:) + :param datadir: Path to the data storage location. + :param ticker_interval: Ticker-interval (e.g. "5m") + :param pairs: List of pairs to load + :param refresh_pairs: Refresh pairs from exchange. + (Note: Requires exchange to be passed as well.) + :param exchange: Exchange object (needed when using "refresh_pairs") + :param timerange: Limit data to be loaded to this timerange + :param fill_up_missing: Fill missing values with "No action"-candles + :param startup_candles: Additional candles to load at the start of the period + :return: dict(:) TODO: refresh_pairs is still used by edge to keep the data uptodate. This should be replaced in the future. Instead, writing the current candles to disk from dataprovider should be implemented, as this would avoid loading ohlcv data twice. diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index aa8a6a882..59130dbc0 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -92,7 +92,6 @@ class Backtesting: # Get maximum required startup period self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) - self.required_startup_s = self.required_startup * timeframe_to_seconds(self.ticker_interval) # Load one (first) strategy self._set_strategy(self.strategylist[0]) @@ -422,11 +421,6 @@ class Backtesting: timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - logger.info('Using indicator startup period: %s ...', self.required_startup) - - # Timerange_startup is timerange - startup-candles - timerange_startup = deepcopy(timerange) - timerange_startup.subtract_start(self.required_startup_s) data = history.load_data( datadir=Path(self.config['datadir']), @@ -453,10 +447,12 @@ class Backtesting: 'Loading backtest data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days ) - if not timerange_startup.starttype: + if not timerange.starttype: # If no startts was defined, we need to move the backtesting start logger.info("Moving start-date by %s candles.", self.required_startup) - timerange.startts = min_date.timestamp + self.required_startup_s + timerange.startts = (min_date.timestamp + + timeframe_to_seconds(self.ticker_interval) + * self.required_startup) timerange.starttype = 'date' for strat in self.strategylist: diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 998edda8a..3353274ef 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -117,7 +117,7 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, - timerange=None, exchange=None, live=False): + timerange=None, exchange=None, live=False, startup_candles=0): tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', pair="UNITTEST/BTC", fill_missing=True)} From 6382a4cd042ed1e84af48493c60e7229986a0e11 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 23 Oct 2019 17:57:38 +0200 Subject: [PATCH 28/50] Implement startup-period to default-strategy --- freqtrade/optimize/backtesting.py | 9 +++++---- freqtrade/strategy/default_strategy.py | 6 +++--- tests/optimize/test_backtesting.py | 10 ++++++++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 59130dbc0..6b82dd601 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -421,7 +421,6 @@ class Backtesting: timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = history.load_data( datadir=Path(self.config['datadir']), pairs=pairs, @@ -447,9 +446,11 @@ class Backtesting: 'Loading backtest data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days ) - if not timerange.starttype: - # If no startts was defined, we need to move the backtesting start - logger.info("Moving start-date by %s candles.", self.required_startup) + if (not timerange.starttype or (self.required_startup + and min_date.timestamp == timerange.startts)): + # If no startts was defined, or test-data starts at the defined test-date + logger.warning("Moving start-date by %s candles to account for startup time.", + self.required_startup) timerange.startts = (min_date.timestamp + timeframe_to_seconds(self.ticker_interval) * self.required_startup) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index b839a9618..0a241691c 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -39,6 +39,9 @@ class DefaultStrategy(IStrategy): 'stoploss_on_exchange': False } + # Count of candles the strategy requires before producing valid signals + startup_candle_count: int = 20 + # Optional time in force for orders order_time_in_force = { 'buy': 'gtc', @@ -105,9 +108,6 @@ class DefaultStrategy(IStrategy): # EMA - Exponential Moving Average dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - return dataframe def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 3353274ef..b14209e2d 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -838,7 +838,10 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): f'Using data directory: {testdatadir} ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', - 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' + 'Backtesting with data from 2017-11-14T21:37:00+00:00 ' + 'up to 2017-11-14T22:58:00+00:00 (0 days)..', + 'Moving start-date by 20 candles to account for startup time.', + 'Loading backtest data from 2017-11-14T21:17:00+00:00 ' 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...' ] @@ -892,7 +895,10 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): f'Using data directory: {testdatadir} ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', - 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' + 'Backtesting with data from 2017-11-14T21:37:00+00:00 ' + 'up to 2017-11-14T22:58:00+00:00 (0 days)..', + 'Moving start-date by 20 candles to account for startup time.', + 'Loading backtest data from 2017-11-14T21:17:00+00:00 ' 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', From 5c2682e2c94828f1cbfa32b9ec6534fae103caf1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 23 Oct 2019 17:58:26 +0200 Subject: [PATCH 29/50] Add startup_candle_count to sample strategy --- freqtrade/strategy/default_strategy.py | 2 +- user_data/strategies/sample_strategy.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 0a241691c..6c343b477 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -39,7 +39,7 @@ class DefaultStrategy(IStrategy): 'stoploss_on_exchange': False } - # Count of candles the strategy requires before producing valid signals + # Number of candles the strategy requires before producing valid signals startup_candle_count: int = 20 # Optional time in force for orders diff --git a/user_data/strategies/sample_strategy.py b/user_data/strategies/sample_strategy.py index 80c30283d..c2fd681d2 100644 --- a/user_data/strategies/sample_strategy.py +++ b/user_data/strategies/sample_strategy.py @@ -59,6 +59,9 @@ class SampleStrategy(IStrategy): sell_profit_only = False ignore_roi_if_buy_signal = False + # Number of candles the strategy requires before producing valid signals + startup_candle_count: int = 20 + # Optional order type mapping. order_types = { 'buy': 'limit', From bd4a23beeb9506b75b4556176ae3eef26924e0ab Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 23 Oct 2019 19:30:01 +0200 Subject: [PATCH 30/50] Refactor start-adjust logic to timerange --- freqtrade/configuration/timerange.py | 27 +++++++++++++++++++++++++++ freqtrade/optimize/backtesting.py | 12 +++--------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index 527402af7..5731631c5 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -1,11 +1,14 @@ """ This module contains the argument manager class """ +import logging import re from typing import Optional import arrow +logger = logging.getLogger(__name__) + class TimeRange: """ @@ -28,9 +31,33 @@ class TimeRange: and self.startts == other.startts and self.stopts == other.stopts) def subtract_start(self, seconds) -> None: + """ + Subtracts from startts if startts is set. + :param seconds: Seconds to subtract from starttime + :return: None (Modifies the object in place) + """ if self.startts: self.startts = self.startts - seconds + def adjust_start_if_necessary(self, ticker_interval_secs: int, startup_candles: int, + min_date: arrow.Arrow) -> None: + """ + Adjust startts by candles. + Applies only if no startup-candles have been available. + :param ticker_interval_secs: Ticker interval in seconds e.g. `timeframe_to_seconds('5m')` + :param startup_candles: Number of candles to move start-date forward + :param min_date: Minimum data date loaded. Key kriterium to decide if start-time + has to be moved + :return: None (Modifies the object in place) + """ + if (not self.starttype or (startup_candles + and min_date.timestamp == self.startts)): + # If no startts was defined, or test-data starts at the defined test-date + logger.warning("Moving start-date by %s candles to account for startup time.", + startup_candles) + self.startts = (min_date.timestamp + ticker_interval_secs * startup_candles) + self.starttype = 'date' + @staticmethod def parse_timerange(text: Optional[str]): """ diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6b82dd601..1d6b328a8 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -446,15 +446,9 @@ class Backtesting: 'Loading backtest data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days ) - if (not timerange.starttype or (self.required_startup - and min_date.timestamp == timerange.startts)): - # If no startts was defined, or test-data starts at the defined test-date - logger.warning("Moving start-date by %s candles to account for startup time.", - self.required_startup) - timerange.startts = (min_date.timestamp - + timeframe_to_seconds(self.ticker_interval) - * self.required_startup) - timerange.starttype = 'date' + # Adjust startts forward if not enough data is available + timerange.adjust_start_if_necessary(timeframe_to_seconds(self.ticker_interval), + self.required_startup, min_date) for strat in self.strategylist: logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) From 5cdae17d19b75b07aed188fd676b056a0ec4e7a2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 23 Oct 2019 19:41:10 +0200 Subject: [PATCH 31/50] Add tests for timerange modifications --- tests/test_timerange.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/test_timerange.py b/tests/test_timerange.py index 4851cbebd..d758092ed 100644 --- a/tests/test_timerange.py +++ b/tests/test_timerange.py @@ -1,10 +1,11 @@ # pragma pylint: disable=missing-docstring, C0103 +import arrow import pytest from freqtrade.configuration import TimeRange -def test_parse_timerange_incorrect() -> None: +def test_parse_timerange_incorrect(): assert TimeRange('date', None, 1274486400, 0) == TimeRange.parse_timerange('20100522-') assert TimeRange(None, 'date', 0, 1274486400) == TimeRange.parse_timerange('-20100522') @@ -28,3 +29,37 @@ def test_parse_timerange_incorrect() -> None: with pytest.raises(Exception, match=r'Incorrect syntax.*'): TimeRange.parse_timerange('-') + + +def test_subtract_start(): + x = TimeRange('date', 'date', 1274486400, 1438214400) + x.subtract_start(300) + assert x.startts == 1274486400 - 300 + + # Do nothing if no startdate exists + x = TimeRange(None, 'date', 0, 1438214400) + x.subtract_start(300) + assert not x.startts + + x = TimeRange('date', None, 1274486400, 0) + x.subtract_start(300) + assert x.startts == 1274486400 - 300 + + +def test_adjust_start_if_necessary(): + min_date = arrow.Arrow(2017, 11, 14, 21, 15, 00) + + x = TimeRange('date', 'date', 1510694100, 1510780500) + # Adjust by 20 candles - min_date == startts + x.adjust_start_if_necessary(300, 20, min_date) + assert x.startts == 1510694100 + (20 * 300) + + x = TimeRange('date', 'date', 1510700100, 1510780500) + # Do nothing, startupe is set and different min_date + x.adjust_start_if_necessary(300, 20, min_date) + assert x.startts == 1510694100 + (20 * 300) + + x = TimeRange(None, 'date', 0, 1510780500) + # Adjust by 20 candles = 20 * 5m + x.adjust_start_if_necessary(300, 20, min_date) + assert x.startts == 1510694100 + (20 * 300) From 86624411c6e5af7c6fa1c1dc9f184f5420ef1172 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 23 Oct 2019 19:54:43 +0200 Subject: [PATCH 32/50] Test trim_dataframe --- tests/data/test_history.py | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 95382768a..d7e0562cc 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -427,6 +427,46 @@ def test_trim_tickerlist(testdatadir) -> None: assert not ticker +def test_trim_dataframe(testdatadir) -> None: + data = history.load_data( + datadir=testdatadir, + ticker_interval='1m', + pairs=['UNITTEST/BTC'] + )['UNITTEST/BTC'] + min_date = int(data.iloc[0]['date'].timestamp()) + max_date = int(data.iloc[-1]['date'].timestamp()) + data_modify = data.copy() + + # Remove first 30 minutes (1800 s) + tr = TimeRange('date', None, min_date + 1800, 0) + data_modify = history.trim_dataframe(data_modify, tr) + assert not data_modify.equals(data) + assert len(data_modify) < len(data) + assert len(data_modify) == len(data) - 30 + assert all(data_modify.iloc[-1] == data.iloc[-1]) + assert all(data_modify.iloc[0] == data.iloc[30]) + + data_modify = data.copy() + # Remove last 30 minutes (1800 s) + tr = TimeRange(None, 'date', 0, max_date - 1800) + data_modify = history.trim_dataframe(data_modify, tr) + assert not data_modify.equals(data) + assert len(data_modify) < len(data) + assert len(data_modify) == len(data) - 30 + assert all(data_modify.iloc[0] == data.iloc[0]) + assert all(data_modify.iloc[-1] == data.iloc[-31]) + + data_modify = data.copy() + # Remove first 25 and last 30 minutes (1800 s) + tr = TimeRange('date', 'date', min_date + 1500, max_date - 1800) + data_modify = history.trim_dataframe(data_modify, tr) + assert not data_modify.equals(data) + assert len(data_modify) < len(data) + assert len(data_modify) == len(data) - 55 + # first row matches 25th original row + assert all(data_modify.iloc[0] == data.iloc[25]) + + def test_file_dump_json_tofile(testdatadir) -> None: file = testdatadir / 'test_{id}.json'.format(id=str(uuid.uuid4())) data = {'bar': 'foo'} From 33164ac78ee2d72647931467abcb8d60e2d5e54e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 23 Oct 2019 20:13:43 +0200 Subject: [PATCH 33/50] Refactor loading of bt data to backtesting ... --- freqtrade/data/history.py | 8 ++++- freqtrade/optimize/backtesting.py | 55 ++++++++++++++++--------------- freqtrade/optimize/hyperopt.py | 25 ++++---------- 3 files changed, 42 insertions(+), 46 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 71ac5c9a7..dfd175b1f 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -183,6 +183,7 @@ def load_data(datadir: Path, timerange: Optional[TimeRange] = None, fill_up_missing: bool = True, startup_candles: int = 0, + fail_without_data: bool = False ) -> Dict[str, DataFrame]: """ Loads ticker history data for a list of pairs @@ -195,6 +196,7 @@ def load_data(datadir: Path, :param timerange: Limit data to be loaded to this timerange :param fill_up_missing: Fill missing values with "No action"-candles :param startup_candles: Additional candles to load at the start of the period + :param fail_without_data: Raise OperationalException if no data is found. :return: dict(:) TODO: refresh_pairs is still used by edge to keep the data uptodate. This should be replaced in the future. Instead, writing the current candles to disk @@ -208,9 +210,13 @@ def load_data(datadir: Path, datadir=datadir, timerange=timerange, refresh_pairs=refresh_pairs, exchange=exchange, - fill_up_missing=fill_up_missing) + fill_up_missing=fill_up_missing, + startup_candles=startup_candles) if hist is not None: result[pair] = hist + + if fail_without_data and not result: + raise OperationalException("No data found. Terminating.") return result diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1d6b328a8..fe31912bc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -105,6 +105,31 @@ class Backtesting: # And the regular "stoploss" function would not apply to that case self.strategy.order_types['stoploss_on_exchange'] = False + def load_bt_data(self): + timerange = TimeRange.parse_timerange(None if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) + + data = history.load_data( + datadir=Path(self.config['datadir']), + pairs=self.config['exchange']['pair_whitelist'], + ticker_interval=self.ticker_interval, + timerange=timerange, + startup_candles=self.required_startup, + fail_without_data=True, + ) + + min_date, max_date = history.get_timeframe(data) + + logger.info( + 'Loading data from %s up to %s (%s days)..', + min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days + ) + # Adjust startts forward if not enough data is available + timerange.adjust_start_if_necessary(timeframe_to_seconds(self.ticker_interval), + self.required_startup, min_date) + + return data, timerange + def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame, skip_nan: bool = False) -> str: """ @@ -414,42 +439,18 @@ class Backtesting: :return: None """ data: Dict[str, Any] = {} - pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - - timerange = TimeRange.parse_timerange(None if self.config.get( - 'timerange') is None else str(self.config.get('timerange'))) - - data = history.load_data( - datadir=Path(self.config['datadir']), - pairs=pairs, - ticker_interval=self.ticker_interval, - timerange=timerange, - startup_candles=self.required_startup - ) - - if not data: - logger.critical("No data found. Terminating.") - return # Use max_open_trades in backtesting, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): max_open_trades = self.config['max_open_trades'] else: logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 + + data, timerange = self.load_bt_data() + all_results = {} - - min_date, max_date = history.get_timeframe(data) - - logger.info( - 'Loading backtest data from %s up to %s (%s days)..', - min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days - ) - # Adjust startts forward if not enough data is available - timerange.adjust_start_if_necessary(timeframe_to_seconds(self.ticker_interval), - self.required_startup, min_date) - for strat in self.strategylist: logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) self._set_strategy(strat) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 07258a048..2264234d4 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -23,7 +23,7 @@ from skopt import Optimizer from skopt.space import Dimension from freqtrade.configuration import TimeRange -from freqtrade.data.history import load_data, get_timeframe +from freqtrade.data.history import load_data, get_timeframe, trim_dataframe from freqtrade.misc import round_dict from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules @@ -379,30 +379,19 @@ class Hyperopt: ) def start(self) -> None: - timerange = TimeRange.parse_timerange(None if self.config.get( - 'timerange') is None else str(self.config.get('timerange'))) - data = load_data( - datadir=Path(self.config['datadir']), - pairs=self.config['exchange']['pair_whitelist'], - ticker_interval=self.backtesting.ticker_interval, - timerange=timerange - ) + data, timerange = self.backtesting.load_bt_data() - if not data: - logger.critical("No data found. Terminating.") - return + preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data) + # Trim startup period from analyzed dataframe + for pair, df in preprocessed.items(): + preprocessed[pair] = trim_dataframe(df, timerange) min_date, max_date = get_timeframe(data) logger.info( 'Hyperopting with data from %s up to %s (%s days)..', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days + min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days ) - - preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data) - dump(preprocessed, self.tickerdata_pickle) # We don't need exchange instance anymore while running hyperopt From 2ba388074e577278a83a17eb92e3980176f75b56 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 23 Oct 2019 20:27:51 +0200 Subject: [PATCH 34/50] Fix small bugs --- freqtrade/configuration/timerange.py | 2 +- tests/optimize/test_backtesting.py | 20 ++++++++------------ tests/optimize/test_hyperopt.py | 7 +++---- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index 5731631c5..df5c937cf 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -51,7 +51,7 @@ class TimeRange: :return: None (Modifies the object in place) """ if (not self.starttype or (startup_candles - and min_date.timestamp == self.startts)): + and min_date.timestamp >= self.startts)): # If no startts was defined, or test-data starts at the defined test-date logger.warning("Moving start-date by %s candles to account for startup time.", startup_candles) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index b14209e2d..ba87848ec 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -117,7 +117,7 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, - timerange=None, exchange=None, live=False, startup_candles=0): + timerange=None, exchange=None, live=False, *args, **kwargs): tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', pair="UNITTEST/BTC", fill_missing=True)} @@ -494,7 +494,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> def get_timeframe(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={})) + mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=None)) mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe) mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) @@ -511,10 +511,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> default_conf['timerange'] = '20180101-20180102' backtesting = Backtesting(default_conf) - backtesting.start() - # check the logs, that will contain the backtest result - - assert log_has('No data found. Terminating.', caplog) + with pytest.raises(OperationalException, match='No data found. Terminating.'): + backtesting.start() def test_backtest(default_conf, fee, mocker, testdatadir) -> None: @@ -838,10 +836,9 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): f'Using data directory: {testdatadir} ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', - 'Backtesting with data from 2017-11-14T21:37:00+00:00 ' + 'Loading data from 2017-11-14T20:57:00+00:00 ' 'up to 2017-11-14T22:58:00+00:00 (0 days)..', - 'Moving start-date by 20 candles to account for startup time.', - 'Loading backtest data from 2017-11-14T21:17:00+00:00 ' + 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...' ] @@ -895,10 +892,9 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): f'Using data directory: {testdatadir} ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', - 'Backtesting with data from 2017-11-14T21:37:00+00:00 ' + 'Loading data from 2017-11-14T20:57:00+00:00 ' 'up to 2017-11-14T22:58:00+00:00 (0 days)..', - 'Moving start-date by 20 candles to account for startup time.', - 'Loading backtest data from 2017-11-14T21:17:00+00:00 ' + 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 052c3ba77..d1448d367 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -228,7 +228,7 @@ def test_start(mocker, default_conf, caplog) -> None: def test_start_no_data(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock(return_value={})) + mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=None)) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -242,9 +242,8 @@ def test_start_no_data(mocker, default_conf, caplog) -> None: '--epochs', '5' ] args = get_args(args) - start_hyperopt(args) - - assert log_has('No data found. Terminating.', caplog) + with pytest.raises(OperationalException, match='No data found. Terminating.'): + start_hyperopt(args) def test_start_filelock(mocker, default_conf, caplog) -> None: From 2bc74882e9940dd6e5c67e94240e7031ed2a9ec1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Oct 2019 10:00:44 +0100 Subject: [PATCH 35/50] Add test for startup_candles --- tests/data/test_history.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index d7e0562cc..057524fb3 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -95,6 +95,23 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> N _clean_test_file(file) +def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None: + ltfmock = mocker.patch('freqtrade.data.history.load_tickerdata_file', + MagicMock(return_value=None)) + timerange = TimeRange('date', None, 1510639620, 0) + history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='1m', + datadir=testdatadir, timerange=timerange, + startup_candles=20, + ) + assert log_has( + 'Using indicator startup period: 20 ...', caplog + ) + assert ltfmock.call_count == 1 + assert ltfmock.call_args_list[0][1]['timerange'] != timerange + # startts is 20 minutes earlier + assert ltfmock.call_args_list[0][1]['timerange'].startts == timerange.startts - 20 * 60 + + def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, default_conf, testdatadir) -> None: """ From c4cb098d14cafe4760551974b6de6c309888166e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Oct 2019 10:17:02 +0100 Subject: [PATCH 36/50] Update documentation with indicator_startup_period --- docs/backtesting.md | 2 ++ docs/strategy-customization.md | 35 ++++++++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 34c5f1fbe..6d21fa2bf 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -72,6 +72,8 @@ The exported trades can be used for [further analysis](#further-backtest-result- freqtrade backtesting --export trades --export-filename=backtest_samplestrategy.json ``` +Please also read about the [strategy startup period](strategy-customization.md#strategy-startup-period). + #### Supplying custom fee value Sometimes your account has certain fee rebates (fee reductions starting with a certain account size or monthly volume), which are not visible to ccxt. diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index ab7dcfc30..ae44d32ea 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -117,6 +117,37 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame Look into the [user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/sample_strategy.py). Then uncomment indicators you need. +### Strategy startup period + +Most indicators have an "instable period", in which they are either not available, or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be. +To account for this, the strategy has an attribute, `startup_candle_count`. +This should be set to the maximum number of candles that the strategy requires to calculate stable indicators. + +In this example strategy, this should be set to 100 (`startup_candle_count = 100`), since the longest needed history is 100 candles. + +``` python + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) +``` + +By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt. + +!!! Warning: + `startup_candle_count` should be below `ohlcv_candle_limit` (which is 500 for most exchanges) - since only this amount of candles will be available during trading operations. + +#### Example + +Let's try to backtest 1 month (January 2019) of 5m candles. + +``` bash +freqtrade backtesting --timerange 20190101-20190201 --ticker-interval 5m +``` + +Since backtesting knows it needs 100 candles to generate valid buy-signals, it'll load data from `20190101 - (100 * 5m)` - which is ~2019-12-31 15:30:00. +If this data is available, Indicators will be calculated with this extended timerange. The startup period (Up to 2019-01-01 00:00:00) will then be removed before starting backtesting. + +!!! Note + If data for the startup-period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-01 08:30:00. + ### Buy signal rules Edit the method `populate_buy_trend()` in your strategy file to update your buy strategy. @@ -267,10 +298,10 @@ class Awesomestrategy(IStrategy): ``` !!! Warning - The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash. + The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash. !!! Note - If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. + If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. ### Additional data (DataProvider) From 223f0cd4d3780a348f1a743329cf671b38ec914d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Oct 2019 10:18:58 +0100 Subject: [PATCH 37/50] Apply startup_period to edge as well --- freqtrade/edge/__init__.py | 3 ++- tests/edge/test_edge.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 2655fbc65..883bf4a0f 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -100,7 +100,8 @@ class Edge: ticker_interval=self.strategy.ticker_interval, refresh_pairs=self._refresh_pairs, exchange=self.exchange, - timerange=self._timerange + timerange=self._timerange, + startup_candles=self.strategy.startup_candle_count, ) if not data: diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index 5e244a97e..e1af50768 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -256,7 +256,7 @@ def test_edge_heartbeat_calculate(mocker, edge_conf): def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, - timerange=None, exchange=None): + timerange=None, exchange=None, *args, **kwargs): hz = 0.1 base = 0.001 From 73f5bff9c5c9487457f529a007cef99faff6f1f0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Oct 2019 10:38:21 +0100 Subject: [PATCH 38/50] Add validation to make sure strategies work on that exchange --- freqtrade/exchange/exchange.py | 10 ++++++++++ freqtrade/resolvers/strategy_resolver.py | 1 + tests/exchange/test_exchange.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 71f0737ef..3fd7d615c 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -228,6 +228,7 @@ class Exchange: self.validate_pairs(config['exchange']['pair_whitelist']) self.validate_ordertypes(config.get('order_types', {})) self.validate_order_time_in_force(config.get('order_time_in_force', {})) + self.validate_required_startup_candles(config.get('startup_candle_count', 0)) # Converts the interval provided in minutes in config to seconds self.markets_refresh_interval: int = exchange_config.get( @@ -443,6 +444,15 @@ class Exchange: raise OperationalException( f'Time in force policies are not supported for {self.name} yet.') + def validate_required_startup_candles(self, startup_candles) -> None: + """ + Checks if required startup_candles is more than ohlcv_candle_limit. + """ + if startup_candles + 5 > self._ft_has['ohlcv_candle_limit']: + raise OperationalException( + f"This strategy requires {startup_candles} candles to start. " + f"{self.name} only provides {self._ft_has['ohlcv_candle_limit']}.") + def exchange_has(self, endpoint: str) -> bool: """ Checks if exchange implements a specific API endpoint. diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index d6fbe9a7a..5bea74027 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -57,6 +57,7 @@ class StrategyResolver(IResolver): ("order_time_in_force", None, False), ("stake_currency", None, False), ("stake_amount", None, False), + ("startup_candle_count", None, False), ("use_sell_signal", True, True), ("sell_profit_only", False, True), ("ignore_roi_if_buy_signal", False, True), diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 1e0a5fdc3..e0e0cc7b1 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -533,6 +533,20 @@ def test_validate_order_types_not_in_config(default_conf, mocker): Exchange(conf) +def test_validate_required_startup_candles(default_conf, mocker, caplog): + api_mock = MagicMock() + default_conf['startup_candle_count'] = 2000 + mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) + + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + + with pytest.raises(OperationalException, match=r'This strategy requires 2000.*'): + Exchange(default_conf) + + def test_exchange_has(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) assert not exchange.exchange_has('ASDFASDF') From 132a4da7cf6ce448877506d28e7fbc9b2b63701d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Oct 2019 10:56:38 +0100 Subject: [PATCH 39/50] Small style fixes and adjusted tests --- docs/strategy-customization.md | 2 +- freqtrade/data/history.py | 2 +- freqtrade/exchange/exchange.py | 1 + freqtrade/optimize/hyperopt.py | 3 +-- freqtrade/strategy/interface.py | 2 +- tests/exchange/test_exchange.py | 8 ++++++-- tests/optimize/test_hyperopt.py | 24 ++++++++++++++++-------- 7 files changed, 27 insertions(+), 15 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index ae44d32ea..4d3d9bce5 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -131,7 +131,7 @@ In this example strategy, this should be set to 100 (`startup_candle_count = 100 By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt. -!!! Warning: +!!! Warning `startup_candle_count` should be below `ohlcv_candle_limit` (which is 500 for most exchanges) - since only this amount of candles will be available during trading operations. #### Example diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index dfd175b1f..c07b58da2 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -147,7 +147,7 @@ def load_pair_history(pair: str, """ timerange_startup = deepcopy(timerange) - if startup_candles: + if startup_candles and timerange_startup: logger.info('Using indicator startup period: %s ...', startup_candles) timerange_startup.subtract_start(timeframe_to_seconds(ticker_interval) * startup_candles) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 3fd7d615c..023e16cc5 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -447,6 +447,7 @@ class Exchange: def validate_required_startup_candles(self, startup_candles) -> None: """ Checks if required startup_candles is more than ohlcv_candle_limit. + Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default. """ if startup_candles + 5 > self._ft_has['ohlcv_candle_limit']: raise OperationalException( diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2264234d4..c576ea6f8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,8 +22,7 @@ from pandas import DataFrame from skopt import Optimizer from skopt.space import Dimension -from freqtrade.configuration import TimeRange -from freqtrade.data.history import load_data, get_timeframe, trim_dataframe +from freqtrade.data.history import get_timeframe, trim_dataframe from freqtrade.misc import round_dict from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 48a70b0ce..d42f8e989 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -424,7 +424,7 @@ class IStrategy(ABC): def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: """ Creates a dataframe and populates indicators for given ticker data - Used by optimize operations only, not during dry / live runs. + Used by optimize operations only, not during dry / live runs. """ return {pair: self.advise_indicators(pair_data, {'pair': pair}) for pair, pair_data in tickerdata.items()} diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index e0e0cc7b1..6805d8e73 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -535,7 +535,6 @@ def test_validate_order_types_not_in_config(default_conf, mocker): def test_validate_required_startup_candles(default_conf, mocker, caplog): api_mock = MagicMock() - default_conf['startup_candle_count'] = 2000 mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) @@ -543,7 +542,12 @@ def test_validate_required_startup_candles(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) - with pytest.raises(OperationalException, match=r'This strategy requires 2000.*'): + default_conf['startup_candle_count'] = 20 + ex = Exchange(default_conf) + assert ex + default_conf['startup_candle_count'] = 600 + + with pytest.raises(OperationalException, match=r'This strategy requires 600.*'): Exchange(default_conf) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index d1448d367..d0c37c40d 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -392,7 +392,8 @@ def test_roi_table_generation(hyperopt) -> None: def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -607,7 +608,8 @@ def test_continue_hyperopt(mocker, default_conf, caplog): def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -644,7 +646,8 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -681,7 +684,8 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -727,7 +731,8 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -> None: mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -756,7 +761,8 @@ def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) - def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -801,7 +807,8 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -852,7 +859,8 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None ]) def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, method, space) -> None: mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) From 2af3ce3ecc5b3af96401ab36014a137a75c6691b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Oct 2019 19:36:00 +0100 Subject: [PATCH 40/50] Improve stoploss documentation - split out offset_is_reached --- docs/stoploss.md | 74 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index f5e2f8df6..3ea8beb72 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -3,74 +3,98 @@ The `stoploss` configuration parameter is loss in percentage that should trigger a sale. For example, value `-0.10` will cause immediate sell if the profit dips below -10% for a given trade. This parameter is optional. -Most of the strategy files already include the optimal `stoploss` -value. This parameter is optional. If you use it in the configuration file, it will take over the -`stoploss` value from the strategy file. +Most of the strategy files already include the optimal `stoploss` value. +Stoploss parameters need to be set in either strategy or configuration file. +Parameters in the configuration will overwrite settings within the strategy. -## Stop Loss support +## Stop Loss Types At this stage the bot contains the following stoploss support modes: 1. static stop loss, defined in either the strategy or configuration. 2. trailing stop loss, defined in the configuration. 3. trailing stop loss, custom positive loss, defined in configuration. +4. trailing stop loss only once the trade has reached a certain offset, !!! Note All stoploss properties can be configured in either Strategy or configuration. Configuration values override strategy values. -Those stoploss modes can be *on exchange* or *off exchange*. If the stoploss is *on exchange* it means a stoploss limit order is placed on the exchange immediately after buy order happens successfuly. This will protect you against sudden crashes in market as the order will be in the queue immediately and if market goes down then the order has more chance of being fulfilled. +Those stoploss modes can be *on exchange* or *off exchange*. If the stoploss is *on exchange* it means a stoploss limit order is placed on the exchange immediately after buy order happens successfully. This will protect you against sudden crashes in market as the order will be in the queue immediately and if market goes down then the order has more chance of being fulfilled. -In case of stoploss on exchange there is another parameter called `stoploss_on_exchange_interval`. This configures the interval in seconds at which the bot will check the stoploss and update it if necessary. As an example in case of trailing stoploss if the order is on the exchange and the market is going up then the bot automatically cancels the previous stoploss order and put a new one with a stop value higher than previous one. It is clear that the bot cannot do it every 5 seconds otherwise it gets banned. So this parameter will tell the bot how often it should update the stoploss order. The default value is 60 (1 minute). +In case of stoploss on exchange there is another parameter called `stoploss_on_exchange_interval`. This configures the interval in seconds at which the bot will check the stoploss and update it if necessary. + +For example, assuming the stoploss is on exchange, and trailing stoploss is enabled, and the market is going up, then the bot automatically cancels the previous stoploss order and puts a new one with a stop value higher than the previous stoploss order. +The bot cannot do this every 5 seconds, otherwise it would get banned by the exchange. +So this parameter will tell the bot how often it should update the stoploss order. The default value is 60 (1 minute). +This same logic will reapply a stoploss order on the exchange should you cancel it accidentally. !!! Note Stoploss on exchange is only supported for Binance as of now. ## Static Stop Loss -This is very simple, basically you define a stop loss of x in your strategy file or alternative in the configuration, which -will overwrite the strategy definition. This will basically try to sell your asset, the second the loss exceeds the defined loss. +This is very simple, you define a stop loss of x. This will try to sell the asset once the loss exceeds the defined loss. ## Trailing Stop Loss -The initial value for this stop loss, is defined in your strategy or configuration. Just as you would define your Stop Loss normally. +The initial value for this is `stoploss`, set either in the strategy or in the configuration file. Just as you would define your Stop loss normally. To enable this Feauture all you have to do is to define the configuration element: ``` json "trailing_stop" : True ``` -This will now activate an algorithm, which automatically moves your stop loss up every time the price of your asset increases. +This will now activate an algorithm, which automatically moves the stop loss up every time the price of your asset increases. -For example, simplified math, +For example, simplified math: -* you buy an asset at a price of 100$ -* your stop loss is defined at 2% -* which means your stop loss, gets triggered once your asset dropped below 98$ -* assuming your asset now increases to 102$ -* your stop loss, will now be 2% of 102$ or 99.96$ -* now your asset drops in value to 101$, your stop loss, will still be 99.96$ +* the bot buys an asset at a price of 100$ +* the stop loss is defined at 2% +* the stop loss would get triggered once the asset dropps below 98$ +* assuming the asset now increases to 102$ +* the stop loss will now be 2% of 102$ or 99.96$ +* now the asset drops in value to 101$, the stop loss, will still be 99.96$, and would trigger at 99.96$. -basically what this means is that your stop loss will be adjusted to be always be 2% of the highest observed price +In summary: The stoploss will be adjusted to be always be 2% of the highest observed price. -### Custom positive loss +### Custom positive stoploss -Due to demand, it is possible to have a default stop loss, when you are in the red with your buy, but once your profit surpasses a certain percentage, -the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you have 1.1% profit, -it will be changed to be only a 1% stop loss, which trails the green candles until it goes below them. +It is also possible to have a default stop loss, when you are in the red with your buy, but once your profit surpasses a certain percentage, the system will utilize a new stop loss, which can have a different value. +For example your default stop loss is 5%, but once you have 1.1% profit, it will be changed to be only a 1% stop loss, which trails the green candles until it goes below them. -Both values can be configured in the main configuration file and requires `"trailing_stop": true` to be set to true. +Both values can be configured in the strategy or configuration file and requires `"trailing_stop": true` to be set to true. ``` json "trailing_stop_positive": 0.01, "trailing_stop_positive_offset": 0.011, - "trailing_only_offset_is_reached": false ``` The 0.01 would translate to a 1% stop loss, once you hit 1.1% profit. You should also make sure to have this value (`trailing_stop_positive_offset`) lower than your minimal ROI, otherwise minimal ROI will apply first and sell your trade. -If `"trailing_only_offset_is_reached": true` then the trailing stoploss is only activated once the offset is reached. Until then, the stoploss remains at the configured`stoploss`. +### Trailing only once offset is reached + +It is also possible to use a static stoploss until the offset is reached, and then trail the trade to take profits once the market goes down again. + +If `"trailing_only_offset_is_reached": true` then the trailing stoploss is only activated once the offset is reached. Until then, the stoploss remains at the configured `stoploss`. +This option can be used with or without `trailing_stop_positive`, but uses `trailing_stop_positive_offset` as offset. + +``` json + "trailing_only_offset_is_reached": true, +``` + +Simplified example: + +```python + stoploss = 0.05 + trailing_stop_positive_offset = 0.03 + trailing_only_offset_is_reached = True +``` + +* the bot buys an asset at a price of 100$ +* the stop loss is defined at 5% +* the stop loss will remain at 95% until profit reaches +3% ## Changing stoploss on open trades From 70ad909b16e15bd7c436a6704f825f2f7c64e3a4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Oct 2019 19:46:05 +0100 Subject: [PATCH 41/50] change samples to python code, and simplify a few things --- docs/stoploss.md | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index 3ea8beb72..7cf885590 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -4,20 +4,18 @@ The `stoploss` configuration parameter is loss in percentage that should trigger For example, value `-0.10` will cause immediate sell if the profit dips below -10% for a given trade. This parameter is optional. Most of the strategy files already include the optimal `stoploss` value. -Stoploss parameters need to be set in either strategy or configuration file. -Parameters in the configuration will overwrite settings within the strategy. + +!!! Info + All stoploss properties mentioned in this file can be set in the Strategy, or in the configuration. Configuration values will override the strategy values. ## Stop Loss Types At this stage the bot contains the following stoploss support modes: -1. static stop loss, defined in either the strategy or configuration. -2. trailing stop loss, defined in the configuration. -3. trailing stop loss, custom positive loss, defined in configuration. -4. trailing stop loss only once the trade has reached a certain offset, - -!!! Note - All stoploss properties can be configured in either Strategy or configuration. Configuration values override strategy values. +1. Static stop loss. +2. Trailing stop loss. +3. Trailing stop loss, custom positive loss. +4. Trailing stop loss only once the trade has reached a certain offset, Those stoploss modes can be *on exchange* or *off exchange*. If the stoploss is *on exchange* it means a stoploss limit order is placed on the exchange immediately after buy order happens successfully. This will protect you against sudden crashes in market as the order will be in the queue immediately and if market goes down then the order has more chance of being fulfilled. @@ -37,11 +35,11 @@ This is very simple, you define a stop loss of x. This will try to sell the asse ## Trailing Stop Loss -The initial value for this is `stoploss`, set either in the strategy or in the configuration file. Just as you would define your Stop loss normally. -To enable this Feauture all you have to do is to define the configuration element: +The initial value for this is `stoploss`, just as you would define your static Stop loss. +To enable trailing stoploss: -``` json -"trailing_stop" : True +``` python +trailing_stop = True ``` This will now activate an algorithm, which automatically moves the stop loss up every time the price of your asset increases. @@ -62,31 +60,36 @@ In summary: The stoploss will be adjusted to be always be 2% of the highest obse It is also possible to have a default stop loss, when you are in the red with your buy, but once your profit surpasses a certain percentage, the system will utilize a new stop loss, which can have a different value. For example your default stop loss is 5%, but once you have 1.1% profit, it will be changed to be only a 1% stop loss, which trails the green candles until it goes below them. -Both values can be configured in the strategy or configuration file and requires `"trailing_stop": true` to be set to true. +Both values require `trailing_stop` to be set to true. -``` json - "trailing_stop_positive": 0.01, - "trailing_stop_positive_offset": 0.011, +``` python + trailing_stop_positive = 0.01 + trailing_stop_positive_offset = 0.011 ``` The 0.01 would translate to a 1% stop loss, once you hit 1.1% profit. +Before this, `stoploss` is used for the trailing stoploss. -You should also make sure to have this value (`trailing_stop_positive_offset`) lower than your minimal ROI, otherwise minimal ROI will apply first and sell your trade. +Read the [next section](#trailing-only-once-offset-is-reached) to keep stoploss at 5% of the entry point. + +!!! Tip + Make sure to have this value (`trailing_stop_positive_offset`) lower than minimal ROI, otherwise minimal ROI will apply first and sell the trade. ### Trailing only once offset is reached -It is also possible to use a static stoploss until the offset is reached, and then trail the trade to take profits once the market goes down again. +It is also possible to use a static stoploss until the offset is reached, and then trail the trade to take profits once the market turns. If `"trailing_only_offset_is_reached": true` then the trailing stoploss is only activated once the offset is reached. Until then, the stoploss remains at the configured `stoploss`. This option can be used with or without `trailing_stop_positive`, but uses `trailing_stop_positive_offset` as offset. -``` json - "trailing_only_offset_is_reached": true, +``` python + trailing_stop_positive_offset = 0.011 + trailing_only_offset_is_reached = true ``` Simplified example: -```python +``` python stoploss = 0.05 trailing_stop_positive_offset = 0.03 trailing_only_offset_is_reached = True From 46b975a49158f5c7caaf827312c85e712222059d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2019 07:40:15 +0000 Subject: [PATCH 42/50] Bump pytest-mock from 1.11.1 to 1.11.2 Bumps [pytest-mock](https://github.com/pytest-dev/pytest-mock) from 1.11.1 to 1.11.2. - [Release notes](https://github.com/pytest-dev/pytest-mock/releases) - [Changelog](https://github.com/pytest-dev/pytest-mock/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-mock/compare/v1.11.1...v1.11.2) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f5cde59e8..ada602360 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,7 +11,7 @@ mypy==0.740 pytest==5.2.1 pytest-asyncio==0.10.0 pytest-cov==2.8.1 -pytest-mock==1.11.1 +pytest-mock==1.11.2 pytest-random-order==1.0.4 # Convert jupyter notebooks to markdown documents From 60b99469b938c2dae88faf2f2b30129d51a163d5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2019 07:40:57 +0000 Subject: [PATCH 43/50] Bump nbconvert from 5.6.0 to 5.6.1 Bumps [nbconvert](https://github.com/jupyter/nbconvert) from 5.6.0 to 5.6.1. - [Release notes](https://github.com/jupyter/nbconvert/releases) - [Commits](https://github.com/jupyter/nbconvert/compare/5.6.0...5.6.1) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f5cde59e8..d9c7467a9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,4 +15,4 @@ pytest-mock==1.11.1 pytest-random-order==1.0.4 # Convert jupyter notebooks to markdown documents -nbconvert==5.6.0 +nbconvert==5.6.1 From 44d0a6f2b89b69ecc2a4bddfdf35ce9093b0521b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2019 07:42:58 +0000 Subject: [PATCH 44/50] Bump ccxt from 1.18.1306 to 1.18.1346 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.18.1306 to 1.18.1346. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/1.18.1306...1.18.1346) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 1e42d8a04..64a43ee62 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.1306 +ccxt==1.18.1346 SQLAlchemy==1.3.10 python-telegram-bot==12.2.0 arrow==0.15.2 From 596a269dfd23b1e687201b1390580da6977d3793 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2019 11:41:44 +0000 Subject: [PATCH 45/50] Bump pytest from 5.2.1 to 5.2.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.2.1 to 5.2.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.2.1...5.2.2) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7e4736372..589ca7c54 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ flake8==3.7.8 flake8-type-annotations==0.1.0 flake8-tidy-imports==3.0.0 mypy==0.740 -pytest==5.2.1 +pytest==5.2.2 pytest-asyncio==0.10.0 pytest-cov==2.8.1 pytest-mock==1.11.2 From 3a6020dcd76ce4b661b8ca7e79e7a64c49391c14 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Oct 2019 12:43:35 +0100 Subject: [PATCH 46/50] small improvements to stoploss doc --- docs/stoploss.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index 7cf885590..105488296 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -15,14 +15,14 @@ At this stage the bot contains the following stoploss support modes: 1. Static stop loss. 2. Trailing stop loss. 3. Trailing stop loss, custom positive loss. -4. Trailing stop loss only once the trade has reached a certain offset, +4. Trailing stop loss only once the trade has reached a certain offset. Those stoploss modes can be *on exchange* or *off exchange*. If the stoploss is *on exchange* it means a stoploss limit order is placed on the exchange immediately after buy order happens successfully. This will protect you against sudden crashes in market as the order will be in the queue immediately and if market goes down then the order has more chance of being fulfilled. In case of stoploss on exchange there is another parameter called `stoploss_on_exchange_interval`. This configures the interval in seconds at which the bot will check the stoploss and update it if necessary. For example, assuming the stoploss is on exchange, and trailing stoploss is enabled, and the market is going up, then the bot automatically cancels the previous stoploss order and puts a new one with a stop value higher than the previous stoploss order. -The bot cannot do this every 5 seconds, otherwise it would get banned by the exchange. +The bot cannot do this every 5 seconds (at each iteration), otherwise it would get banned by the exchange. So this parameter will tell the bot how often it should update the stoploss order. The default value is 60 (1 minute). This same logic will reapply a stoploss order on the exchange should you cancel it accidentally. @@ -31,7 +31,7 @@ This same logic will reapply a stoploss order on the exchange should you cancel ## Static Stop Loss -This is very simple, you define a stop loss of x. This will try to sell the asset once the loss exceeds the defined loss. +This is very simple, you define a stop loss of x (as a ratio of price, i.e. x * 100% of price). This will try to sell the asset once the loss exceeds the defined loss. ## Trailing Stop Loss @@ -51,7 +51,7 @@ For example, simplified math: * the stop loss would get triggered once the asset dropps below 98$ * assuming the asset now increases to 102$ * the stop loss will now be 2% of 102$ or 99.96$ -* now the asset drops in value to 101$, the stop loss, will still be 99.96$, and would trigger at 99.96$. +* now the asset drops in value to 101$, the stop loss will still be 99.96$ and would trigger at 99.96$. In summary: The stoploss will be adjusted to be always be 2% of the highest observed price. From 73343b338732da04056f372a47a40e8dec254ed5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Oct 2019 12:53:12 +0100 Subject: [PATCH 47/50] Address feedback --- docs/configuration.md | 2 +- tests/conftest.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 90f2687d0..bfe900d3e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -75,7 +75,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.key` | '' | API key to use for the exchange. Only required when you are in production mode. ***Keep it in secrete, do not disclose publicly.*** | `exchange.secret` | '' | API secret to use for the exchange. Only required when you are in production mode. ***Keep it in secrete, do not disclose publicly.*** | `exchange.password` | '' | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests. ***Keep it in secrete, do not disclose publicly.*** -| `exchange.pair_whitelist` | [] | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used when using VolumePairList (see [below](#dynamic-pairlists)). +| `exchange.pair_whitelist` | [] | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used by VolumePairList (see [below](#dynamic-pairlists)). | `exchange.pair_blacklist` | [] | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#dynamic-pairlists)). | `exchange.ccxt_config` | None | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.ccxt_async_config` | None | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) diff --git a/tests/conftest.py b/tests/conftest.py index 0d0511751..4feae6a60 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -129,8 +129,7 @@ def patch_freqtradebot(mocker, config) -> None: patch_exchange(mocker) mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) - mocker.patch('freqtrade.freqtradebot.FreqtradeBot._refresh_whitelist', - MagicMock(return_value=config['exchange']['pair_whitelist'])) + patch_whitelist(mocker, config) def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: From 61c037f2cf0bb6a43026bc43ad3e2ed4312d0d2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Oct 2019 13:05:54 +0100 Subject: [PATCH 48/50] Fix some typos and comment mistakes --- docs/strategy-customization.md | 14 +++++++------- freqtrade/configuration/timerange.py | 2 +- freqtrade/data/history.py | 2 +- tests/test_timerange.py | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 4d3d9bce5..cef362ffd 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -119,8 +119,8 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame ### Strategy startup period -Most indicators have an "instable period", in which they are either not available, or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be. -To account for this, the strategy has an attribute, `startup_candle_count`. +Most indicators have an instable startup period, in which they are either not available, or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be. +To account for this, the strategy can be assigned the `startup_candle_count` attribute. This should be set to the maximum number of candles that the strategy requires to calculate stable indicators. In this example strategy, this should be set to 100 (`startup_candle_count = 100`), since the longest needed history is 100 candles. @@ -132,21 +132,21 @@ In this example strategy, this should be set to 100 (`startup_candle_count = 100 By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt. !!! Warning - `startup_candle_count` should be below `ohlcv_candle_limit` (which is 500 for most exchanges) - since only this amount of candles will be available during trading operations. + `startup_candle_count` should be below `ohlcv_candle_limit` (which is 500 for most exchanges) - since only this amount of candles will be available during Dry-Run/Live Trade operations. #### Example -Let's try to backtest 1 month (January 2019) of 5m candles. +Let's try to backtest 1 month (January 2019) of 5m candles using the an example strategy with EMA100, as above. ``` bash freqtrade backtesting --timerange 20190101-20190201 --ticker-interval 5m ``` -Since backtesting knows it needs 100 candles to generate valid buy-signals, it'll load data from `20190101 - (100 * 5m)` - which is ~2019-12-31 15:30:00. -If this data is available, Indicators will be calculated with this extended timerange. The startup period (Up to 2019-01-01 00:00:00) will then be removed before starting backtesting. +Assuming `startup_candle_count` is set to 100, backtesting knows it needs 100 candles to generate valid buy signals. It will load data from `20190101 - (100 * 5m)` - which is ~2019-12-31 15:30:00. +If this data is available, indicators will be calculated with this extended timerange. The instable startup period (up to 2019-01-01 00:00:00) will then be removed before starting backtesting. !!! Note - If data for the startup-period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-01 08:30:00. + If data for the startup period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-01 08:30:00. ### Buy signal rules diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index df5c937cf..156f0e1e2 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -52,7 +52,7 @@ class TimeRange: """ if (not self.starttype or (startup_candles and min_date.timestamp >= self.startts)): - # If no startts was defined, or test-data starts at the defined test-date + # If no startts was defined, or backtest-data starts at the defined backtest-date logger.warning("Moving start-date by %s candles to account for startup time.", startup_candles) self.startts = (min_date.timestamp + ticker_interval_secs * startup_candles) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index c07b58da2..412b086c0 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -147,7 +147,7 @@ def load_pair_history(pair: str, """ timerange_startup = deepcopy(timerange) - if startup_candles and timerange_startup: + if startup_candles > 0 and timerange_startup: logger.info('Using indicator startup period: %s ...', startup_candles) timerange_startup.subtract_start(timeframe_to_seconds(ticker_interval) * startup_candles) diff --git a/tests/test_timerange.py b/tests/test_timerange.py index d758092ed..5c35535f0 100644 --- a/tests/test_timerange.py +++ b/tests/test_timerange.py @@ -55,7 +55,7 @@ def test_adjust_start_if_necessary(): assert x.startts == 1510694100 + (20 * 300) x = TimeRange('date', 'date', 1510700100, 1510780500) - # Do nothing, startupe is set and different min_date + # Do nothing, startup is set and different min_date x.adjust_start_if_necessary(300, 20, min_date) assert x.startts == 1510694100 + (20 * 300) From e82460bde67475968c2b934b29ab8d9844fc8e82 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Oct 2019 14:24:12 +0100 Subject: [PATCH 49/50] Fix create_cum_profit to work with trades that don't open on candle opens --- freqtrade/data/btanalysis.py | 12 +++++++++--- freqtrade/plot/plotting.py | 10 +++++----- tests/data/test_btanalysis.py | 2 +- tests/test_plotting.py | 4 ++-- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 17abae3b6..0f5d395ff 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -150,15 +150,21 @@ def combine_tickers_with_mean(tickers: Dict[str, pd.DataFrame], column: str = "c return df_comb -def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str) -> pd.DataFrame: +def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, + timeframe: str) -> pd.DataFrame: """ Adds a column `col_name` with the cumulative profit for the given trades array. :param df: DataFrame with date index :param trades: DataFrame containing trades (requires columns close_time and profitperc) + :param col_name: Column name that will be assigned the results + :param timeframe: Timeframe used during the operations :return: Returns df with one additional column, col_name, containing the cumulative profit. """ - # Use groupby/sum().cumsum() to avoid errors when multiple trades sold at the same candle. - df[col_name] = trades.groupby('close_time')['profitperc'].sum().cumsum() + from freqtrade.exchange import timeframe_to_minutes + ticker_minutes = timeframe_to_minutes(timeframe) + # Resample to ticker_interval to make sure trades match candles + _trades_sum = trades.resample(f'{ticker_minutes}min', on='close_time')[['profitperc']].sum() + df.loc[:, col_name] = _trades_sum.cumsum() # Set first value to 0 df.loc[df.iloc[0].name, col_name] = 0 # FFill to get continuous diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 6bd5993b6..bbdb52ca1 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -264,12 +264,12 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], - trades: pd.DataFrame) -> go.Figure: + trades: pd.DataFrame, timeframe: str) -> go.Figure: # Combine close-values for all pairs, rename columns to "pair" df_comb = combine_tickers_with_mean(tickers, "close") # Add combined cumulative profit - df_comb = create_cum_profit(df_comb, trades, 'cum_profit') + df_comb = create_cum_profit(df_comb, trades, 'cum_profit', timeframe) # Plot the pairs average close prices, and total profit growth avgclose = go.Scatter( @@ -293,7 +293,7 @@ def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], for pair in pairs: profit_col = f'cum_profit_{pair}' - df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col) + df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col, timeframe) fig = add_profit(fig, 3, df_comb, profit_col, f"Profit {pair}") @@ -382,9 +382,9 @@ def plot_profit(config: Dict[str, Any]) -> None: ) # Filter trades to relevant pairs trades = trades[trades['pair'].isin(plot_elements["pairs"])] - # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend - fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], trades) + fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], + trades, config.get('ticker_interval', '5m')) store_plot_file(fig, filename='freqtrade-profit-plot.html', directory=config['user_data_dir'] / "plot", auto_open=True) diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 4068e00e4..a7d646823 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -130,7 +130,7 @@ def test_create_cum_profit(testdatadir): cum_profits = create_cum_profit(df.set_index('date'), bt_data[bt_data["pair"] == 'POWR/BTC'], - "cum_profits") + "cum_profits", timeframe="5m") assert "cum_profits" in cum_profits.columns assert cum_profits.iloc[0]['cum_profits'] == 0 assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005 diff --git a/tests/test_plotting.py b/tests/test_plotting.py index a39b2b76e..1c7d1b392 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -234,7 +234,7 @@ def test_add_profit(testdatadir): cum_profits = create_cum_profit(df.set_index('date'), bt_data[bt_data["pair"] == 'POWR/BTC'], - "cum_profits") + "cum_profits", timeframe="5m") fig1 = add_profit(fig, row=2, data=cum_profits, column='cum_profits', name='Profits') figure = fig1.layout.figure @@ -256,7 +256,7 @@ def test_generate_profit_graph(testdatadir): ) trades = trades[trades['pair'].isin(pairs)] - fig = generate_profit_graph(pairs, tickers, trades) + fig = generate_profit_graph(pairs, tickers, trades, timeframe="5m") assert isinstance(fig, go.Figure) assert fig.layout.title.text == "Freqtrade Profit plot" From 069da224bc28ce199a76908748120c84db2b54a7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Oct 2019 14:30:01 +0100 Subject: [PATCH 50/50] Add test to verify this is correct --- tests/data/test_btanalysis.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index a7d646823..a04a2c529 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock import pytest from arrow import Arrow -from pandas import DataFrame, to_datetime +from pandas import DataFrame, DateOffset, to_datetime from freqtrade.configuration import TimeRange from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, @@ -134,3 +134,21 @@ def test_create_cum_profit(testdatadir): assert "cum_profits" in cum_profits.columns assert cum_profits.iloc[0]['cum_profits'] == 0 assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005 + + +def test_create_cum_profit1(testdatadir): + filename = testdatadir / "backtest-result_test.json" + bt_data = load_backtest_data(filename) + # Move close-time to "off" the candle, to make sure the logic still works + bt_data.loc[:, 'close_time'] = bt_data.loc[:, 'close_time'] + DateOffset(seconds=20) + timerange = TimeRange.parse_timerange("20180110-20180112") + + df = load_pair_history(pair="POWR/BTC", ticker_interval='5m', + datadir=testdatadir, timerange=timerange) + + cum_profits = create_cum_profit(df.set_index('date'), + bt_data[bt_data["pair"] == 'POWR/BTC'], + "cum_profits", timeframe="5m") + assert "cum_profits" in cum_profits.columns + assert cum_profits.iloc[0]['cum_profits'] == 0 + assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005