Merge pull request #3215 from freqtrade/backtest_use_pairlists

Backtest use pairlists

docs/backtesting.md

@@ -17,8 +17,12 @@ For details on downloading, please refer to the [Data Downloading](data-download
 
 The result of backtesting will confirm if your bot has better odds of making a profit than a loss.
 
-!!! Tip "Using dynamic pairlists for backtesting"
-    While using dynamic pairlists during backtesting is not possible, a dynamic pairlist using current data can be generated via the [`test-pairlist`](utils.md#test-pairlist) command, and needs to be specified as `"pair_whitelist"` attribute in the configuration.
+!!! Warning "Using dynamic pairlists for backtesting"
+    Using dynamic pairlists is possible, however it relies on the current market conditions - which will not reflect the historic status of the pairlist.
+    Also, when using pairlists other than StaticPairlist, reproducability of backtesting-results cannot be guaranteed.
+    Please read the [pairlists documentation](configuration.md#pairlists) for more information.
+
+    To achieve reproducible results, best generate a pairlist via the [`test-pairlist`](utils.md#test-pairlist) command and use that as static pairlist.
 
 ### Run a backtesting against the currencies listed in your config file
 

docs/configuration.md

@@ -544,7 +544,6 @@ A fixed slot (mirroring `bid_strategy.order_book_top`) can be defined by setting
     Using `ask_strategy.order_book_max` higher than 1 will result in improper dry-run results (significantly better than real orders executed on exchange), since dry-run assumes orders to be filled almost instantly.
     It is therefore advised to not use this setting for dry-runs.
 
-
 #### Sell price without Orderbook enabled
 
 When not using orderbook (`ask_strategy.use_order_book=False`), the price at the `ask_strategy.price_side` side (defaults to `"ask"`) from the ticker will be used as the sell price.
@@ -622,6 +621,7 @@ Min price precision is 8 decimals. If price is 0.00000011 - one step would be 0.
 These pairs are dangerous since it may be impossible to place the desired stoploss - and often result in high losses.
 
 #### Spread Filter
+
 Removes pairs that have a difference between asks and bids above the specified ratio (default `0.005`).
 Example:
 If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027 the ratio is calculated as: `1 - bid/ask ~= 0.037` which is `> 0.005` 

freqtrade/optimize/backtesting.py

@@ -20,6 +20,7 @@ from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
 from freqtrade.optimize.optimize_reports import (show_backtest_results,
                                                  store_backtest_result)
+from freqtrade.pairlist.pairlistmanager import PairListManager
 from freqtrade.persistence import Trade
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.state import RunMode
@@ -63,10 +64,19 @@ class Backtesting:
         self.strategylist: List[IStrategy] = []
         self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
 
+        self.pairlists = PairListManager(self.exchange, self.config)
+        if 'VolumePairList' in self.pairlists.name_list:
+            raise OperationalException("VolumePairList not allowed for backtesting.")
+
+        self.pairlists.refresh_pairlist()
+
+        if len(self.pairlists.whitelist) == 0:
+            raise OperationalException("No pair in whitelist.")
+
         if config.get('fee'):
             self.fee = config['fee']
         else:
-            self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0])
+            self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0])
 
         if self.config.get('runmode') != RunMode.HYPEROPT:
             self.dataprovider = DataProvider(self.config, self.exchange)
@@ -111,7 +121,7 @@ class Backtesting:
 
         data = history.load_data(
             datadir=self.config['datadir'],
-            pairs=self.config['exchange']['pair_whitelist'],
+            pairs=self.pairlists.whitelist,
             timeframe=self.timeframe,
             timerange=timerange,
             startup_candles=self.required_startup,

freqtrade/optimize/hyperopt.py

@@ -626,6 +626,7 @@ class Hyperopt:
 
         # We don't need exchange instance anymore while running hyperopt
         self.backtesting.exchange = None  # type: ignore
+        self.backtesting.pairlists = None  # type: ignore
 
         self.trials = self.load_previous_results(self.trials_file)
 

tests/optimize/test_backtesting.py

@@ -2,7 +2,7 @@
 
 import random
 from pathlib import Path
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, PropertyMock
 
 import numpy as np
 import pandas as pd
@@ -10,8 +10,9 @@ import pytest
 from arrow import Arrow
 
 from freqtrade import constants
+from freqtrade.commands.optimize_commands import (setup_optimize_configuration,
+                                                  start_backtesting)
 from freqtrade.configuration import TimeRange
-from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_backtesting
 from freqtrade.data import history
 from freqtrade.data.btanalysis import evaluate_result_multi
 from freqtrade.data.converter import clean_ohlcv_dataframe
@@ -333,8 +334,9 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
     patch_exchange(mocker)
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest')
     mocker.patch('freqtrade.optimize.backtesting.show_backtest_results')
+    mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist',
+                 PropertyMock(return_value=['UNITTEST/BTC']))
 
-    default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
     default_conf['ticker_interval'] = '1m'
     default_conf['datadir'] = testdatadir
     default_conf['export'] = None
@@ -362,9 +364,9 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
     mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
     patch_exchange(mocker)
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest')
-    mocker.patch('freqtrade.optimize.backtesting.show_backtest_results')
+    mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist',
+                 PropertyMock(return_value=['UNITTEST/BTC']))
 
-    default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
     default_conf['ticker_interval'] = "1m"
     default_conf['datadir'] = testdatadir
     default_conf['export'] = None
@@ -375,6 +377,29 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
         backtesting.start()
 
 
+def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> None:
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
+    mocker.patch('freqtrade.data.history.history_utils.load_pair_history',
+                 MagicMock(return_value=pd.DataFrame()))
+    mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
+    patch_exchange(mocker)
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest')
+    mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist',
+                 PropertyMock(return_value=[]))
+
+    default_conf['ticker_interval'] = "1m"
+    default_conf['datadir'] = testdatadir
+    default_conf['export'] = None
+    default_conf['timerange'] = '20180101-20180102'
+
+    with pytest.raises(OperationalException, match='No pair in whitelist.'):
+        Backtesting(default_conf)
+
+    default_conf['pairlists'] = [{"method": "VolumePairList", "number_assets": 5}]
+    with pytest.raises(OperationalException, match='VolumePairList not allowed for backtesting.'):
+        Backtesting(default_conf)
+
+
 def test_backtest(default_conf, fee, mocker, testdatadir) -> None:
     default_conf['ask_strategy']['use_sell_signal'] = False
     mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
@@ -585,12 +610,12 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
 
 
 def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
-    default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
 
     patch_exchange(mocker)
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
     mocker.patch('freqtrade.optimize.backtesting.show_backtest_results', MagicMock())
-
+    mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist',
+                 PropertyMock(return_value=['UNITTEST/BTC']))
     patched_configuration_load_config_file(mocker, default_conf)
 
     args = [
@@ -625,10 +650,11 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
 
 
 def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
-    default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
 
     patch_exchange(mocker)
     backtestmock = MagicMock()
+    mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist',
+                 PropertyMock(return_value=['UNITTEST/BTC']))
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
     gen_table_mock = MagicMock()
     mocker.patch('freqtrade.optimize.optimize_reports.generate_text_table', gen_table_mock)