From 29fed37df3e92de97cc9a0a75651207fd37c9c3a Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Thu, 13 May 2021 09:47:28 +0300 Subject: [PATCH 1/2] Fix exception when few pairs with no data do not result in aborting backtest. Exception is triggered by backtesting 20210301-20210501 range with BAKE/USDT pair (binance). Pair data starts on 2021-04-30 12:00:00 and after adjusting for startup candles pair dataframe is empty. Solution: Since there are other pairs with enough data - skip pairs with no data and issue a warning. Exception: ``` Traceback (most recent call last): File "/home/rk/src/freqtrade/freqtrade/main.py", line 37, in main return_code = args['func'](args) File "/home/rk/src/freqtrade/freqtrade/commands/optimize_commands.py", line 53, in start_backtesting backtesting.start() File "/home/rk/src/freqtrade/freqtrade/optimize/backtesting.py", line 502, in start min_date, max_date = self.backtest_one_strategy(strat, data, timerange) File "/home/rk/src/freqtrade/freqtrade/optimize/backtesting.py", line 474, in backtest_one_strategy results = self.backtest( File "/home/rk/src/freqtrade/freqtrade/optimize/backtesting.py", line 365, in backtest data: Dict = self._get_ohlcv_as_lists(processed) File "/home/rk/src/freqtrade/freqtrade/optimize/backtesting.py", line 199, in _get_ohlcv_as_lists pair_data.loc[:, 'buy'] = 0 # cleanup from previous run File "/home/rk/src/freqtrade/venv/lib/python3.9/site-packages/pandas/core/indexing.py", line 692, in __setitem__ iloc._setitem_with_indexer(indexer, value, self.name) File "/home/rk/src/freqtrade/venv/lib/python3.9/site-packages/pandas/core/indexing.py", line 1587, in _setitem_with_indexer raise ValueError( ValueError: cannot set a frame with no defined index and a scalar ``` --- freqtrade/optimize/backtesting.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e057d8189..450b88f3b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -9,7 +9,7 @@ from copy import deepcopy from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple -from pandas import DataFrame, NaT +from pandas import DataFrame from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency from freqtrade.constants import DATETIME_PRINT_FORMAT @@ -457,13 +457,21 @@ class Backtesting: preprocessed = self.strategy.ohlcvdata_to_dataframe(data) # Trim startup period from analyzed dataframe - for pair, df in preprocessed.items(): - preprocessed[pair] = trim_dataframe(df, timerange, - startup_candles=self.required_startup) - min_date, max_date = history.get_timerange(preprocessed) - if min_date is NaT or max_date is NaT: + for pair in list(preprocessed): + df = preprocessed[pair] + df = trim_dataframe(df, timerange, startup_candles=self.required_startup) + if len(df) > 0: + preprocessed[pair] = df + else: + logger.warning(f'{pair} has no data left after adjusting for startup candles, ' + f'skipping.') + del preprocessed[pair] + + if not preprocessed: raise OperationalException( - "No data left after adjusting for startup candles. ") + "No data left after adjusting for startup candles.") + + min_date, max_date = history.get_timerange(preprocessed) logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' f'({(max_date - min_date).days} days).') From 2d5f465f1b4646f23244692abf0c719cc3ba1cb5 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Thu, 13 May 2021 11:49:12 +0300 Subject: [PATCH 2/2] Fix protections being loaded multiple times for first strategy when backtesting. --- freqtrade/optimize/backtesting.py | 2 -- freqtrade/optimize/hyperopt.py | 1 + tests/optimize/test_backtest_detail.py | 1 + tests/optimize/test_backtesting.py | 16 ++++++++++++++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 450b88f3b..f05a0d88b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -115,8 +115,6 @@ class Backtesting: # Get maximum required startup period self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) - # Load one (first) strategy - self._set_strategy(self.strategylist[0]) def __del__(self): LoggingMixin.show_output = True diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5ccf02d01..5ba242bd1 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -79,6 +79,7 @@ class Hyperopt: self.custom_hyperopt = HyperOptAuto(self.config) else: self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config) + self.backtesting._set_strategy(self.backtesting.strategylist[0]) self.custom_hyperopt.strategy = self.backtesting.strategy self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 7da7709c6..ca2baaf33 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -493,6 +493,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: patch_exchange(mocker) frame = _build_backtest_dataframe(data.data) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.advise_buy = lambda a, m: frame backtesting.strategy.advise_sell = lambda a, m: frame caplog.set_level(logging.DEBUG) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 2ebea564b..03a65b159 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -83,6 +83,7 @@ def simple_backtest(config, contour, mocker, testdatadir) -> None: patch_exchange(mocker) config['timeframe'] = '1m' backtesting = Backtesting(config) + backtesting._set_strategy(backtesting.strategylist[0]) data = load_data_test(contour, testdatadir) processed = backtesting.strategy.ohlcvdata_to_dataframe(data) @@ -106,6 +107,7 @@ def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC'): data = trim_dictlist(data, -201) patch_exchange(mocker) backtesting = Backtesting(conf) + backtesting._set_strategy(backtesting.strategylist[0]) processed = backtesting.strategy.ohlcvdata_to_dataframe(data) min_date, max_date = get_timerange(processed) return { @@ -285,6 +287,7 @@ def test_backtesting_init(mocker, default_conf, order_types) -> None: patch_exchange(mocker) get_fee = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5)) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) assert backtesting.config == default_conf assert backtesting.timeframe == '5m' assert callable(backtesting.strategy.ohlcvdata_to_dataframe) @@ -315,11 +318,13 @@ def test_data_with_fee(default_conf, mocker, testdatadir) -> None: fee_mock = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5)) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) assert backtesting.fee == 0.1234 assert fee_mock.call_count == 0 default_conf['fee'] = 0.0 backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) assert backtesting.fee == 0.0 assert fee_mock.call_count == 0 @@ -330,6 +335,7 @@ def test_data_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: data = history.load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, fill_up_missing=True) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) processed = backtesting.strategy.ohlcvdata_to_dataframe(data) assert len(processed['UNITTEST/BTC']) == 102 @@ -361,6 +367,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: default_conf['timerange'] = '-1510694220' backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.bot_loop_start = MagicMock() backtesting.start() # check the logs, that will contain the backtest result @@ -393,6 +400,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> default_conf['timerange'] = '20180101-20180102' backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) with pytest.raises(OperationalException, match='No data found. Terminating.'): backtesting.start() @@ -465,6 +473,7 @@ def test_backtest__enter_trade(default_conf, fee, mocker) -> None: default_conf['stake_amount'] = 'unlimited' default_conf['max_open_trades'] = 2 backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) pair = 'UNITTEST/BTC' row = [ pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0), @@ -508,6 +517,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) patch_exchange(mocker) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) pair = 'UNITTEST/BTC' timerange = TimeRange('date', None, 1517227800, 0) data = history.load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], @@ -570,6 +580,7 @@ def test_backtest_1min_timeframe(default_conf, fee, mocker, testdatadir) -> None mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) patch_exchange(mocker) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) # Run a backtesting for an exiting 1min timeframe timerange = TimeRange.parse_timerange('1510688220-1510700340') @@ -591,6 +602,7 @@ def test_backtest_1min_timeframe(default_conf, fee, mocker, testdatadir) -> None def test_processed(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) dict_of_tickerrows = load_data_test('raise', testdatadir) dataframes = backtesting.strategy.ohlcvdata_to_dataframe(dict_of_tickerrows) @@ -661,6 +673,7 @@ def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir): backtest_conf = _make_backtest_conf(mocker, conf=default_conf, datadir=testdatadir) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.advise_buy = fun # Override backtesting.strategy.advise_sell = fun # Override result = backtesting.backtest(**backtest_conf) @@ -676,6 +689,7 @@ def test_backtest_only_sell(mocker, default_conf, testdatadir): backtest_conf = _make_backtest_conf(mocker, conf=default_conf, datadir=testdatadir) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.advise_buy = fun # Override backtesting.strategy.advise_sell = fun # Override result = backtesting.backtest(**backtest_conf) @@ -689,6 +703,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): pair='UNITTEST/BTC', datadir=testdatadir) default_conf['timeframe'] = '1m' backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.advise_buy = _trend_alternate # Override backtesting.strategy.advise_sell = _trend_alternate # Override result = backtesting.backtest(**backtest_conf) @@ -731,6 +746,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) default_conf['timeframe'] = '5m' backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.advise_buy = _trend_alternate_hold # Override backtesting.strategy.advise_sell = _trend_alternate_hold # Override