From 365479f5e0a19a2e08e4f3309ce342d58c401d27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 13 Jul 2021 19:36:15 +0200 Subject: [PATCH] Remove startup-candles after populating buy/sell signals closes #5242 --- freqtrade/optimize/backtesting.py | 29 ++++++++++++++------------ freqtrade/optimize/hyperopt.py | 7 +++---- tests/optimize/test_backtest_detail.py | 1 + tests/optimize/test_backtesting.py | 1 + 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ebd4135d9..f52e9a49f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -15,7 +15,7 @@ from freqtrade.configuration import TimeRange, remove_credentials, validate_conf from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.data import history from freqtrade.data.btanalysis import trade_list_to_dataframe -from freqtrade.data.converter import trim_dataframes +from freqtrade.data.converter import trim_dataframe, trim_dataframes from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import BacktestState, SellType from freqtrade.exceptions import DependencyException, OperationalException @@ -116,6 +116,9 @@ class Backtesting: self.wallets = Wallets(self.config, self.exchange, log=False) + self.timerange = TimeRange.parse_timerange( + None if self.config.get('timerange') is None else str(self.config.get('timerange'))) + # Get maximum required startup period self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) @@ -154,14 +157,11 @@ class Backtesting: """ self.progress.init_step(BacktestState.DATALOAD, 1) - timerange = TimeRange.parse_timerange(None if self.config.get( - 'timerange') is None else str(self.config.get('timerange'))) - data = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, timeframe=self.timeframe, - timerange=timerange, + timerange=self.timerange, startup_candles=self.required_startup, fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), @@ -174,11 +174,11 @@ class Backtesting: f'({(max_date - min_date).days} days).') # Adjust startts forward if not enough data is available - timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe), - self.required_startup, min_date) + self.timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe), + self.required_startup, min_date) self.progress.set_new_value(1) - return data, timerange + return data, self.timerange def prepare_backtest(self, enable_protections): """ @@ -223,7 +223,9 @@ class Backtesting: df_analyzed = self.strategy.advise_sell( self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() - + # Trim startup period from analyzed dataframe + df_analyzed = trim_dataframe(df_analyzed, self.timerange, + startup_candles=self.required_startup) # To avoid using data from future, we use buy/sell signals shifted # from the previous candle df_analyzed.loc[:, 'buy'] = df_analyzed.loc[:, 'buy'].shift(1) @@ -537,14 +539,15 @@ class Backtesting: preprocessed = self.strategy.ohlcvdata_to_dataframe(data) # Trim startup period from analyzed dataframe - preprocessed = trim_dataframes(preprocessed, timerange, self.required_startup) + preprocessed_tmp = trim_dataframes(preprocessed, timerange, self.required_startup) - if not preprocessed: + if not preprocessed_tmp: raise OperationalException( "No data left after adjusting for startup candles.") - min_date, max_date = history.get_timerange(preprocessed) - + # Use preprocessed_tmp for date generation (the trimmed dataframe). + # Backtesting will re-trim the dataframes after buy/sell signal generation. + min_date, max_date = history.get_timerange(preprocessed_tmp) logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' f'({(max_date - min_date).days} days).') diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 80ae8886e..cb6884385 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -378,16 +378,15 @@ class Hyperopt: preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data) - # Trim startup period from analyzed dataframe + # Trim startup period from analyzed dataframe to get correct dates for output. processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup) - self.min_date, self.max_date = get_timerange(processed) logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} ' f'({(self.max_date - self.min_date).days} days)..') - - dump(processed, self.data_pickle_file) + # Store non-trimmed data - will be trimmed after signal generation. + dump(preprocessed, self.data_pickle_file) def start(self) -> None: self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None)) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 0bf197739..af3c1317b 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -575,6 +575,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: frame = _build_backtest_dataframe(data.data) backtesting = Backtesting(default_conf) backtesting._set_strategy(backtesting.strategylist[0]) + backtesting.required_startup = 0 backtesting.strategy.advise_buy = lambda a, m: frame backtesting.strategy.advise_sell = lambda a, m: frame backtesting.strategy.use_custom_stoploss = data.use_custom_stoploss diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 77b9e23fa..2998157a5 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -727,6 +727,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): pair='UNITTEST/BTC', datadir=testdatadir) default_conf['timeframe'] = '1m' backtesting = Backtesting(default_conf) + backtesting.required_startup = 0 backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.advise_buy = _trend_alternate # Override backtesting.strategy.advise_sell = _trend_alternate # Override