diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d4b51d04d..78e6fb002 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -246,6 +246,9 @@ class Backtesting: Helper function to convert a processed dataframes into lists for performance reasons. Used by backtest() - so keep this optimized for performance. + + :param processed: a processed dictionary with format {pair, data}, which gets cleared to + optimize memory usage! """ # Every change to this headers list must evaluate further usages of the resulting tuple # and eventually change the constants for indexes at the top @@ -254,7 +257,8 @@ class Backtesting: self.progress.init_step(BacktestState.CONVERT, len(processed)) # Create dict with data - for pair, pair_data in processed.items(): + for pair in processed.keys(): + pair_data = processed[pair] self.check_abort() self.progress.increment() if not pair_data.empty: @@ -283,6 +287,9 @@ class Backtesting: # Convert from Pandas to list for performance reasons # (Looping Pandas is slow.) data[pair] = df_analyzed[headers].values.tolist() + + # Do not hold on to old data to reduce memory usage + processed[pair] = pair_data = None return data def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple, @@ -519,7 +526,8 @@ class Backtesting: Of course try to not have ugly code. By some accessor are sometime slower than functions. Avoid extensive logging in this method and functions it calls. - :param processed: a processed dictionary with format {pair, data} + :param processed: a processed dictionary with format {pair, data}, which gets cleared to + optimize memory usage! :param start_date: backtesting timerange start datetime :param end_date: backtesting timerange end datetime :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index f5e182c1d..6290c3c55 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument import random +from copy import deepcopy from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import MagicMock, PropertyMock @@ -648,7 +649,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: processed = backtesting.strategy.advise_all_indicators(data) min_date, max_date = get_timerange(processed) result = backtesting.backtest( - processed=processed, + processed=deepcopy(processed), start_date=min_date, end_date=max_date, max_open_trades=10, @@ -887,7 +888,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) processed = backtesting.strategy.advise_all_indicators(data) min_date, max_date = get_timerange(processed) backtest_conf = { - 'processed': processed, + 'processed': deepcopy(processed), 'start_date': min_date, 'end_date': max_date, 'max_open_trades': 3, @@ -909,7 +910,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) 'NXT/BTC', '5m')[0]) == len(data['NXT/BTC']) - 1 - backtesting.strategy.startup_candle_count backtest_conf = { - 'processed': processed, + 'processed': deepcopy(processed), 'start_date': min_date, 'end_date': max_date, 'max_open_trades': 1,