Merge pull request #6133 from rokups/rk/reduce-memory-usage

Reduce memory usage by not holding on to no longer needed data

freqtrade/optimize/backtesting.py

@@ -246,6 +246,9 @@ class Backtesting:
         Helper function to convert a processed dataframes into lists for performance reasons.
 
         Used by backtest() - so keep this optimized for performance.
+
+        :param processed: a processed dictionary with format {pair, data}, which gets cleared to
+        optimize memory usage!
         """
         # Every change to this headers list must evaluate further usages of the resulting tuple
         # and eventually change the constants for indexes at the top
@@ -254,7 +257,8 @@ class Backtesting:
         self.progress.init_step(BacktestState.CONVERT, len(processed))
 
         # Create dict with data
-        for pair, pair_data in processed.items():
+        for pair in processed.keys():
+            pair_data = processed[pair]
             self.check_abort()
             self.progress.increment()
             if not pair_data.empty:
@@ -283,6 +287,9 @@ class Backtesting:
             # Convert from Pandas to list for performance reasons
             # (Looping Pandas is slow.)
             data[pair] = df_analyzed[headers].values.tolist()
+
+            # Do not hold on to old data to reduce memory usage
+            processed[pair] = pair_data = None
         return data
 
     def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple,
@@ -519,7 +526,8 @@ class Backtesting:
         Of course try to not have ugly code. By some accessor are sometime slower than functions.
         Avoid extensive logging in this method and functions it calls.
 
-        :param processed: a processed dictionary with format {pair, data}
+        :param processed: a processed dictionary with format {pair, data}, which gets cleared to
+        optimize memory usage!
         :param start_date: backtesting timerange start datetime
         :param end_date: backtesting timerange end datetime
         :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited

tests/optimize/test_backtesting.py

@@ -1,6 +1,7 @@
 # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument
 
 import random
+from copy import deepcopy
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from unittest.mock import MagicMock, PropertyMock
@@ -648,7 +649,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None:
     processed = backtesting.strategy.advise_all_indicators(data)
     min_date, max_date = get_timerange(processed)
     result = backtesting.backtest(
-        processed=processed,
+        processed=deepcopy(processed),
         start_date=min_date,
         end_date=max_date,
         max_open_trades=10,
@@ -887,7 +888,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
     processed = backtesting.strategy.advise_all_indicators(data)
     min_date, max_date = get_timerange(processed)
     backtest_conf = {
-        'processed': processed,
+        'processed': deepcopy(processed),
         'start_date': min_date,
         'end_date': max_date,
         'max_open_trades': 3,
@@ -909,7 +910,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
         'NXT/BTC', '5m')[0]) == len(data['NXT/BTC']) - 1 - backtesting.strategy.startup_candle_count
 
     backtest_conf = {
-        'processed': processed,
+        'processed': deepcopy(processed),
         'start_date': min_date,
         'end_date': max_date,
         'max_open_trades': 1,