Reduce memory usage by not holding on to no longer needed data.

This commit is contained in:
Rokas Kupstys 2021-12-29 12:09:01 +02:00
parent 78ccaae318
commit a715083fc0
2 changed files with 14 additions and 5 deletions

View File

@ -246,6 +246,9 @@ class Backtesting:
Helper function to convert a processed dataframes into lists for performance reasons.
Used by backtest() - so keep this optimized for performance.
:param processed: a processed dictionary with format {pair, data}, which gets cleared to
optimize memory usage!
"""
# Every change to this headers list must evaluate further usages of the resulting tuple
# and eventually change the constants for indexes at the top
@ -254,7 +257,8 @@ class Backtesting:
self.progress.init_step(BacktestState.CONVERT, len(processed))
# Create dict with data
for pair, pair_data in processed.items():
for pair in processed.keys():
pair_data = processed[pair]
self.check_abort()
self.progress.increment()
if not pair_data.empty:
@ -283,6 +287,9 @@ class Backtesting:
# Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.)
data[pair] = df_analyzed[headers].values.tolist()
# Do not hold on to old data to reduce memory usage
processed[pair] = pair_data = None
return data
def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple,
@ -519,7 +526,8 @@ class Backtesting:
Of course try to not have ugly code. By some accessor are sometime slower than functions.
Avoid extensive logging in this method and functions it calls.
:param processed: a processed dictionary with format {pair, data}
:param processed: a processed dictionary with format {pair, data}, which gets cleared to
optimize memory usage!
:param start_date: backtesting timerange start datetime
:param end_date: backtesting timerange end datetime
:param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited

View File

@ -1,6 +1,7 @@
# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument
import random
from copy import deepcopy
from datetime import datetime, timedelta, timezone
from pathlib import Path
from unittest.mock import MagicMock, PropertyMock
@ -648,7 +649,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None:
processed = backtesting.strategy.advise_all_indicators(data)
min_date, max_date = get_timerange(processed)
result = backtesting.backtest(
processed=processed,
processed=deepcopy(processed),
start_date=min_date,
end_date=max_date,
max_open_trades=10,
@ -887,7 +888,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
processed = backtesting.strategy.advise_all_indicators(data)
min_date, max_date = get_timerange(processed)
backtest_conf = {
'processed': processed,
'processed': deepcopy(processed),
'start_date': min_date,
'end_date': max_date,
'max_open_trades': 3,
@ -909,7 +910,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
'NXT/BTC', '5m')[0]) == len(data['NXT/BTC']) - 1 - backtesting.strategy.startup_candle_count
backtest_conf = {
'processed': processed,
'processed': deepcopy(processed),
'start_date': min_date,
'end_date': max_date,
'max_open_trades': 1,