adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #6133 from rokups/rk/reduce-memory-usage

Browse Source 
Reduce memory usage by not holding on to no longer needed data
This commit is contained in:
Matthias 2021-12-31 11:36:52 +01:00 committed by GitHub
commit 880ee016a4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
freqtrade/optimize/backtesting.py
View File

@ -246,6 +246,9 @@ class Backtesting:
Helper function to convert a processed dataframes into lists for performance reasons. Helper function to convert a processed dataframes into lists for performance reasons.
Used by backtest() - so keep this optimized for performance. Used by backtest() - so keep this optimized for performance.
:param processed: a processed dictionary with format {pair, data}, which gets cleared to
optimize memory usage!
""" """
# Every change to this headers list must evaluate further usages of the resulting tuple # Every change to this headers list must evaluate further usages of the resulting tuple
# and eventually change the constants for indexes at the top # and eventually change the constants for indexes at the top
@ -254,7 +257,8 @@ class Backtesting:
self.progress.init_step(BacktestState.CONVERT, len(processed)) self.progress.init_step(BacktestState.CONVERT, len(processed))
# Create dict with data # Create dict with data
for pair, pair_data in processed.items(): for pair in processed.keys():
pair_data = processed[pair]
self.check_abort() self.check_abort()
self.progress.increment() self.progress.increment()
if not pair_data.empty: if not pair_data.empty:
@ -283,6 +287,9 @@ class Backtesting:
# Convert from Pandas to list for performance reasons # Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.) # (Looping Pandas is slow.)
data[pair] = df_analyzed[headers].values.tolist() data[pair] = df_analyzed[headers].values.tolist()
# Do not hold on to old data to reduce memory usage
processed[pair] = pair_data = None
return data return data
def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple, def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple,
@ -519,7 +526,8 @@ class Backtesting:
Of course try to not have ugly code. By some accessor are sometime slower than functions. Of course try to not have ugly code. By some accessor are sometime slower than functions.
Avoid extensive logging in this method and functions it calls. Avoid extensive logging in this method and functions it calls.
:param processed: a processed dictionary with format {pair, data} :param processed: a processed dictionary with format {pair, data}, which gets cleared to
optimize memory usage!
:param start_date: backtesting timerange start datetime :param start_date: backtesting timerange start datetime
:param end_date: backtesting timerange end datetime :param end_date: backtesting timerange end datetime
:param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited

7
tests/optimize/test_backtesting.py
View File

@ -1,6 +1,7 @@
# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument
import random import random
from copy import deepcopy
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from pathlib import Path from pathlib import Path
from unittest.mock import MagicMock, PropertyMock from unittest.mock import MagicMock, PropertyMock
@ -648,7 +649,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None:
processed = backtesting.strategy.advise_all_indicators(data) processed = backtesting.strategy.advise_all_indicators(data)
min_date, max_date = get_timerange(processed) min_date, max_date = get_timerange(processed)
result = backtesting.backtest( result = backtesting.backtest(
processed=processed, processed=deepcopy(processed),
start_date=min_date, start_date=min_date,
end_date=max_date, end_date=max_date,
max_open_trades=10, max_open_trades=10,
@ -887,7 +888,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
processed = backtesting.strategy.advise_all_indicators(data) processed = backtesting.strategy.advise_all_indicators(data)
min_date, max_date = get_timerange(processed) min_date, max_date = get_timerange(processed)
backtest_conf = { backtest_conf = {
'processed': processed, 'processed': deepcopy(processed),
'start_date': min_date, 'start_date': min_date,
'end_date': max_date, 'end_date': max_date,
'max_open_trades': 3, 'max_open_trades': 3,
@ -909,7 +910,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
'NXT/BTC', '5m')[0]) == len(data['NXT/BTC']) - 1 - backtesting.strategy.startup_candle_count 'NXT/BTC', '5m')[0]) == len(data['NXT/BTC']) - 1 - backtesting.strategy.startup_candle_count
backtest_conf = { backtest_conf = {
'processed': processed, 'processed': deepcopy(processed),
'start_date': min_date, 'start_date': min_date,
'end_date': max_date, 'end_date': max_date,
'max_open_trades': 1, 'max_open_trades': 1,