From ace184a281f3ef021e5ced5342b6e67ac785375d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 13 Mar 2021 10:16:32 +0100 Subject: [PATCH] Improve backtest performance --- freqtrade/optimize/backtesting.py | 4 +++- freqtrade/persistence/models.py | 32 ++++++++++++++++++++++++++++--- freqtrade/wallets.py | 11 ++++++++--- tests/conftest.py | 2 +- tests/test_persistence.py | 2 +- 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 575ad486a..f2cf0d0dc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -377,7 +377,7 @@ class Backtesting: open_trade_count += 1 # logger.debug(f"{pair} - Emulate creation of new trade: {trade}.") open_trades[pair].append(trade) - LocalTrade.trades.append(trade) + LocalTrade.add_bt_trade(trade) for trade in open_trades[pair]: # also check the buying candle for sell conditions. @@ -387,6 +387,8 @@ class Backtesting: # logger.debug(f"{pair} - Backtesting sell {trade}") open_trade_count -= 1 open_trades[pair].remove(trade) + + LocalTrade.close_bt_trade(trade) trades.append(trade_entry) if enable_protections: self.protections.stop_per_pair(pair, row[DATE_IDX]) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 8797c7007..f42b832eb 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -208,6 +208,8 @@ class LocalTrade(): use_db: bool = False # Trades container for backtesting trades: List['LocalTrade'] = [] + trades_open: List['LocalTrade'] = [] + total_profit: float = 0 id: int = 0 @@ -350,6 +352,8 @@ class LocalTrade(): Resets all trades. Only active for backtesting mode. """ LocalTrade.trades = [] + LocalTrade.trades_open = [] + LocalTrade.total_profit = 0 def adjust_min_max_rates(self, current_price: float) -> None: """ @@ -599,7 +603,17 @@ class LocalTrade(): """ # Offline mode - without database - sel_trades = [trade for trade in LocalTrade.trades] + if is_open is not None: + if is_open: + sel_trades = LocalTrade.trades_open + else: + sel_trades = LocalTrade.trades + + else: + # Not used during backtesting, but might be used by a strategy + sel_trades = [trade for trade in LocalTrade.trades + LocalTrade.trades_open + if trade.is_open == is_open] + if pair: sel_trades = [trade for trade in sel_trades if trade.pair == pair] if open_date: @@ -607,10 +621,22 @@ class LocalTrade(): if close_date: sel_trades = [trade for trade in sel_trades if trade.close_date and trade.close_date > close_date] - if is_open is not None: - sel_trades = [trade for trade in sel_trades if trade.is_open == is_open] + return sel_trades + @staticmethod + def close_bt_trade(trade): + LocalTrade.trades_open.remove(trade) + LocalTrade.trades.append(trade) + LocalTrade.total_profit += trade.close_profit_abs + + @staticmethod + def add_bt_trade(trade): + if trade.is_open: + LocalTrade.trades_open.append(trade) + else: + LocalTrade.trades.append(trade) + @staticmethod def get_open_trades() -> List[Any]: """ diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 553f7c61d..575fe1b67 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -10,7 +10,7 @@ import arrow from freqtrade.constants import UNLIMITED_STAKE_AMOUNT from freqtrade.exceptions import DependencyException from freqtrade.exchange import Exchange -from freqtrade.persistence import Trade +from freqtrade.persistence import LocalTrade, Trade from freqtrade.state import RunMode @@ -66,9 +66,14 @@ class Wallets: """ # Recreate _wallets to reset closed trade balances _wallets = {} - closed_trades = Trade.get_trades_proxy(is_open=False) open_trades = Trade.get_trades_proxy(is_open=True) - tot_profit = sum([trade.close_profit_abs for trade in closed_trades]) + # If not backtesting... + # TODO: potentially remove the ._log workaround to determine backtest mode. + if self._log: + closed_trades = Trade.get_trades_proxy(is_open=False) + tot_profit = sum([trade.close_profit_abs for trade in closed_trades]) + else: + tot_profit = LocalTrade.total_profit tot_in_trades = sum([trade.stake_amount for trade in open_trades]) current_stake = self.start_cap + tot_profit - tot_in_trades diff --git a/tests/conftest.py b/tests/conftest.py index 75632e4c7..3522ef02d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -199,7 +199,7 @@ def create_mock_trades(fee, use_db: bool = True): if use_db: Trade.session.add(trade) else: - LocalTrade.trades.append(trade) + LocalTrade.add_bt_trade(trade) # Simulate dry_run entries trade = mock_trade_1(fee) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 1a8124b00..8c89c98ed 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1196,6 +1196,6 @@ def test_Trade_object_idem(): # Fails if only a column is added without corresponding parent field for item in localtrade: if (not item.startswith('__') - and item not in ('trades', ) + and item not in ('trades', 'trades_open', 'total_profit') and type(getattr(LocalTrade, item)) not in (property, FunctionType)): assert item in trade