Merge pull request #4535 from freqtrade/fix/backtestperformance

Improve backtest performance after enabling compounding

freqtrade/optimize/backtesting.py

@@ -377,7 +377,7 @@ class Backtesting:
                         open_trade_count += 1
                         # logger.debug(f"{pair} - Emulate creation of new trade: {trade}.")
                         open_trades[pair].append(trade)
-                        LocalTrade.trades.append(trade)
+                        LocalTrade.add_bt_trade(trade)
 
                 for trade in open_trades[pair]:
                     # also check the buying candle for sell conditions.
@@ -387,6 +387,8 @@ class Backtesting:
                         # logger.debug(f"{pair} - Backtesting sell {trade}")
                         open_trade_count -= 1
                         open_trades[pair].remove(trade)
+
+                        LocalTrade.close_bt_trade(trade)
                         trades.append(trade_entry)
                         if enable_protections:
                             self.protections.stop_per_pair(pair, row[DATE_IDX])

freqtrade/persistence/models.py

@@ -208,6 +208,8 @@ class LocalTrade():
     use_db: bool = False
     # Trades container for backtesting
     trades: List['LocalTrade'] = []
+    trades_open: List['LocalTrade'] = []
+    total_profit: float = 0
 
     id: int = 0
 
@@ -350,6 +352,8 @@ class LocalTrade():
         Resets all trades. Only active for backtesting mode.
         """
         LocalTrade.trades = []
+        LocalTrade.trades_open = []
+        LocalTrade.total_profit = 0
 
     def adjust_min_max_rates(self, current_price: float) -> None:
         """
@@ -599,7 +603,16 @@ class LocalTrade():
         """
 
         # Offline mode - without database
-        sel_trades = [trade for trade in LocalTrade.trades]
+        if is_open is not None:
+            if is_open:
+                sel_trades = LocalTrade.trades_open
+            else:
+                sel_trades = LocalTrade.trades
+
+        else:
+            # Not used during backtesting, but might be used by a strategy
+            sel_trades = [trade for trade in LocalTrade.trades + LocalTrade.trades_open]
+
         if pair:
             sel_trades = [trade for trade in sel_trades if trade.pair == pair]
         if open_date:
@@ -607,10 +620,22 @@ class LocalTrade():
         if close_date:
             sel_trades = [trade for trade in sel_trades if trade.close_date
                           and trade.close_date > close_date]
-        if is_open is not None:
-            sel_trades = [trade for trade in sel_trades if trade.is_open == is_open]
+
         return sel_trades
 
+    @staticmethod
+    def close_bt_trade(trade):
+        LocalTrade.trades_open.remove(trade)
+        LocalTrade.trades.append(trade)
+        LocalTrade.total_profit += trade.close_profit_abs
+
+    @staticmethod
+    def add_bt_trade(trade):
+        if trade.is_open:
+            LocalTrade.trades_open.append(trade)
+        else:
+            LocalTrade.trades.append(trade)
+
     @staticmethod
     def get_open_trades() -> List[Any]:
         """

freqtrade/wallets.py

@@ -10,7 +10,7 @@ import arrow
 from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
 from freqtrade.exceptions import DependencyException
 from freqtrade.exchange import Exchange
-from freqtrade.persistence import Trade
+from freqtrade.persistence import LocalTrade, Trade
 from freqtrade.state import RunMode
 
 
@@ -66,9 +66,14 @@ class Wallets:
         """
         # Recreate _wallets to reset closed trade balances
         _wallets = {}
-        closed_trades = Trade.get_trades_proxy(is_open=False)
         open_trades = Trade.get_trades_proxy(is_open=True)
+        # If not backtesting...
+        # TODO: potentially remove the ._log workaround to determine backtest mode.
+        if self._log:
+            closed_trades = Trade.get_trades_proxy(is_open=False)
             tot_profit = sum([trade.close_profit_abs for trade in closed_trades])
+        else:
+            tot_profit = LocalTrade.total_profit
         tot_in_trades = sum([trade.stake_amount for trade in open_trades])
 
         current_stake = self.start_cap + tot_profit - tot_in_trades

tests/conftest.py

@@ -199,7 +199,7 @@ def create_mock_trades(fee, use_db: bool = True):
         if use_db:
             Trade.session.add(trade)
         else:
-            LocalTrade.trades.append(trade)
+            LocalTrade.add_bt_trade(trade)
 
     # Simulate dry_run entries
     trade = mock_trade_1(fee)

tests/conftest_trades.py

@@ -29,6 +29,7 @@ def mock_trade_1(fee):
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         is_open=True,
+        open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=17),
         open_rate=0.123,
         exchange='bittrex',
         open_order_id='dry_run_buy_12345',
@@ -183,6 +184,7 @@ def mock_trade_4(fee):
         amount_requested=124.0,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
+        open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=14),
         is_open=True,
         open_rate=0.123,
         exchange='bittrex',
@@ -234,6 +236,7 @@ def mock_trade_5(fee):
         amount_requested=124.0,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
+        open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=12),
         is_open=True,
         open_rate=0.123,
         exchange='bittrex',
@@ -284,6 +287,7 @@ def mock_trade_6(fee):
         stake_amount=0.001,
         amount=2.0,
         amount_requested=2.0,
+        open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=5),
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         is_open=True,

tests/test_persistence.py

@@ -1,5 +1,6 @@
 # pragma pylint: disable=missing-docstring, C0103
 import logging
+from datetime import datetime, timedelta, timezone
 from types import FunctionType
 from unittest.mock import MagicMock
 
@@ -1044,6 +1045,7 @@ def test_fee_updated(fee):
 def test_total_open_trades_stakes(fee, use_db):
 
     Trade.use_db = use_db
+    Trade.reset_trades()
     res = Trade.total_open_trades_stakes()
     assert res == 0
     create_mock_trades(fee, use_db)
@@ -1053,6 +1055,26 @@ def test_total_open_trades_stakes(fee, use_db):
     Trade.use_db = True
 
 
+@pytest.mark.usefixtures("init_persistence")
+@pytest.mark.parametrize('use_db', [True, False])
+def test_get_trades_proxy(fee, use_db):
+    Trade.use_db = use_db
+    Trade.reset_trades()
+    create_mock_trades(fee, use_db)
+    trades = Trade.get_trades_proxy()
+    assert len(trades) == 6
+
+    assert isinstance(trades[0], Trade)
+
+    assert len(Trade.get_trades_proxy(is_open=True)) == 4
+    assert len(Trade.get_trades_proxy(is_open=False)) == 2
+    opendate = datetime.now(tz=timezone.utc) - timedelta(minutes=15)
+
+    assert len(Trade.get_trades_proxy(open_date=opendate)) == 3
+
+    Trade.use_db = True
+
+
 @pytest.mark.usefixtures("init_persistence")
 def test_get_overall_performance(fee):
 
@@ -1196,6 +1218,6 @@ def test_Trade_object_idem():
     # Fails if only a column is added without corresponding parent field
     for item in localtrade:
         if (not item.startswith('__')
-                and item not in ('trades', )
+                and item not in ('trades', 'trades_open', 'total_profit')
                 and type(getattr(LocalTrade, item)) not in (property, FunctionType)):
             assert item in trade