From d1acc8092cef3bd2a2035642640a7cab92429dcd Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 13 Mar 2021 10:16:32 +0100 Subject: [PATCH 1/2] Improve backtest performance --- freqtrade/optimize/backtesting.py | 4 +++- freqtrade/persistence/models.py | 32 ++++++++++++++++++++++++++++--- freqtrade/wallets.py | 11 ++++++++--- tests/conftest.py | 2 +- tests/test_persistence.py | 2 +- 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 575ad486a..f2cf0d0dc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -377,7 +377,7 @@ class Backtesting: open_trade_count += 1 # logger.debug(f"{pair} - Emulate creation of new trade: {trade}.") open_trades[pair].append(trade) - LocalTrade.trades.append(trade) + LocalTrade.add_bt_trade(trade) for trade in open_trades[pair]: # also check the buying candle for sell conditions. @@ -387,6 +387,8 @@ class Backtesting: # logger.debug(f"{pair} - Backtesting sell {trade}") open_trade_count -= 1 open_trades[pair].remove(trade) + + LocalTrade.close_bt_trade(trade) trades.append(trade_entry) if enable_protections: self.protections.stop_per_pair(pair, row[DATE_IDX]) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index ab714ae8b..41a5a99ff 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -208,6 +208,8 @@ class LocalTrade(): use_db: bool = False # Trades container for backtesting trades: List['LocalTrade'] = [] + trades_open: List['LocalTrade'] = [] + total_profit: float = 0 id: int = 0 @@ -350,6 +352,8 @@ class LocalTrade(): Resets all trades. Only active for backtesting mode. """ LocalTrade.trades = [] + LocalTrade.trades_open = [] + LocalTrade.total_profit = 0 def adjust_min_max_rates(self, current_price: float) -> None: """ @@ -599,7 +603,17 @@ class LocalTrade(): """ # Offline mode - without database - sel_trades = [trade for trade in LocalTrade.trades] + if is_open is not None: + if is_open: + sel_trades = LocalTrade.trades_open + else: + sel_trades = LocalTrade.trades + + else: + # Not used during backtesting, but might be used by a strategy + sel_trades = [trade for trade in LocalTrade.trades + LocalTrade.trades_open + if trade.is_open == is_open] + if pair: sel_trades = [trade for trade in sel_trades if trade.pair == pair] if open_date: @@ -607,10 +621,22 @@ class LocalTrade(): if close_date: sel_trades = [trade for trade in sel_trades if trade.close_date and trade.close_date > close_date] - if is_open is not None: - sel_trades = [trade for trade in sel_trades if trade.is_open == is_open] + return sel_trades + @staticmethod + def close_bt_trade(trade): + LocalTrade.trades_open.remove(trade) + LocalTrade.trades.append(trade) + LocalTrade.total_profit += trade.close_profit_abs + + @staticmethod + def add_bt_trade(trade): + if trade.is_open: + LocalTrade.trades_open.append(trade) + else: + LocalTrade.trades.append(trade) + @staticmethod def get_open_trades() -> List[Any]: """ diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 553f7c61d..575fe1b67 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -10,7 +10,7 @@ import arrow from freqtrade.constants import UNLIMITED_STAKE_AMOUNT from freqtrade.exceptions import DependencyException from freqtrade.exchange import Exchange -from freqtrade.persistence import Trade +from freqtrade.persistence import LocalTrade, Trade from freqtrade.state import RunMode @@ -66,9 +66,14 @@ class Wallets: """ # Recreate _wallets to reset closed trade balances _wallets = {} - closed_trades = Trade.get_trades_proxy(is_open=False) open_trades = Trade.get_trades_proxy(is_open=True) - tot_profit = sum([trade.close_profit_abs for trade in closed_trades]) + # If not backtesting... + # TODO: potentially remove the ._log workaround to determine backtest mode. + if self._log: + closed_trades = Trade.get_trades_proxy(is_open=False) + tot_profit = sum([trade.close_profit_abs for trade in closed_trades]) + else: + tot_profit = LocalTrade.total_profit tot_in_trades = sum([trade.stake_amount for trade in open_trades]) current_stake = self.start_cap + tot_profit - tot_in_trades diff --git a/tests/conftest.py b/tests/conftest.py index 498d65b0a..801ffad2f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -191,7 +191,7 @@ def create_mock_trades(fee, use_db: bool = True): if use_db: Trade.session.add(trade) else: - LocalTrade.trades.append(trade) + LocalTrade.add_bt_trade(trade) # Simulate dry_run entries trade = mock_trade_1(fee) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 1a8124b00..8c89c98ed 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1196,6 +1196,6 @@ def test_Trade_object_idem(): # Fails if only a column is added without corresponding parent field for item in localtrade: if (not item.startswith('__') - and item not in ('trades', ) + and item not in ('trades', 'trades_open', 'total_profit') and type(getattr(LocalTrade, item)) not in (property, FunctionType)): assert item in trade From 0320c8dc92d4e4acb4f61310f202ba6db2be840a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 13 Mar 2021 15:46:20 +0100 Subject: [PATCH 2/2] Improve tests for trades_proxy --- freqtrade/persistence/models.py | 3 +-- tests/conftest_trades.py | 4 ++++ tests/test_persistence.py | 22 ++++++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 41a5a99ff..ed8a2259b 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -611,8 +611,7 @@ class LocalTrade(): else: # Not used during backtesting, but might be used by a strategy - sel_trades = [trade for trade in LocalTrade.trades + LocalTrade.trades_open - if trade.is_open == is_open] + sel_trades = [trade for trade in LocalTrade.trades + LocalTrade.trades_open] if pair: sel_trades = [trade for trade in sel_trades if trade.pair == pair] diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index 1d775830d..8e4be9165 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -29,6 +29,7 @@ def mock_trade_1(fee): fee_open=fee.return_value, fee_close=fee.return_value, is_open=True, + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=17), open_rate=0.123, exchange='bittrex', open_order_id='dry_run_buy_12345', @@ -183,6 +184,7 @@ def mock_trade_4(fee): amount_requested=124.0, fee_open=fee.return_value, fee_close=fee.return_value, + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=14), is_open=True, open_rate=0.123, exchange='bittrex', @@ -234,6 +236,7 @@ def mock_trade_5(fee): amount_requested=124.0, fee_open=fee.return_value, fee_close=fee.return_value, + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=12), is_open=True, open_rate=0.123, exchange='bittrex', @@ -284,6 +287,7 @@ def mock_trade_6(fee): stake_amount=0.001, amount=2.0, amount_requested=2.0, + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=5), fee_open=fee.return_value, fee_close=fee.return_value, is_open=True, diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 8c89c98ed..ab900cbb8 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring, C0103 import logging +from datetime import datetime, timedelta, timezone from types import FunctionType from unittest.mock import MagicMock @@ -1044,6 +1045,7 @@ def test_fee_updated(fee): def test_total_open_trades_stakes(fee, use_db): Trade.use_db = use_db + Trade.reset_trades() res = Trade.total_open_trades_stakes() assert res == 0 create_mock_trades(fee, use_db) @@ -1053,6 +1055,26 @@ def test_total_open_trades_stakes(fee, use_db): Trade.use_db = True +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize('use_db', [True, False]) +def test_get_trades_proxy(fee, use_db): + Trade.use_db = use_db + Trade.reset_trades() + create_mock_trades(fee, use_db) + trades = Trade.get_trades_proxy() + assert len(trades) == 6 + + assert isinstance(trades[0], Trade) + + assert len(Trade.get_trades_proxy(is_open=True)) == 4 + assert len(Trade.get_trades_proxy(is_open=False)) == 2 + opendate = datetime.now(tz=timezone.utc) - timedelta(minutes=15) + + assert len(Trade.get_trades_proxy(open_date=opendate)) == 3 + + Trade.use_db = True + + @pytest.mark.usefixtures("init_persistence") def test_get_overall_performance(fee):