Merge pull request #3803 from freqtrade/bt_params

Backtesting - handle mdifferent max_open_trades per strategy

freqtrade/optimize/backtesting.py

@@ -380,12 +380,6 @@ class Backtesting:
         logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
         logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
 
-        # Use max_open_trades in backtesting, except --disable-max-market-positions is set
-        if self.config.get('use_max_market_positions', True):
-            max_open_trades = self.config['max_open_trades']
-        else:
-            logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...')
-            max_open_trades = 0
         position_stacking = self.config.get('position_stacking', False)
 
         data, timerange = self.load_bt_data()
@@ -395,6 +389,15 @@ class Backtesting:
             logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
             self._set_strategy(strat)
 
+            # Use max_open_trades in backtesting, except --disable-max-market-positions is set
+            if self.config.get('use_max_market_positions', True):
+                # Must come from strategy config, as the strategy may modify this setting.
+                max_open_trades = self.strategy.config['max_open_trades']
+            else:
+                logger.info(
+                    'Ignoring max_open_trades (--disable-max-market-positions was used) ...')
+                max_open_trades = 0
+
             # need to reprocess data every time to populate signals
             preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
 
@@ -407,7 +410,7 @@ class Backtesting:
                         f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
                         f'({(max_date - min_date).days} days)..')
             # Execute backtest and print results
-            all_results[self.strategy.get_strategy_name()] = self.backtest(
+            results = self.backtest(
                 processed=preprocessed,
                 stake_amount=self.config['stake_amount'],
                 start_date=min_date,
@@ -415,9 +418,13 @@ class Backtesting:
                 max_open_trades=max_open_trades,
                 position_stacking=position_stacking,
             )
+            all_results[self.strategy.get_strategy_name()] = {
+                'results': results,
+                'config': self.strategy.config,
+            }
+
+        stats = generate_backtest_stats(data, all_results, min_date=min_date, max_date=max_date)
 
-        stats = generate_backtest_stats(self.config, data, all_results,
-                                        min_date=min_date, max_date=max_date)
         if self.config.get('export', False):
             store_backtest_stats(self.config['exportfilename'], stats)
 

freqtrade/optimize/optimize_reports.py

@@ -1,7 +1,7 @@
 import logging
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Union
 
 from arrow import Arrow
 from pandas import DataFrame
@@ -143,19 +143,18 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
     return tabular_data
 
 
-def generate_strategy_metrics(stake_currency: str, max_open_trades: int,
+def generate_strategy_metrics(all_results: Dict) -> List[Dict]:
-                              all_results: Dict) -> List[Dict]:
     """
     Generate summary per strategy
-    :param stake_currency: stake-currency - used to correctly name headers
-    :param max_open_trades: Maximum allowed open trades used for backtest
     :param all_results: Dict of <Strategyname: BacktestResult> containing results for all strategies
     :return: List of Dicts containing the metrics per Strategy
     """
 
     tabular_data = []
     for strategy, results in all_results.items():
-        tabular_data.append(_generate_result_line(results, max_open_trades, strategy))
+        tabular_data.append(_generate_result_line(
+            results['results'], results['config']['max_open_trades'], strategy)
+            )
     return tabular_data
 
 
@@ -219,25 +218,29 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
     }
 
 
-def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame],
+def generate_backtest_stats(btdata: Dict[str, DataFrame],
-                            all_results: Dict[str, DataFrame],
+                            all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]],
                             min_date: Arrow, max_date: Arrow
                             ) -> Dict[str, Any]:
     """
-    :param config: Configuration object used for backtest
     :param btdata: Backtest data
-    :param all_results: backtest result - dictionary with { Strategy: results}.
+    :param all_results: backtest result - dictionary in the form:
+                     { Strategy: {'results: results, 'config: config}}.
     :param min_date: Backtest start date
     :param max_date: Backtest end date
     :return:
     Dictionary containing results per strategy and a stratgy summary.
     """
-    stake_currency = config['stake_currency']
-    max_open_trades = config['max_open_trades']
     result: Dict[str, Any] = {'strategy': {}}
     market_change = calculate_market_change(btdata, 'close')
 
-    for strategy, results in all_results.items():
+    for strategy, content in all_results.items():
+        results: Dict[str, DataFrame] = content['results']
+        if not isinstance(results, DataFrame):
+            continue
+        config = content['config']
+        max_open_trades = config['max_open_trades']
+        stake_currency = config['stake_currency']
 
         pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
                                              max_open_trades=max_open_trades,
@@ -277,6 +280,16 @@ def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame],
             'max_open_trades': (config['max_open_trades']
                                 if config['max_open_trades'] != float('inf') else -1),
             'timeframe': config['timeframe'],
+            # Parameters relevant for backtesting
+            'stoploss': config['stoploss'],
+            'trailing_stop': config.get('trailing_stop', False),
+            'trailing_stop_positive': config.get('trailing_stop_positive'),
+            'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0),
+            'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False),
+            'minimal_roi': config['minimal_roi'],
+            'use_sell_signal': config['ask_strategy']['use_sell_signal'],
+            'sell_profit_only': config['ask_strategy']['sell_profit_only'],
+            'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'],
             **daily_stats,
         }
         result['strategy'][strategy] = strat_stats
@@ -300,9 +313,7 @@ def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame],
                 'drawdown_end_ts': 0,
             })
 
-    strategy_results = generate_strategy_metrics(stake_currency=stake_currency,
+    strategy_results = generate_strategy_metrics(all_results=all_results)
-                                                 max_open_trades=max_open_trades,
-                                                 all_results=all_results)
 
     result['strategy_comparison'] = strategy_results
 

tests/optimize/test_backtesting.py

@@ -14,7 +14,7 @@ from freqtrade.commands.optimize_commands import (setup_optimize_configuration,
                                                   start_backtesting)
 from freqtrade.configuration import TimeRange
 from freqtrade.data import history
-from freqtrade.data.btanalysis import evaluate_result_multi
+from freqtrade.data.btanalysis import BT_DATA_COLUMNS, evaluate_result_multi
 from freqtrade.data.converter import clean_ohlcv_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.history import get_timerange
@@ -694,7 +694,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
 def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
 
     patch_exchange(mocker)
-    backtestmock = MagicMock()
+    backtestmock = MagicMock(return_value=pd.DataFrame(columns=BT_DATA_COLUMNS + ['profit_abs']))
     mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist',
                  PropertyMock(return_value=['UNITTEST/BTC']))
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)

tests/optimize/test_optimize_reports.py

@@ -5,7 +5,6 @@ from pathlib import Path
 import pandas as pd
 import pytest
 from arrow import Arrow
-
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import LAST_BT_RESULT_FN
 from freqtrade.data import history
@@ -22,11 +21,12 @@ from freqtrade.optimize.optimize_reports import (generate_backtest_stats,
                                                  text_table_bt_results,
                                                  text_table_sell_reason,
                                                  text_table_strategy)
+from freqtrade.resolvers.strategy_resolver import StrategyResolver
 from freqtrade.strategy.interface import SellType
 from tests.data.test_history import _backup_file, _clean_test_file
 
 
-def test_text_table_bt_results(default_conf, mocker):
+def test_text_table_bt_results():
 
     results = pd.DataFrame(
         {
@@ -57,7 +57,11 @@ def test_text_table_bt_results(default_conf, mocker):
 
 
 def test_generate_backtest_stats(default_conf, testdatadir):
-    results = {'DefStrat': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC",
+    default_conf.update({'strategy': 'DefaultStrategy'})
+    StrategyResolver.load_strategy(default_conf)
+
+    results = {'DefStrat': {
+        'results': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC",
                                           "UNITTEST/BTC", "UNITTEST/BTC"],
                                  "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780],
                                  "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003],
@@ -75,14 +79,16 @@ def test_generate_backtest_stats(default_conf, testdatadir):
                                  "open_at_end": [False, False, False, True],
                                  "sell_reason": [SellType.ROI, SellType.STOP_LOSS,
                                                  SellType.ROI, SellType.FORCE_SELL]
-                                         })}
+                                 }),
+        'config': default_conf}
+        }
     timerange = TimeRange.parse_timerange('1510688220-1510700340')
     min_date = Arrow.fromtimestamp(1510688220)
     max_date = Arrow.fromtimestamp(1510700340)
     btdata = history.load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange,
                                fill_up_missing=True)
 
-    stats = generate_backtest_stats(default_conf, btdata, results, min_date, max_date)
+    stats = generate_backtest_stats(btdata, results, min_date, max_date)
     assert isinstance(stats, dict)
     assert 'strategy' in stats
     assert 'DefStrat' in stats['strategy']
@@ -90,11 +96,12 @@ def test_generate_backtest_stats(default_conf, testdatadir):
     strat_stats = stats['strategy']['DefStrat']
     assert strat_stats['backtest_start'] == min_date.datetime
     assert strat_stats['backtest_end'] == max_date.datetime
-    assert strat_stats['total_trades'] == len(results['DefStrat'])
+    assert strat_stats['total_trades'] == len(results['DefStrat']['results'])
     # Above sample had no loosing trade
     assert strat_stats['max_drawdown'] == 0.0
 
-    results = {'DefStrat': pd.DataFrame(
+    results = {'DefStrat': {
+        'results': pd.DataFrame(
             {"pair": ["UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC"],
              "profit_percent": [0.003312, 0.010801, -0.013803, 0.002780],
              "profit_abs": [0.000003, 0.000011, -0.000014, 0.000003],
@@ -112,7 +119,9 @@ def test_generate_backtest_stats(default_conf, testdatadir):
              "open_at_end": [False, False, False, True],
              "sell_reason": [SellType.ROI, SellType.STOP_LOSS,
                              SellType.ROI, SellType.FORCE_SELL]
-         })}
+             }),
+        'config': default_conf}
+    }
 
     assert strat_stats['max_drawdown'] == 0.0
     assert strat_stats['drawdown_start'] == Arrow.fromtimestamp(0).datetime
@@ -165,7 +174,7 @@ def test_store_backtest_stats(testdatadir, mocker):
     assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / 'testresult'))
 
 
-def test_generate_pair_metrics(default_conf, mocker):
+def test_generate_pair_metrics():
 
     results = pd.DataFrame(
         {
@@ -213,7 +222,7 @@ def test_generate_daily_stats(testdatadir):
     assert res['losing_days'] == 0
 
 
-def test_text_table_sell_reason(default_conf):
+def test_text_table_sell_reason():
 
     results = pd.DataFrame(
         {
@@ -245,7 +254,7 @@ def test_text_table_sell_reason(default_conf):
                                   stake_currency='BTC') == result_str
 
 
-def test_generate_sell_reason_stats(default_conf):
+def test_generate_sell_reason_stats():
 
     results = pd.DataFrame(
         {
@@ -280,9 +289,10 @@ def test_generate_sell_reason_stats(default_conf):
     assert stop_result['profit_mean_pct'] == round(stop_result['profit_mean'] * 100, 2)
 
 
-def test_text_table_strategy(default_conf, mocker):
+def test_text_table_strategy(default_conf):
+    default_conf['max_open_trades'] = 2
     results = {}
-    results['TestStrategy1'] = pd.DataFrame(
+    results['TestStrategy1'] = {'results': pd.DataFrame(
         {
             'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'],
             'profit_percent': [0.1, 0.2, 0.3],
@@ -293,8 +303,8 @@ def test_text_table_strategy(default_conf, mocker):
             'losses': [0, 0, 1],
             'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
         }
-    )
+    ), 'config': default_conf}
-    results['TestStrategy2'] = pd.DataFrame(
+    results['TestStrategy2'] = {'results': pd.DataFrame(
         {
             'pair': ['LTC/BTC', 'LTC/BTC', 'LTC/BTC'],
             'profit_percent': [0.4, 0.2, 0.3],
@@ -305,7 +315,7 @@ def test_text_table_strategy(default_conf, mocker):
             'losses': [0, 0, 1],
             'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
         }
-    )
+    ), 'config': default_conf}
 
     result_str = (
         '|      Strategy |   Buys |   Avg Profit % |   Cum Profit % |   Tot'
@@ -318,14 +328,12 @@ def test_text_table_strategy(default_conf, mocker):
         '          45.00 |        0:20:00 |      3 |       0 |        0 |'
     )
 
-    strategy_results = generate_strategy_metrics(stake_currency='BTC',
+    strategy_results = generate_strategy_metrics(all_results=results)
-                                                 max_open_trades=2,
-                                                 all_results=results)
 
     assert text_table_strategy(strategy_results, 'BTC') == result_str
 
 
-def test_generate_edge_table(edge_conf, mocker):
+def test_generate_edge_table():
 
     results = {}
     results['ETH/BTC'] = PairInfo(-0.01, 0.60, 2, 1, 3, 10, 60)