Merge pull request #4215 from freqtrade/refactor/backtest

Small backtest refactor, introduce calling `bot_loop_start` in backtesting

docs/bot-basics.md

@@ -49,8 +49,9 @@ This loop will be repeated again and again until the bot is stopped.
 [backtesting](backtesting.md) or [hyperopt](hyperopt.md) do only part of the above logic, since most of the trading operations are fully simulated.
 
 * Load historic data for configured pairlist.
-* Calculate indicators (calls `populate_indicators()`).
+* Calls `bot_loop_start()` once.
-* Calls `populate_buy_trend()` and `populate_sell_trend()`
+* Calculate indicators (calls `populate_indicators()` once per pair).
+* Calculate buy / sell signals (calls `populate_buy_trend()` and `populate_sell_trend()` once per pair)
 * Loops per candle simulating entry and exit points.
 * Generate backtest report output
 

freqtrade/optimize/backtesting.py

@@ -6,7 +6,7 @@ This module contains the backtesting logic
 import logging
 from collections import defaultdict
 from copy import deepcopy
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, NamedTuple, Optional, Tuple
 
 from pandas import DataFrame
@@ -26,6 +26,7 @@ from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType
+from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 
 
 logger = logging.getLogger(__name__)
@@ -76,6 +77,8 @@ class Backtesting:
         # Reset keys for backtesting
         remove_credentials(self.config)
         self.strategylist: List[IStrategy] = []
+        self.all_results: Dict[str, Dict] = {}
+
         self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
 
         dataprovider = DataProvider(self.config, self.exchange)
@@ -150,6 +153,10 @@ class Backtesting:
         self.strategy.order_types['stoploss_on_exchange'] = False
 
     def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]:
+        """
+        Loads backtest data and returns the data combined with the timerange
+        as tuple.
+        """
         timerange = TimeRange.parse_timerange(None if self.config.get(
             'timerange') is None else str(self.config.get('timerange')))
 
@@ -424,6 +431,53 @@ class Backtesting:
 
         return DataFrame.from_records(trades, columns=BacktestResult._fields)
 
+    def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange):
+        logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
+        backtest_start_time = datetime.now(timezone.utc)
+        self._set_strategy(strat)
+
+        strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)()
+
+        # Use max_open_trades in backtesting, except --disable-max-market-positions is set
+        if self.config.get('use_max_market_positions', True):
+            # Must come from strategy config, as the strategy may modify this setting.
+            max_open_trades = self.strategy.config['max_open_trades']
+        else:
+            logger.info(
+                'Ignoring max_open_trades (--disable-max-market-positions was used) ...')
+            max_open_trades = 0
+
+        # need to reprocess data every time to populate signals
+        preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
+
+        # Trim startup period from analyzed dataframe
+        for pair, df in preprocessed.items():
+            preprocessed[pair] = trim_dataframe(df, timerange)
+        min_date, max_date = history.get_timerange(preprocessed)
+
+        logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
+                    f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
+                    f'({(max_date - min_date).days} days)..')
+        # Execute backtest and store results
+        results = self.backtest(
+            processed=preprocessed,
+            stake_amount=self.config['stake_amount'],
+            start_date=min_date.datetime,
+            end_date=max_date.datetime,
+            max_open_trades=max_open_trades,
+            position_stacking=self.config.get('position_stacking', False),
+            enable_protections=self.config.get('enable_protections', False),
+        )
+        backtest_end_time = datetime.now(timezone.utc)
+        self.all_results[self.strategy.get_strategy_name()] = {
+            'results': results,
+            'config': self.strategy.config,
+            'locks': PairLocks.locks,
+            'backtest_start_time': int(backtest_start_time.timestamp()),
+            'backtest_end_time': int(backtest_end_time.timestamp()),
+        }
+        return min_date, max_date
+
     def start(self) -> None:
         """
         Run backtesting end-to-end
@@ -431,55 +485,15 @@ class Backtesting:
         """
         data: Dict[str, Any] = {}
 
-        logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
-        logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
-
-        position_stacking = self.config.get('position_stacking', False)
-
         data, timerange = self.load_bt_data()
 
-        all_results = {}
+        min_date = None
+        max_date = None
         for strat in self.strategylist:
-            logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
+            min_date, max_date = self.backtest_one_strategy(strat, data, timerange)
-            self._set_strategy(strat)
 
-            # Use max_open_trades in backtesting, except --disable-max-market-positions is set
+        stats = generate_backtest_stats(data, self.all_results,
-            if self.config.get('use_max_market_positions', True):
+                                        min_date=min_date, max_date=max_date)
-                # Must come from strategy config, as the strategy may modify this setting.
-                max_open_trades = self.strategy.config['max_open_trades']
-            else:
-                logger.info(
-                    'Ignoring max_open_trades (--disable-max-market-positions was used) ...')
-                max_open_trades = 0
-
-            # need to reprocess data every time to populate signals
-            preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
-
-            # Trim startup period from analyzed dataframe
-            for pair, df in preprocessed.items():
-                preprocessed[pair] = trim_dataframe(df, timerange)
-            min_date, max_date = history.get_timerange(preprocessed)
-
-            logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
-                        f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
-                        f'({(max_date - min_date).days} days)..')
-            # Execute backtest and print results
-            results = self.backtest(
-                processed=preprocessed,
-                stake_amount=self.config['stake_amount'],
-                start_date=min_date.datetime,
-                end_date=max_date.datetime,
-                max_open_trades=max_open_trades,
-                position_stacking=position_stacking,
-                enable_protections=self.config.get('enable_protections', False),
-            )
-            all_results[self.strategy.get_strategy_name()] = {
-                'results': results,
-                'config': self.strategy.config,
-                'locks': PairLocks.locks,
-            }
-
-        stats = generate_backtest_stats(data, all_results, min_date=min_date, max_date=max_date)
 
         if self.config.get('export', False):
             store_backtest_stats(self.config['exportfilename'], stats)

freqtrade/optimize/hyperopt.py

@@ -650,7 +650,7 @@ class Hyperopt:
         # Trim startup period from analyzed dataframe
         for pair, df in preprocessed.items():
             preprocessed[pair] = trim_dataframe(df, timerange)
-        min_date, max_date = get_timerange(data)
+        min_date, max_date = get_timerange(preprocessed)
 
         logger.info(f'Hyperopting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '

freqtrade/optimize/optimize_reports.py

@@ -282,6 +282,9 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
             'backtest_end_ts': max_date.int_timestamp * 1000,
             'backtest_days': backtest_days,
 
+            'backtest_run_start_ts': content['backtest_start_time'],
+            'backtest_run_end_ts': content['backtest_end_time'],
+
             'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0,
             'market_change': market_change,
             'pairlist': list(btdata.keys()),
@@ -290,6 +293,9 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
             'max_open_trades': (config['max_open_trades']
                                 if config['max_open_trades'] != float('inf') else -1),
             'timeframe': config['timeframe'],
+            'timerange': config.get('timerange', ''),
+            'enable_protections': config.get('enable_protections', False),
+            'strategy_name': strategy,
             # Parameters relevant for backtesting
             'stoploss': config['stoploss'],
             'trailing_stop': config.get('trailing_stop', False),

tests/optimize/test_backtesting.py

@@ -350,17 +350,17 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
     default_conf['timerange'] = '-1510694220'
 
     backtesting = Backtesting(default_conf)
+    backtesting.strategy.bot_loop_start = MagicMock()
     backtesting.start()
     # check the logs, that will contain the backtest result
     exists = [
-        'Using stake_currency: BTC ...',
-        'Using stake_amount: 0.001 ...',
         'Backtesting with data from 2017-11-14 21:17:00 '
         'up to 2017-11-14 22:59:00 (0 days)..'
     ]
     for line in exists:
         assert log_has(line, caplog)
     assert backtesting.strategy.dp._pairlists is not None
+    assert backtesting.strategy.bot_loop_start.call_count == 1
 
 
 def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None:
@@ -722,8 +722,6 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
         'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
         'Parameter --timerange detected: 1510694220-1510700340 ...',
         f'Using data directory: {testdatadir} ...',
-        'Using stake_currency: BTC ...',
-        'Using stake_amount: 0.001 ...',
         'Loading data from 2017-11-14 20:57:00 '
         'up to 2017-11-14 22:58:00 (0 days)..',
         'Backtesting with data from 2017-11-14 21:17:00 '
@@ -786,8 +784,6 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
         'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
         'Parameter --timerange detected: 1510694220-1510700340 ...',
         f'Using data directory: {testdatadir} ...',
-        'Using stake_currency: BTC ...',
-        'Using stake_amount: 0.001 ...',
         'Loading data from 2017-11-14 20:57:00 '
         'up to 2017-11-14 22:58:00 (0 days)..',
         'Backtesting with data from 2017-11-14 21:17:00 '
@@ -865,8 +861,6 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat
         'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
         'Parameter --timerange detected: 1510694220-1510700340 ...',
         f'Using data directory: {testdatadir} ...',
-        'Using stake_currency: BTC ...',
-        'Using stake_amount: 0.001 ...',
         'Loading data from 2017-11-14 20:57:00 '
         'up to 2017-11-14 22:58:00 (0 days)..',
         'Backtesting with data from 2017-11-14 21:17:00 '

tests/optimize/test_optimize_reports.py

@@ -77,7 +77,10 @@ def test_generate_backtest_stats(default_conf, testdatadir):
                                                  SellType.ROI, SellType.FORCE_SELL]
                                  }),
         'config': default_conf,
-        'locks': []}
+        'locks': [],
+        'backtest_start_time': Arrow.utcnow().int_timestamp,
+        'backtest_end_time': Arrow.utcnow().int_timestamp,
+        }
         }
     timerange = TimeRange.parse_timerange('1510688220-1510700340')
     min_date = Arrow.fromtimestamp(1510688220)