Rework backtesting --no-cahche to --cache=[none, day, week, month].

Fix an issue where config modification during runtime would prevent use of cached results.

freqtrade/commands/arguments.py

@@ -24,7 +24,7 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
 ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
                                         "enable_protections", "dry_run_wallet", "timeframe_detail",
                                         "strategy_list", "export", "exportfilename",
-                                        "backtest_breakdown", "no_backtest_cache"]
+                                        "backtest_breakdown", "backtest_cache"]
 
 ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
                                         "position_stacking", "use_max_market_positions",

freqtrade/commands/cli_options.py

@@ -205,10 +205,12 @@ AVAILABLE_CLI_OPTIONS = {
         nargs='+',
         choices=constants.BACKTEST_BREAKDOWNS
     ),
-    "no_backtest_cache": Arg(
+    "backtest_cache": Arg(
-        '--no-cache',
+        '--cache',
-        help='Do not reuse cached backtest results.',
+        help='Load a cached backtest result no older than specified age.',
-        action='store_true'
+        metavar='AGE',
+        default=constants.BACKTEST_CACHE_DEFAULT,
+        choices=constants.BACKTEST_CACHE_AGE,
     ),
     # Edge
     "stoploss_range": Arg(

freqtrade/configuration/configuration.py

@@ -276,8 +276,8 @@ class Configuration:
         self._args_to_config(config, argname='backtest_breakdown',
                              logstring='Parameter --breakdown detected ...')
 
-        self._args_to_config(config, argname='no_backtest_cache',
+        self._args_to_config(config, argname='backtest_cache',
-                             logstring='Parameter --no-cache detected ...')
+                             logstring='Parameter --cache={} detected ...')
 
         self._args_to_config(config, argname='disableparamexport',
                              logstring='Parameter --disableparamexport detected: {} ...')

freqtrade/constants.py

@@ -34,6 +34,9 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
 AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
 AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
 BACKTEST_BREAKDOWNS = ['day', 'week', 'month']
+BACKTEST_CACHE_AGE = ['none', 'day', 'week', 'month']
+BACKTEST_CACHE_DEFAULT = 'day'
+BACKTEST_CACHE_NONE = 'none'
 DRY_RUN_WALLET = 1000
 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S'
 MATH_CLOSE_PREC = 1e-14  # Precision used for float comparisons

freqtrade/data/btanalysis.py

@@ -3,6 +3,7 @@ Helpers when analyzing backtest data
 """
 import logging
 from copy import copy
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union
 
@@ -143,12 +144,24 @@ def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]:
     return data
 
 
-def find_existing_backtest_stats(dirname: Union[Path, str],
+def _load_and_merge_backtest_result(strategy_name: str, filename: Path, results: Dict[str, Any]):
-                                 run_ids: Dict[str, str]) -> Dict[str, Any]:
+    bt_data = load_backtest_stats(filename)
+    for k in ('metadata', 'strategy'):
+        results[k][strategy_name] = bt_data[k][strategy_name]
+    comparison = bt_data['strategy_comparison']
+    for i in range(len(comparison)):
+        if comparison[i]['key'] == strategy_name:
+            results['strategy_comparison'].append(comparison[i])
+            break
+
+
+def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, str],
+                                 min_backtest_date: datetime = None) -> Dict[str, Any]:
     """
     Find existing backtest stats that match specified run IDs and load them.
     :param dirname: pathlib.Path object, or string pointing to the file.
     :param run_ids: {strategy_name: id_string} dictionary.
+    :param min_backtest_date: do not load a backtest older than specified date.
     :return: results dict.
     """
     # Copy so we can modify this dict without affecting parent scope.
@@ -169,18 +182,27 @@ def find_existing_backtest_stats(dirname: Union[Path, str],
             break
 
         for strategy_name, run_id in list(run_ids.items()):
-            if metadata.get(strategy_name, {}).get('run_id') == run_id:
+            strategy_metadata = metadata.get(strategy_name, None)
-                # TODO: load_backtest_stats() may load an old version of backtest which is
+            if not strategy_metadata:
-                #  incompatible with current version.
+                # This strategy is not present in analyzed backtest.
+                continue
+
+            if min_backtest_date is not None:
+                try:
+                    backtest_date = strategy_metadata['backtest_start_time']
+                except KeyError:
+                    # Older metadata format without backtest time, too old to consider.
+                    return results
+                backtest_date = datetime.fromtimestamp(backtest_date, tz=timezone.utc)
+                if backtest_date < min_backtest_date:
+                    # Do not use a cached result for this strategy as first result is too old.
+                    del run_ids[strategy_name]
+                    continue
+
+            if strategy_metadata['run_id'] == run_id:
                 del run_ids[strategy_name]
-                bt_data = load_backtest_stats(filename)
+                _load_and_merge_backtest_result(strategy_name, filename, results)
-                for k in ('metadata', 'strategy'):
+
-                    results[k][strategy_name] = bt_data[k][strategy_name]
-                comparison = bt_data['strategy_comparison']
-                for i in range(len(comparison)):
-                    if comparison[i]['key'] == strategy_name:
-                        results['strategy_comparison'].append(comparison[i])
-                        break
         if len(run_ids) == 0:
             break
     return results

freqtrade/optimize/backtesting.py

@@ -11,6 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple
 
 from pandas import DataFrame
 
+from freqtrade import constants
 from freqtrade.configuration import TimeRange, validate_config_consistency
 from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data import history
@@ -64,6 +65,7 @@ class Backtesting:
         self.results: Dict[str, Any] = {}
 
         config['dry_run'] = True
+        self.run_ids: Dict[str, str] = {}
         self.strategylist: List[IStrategy] = []
         self.all_results: Dict[str, Dict] = {}
 
@@ -728,7 +730,7 @@ class Backtesting:
         )
         backtest_end_time = datetime.now(timezone.utc)
         results.update({
-            'run_id': get_strategy_run_id(strat),
+            'run_id': self.run_ids.get(strat.get_strategy_name(), ''),
             'backtest_start_time': int(backtest_start_time.timestamp()),
             'backtest_end_time': int(backtest_end_time.timestamp()),
         })
@@ -736,6 +738,20 @@ class Backtesting:
 
         return min_date, max_date
 
+    def _get_min_cached_backtest_date(self):
+        min_backtest_date = None
+        backtest_cache_age = self.config.get('backtest_cache', constants.BACKTEST_CACHE_DEFAULT)
+        if self.timerange.stopts == 0 or datetime.fromtimestamp(
+           self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc):
+            logger.warning('Backtest result caching disabled due to use of open-ended timerange.')
+        elif backtest_cache_age == 'day':
+            min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(days=1)
+        elif backtest_cache_age == 'week':
+            min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(weeks=1)
+        elif backtest_cache_age == 'month':
+            min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(weeks=4)
+        return min_backtest_date
+
     def start(self) -> None:
         """
         Run backtesting end-to-end
@@ -747,21 +763,17 @@ class Backtesting:
         self.load_bt_data_detail()
         logger.info("Dataload complete. Calculating indicators")
 
-        run_ids = {
+        self.run_ids = {
             strategy.get_strategy_name(): get_strategy_run_id(strategy)
             for strategy in self.strategylist
         }
 
         # Load previous result that will be updated incrementally.
         # This can be circumvented in certain instances in combination with downloading more data
-        if self.timerange.stopts == 0 or datetime.fromtimestamp(
+        min_backtest_date = self._get_min_cached_backtest_date()
-           self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc):
+        if min_backtest_date is not None:
-            self.config['no_backtest_cache'] = True
-            logger.warning('Backtest result caching disabled due to use of open-ended timerange.')
-
-        if not self.config.get('no_backtest_cache', False):
             self.results = find_existing_backtest_stats(
-                self.config['user_data_dir'] / 'backtest_results', run_ids)
+                self.config['user_data_dir'] / 'backtest_results', self.run_ids, min_backtest_date)
 
         for strat in self.strategylist:
             if self.results and strat.get_strategy_name() in self.results['strategy']:

freqtrade/optimize/optimize_reports.py

@@ -527,7 +527,8 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
         strat_stats = generate_strategy_stats(pairlist, strategy, content,
                                               min_date, max_date, market_change=market_change)
         metadata[strategy] = {
-            'run_id': content['run_id']
+            'run_id': content['run_id'],
+            'backtest_start_time': content['backtest_start_time'],
         }
         result['strategy'][strategy] = strat_stats
 

tests/optimize/test_backtesting.py

@@ -11,6 +11,7 @@ import pandas as pd
 import pytest
 from arrow import Arrow
 
+from freqtrade import constants
 from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_backtesting
 from freqtrade.configuration import TimeRange
 from freqtrade.data import history
@@ -1242,8 +1243,11 @@ def test_backtest_start_multi_strat_nomock_detail(default_conf, mocker,
 
 
 @pytest.mark.filterwarnings("ignore:deprecated")
-def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testdatadir):
+@pytest.mark.parametrize('run_id', ['2', 'changed'])
-
+@pytest.mark.parametrize('start_delta', [{'days': 0}, {'days': 1}, {'weeks': 1}, {'weeks': 4}])
+@pytest.mark.parametrize('cache', constants.BACKTEST_CACHE_AGE)
+def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testdatadir, run_id,
+                                            start_delta, cache):
     default_conf.update({
         "use_sell_signal": True,
         "sell_profit_only": False,
@@ -1263,9 +1267,19 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
     mocker.patch('freqtrade.optimize.backtesting.show_backtest_results', MagicMock())
 
+    now = min_backtest_date = datetime.now(tz=timezone.utc)
+    start_time = now - timedelta(**start_delta) + timedelta(hours=1)
+    if cache == 'none':
+        min_backtest_date = now + timedelta(days=1)
+    elif cache == 'day':
+        min_backtest_date = now - timedelta(days=1)
+    elif cache == 'week':
+        min_backtest_date = now - timedelta(weeks=1)
+    elif cache == 'month':
+        min_backtest_date = now - timedelta(weeks=4)
     load_backtest_metadata = MagicMock(return_value={
-        'StrategyTestV2': {'run_id': '1'},
+        'StrategyTestV2': {'run_id': '1', 'backtest_start_time': now.timestamp()},
-        'TestStrategyLegacyV1': {'run_id': 'changed'}
+        'TestStrategyLegacyV1': {'run_id': run_id, 'backtest_start_time': start_time.timestamp()}
     })
     load_backtest_stats = MagicMock(side_effect=[
         {
@@ -1279,7 +1293,8 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
             'strategy_comparison': [{'key': 'TestStrategyLegacyV1'}]
         }
     ])
-    mocker.patch('pathlib.Path.glob', return_value=['not important'])
+    mocker.patch('pathlib.Path.glob', return_value=[
+        Path(datetime.strftime(datetime.now(), 'backtest-result-%Y-%m-%d_%H-%M-%S.json'))])
     mocker.patch.multiple('freqtrade.data.btanalysis',
                           load_backtest_metadata=load_backtest_metadata,
                           load_backtest_stats=load_backtest_stats)
@@ -1296,29 +1311,49 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
         '--timerange', '1510694220-1510700340',
         '--enable-position-stacking',
         '--disable-max-market-positions',
+        '--cache', cache,
         '--strategy-list',
         'StrategyTestV2',
         'TestStrategyLegacyV1',
     ]
     args = get_args(args)
     start_backtesting(args)
-    # 1 backtest, 1 loaded from cache
-    assert backtestmock.call_count == 1
 
     # check the logs, that will contain the backtest result
     exists = [
         'Parameter -i/--timeframe detected ... Using timeframe: 1m ...',
-        'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
         'Parameter --timerange detected: 1510694220-1510700340 ...',
         f'Using data directory: {testdatadir} ...',
         'Loading data from 2017-11-14 20:57:00 '
         'up to 2017-11-14 22:58:00 (0 days).',
-        'Backtesting with data from 2017-11-14 21:17:00 '
-        'up to 2017-11-14 22:58:00 (0 days).',
         'Parameter --enable-position-stacking detected ...',
-        'Reusing result of previous backtest for StrategyTestV2',
-        'Running backtesting for Strategy TestStrategyLegacyV1',
     ]
 
     for line in exists:
         assert log_has(line, caplog)
+
+    if cache == 'none':
+        assert backtestmock.call_count == 2
+        exists = [
+            'Running backtesting for Strategy StrategyTestV2',
+            'Running backtesting for Strategy TestStrategyLegacyV1',
+            'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
+            'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).',
+        ]
+    elif run_id == '2' and min_backtest_date < start_time:
+        assert backtestmock.call_count == 0
+        exists = [
+            'Reusing result of previous backtest for StrategyTestV2',
+            'Reusing result of previous backtest for TestStrategyLegacyV1',
+        ]
+    else:
+        exists = [
+            'Reusing result of previous backtest for StrategyTestV2',
+            'Running backtesting for Strategy TestStrategyLegacyV1',
+            'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
+            'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).',
+        ]
+        assert backtestmock.call_count == 1
+
+    for line in exists:
+        assert log_has(line, caplog)