diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 119a45662..290865a04 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -24,7 +24,7 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", "enable_protections", "dry_run_wallet", "timeframe_detail", "strategy_list", "export", "exportfilename", - "backtest_breakdown", "no_backtest_cache"] + "backtest_breakdown", "backtest_cache"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "position_stacking", "use_max_market_positions", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 0fb93f0b8..87266fd68 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -205,10 +205,12 @@ AVAILABLE_CLI_OPTIONS = { nargs='+', choices=constants.BACKTEST_BREAKDOWNS ), - "no_backtest_cache": Arg( - '--no-cache', - help='Do not reuse cached backtest results.', - action='store_true' + "backtest_cache": Arg( + '--cache', + help='Load a cached backtest result no older than specified age.', + metavar='AGE', + default=constants.BACKTEST_CACHE_DEFAULT, + choices=constants.BACKTEST_CACHE_AGE, ), # Edge "stoploss_range": Arg( diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 066097916..3ac2e3ddd 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -276,8 +276,8 @@ class Configuration: self._args_to_config(config, argname='backtest_breakdown', logstring='Parameter --breakdown detected ...') - self._args_to_config(config, argname='no_backtest_cache', - logstring='Parameter --no-cache detected ...') + self._args_to_config(config, argname='backtest_cache', + logstring='Parameter --cache={} detected ...') self._args_to_config(config, argname='disableparamexport', logstring='Parameter --disableparamexport detected: {} ...') diff --git a/freqtrade/constants.py b/freqtrade/constants.py index f15759ea5..a06666695 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -34,6 +34,9 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] BACKTEST_BREAKDOWNS = ['day', 'week', 'month'] +BACKTEST_CACHE_AGE = ['none', 'day', 'week', 'month'] +BACKTEST_CACHE_DEFAULT = 'day' +BACKTEST_CACHE_NONE = 'none' DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 27ce8e0ba..1a4d2b1d1 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -3,6 +3,7 @@ Helpers when analyzing backtest data """ import logging from copy import copy +from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union @@ -143,12 +144,24 @@ def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]: return data -def find_existing_backtest_stats(dirname: Union[Path, str], - run_ids: Dict[str, str]) -> Dict[str, Any]: +def _load_and_merge_backtest_result(strategy_name: str, filename: Path, results: Dict[str, Any]): + bt_data = load_backtest_stats(filename) + for k in ('metadata', 'strategy'): + results[k][strategy_name] = bt_data[k][strategy_name] + comparison = bt_data['strategy_comparison'] + for i in range(len(comparison)): + if comparison[i]['key'] == strategy_name: + results['strategy_comparison'].append(comparison[i]) + break + + +def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, str], + min_backtest_date: datetime = None) -> Dict[str, Any]: """ Find existing backtest stats that match specified run IDs and load them. :param dirname: pathlib.Path object, or string pointing to the file. :param run_ids: {strategy_name: id_string} dictionary. + :param min_backtest_date: do not load a backtest older than specified date. :return: results dict. """ # Copy so we can modify this dict without affecting parent scope. @@ -169,18 +182,27 @@ def find_existing_backtest_stats(dirname: Union[Path, str], break for strategy_name, run_id in list(run_ids.items()): - if metadata.get(strategy_name, {}).get('run_id') == run_id: - # TODO: load_backtest_stats() may load an old version of backtest which is - # incompatible with current version. + strategy_metadata = metadata.get(strategy_name, None) + if not strategy_metadata: + # This strategy is not present in analyzed backtest. + continue + + if min_backtest_date is not None: + try: + backtest_date = strategy_metadata['backtest_start_time'] + except KeyError: + # Older metadata format without backtest time, too old to consider. + return results + backtest_date = datetime.fromtimestamp(backtest_date, tz=timezone.utc) + if backtest_date < min_backtest_date: + # Do not use a cached result for this strategy as first result is too old. + del run_ids[strategy_name] + continue + + if strategy_metadata['run_id'] == run_id: del run_ids[strategy_name] - bt_data = load_backtest_stats(filename) - for k in ('metadata', 'strategy'): - results[k][strategy_name] = bt_data[k][strategy_name] - comparison = bt_data['strategy_comparison'] - for i in range(len(comparison)): - if comparison[i]['key'] == strategy_name: - results['strategy_comparison'].append(comparison[i]) - break + _load_and_merge_backtest_result(strategy_name, filename, results) + if len(run_ids) == 0: break return results diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a4a5fd140..b98ea1999 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -11,6 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame +from freqtrade import constants from freqtrade.configuration import TimeRange, validate_config_consistency from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.data import history @@ -64,6 +65,7 @@ class Backtesting: self.results: Dict[str, Any] = {} config['dry_run'] = True + self.run_ids: Dict[str, str] = {} self.strategylist: List[IStrategy] = [] self.all_results: Dict[str, Dict] = {} @@ -728,7 +730,7 @@ class Backtesting: ) backtest_end_time = datetime.now(timezone.utc) results.update({ - 'run_id': get_strategy_run_id(strat), + 'run_id': self.run_ids.get(strat.get_strategy_name(), ''), 'backtest_start_time': int(backtest_start_time.timestamp()), 'backtest_end_time': int(backtest_end_time.timestamp()), }) @@ -736,6 +738,20 @@ class Backtesting: return min_date, max_date + def _get_min_cached_backtest_date(self): + min_backtest_date = None + backtest_cache_age = self.config.get('backtest_cache', constants.BACKTEST_CACHE_DEFAULT) + if self.timerange.stopts == 0 or datetime.fromtimestamp( + self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc): + logger.warning('Backtest result caching disabled due to use of open-ended timerange.') + elif backtest_cache_age == 'day': + min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(days=1) + elif backtest_cache_age == 'week': + min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(weeks=1) + elif backtest_cache_age == 'month': + min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(weeks=4) + return min_backtest_date + def start(self) -> None: """ Run backtesting end-to-end @@ -747,21 +763,17 @@ class Backtesting: self.load_bt_data_detail() logger.info("Dataload complete. Calculating indicators") - run_ids = { + self.run_ids = { strategy.get_strategy_name(): get_strategy_run_id(strategy) for strategy in self.strategylist } # Load previous result that will be updated incrementally. # This can be circumvented in certain instances in combination with downloading more data - if self.timerange.stopts == 0 or datetime.fromtimestamp( - self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc): - self.config['no_backtest_cache'] = True - logger.warning('Backtest result caching disabled due to use of open-ended timerange.') - - if not self.config.get('no_backtest_cache', False): + min_backtest_date = self._get_min_cached_backtest_date() + if min_backtest_date is not None: self.results = find_existing_backtest_stats( - self.config['user_data_dir'] / 'backtest_results', run_ids) + self.config['user_data_dir'] / 'backtest_results', self.run_ids, min_backtest_date) for strat in self.strategylist: if self.results and strat.get_strategy_name() in self.results['strategy']: diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 46930d7b1..859238af3 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -527,7 +527,8 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], strat_stats = generate_strategy_stats(pairlist, strategy, content, min_date, max_date, market_change=market_change) metadata[strategy] = { - 'run_id': content['run_id'] + 'run_id': content['run_id'], + 'backtest_start_time': content['backtest_start_time'], } result['strategy'][strategy] = strat_stats diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 7dd0abd4a..edd0faba3 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -11,6 +11,7 @@ import pandas as pd import pytest from arrow import Arrow +from freqtrade import constants from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_backtesting from freqtrade.configuration import TimeRange from freqtrade.data import history @@ -1242,8 +1243,11 @@ def test_backtest_start_multi_strat_nomock_detail(default_conf, mocker, @pytest.mark.filterwarnings("ignore:deprecated") -def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testdatadir): - +@pytest.mark.parametrize('run_id', ['2', 'changed']) +@pytest.mark.parametrize('start_delta', [{'days': 0}, {'days': 1}, {'weeks': 1}, {'weeks': 4}]) +@pytest.mark.parametrize('cache', constants.BACKTEST_CACHE_AGE) +def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testdatadir, run_id, + start_delta, cache): default_conf.update({ "use_sell_signal": True, "sell_profit_only": False, @@ -1263,9 +1267,19 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) mocker.patch('freqtrade.optimize.backtesting.show_backtest_results', MagicMock()) + now = min_backtest_date = datetime.now(tz=timezone.utc) + start_time = now - timedelta(**start_delta) + timedelta(hours=1) + if cache == 'none': + min_backtest_date = now + timedelta(days=1) + elif cache == 'day': + min_backtest_date = now - timedelta(days=1) + elif cache == 'week': + min_backtest_date = now - timedelta(weeks=1) + elif cache == 'month': + min_backtest_date = now - timedelta(weeks=4) load_backtest_metadata = MagicMock(return_value={ - 'StrategyTestV2': {'run_id': '1'}, - 'TestStrategyLegacyV1': {'run_id': 'changed'} + 'StrategyTestV2': {'run_id': '1', 'backtest_start_time': now.timestamp()}, + 'TestStrategyLegacyV1': {'run_id': run_id, 'backtest_start_time': start_time.timestamp()} }) load_backtest_stats = MagicMock(side_effect=[ { @@ -1279,7 +1293,8 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda 'strategy_comparison': [{'key': 'TestStrategyLegacyV1'}] } ]) - mocker.patch('pathlib.Path.glob', return_value=['not important']) + mocker.patch('pathlib.Path.glob', return_value=[ + Path(datetime.strftime(datetime.now(), 'backtest-result-%Y-%m-%d_%H-%M-%S.json'))]) mocker.patch.multiple('freqtrade.data.btanalysis', load_backtest_metadata=load_backtest_metadata, load_backtest_stats=load_backtest_stats) @@ -1296,29 +1311,49 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda '--timerange', '1510694220-1510700340', '--enable-position-stacking', '--disable-max-market-positions', + '--cache', cache, '--strategy-list', 'StrategyTestV2', 'TestStrategyLegacyV1', ] args = get_args(args) start_backtesting(args) - # 1 backtest, 1 loaded from cache - assert backtestmock.call_count == 1 # check the logs, that will contain the backtest result exists = [ 'Parameter -i/--timeframe detected ... Using timeframe: 1m ...', - 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', 'Loading data from 2017-11-14 20:57:00 ' 'up to 2017-11-14 22:58:00 (0 days).', - 'Backtesting with data from 2017-11-14 21:17:00 ' - 'up to 2017-11-14 22:58:00 (0 days).', 'Parameter --enable-position-stacking detected ...', - 'Reusing result of previous backtest for StrategyTestV2', - 'Running backtesting for Strategy TestStrategyLegacyV1', ] for line in exists: assert log_has(line, caplog) + + if cache == 'none': + assert backtestmock.call_count == 2 + exists = [ + 'Running backtesting for Strategy StrategyTestV2', + 'Running backtesting for Strategy TestStrategyLegacyV1', + 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', + 'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).', + ] + elif run_id == '2' and min_backtest_date < start_time: + assert backtestmock.call_count == 0 + exists = [ + 'Reusing result of previous backtest for StrategyTestV2', + 'Reusing result of previous backtest for TestStrategyLegacyV1', + ] + else: + exists = [ + 'Reusing result of previous backtest for StrategyTestV2', + 'Running backtesting for Strategy TestStrategyLegacyV1', + 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', + 'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).', + ] + assert backtestmock.call_count == 1 + + for line in exists: + assert log_has(line, caplog)