adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework backtesting --no-cahche to --cache=[none, day, week, month].

Browse Source 
Fix an issue where config modification during runtime would prevent use of cached results.
This commit is contained in:
Rokas Kupstys
2022-01-18 11:00:51 +02:00
parent 3c06d31bbf
commit 5fffc5033a
8 changed files with 117 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
freqtrade/commands/arguments.py
View File

@@ -24,7 +24,7 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
"enable_protections", "dry_run_wallet", "timeframe_detail",
"strategy_list", "export", "exportfilename",
"backtest_breakdown", "no_backtest_cache"]
"backtest_breakdown", "backtest_cache"]
ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
"position_stacking", "use_max_market_positions",

10
freqtrade/commands/cli_options.py
View File

@@ -205,10 +205,12 @@ AVAILABLE_CLI_OPTIONS = {
nargs='+',
choices=constants.BACKTEST_BREAKDOWNS
),
"no_backtest_cache": Arg(
'--no-cache',
help='Do not reuse cached backtest results.',
action='store_true'
"backtest_cache": Arg(
'--cache',
help='Load a cached backtest result no older than specified age.',
metavar='AGE',
default=constants.BACKTEST_CACHE_DEFAULT,
choices=constants.BACKTEST_CACHE_AGE,
),
# Edge
"stoploss_range": Arg(

4
freqtrade/configuration/configuration.py
View File

@@ -276,8 +276,8 @@ class Configuration:
self._args_to_config(config, argname='backtest_breakdown',
logstring='Parameter --breakdown detected ...')
self._args_to_config(config, argname='no_backtest_cache',
logstring='Parameter --no-cache detected ...')
self._args_to_config(config, argname='backtest_cache',
logstring='Parameter --cache={} detected ...')
self._args_to_config(config, argname='disableparamexport',
logstring='Parameter --disableparamexport detected: {} ...')

3
freqtrade/constants.py
View File

@@ -34,6 +34,9 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
BACKTEST_BREAKDOWNS = ['day', 'week', 'month']
BACKTEST_CACHE_AGE = ['none', 'day', 'week', 'month']
BACKTEST_CACHE_DEFAULT = 'day'
BACKTEST_CACHE_NONE = 'none'
DRY_RUN_WALLET = 1000
DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S'
MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons

48
freqtrade/data/btanalysis.py
View File

@@ -3,6 +3,7 @@ Helpers when analyzing backtest data
"""
import logging
from copy import copy
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union
@@ -143,12 +144,24 @@ def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]:
return data
def find_existing_backtest_stats(dirname: Union[Path, str],
run_ids: Dict[str, str]) -> Dict[str, Any]:
def _load_and_merge_backtest_result(strategy_name: str, filename: Path, results: Dict[str, Any]):
bt_data = load_backtest_stats(filename)
for k in ('metadata', 'strategy'):
results[k][strategy_name] = bt_data[k][strategy_name]
comparison = bt_data['strategy_comparison']
for i in range(len(comparison)):
if comparison[i]['key'] == strategy_name:
results['strategy_comparison'].append(comparison[i])
break
def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, str],
min_backtest_date: datetime = None) -> Dict[str, Any]:
"""
Find existing backtest stats that match specified run IDs and load them.
:param dirname: pathlib.Path object, or string pointing to the file.
:param run_ids: {strategy_name: id_string} dictionary.
:param min_backtest_date: do not load a backtest older than specified date.
:return: results dict.
"""
# Copy so we can modify this dict without affecting parent scope.
@@ -169,18 +182,27 @@ def find_existing_backtest_stats(dirname: Union[Path, str],
break
for strategy_name, run_id in list(run_ids.items()):
if metadata.get(strategy_name, {}).get('run_id') == run_id:
# TODO: load_backtest_stats() may load an old version of backtest which is
# incompatible with current version.
strategy_metadata = metadata.get(strategy_name, None)
if not strategy_metadata:
# This strategy is not present in analyzed backtest.
continue
if min_backtest_date is not None:
try:
backtest_date = strategy_metadata['backtest_start_time']
except KeyError:
# Older metadata format without backtest time, too old to consider.
return results
backtest_date = datetime.fromtimestamp(backtest_date, tz=timezone.utc)
if backtest_date < min_backtest_date:
# Do not use a cached result for this strategy as first result is too old.
del run_ids[strategy_name]
continue
if strategy_metadata['run_id'] == run_id:
del run_ids[strategy_name]
bt_data = load_backtest_stats(filename)
for k in ('metadata', 'strategy'):
results[k][strategy_name] = bt_data[k][strategy_name]
comparison = bt_data['strategy_comparison']
for i in range(len(comparison)):
if comparison[i]['key'] == strategy_name:
results['strategy_comparison'].append(comparison[i])
break
_load_and_merge_backtest_result(strategy_name, filename, results)
if len(run_ids) == 0:
break
return results

30
freqtrade/optimize/backtesting.py
View File

@@ -11,6 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple
from pandas import DataFrame
from freqtrade import constants
from freqtrade.configuration import TimeRange, validate_config_consistency
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.data import history
@@ -64,6 +65,7 @@ class Backtesting:
self.results: Dict[str, Any] = {}
config['dry_run'] = True
self.run_ids: Dict[str, str] = {}
self.strategylist: List[IStrategy] = []
self.all_results: Dict[str, Dict] = {}
@@ -728,7 +730,7 @@ class Backtesting:
)
backtest_end_time = datetime.now(timezone.utc)
results.update({
'run_id': get_strategy_run_id(strat),
'run_id': self.run_ids.get(strat.get_strategy_name(), ''),
'backtest_start_time': int(backtest_start_time.timestamp()),
'backtest_end_time': int(backtest_end_time.timestamp()),
})
@@ -736,6 +738,20 @@ class Backtesting:
return min_date, max_date
def _get_min_cached_backtest_date(self):
min_backtest_date = None
backtest_cache_age = self.config.get('backtest_cache', constants.BACKTEST_CACHE_DEFAULT)
if self.timerange.stopts == 0 or datetime.fromtimestamp(
self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc):
logger.warning('Backtest result caching disabled due to use of open-ended timerange.')
elif backtest_cache_age == 'day':
min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(days=1)
elif backtest_cache_age == 'week':
min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(weeks=1)
elif backtest_cache_age == 'month':
min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(weeks=4)
return min_backtest_date
def start(self) -> None:
"""
Run backtesting end-to-end
@@ -747,21 +763,17 @@ class Backtesting:
self.load_bt_data_detail()
logger.info("Dataload complete. Calculating indicators")
run_ids = {
self.run_ids = {
strategy.get_strategy_name(): get_strategy_run_id(strategy)
for strategy in self.strategylist
}
# Load previous result that will be updated incrementally.
# This can be circumvented in certain instances in combination with downloading more data
if self.timerange.stopts == 0 or datetime.fromtimestamp(
self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc):
self.config['no_backtest_cache'] = True
logger.warning('Backtest result caching disabled due to use of open-ended timerange.')
if not self.config.get('no_backtest_cache', False):
min_backtest_date = self._get_min_cached_backtest_date()
if min_backtest_date is not None:
self.results = find_existing_backtest_stats(
self.config['user_data_dir'] / 'backtest_results', run_ids)
self.config['user_data_dir'] / 'backtest_results', self.run_ids, min_backtest_date)
for strat in self.strategylist:
if self.results and strat.get_strategy_name() in self.results['strategy']:

3
freqtrade/optimize/optimize_reports.py
View File

@@ -527,7 +527,8 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
strat_stats = generate_strategy_stats(pairlist, strategy, content,
min_date, max_date, market_change=market_change)
metadata[strategy] = {
'run_id': content['run_id']
'run_id': content['run_id'],
'backtest_start_time': content['backtest_start_time'],
}
result['strategy'][strategy] = strat_stats

59
tests/optimize/test_backtesting.py
View File

@@ -11,6 +11,7 @@ import pandas as pd
import pytest
from arrow import Arrow
from freqtrade import constants
from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_backtesting
from freqtrade.configuration import TimeRange
from freqtrade.data import history
@@ -1242,8 +1243,11 @@ def test_backtest_start_multi_strat_nomock_detail(default_conf, mocker,
@pytest.mark.filterwarnings("ignore:deprecated")
def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testdatadir):
@pytest.mark.parametrize('run_id', ['2', 'changed'])
@pytest.mark.parametrize('start_delta', [{'days': 0}, {'days': 1}, {'weeks': 1}, {'weeks': 4}])
@pytest.mark.parametrize('cache', constants.BACKTEST_CACHE_AGE)
def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testdatadir, run_id,
start_delta, cache):
default_conf.update({
"use_sell_signal": True,
"sell_profit_only": False,
@@ -1263,9 +1267,19 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
mocker.patch('freqtrade.optimize.backtesting.show_backtest_results', MagicMock())
now = min_backtest_date = datetime.now(tz=timezone.utc)
start_time = now - timedelta(**start_delta) + timedelta(hours=1)
if cache == 'none':
min_backtest_date = now + timedelta(days=1)
elif cache == 'day':
min_backtest_date = now - timedelta(days=1)
elif cache == 'week':
min_backtest_date = now - timedelta(weeks=1)
elif cache == 'month':
min_backtest_date = now - timedelta(weeks=4)
load_backtest_metadata = MagicMock(return_value={
'StrategyTestV2': {'run_id': '1'},
'TestStrategyLegacyV1': {'run_id': 'changed'}
'StrategyTestV2': {'run_id': '1', 'backtest_start_time': now.timestamp()},
'TestStrategyLegacyV1': {'run_id': run_id, 'backtest_start_time': start_time.timestamp()}
})
load_backtest_stats = MagicMock(side_effect=[
{
@@ -1279,7 +1293,8 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
'strategy_comparison': [{'key': 'TestStrategyLegacyV1'}]
}
])
mocker.patch('pathlib.Path.glob', return_value=['not important'])
mocker.patch('pathlib.Path.glob', return_value=[
Path(datetime.strftime(datetime.now(), 'backtest-result-%Y-%m-%d_%H-%M-%S.json'))])
mocker.patch.multiple('freqtrade.data.btanalysis',
load_backtest_metadata=load_backtest_metadata,
load_backtest_stats=load_backtest_stats)
@@ -1296,29 +1311,49 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda
'--timerange', '1510694220-1510700340',
'--enable-position-stacking',
'--disable-max-market-positions',
'--cache', cache,
'--strategy-list',
'StrategyTestV2',
'TestStrategyLegacyV1',
]
args = get_args(args)
start_backtesting(args)
# 1 backtest, 1 loaded from cache
assert backtestmock.call_count == 1
# check the logs, that will contain the backtest result
exists = [
'Parameter -i/--timeframe detected ... Using timeframe: 1m ...',
'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
'Parameter --timerange detected: 1510694220-1510700340 ...',
f'Using data directory: {testdatadir} ...',
'Loading data from 2017-11-14 20:57:00 '
'up to 2017-11-14 22:58:00 (0 days).',
'Backtesting with data from 2017-11-14 21:17:00 '
'up to 2017-11-14 22:58:00 (0 days).',
'Parameter --enable-position-stacking detected ...',
'Reusing result of previous backtest for StrategyTestV2',
'Running backtesting for Strategy TestStrategyLegacyV1',
]
for line in exists:
assert log_has(line, caplog)
if cache == 'none':
assert backtestmock.call_count == 2
exists = [
'Running backtesting for Strategy StrategyTestV2',
'Running backtesting for Strategy TestStrategyLegacyV1',
'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).',
]
elif run_id == '2' and min_backtest_date < start_time:
assert backtestmock.call_count == 0
exists = [
'Reusing result of previous backtest for StrategyTestV2',
'Reusing result of previous backtest for TestStrategyLegacyV1',
]
else:
exists = [
'Reusing result of previous backtest for StrategyTestV2',
'Running backtesting for Strategy TestStrategyLegacyV1',
'Ignoring max_open_trades (--disable-max-market-positions was used) ...',
'Backtesting with data from 2017-11-14 21:17:00 up to 2017-11-14 22:58:00 (0 days).',
]
assert backtestmock.call_count == 1
for line in exists:
assert log_has(line, caplog)