From 16861db653ec8166f73fc8480894f186a137e7bd Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Thu, 6 Jan 2022 11:53:11 +0200 Subject: [PATCH 01/11] Implement previous backtest result reuse when config and strategy did not change. --- docs/backtesting.md | 5 ++ freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 5 ++ freqtrade/configuration/configuration.py | 3 ++ freqtrade/data/btanalysis.py | 67 +++++++++++++++++++++++- freqtrade/misc.py | 33 +++++++++++- freqtrade/optimize/backtesting.py | 51 +++++++++++++++--- freqtrade/optimize/optimize_reports.py | 19 ++++++- tests/optimize/test_optimize_reports.py | 10 ++-- 9 files changed, 179 insertions(+), 16 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 001941993..ee930db34 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -76,6 +76,7 @@ optional arguments: _today.json` --breakdown {day,week,month} [{day,week,month} ...] Show backtesting breakdown per [day, week, month]. + --no-cache Do not reuse cached backtest results. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -457,6 +458,10 @@ freqtrade backtesting --strategy MyAwesomeStrategy --breakdown day month The output will show a table containing the realized absolute Profit (in stake currency) for the given timeperiod, as well as wins, draws and losses that materialized (closed) on this day. +### Backtest result caching + +To save time, by default backtest will reuse a cached result when backtested strategy and config match that of previous backtest. To force a new backtest despite existing result for identical run specify `--no-cache` parameter. + ### Further backtest-result analysis To further analyze your backtest results, you can [export the trades](#exporting-trades-to-file). diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 032f7dd51..119a45662 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -24,7 +24,7 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", "enable_protections", "dry_run_wallet", "timeframe_detail", "strategy_list", "export", "exportfilename", - "backtest_breakdown"] + "backtest_breakdown", "no_backtest_cache"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "position_stacking", "use_max_market_positions", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 6aa4ed363..0fb93f0b8 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -205,6 +205,11 @@ AVAILABLE_CLI_OPTIONS = { nargs='+', choices=constants.BACKTEST_BREAKDOWNS ), + "no_backtest_cache": Arg( + '--no-cache', + help='Do not reuse cached backtest results.', + action='store_true' + ), # Edge "stoploss_range": Arg( '--stoplosses', diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index f5a674878..066097916 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -276,6 +276,9 @@ class Configuration: self._args_to_config(config, argname='backtest_breakdown', logstring='Parameter --breakdown detected ...') + self._args_to_config(config, argname='no_backtest_cache', + logstring='Parameter --no-cache detected ...') + self._args_to_config(config, argname='disableparamexport', logstring='Parameter --disableparamexport detected: {} ...') diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 10dba8683..27ce8e0ba 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -2,6 +2,7 @@ Helpers when analyzing backtest data """ import logging +from copy import copy from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union @@ -10,7 +11,7 @@ import pandas as pd from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.exceptions import OperationalException -from freqtrade.misc import json_load +from freqtrade.misc import get_backtest_metadata_filename, json_load from freqtrade.persistence import LocalTrade, Trade, init_db @@ -102,6 +103,23 @@ def get_latest_hyperopt_file(directory: Union[Path, str], predef_filename: str = return directory / get_latest_hyperopt_filename(directory) +def load_backtest_metadata(filename: Union[Path, str]) -> Dict[str, Any]: + """ + Read metadata dictionary from backtest results file without reading and deserializing entire + file. + :param filename: path to backtest results file. + :return: metadata dict or None if metadata is not present. + """ + filename = get_backtest_metadata_filename(filename) + try: + with filename.open() as fp: + return json_load(fp) + except FileNotFoundError: + return {} + except Exception as e: + raise OperationalException('Unexpected error while loading backtest metadata.') from e + + def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]: """ Load backtest statistics file. @@ -118,9 +136,56 @@ def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]: with filename.open() as file: data = json_load(file) + # Legacy list format does not contain metadata. + if isinstance(data, dict): + data['metadata'] = load_backtest_metadata(filename) + return data +def find_existing_backtest_stats(dirname: Union[Path, str], + run_ids: Dict[str, str]) -> Dict[str, Any]: + """ + Find existing backtest stats that match specified run IDs and load them. + :param dirname: pathlib.Path object, or string pointing to the file. + :param run_ids: {strategy_name: id_string} dictionary. + :return: results dict. + """ + # Copy so we can modify this dict without affecting parent scope. + run_ids = copy(run_ids) + dirname = Path(dirname) + results: Dict[str, Any] = { + 'metadata': {}, + 'strategy': {}, + 'strategy_comparison': [], + } + + # Weird glob expression here avoids including .meta.json files. + for filename in reversed(sorted(dirname.glob('backtest-result-*-[0-9][0-9].json'))): + metadata = load_backtest_metadata(filename) + if not metadata: + # Files are sorted from newest to oldest. When file without metadata is encountered it + # is safe to assume older files will also not have any metadata. + break + + for strategy_name, run_id in list(run_ids.items()): + if metadata.get(strategy_name, {}).get('run_id') == run_id: + # TODO: load_backtest_stats() may load an old version of backtest which is + # incompatible with current version. + del run_ids[strategy_name] + bt_data = load_backtest_stats(filename) + for k in ('metadata', 'strategy'): + results[k][strategy_name] = bt_data[k][strategy_name] + comparison = bt_data['strategy_comparison'] + for i in range(len(comparison)): + if comparison[i]['key'] == strategy_name: + results['strategy_comparison'].append(comparison[i]) + break + if len(run_ids) == 0: + break + return results + + def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = None) -> pd.DataFrame: """ Load backtest data file. diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 6f439866b..f09e5ee47 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -2,11 +2,13 @@ Various tool function for Freqtrade and scripts """ import gzip +import hashlib import logging import re +from copy import deepcopy from datetime import datetime from pathlib import Path -from typing import Any, Iterator, List +from typing import Any, Iterator, List, Union from typing.io import IO from urllib.parse import urlparse @@ -228,3 +230,32 @@ def parse_db_uri_for_logging(uri: str): return uri pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0] return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@') + + +def get_strategy_run_id(strategy) -> str: + """ + Generate unique identification hash for a backtest run. Identical config and strategy file will + always return an identical hash. + :param strategy: strategy object. + :return: hex string id. + """ + digest = hashlib.sha1() + config = deepcopy(strategy.config) + + # Options that have no impact on results of individual backtest. + not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server') + for k in not_important_keys: + if k in config: + del config[k] + + digest.update(rapidjson.dumps(config, default=str, + number_mode=rapidjson.NM_NATIVE).encode('utf-8')) + with open(strategy.__file__, 'rb') as fp: + digest.update(fp.read()) + return digest.hexdigest().lower() + + +def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path: + """Return metadata filename for specified backtest results file.""" + filename = Path(filename) + return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 754b46d81..950531637 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -14,12 +14,13 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange, validate_config_consistency from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.data import history -from freqtrade.data.btanalysis import trade_list_to_dataframe +from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe from freqtrade.data.converter import trim_dataframe, trim_dataframes from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import BacktestState, SellType from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds +from freqtrade.misc import get_strategy_run_id from freqtrade.mixins import LoggingMixin from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, @@ -60,7 +61,7 @@ class Backtesting: LoggingMixin.show_output = False self.config = config - self.results: Optional[Dict[str, Any]] = None + self.results: Dict[str, Any] = {} config['dry_run'] = True self.strategylist: List[IStrategy] = [] @@ -727,6 +728,7 @@ class Backtesting: ) backtest_end_time = datetime.now(timezone.utc) results.update({ + 'run_id': get_strategy_run_id(strat), 'backtest_start_time': int(backtest_start_time.timestamp()), 'backtest_end_time': int(backtest_end_time.timestamp()), }) @@ -745,15 +747,50 @@ class Backtesting: self.load_bt_data_detail() logger.info("Dataload complete. Calculating indicators") - for strat in self.strategylist: - min_date, max_date = self.backtest_one_strategy(strat, data, timerange) - if len(self.strategylist) > 0: + run_ids = { + strategy.get_strategy_name(): get_strategy_run_id(strategy) + for strategy in self.strategylist + } - self.results = generate_backtest_stats(data, self.all_results, - min_date=min_date, max_date=max_date) + # Load previous result that will be updated incrementally. + if self.config.get('timerange', '-').endswith('-'): + self.config['no_backtest_cache'] = True + logger.warning('Backtest result caching disabled due to use of open-ended timerange.') + + if not self.config.get('no_backtest_cache', False): + self.results = find_existing_backtest_stats( + self.config['user_data_dir'] / 'backtest_results', run_ids) + + for strat in self.strategylist: + if self.results and strat.get_strategy_name() in self.results['strategy']: + # When previous result hash matches - reuse that result and skip backtesting. + logger.info(f'Reusing result of previous backtest for {strat.get_strategy_name()}') + continue + min_date, max_date = self.backtest_one_strategy(strat, data, timerange) + + # Update old results with new ones. + if len(self.all_results) > 0: + results = generate_backtest_stats( + data, self.all_results, min_date=min_date, max_date=max_date) + if self.results: + self.results['metadata'].update(results['metadata']) + self.results['strategy'].update(results['strategy']) + self.results['strategy_comparison'].extend(results['strategy_comparison']) + else: + self.results = results if self.config.get('export', 'none') == 'trades': store_backtest_stats(self.config['exportfilename'], self.results) + # Results may be mixed up now. Sort them so they follow --strategy-list order. + if 'strategy_list' in self.config and len(self.results) > 0: + self.results['strategy_comparison'] = sorted( + self.results['strategy_comparison'], + key=lambda c: self.config['strategy_list'].index(c['key'])) + self.results['strategy'] = dict( + sorted(self.results['strategy'].items(), + key=lambda kv: self.config['strategy_list'].index(kv[0]))) + + if len(self.strategylist) > 0: # Show backtest results show_backtest_results(self.config, self.results) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index d0ffe49a9..46930d7b1 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -11,7 +11,8 @@ from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change, calculate_max_drawdown) -from freqtrade.misc import decimals_per_coin, file_dump_json, round_coin_value +from freqtrade.misc import (decimals_per_coin, file_dump_json, get_backtest_metadata_filename, + round_coin_value) logger = logging.getLogger(__name__) @@ -33,6 +34,11 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N recordfilename.parent, f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}' ).with_suffix(recordfilename.suffix) + + # Store metadata separately. + file_dump_json(get_backtest_metadata_filename(filename), stats['metadata']) + del stats['metadata'] + file_dump_json(filename, stats) latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN) @@ -509,16 +515,25 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], :param max_date: Backtest end date :return: Dictionary containing results per strategy and a strategy summary. """ - result: Dict[str, Any] = {'strategy': {}} + result: Dict[str, Any] = { + 'metadata': {}, + 'strategy': {}, + 'strategy_comparison': [], + } market_change = calculate_market_change(btdata, 'close') + metadata = {} pairlist = list(btdata.keys()) for strategy, content in all_results.items(): strat_stats = generate_strategy_stats(pairlist, strategy, content, min_date, max_date, market_change=market_change) + metadata[strategy] = { + 'run_id': content['run_id'] + } result['strategy'][strategy] = strat_stats strategy_results = generate_strategy_comparison(bt_stats=result['strategy']) + result['metadata'] = metadata result['strategy_comparison'] = strategy_results return result diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index ed939d6b0..68257f4d8 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -84,6 +84,7 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): 'rejected_signals': 20, 'backtest_start_time': Arrow.utcnow().int_timestamp, 'backtest_end_time': Arrow.utcnow().int_timestamp, + 'run_id': '123', } } timerange = TimeRange.parse_timerange('1510688220-1510700340') @@ -132,6 +133,7 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): 'rejected_signals': 20, 'backtest_start_time': Arrow.utcnow().int_timestamp, 'backtest_end_time': Arrow.utcnow().int_timestamp, + 'run_id': '124', } } @@ -178,16 +180,16 @@ def test_store_backtest_stats(testdatadir, mocker): dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_json') - store_backtest_stats(testdatadir, {}) + store_backtest_stats(testdatadir, {'metadata': {}}) - assert dump_mock.call_count == 2 + assert dump_mock.call_count == 3 assert isinstance(dump_mock.call_args_list[0][0][0], Path) assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir/'backtest-result')) dump_mock.reset_mock() filename = testdatadir / 'testresult.json' - store_backtest_stats(filename, {}) - assert dump_mock.call_count == 2 + store_backtest_stats(filename, {'metadata': {}}) + assert dump_mock.call_count == 3 assert isinstance(dump_mock.call_args_list[0][0][0], Path) # result will be testdatadir / testresult-.json assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / 'testresult')) From 526ed7fa9a344bcd881976324c92be9b2b51cb05 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Fri, 7 Jan 2022 13:10:14 +0200 Subject: [PATCH 02/11] Add test_backtest_start_multi_strat_caching test flexing backtest result caching. --- tests/optimize/test_backtesting.py | 85 ++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 6290c3c55..5fd482340 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1239,3 +1239,88 @@ def test_backtest_start_multi_strat_nomock_detail(default_conf, mocker, assert 'BACKTESTING REPORT' in captured.out assert 'SELL REASON STATS' in captured.out assert 'LEFT OPEN TRADES REPORT' in captured.out + + +@pytest.mark.filterwarnings("ignore:deprecated") +def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testdatadir): + + default_conf.update({ + "use_sell_signal": True, + "sell_profit_only": False, + "sell_profit_offset": 0.0, + "ignore_roi_if_buy_signal": False, + }) + patch_exchange(mocker) + backtestmock = MagicMock(return_value={ + 'results': pd.DataFrame(columns=BT_DATA_COLUMNS), + 'config': default_conf, + 'locks': [], + 'rejected_signals': 20, + 'final_balance': 1000, + }) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) + mocker.patch('freqtrade.optimize.backtesting', show_backtest_result=MagicMock()) + + path_glob = MagicMock(return_value=['not important']) + load_backtest_metadata = MagicMock(return_value={ + 'StrategyTestV2': {'run_id': '1'}, + 'TestStrategyLegacyV1': {'run_id': 'changed'} + }) + load_backtest_stats = MagicMock(side_effect=[ + { + 'metadata': {'StrategyTestV2': {'run_id': '1'}}, + 'strategy': {'StrategyTestV2': {}}, + 'strategy_comparison': [{'key': 'StrategyTestV2'}] + }, + { + 'metadata': {'TestStrategyLegacyV1': {'run_id': '2'}}, + 'strategy': {'TestStrategyLegacyV1': {}}, + 'strategy_comparison': [{'key': 'TestStrategyLegacyV1'}] + } + ]) + get_strategy_run_id = MagicMock(side_effect=['1', '2', '2']) + mocker.patch('pathlib.Path.glob', path_glob) + mocker.patch.multiple('freqtrade.data.btanalysis', + load_backtest_metadata=load_backtest_metadata, + load_backtest_stats=load_backtest_stats) + mocker.patch('freqtrade.misc.get_strategy_run_id', get_strategy_run_id) + + patched_configuration_load_config_file(mocker, default_conf) + + args = [ + 'backtesting', + '--config', 'config.json', + '--datadir', str(testdatadir), + '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), + '--timeframe', '1m', + '--timerange', '1510694220-1510700340', + '--enable-position-stacking', + '--disable-max-market-positions', + '--strategy-list', + 'StrategyTestV2', + 'TestStrategyLegacyV1', + ] + args = get_args(args) + start_backtesting(args) + # 1 backtest, 1 loaded from cache + assert backtestmock.call_count == 1 + + # check the logs, that will contain the backtest result + exists = [ + 'Parameter -i/--timeframe detected ... Using timeframe: 1m ...', + 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', + 'Parameter --timerange detected: 1510694220-1510700340 ...', + f'Using data directory: {testdatadir} ...', + 'Loading data from 2017-11-14 20:57:00 ' + 'up to 2017-11-14 22:58:00 (0 days).', + 'Backtesting with data from 2017-11-14 21:17:00 ' + 'up to 2017-11-14 22:58:00 (0 days).', + 'Parameter --enable-position-stacking detected ...', + 'Reusing result of previous backtest for StrategyTestV2', + 'Running backtesting for Strategy TestStrategyLegacyV1', + ] + + for line in exists: + assert log_has(line, caplog) From 9becce9897cf9941429232f92aa975001b4e5bca Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 8 Jan 2022 08:41:09 +0100 Subject: [PATCH 03/11] Update failing test --- tests/optimize/test_backtesting.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 5fd482340..7dd0abd4a 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1261,9 +1261,8 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) - mocker.patch('freqtrade.optimize.backtesting', show_backtest_result=MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.show_backtest_results', MagicMock()) - path_glob = MagicMock(return_value=['not important']) load_backtest_metadata = MagicMock(return_value={ 'StrategyTestV2': {'run_id': '1'}, 'TestStrategyLegacyV1': {'run_id': 'changed'} @@ -1280,12 +1279,11 @@ def test_backtest_start_multi_strat_caching(default_conf, mocker, caplog, testda 'strategy_comparison': [{'key': 'TestStrategyLegacyV1'}] } ]) - get_strategy_run_id = MagicMock(side_effect=['1', '2', '2']) - mocker.patch('pathlib.Path.glob', path_glob) + mocker.patch('pathlib.Path.glob', return_value=['not important']) mocker.patch.multiple('freqtrade.data.btanalysis', load_backtest_metadata=load_backtest_metadata, load_backtest_stats=load_backtest_stats) - mocker.patch('freqtrade.misc.get_strategy_run_id', get_strategy_run_id) + mocker.patch('freqtrade.optimize.backtesting.get_strategy_run_id', side_effect=['1', '2', '2']) patched_configuration_load_config_file(mocker, default_conf) From 2b7405470aebf4860c2fb50b76236399f4483aab Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 15 Jan 2022 17:30:20 +0200 Subject: [PATCH 04/11] Fix timerange check. --- freqtrade/optimize/backtesting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 950531637..398a35893 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -753,7 +753,8 @@ class Backtesting: } # Load previous result that will be updated incrementally. - if self.config.get('timerange', '-').endswith('-'): + if self.timerange.stopts == 0 or datetime.fromtimestamp( + self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc): self.config['no_backtest_cache'] = True logger.warning('Backtest result caching disabled due to use of open-ended timerange.') From 062d00e8f2aea8f19933b0b978c60b9e32d0ae06 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 15 Jan 2022 17:31:16 +0200 Subject: [PATCH 05/11] Fix @informative decorator failing with edge. --- freqtrade/optimize/edge_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index f211da750..cc9bafb0b 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -34,7 +34,7 @@ class EdgeCli: self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) self.strategy = StrategyResolver.load_strategy(self.config) - self.strategy.dp = DataProvider(config, None) + self.strategy.dp = DataProvider(config, self.exchange) validate_config_consistency(self.config) From 6c4b261469aef755c94be672d3eba807e50860da Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jan 2022 08:04:39 +0100 Subject: [PATCH 06/11] Convert nan to None in get_signal. --- freqtrade/strategy/interface.py | 3 +++ tests/strategy/test_interface.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 36061dc20..c8fb24da1 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -654,6 +654,9 @@ class IStrategy(ABC, HyperStrategyMixin): buy_tag = latest.get(SignalTagType.BUY_TAG.value, None) exit_tag = latest.get(SignalTagType.EXIT_TAG.value, None) + # Tags can be None, which does not resolve to False. + buy_tag = buy_tag if isinstance(buy_tag, str) else None + exit_tag = exit_tag if isinstance(exit_tag, str) else None logger.debug('trigger: %s (pair=%s) buy=%s sell=%s', latest['date'], pair, str(buy), str(sell)) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index f57a9f34e..fd1c2753f 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -36,6 +36,10 @@ def test_returns_latest_signal(ohlcv_history): mocked_history = ohlcv_history.copy() mocked_history['sell'] = 0 mocked_history['buy'] = 0 + # Set tags in lines that don't matter to test nan in the sell line + mocked_history.loc[0, 'buy_tag'] = 'wrong_line' + mocked_history.loc[0, 'exit_tag'] = 'wrong_line' + mocked_history.loc[1, 'sell'] = 1 assert _STRATEGY.get_signal('ETH/BTC', '5m', mocked_history) == (False, True, None, None) From b96b0f89bd0e0e687d5509a7036e15abebbd6a1c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jan 2022 13:16:49 +0100 Subject: [PATCH 07/11] improved unfilledtimeout defaults --- config_examples/config_binance.example.json | 4 +++- config_examples/config_bittrex.example.json | 4 +++- config_examples/config_ftx.example.json | 4 +++- config_examples/config_full.example.json | 2 +- config_examples/config_kraken.example.json | 4 +++- freqtrade/templates/base_config.json.j2 | 3 ++- 6 files changed, 15 insertions(+), 6 deletions(-) diff --git a/config_examples/config_binance.example.json b/config_examples/config_binance.example.json index d59ff96cb..c6faf506c 100644 --- a/config_examples/config_binance.example.json +++ b/config_examples/config_binance.example.json @@ -9,7 +9,9 @@ "cancel_open_orders_on_exit": false, "unfilledtimeout": { "buy": 10, - "sell": 30 + "sell": 10, + "exit_timeout_count": 0, + "unit": "minutes" }, "bid_strategy": { "ask_last_balance": 0.0, diff --git a/config_examples/config_bittrex.example.json b/config_examples/config_bittrex.example.json index 4352d8822..9fe99c835 100644 --- a/config_examples/config_bittrex.example.json +++ b/config_examples/config_bittrex.example.json @@ -9,7 +9,9 @@ "cancel_open_orders_on_exit": false, "unfilledtimeout": { "buy": 10, - "sell": 30 + "sell": 10, + "exit_timeout_count": 0, + "unit": "minutes" }, "bid_strategy": { "use_order_book": true, diff --git a/config_examples/config_ftx.example.json b/config_examples/config_ftx.example.json index 4d9633cc0..4f7c2af54 100644 --- a/config_examples/config_ftx.example.json +++ b/config_examples/config_ftx.example.json @@ -9,7 +9,9 @@ "cancel_open_orders_on_exit": false, "unfilledtimeout": { "buy": 10, - "sell": 30 + "sell": 10, + "exit_timeout_count": 0, + "unit": "minutes" }, "bid_strategy": { "ask_last_balance": 0.0, diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index 81a034a21..5202954f4 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -28,7 +28,7 @@ "stoploss": -0.10, "unfilledtimeout": { "buy": 10, - "sell": 30, + "sell": 10, "exit_timeout_count": 0, "unit": "minutes" }, diff --git a/config_examples/config_kraken.example.json b/config_examples/config_kraken.example.json index 32def895c..5ac3a9255 100644 --- a/config_examples/config_kraken.example.json +++ b/config_examples/config_kraken.example.json @@ -9,7 +9,9 @@ "cancel_open_orders_on_exit": false, "unfilledtimeout": { "buy": 10, - "sell": 30 + "sell": 10, + "exit_timeout_count": 0, + "unit": "minutes" }, "bid_strategy": { "use_order_book": true, diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index e2fa1c63e..c91715b1f 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -15,7 +15,8 @@ "cancel_open_orders_on_exit": false, "unfilledtimeout": { "buy": 10, - "sell": 30, + "sell": 10, + "exit_timeout_count": 0, "unit": "minutes" }, "bid_strategy": { From 69c00db7cdb2e0ee9889a8e649a02f4935164953 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jan 2022 13:39:50 +0100 Subject: [PATCH 08/11] Only show /balance % improvement if trades have been made --- freqtrade/rpc/rpc.py | 2 ++ freqtrade/rpc/telegram.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f53bc7d94..e568fca8c 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -592,6 +592,7 @@ class RPC: value = self._fiat_converter.convert_amount( total, stake_currency, fiat_display_currency) if self._fiat_converter else 0 + trade_count = len(Trade.get_trades_proxy()) starting_capital_ratio = 0.0 starting_capital_ratio = (total / starting_capital) - 1 if starting_capital else 0.0 starting_cap_fiat_ratio = (value / starting_cap_fiat) - 1 if starting_cap_fiat else 0.0 @@ -608,6 +609,7 @@ class RPC: 'starting_capital_fiat': starting_cap_fiat, 'starting_capital_fiat_ratio': starting_cap_fiat_ratio, 'starting_capital_fiat_pct': round(starting_cap_fiat_ratio * 100, 2), + 'trade_count': trade_count, 'note': 'Simulated balances' if self._freqtrade.config['dry_run'] else '' } diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 61a0b1f65..716694a81 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -765,14 +765,17 @@ class Telegram(RPCHandler): f"(< {balance_dust_level} {result['stake']}):*\n" f"\t`Est. {result['stake']}: " f"{round_coin_value(total_dust_balance, result['stake'], False)}`\n") + tc = result['trade_count'] > 0 + stake_improve = f" `({result['starting_capital_ratio']:.2%})`" if tc else '' + fiat_val = f" `({result['starting_capital_fiat_ratio']:.2%})`" if tc else '' output += ("\n*Estimated Value*:\n" f"\t`{result['stake']}: " f"{round_coin_value(result['total'], result['stake'], False)}`" - f" `({result['starting_capital_ratio']:.2%})`\n" + f"{stake_improve}\n" f"\t`{result['symbol']}: " f"{round_coin_value(result['value'], result['symbol'], False)}`" - f" `({result['starting_capital_fiat_ratio']:.2%})`\n") + f"{fiat_val}\n") self._send_msg(output, reload_able=True, callback_path="update_balance", query=update.callback_query) except RPCException as e: From d08885ed92f390c5ded5799b243a5ba5f18d92e0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jan 2022 15:37:00 +0100 Subject: [PATCH 09/11] Fix empty "/log" endpoint in certain moments --- freqtrade/loggers.py | 16 +++++++++++++++- tests/test_configuration.py | 6 +++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index 5c5831695..e5b6ddbe9 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -7,11 +7,25 @@ from typing import Any, Dict from freqtrade.exceptions import OperationalException +class FTBufferingHandler(BufferingHandler): + def flush(self): + """ + Override Flush behaviour - we keep half of the configured capacity + otherwise, we have moments with "empty" logs. + """ + self.acquire() + try: + # Keep half of the records in buffer. + self.buffer = self.buffer[-int(self.capacity / 2):] + finally: + self.release() + + logger = logging.getLogger(__name__) LOGFORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' # Initialize bufferhandler - will be used for /log endpoints -bufferHandler = BufferingHandler(1000) +bufferHandler = FTBufferingHandler(1000) bufferHandler.setFormatter(Formatter(LOGFORMAT)) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 57add66bf..0a6935649 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -22,7 +22,7 @@ from freqtrade.configuration.load_config import load_config_file, load_file, log from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL, ENV_VAR_PREFIX from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.loggers import _set_loggers, setup_logging, setup_logging_pre +from freqtrade.loggers import FTBufferingHandler, _set_loggers, setup_logging, setup_logging_pre from tests.conftest import log_has, log_has_re, patched_configuration_load_config_file @@ -686,7 +686,7 @@ def test_set_loggers_syslog(): assert len(logger.handlers) == 3 assert [x for x in logger.handlers if type(x) == logging.handlers.SysLogHandler] assert [x for x in logger.handlers if type(x) == logging.StreamHandler] - assert [x for x in logger.handlers if type(x) == logging.handlers.BufferingHandler] + assert [x for x in logger.handlers if type(x) == FTBufferingHandler] # setting up logging again should NOT cause the loggers to be added a second time. setup_logging(config) assert len(logger.handlers) == 3 @@ -709,7 +709,7 @@ def test_set_loggers_Filehandler(tmpdir): assert len(logger.handlers) == 3 assert [x for x in logger.handlers if type(x) == logging.handlers.RotatingFileHandler] assert [x for x in logger.handlers if type(x) == logging.StreamHandler] - assert [x for x in logger.handlers if type(x) == logging.handlers.BufferingHandler] + assert [x for x in logger.handlers if type(x) == FTBufferingHandler] # setting up logging again should NOT cause the loggers to be added a second time. setup_logging(config) assert len(logger.handlers) == 3 From 2bcfc0c90ce83a16f64b959d954390aadd708e86 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jan 2022 18:01:05 +0100 Subject: [PATCH 10/11] Add warning about cache problems --- docs/backtesting.md | 4 ++++ freqtrade/optimize/backtesting.py | 1 + 2 files changed, 5 insertions(+) diff --git a/docs/backtesting.md b/docs/backtesting.md index ee930db34..eae6ac4a9 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -462,6 +462,10 @@ The output will show a table containing the realized absolute Profit (in stake c To save time, by default backtest will reuse a cached result when backtested strategy and config match that of previous backtest. To force a new backtest despite existing result for identical run specify `--no-cache` parameter. +!!! Warning + Caching is automatically disabled for open-ended timeranges (`--timerange 20210101-`), as freqtrade cannot ensure reliably that the underlying data didn't change. It can also use cached results where it shouldn't if the original backtest had missing data at the end, which was fixed by downloading more data. + In this instance, please use `--no-cache` once to get a fresh backtest. + ### Further backtest-result analysis To further analyze your backtest results, you can [export the trades](#exporting-trades-to-file). diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 398a35893..a4a5fd140 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -753,6 +753,7 @@ class Backtesting: } # Load previous result that will be updated incrementally. + # This can be circumvented in certain instances in combination with downloading more data if self.timerange.stopts == 0 or datetime.fromtimestamp( self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc): self.config['no_backtest_cache'] = True From 3cdb672ac38c1ffc4a9cf6c6f64ddfb8ac47977f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Jan 2022 19:11:20 +0100 Subject: [PATCH 11/11] Improve test coverage --- tests/data/test_btanalysis.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 9f0b5aef7..eed3532f8 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -13,7 +13,8 @@ from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelis calculate_underwater, combine_dataframes_with_mean, create_cum_profit, extract_trades_of_period, get_latest_backtest_filename, get_latest_hyperopt_file, - load_backtest_data, load_trades, load_trades_from_db) + load_backtest_data, load_backtest_metadata, load_trades, + load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history from freqtrade.exceptions import OperationalException from tests.conftest import create_mock_trades @@ -40,7 +41,7 @@ def test_get_latest_backtest_filename(testdatadir, mocker): get_latest_backtest_filename(testdatadir) -def test_get_latest_hyperopt_file(testdatadir, mocker): +def test_get_latest_hyperopt_file(testdatadir): res = get_latest_hyperopt_file(testdatadir / 'does_not_exist', 'testfile.pickle') assert res == testdatadir / 'does_not_exist/testfile.pickle' @@ -51,6 +52,17 @@ def test_get_latest_hyperopt_file(testdatadir, mocker): assert res == testdatadir.parent / "hyperopt_results.pickle" +def test_load_backtest_metadata(mocker, testdatadir): + res = load_backtest_metadata(testdatadir / 'nonexistant.file.json') + assert res == {} + + mocker.patch('freqtrade.data.btanalysis.get_backtest_metadata_filename') + mocker.patch('freqtrade.data.btanalysis.json_load', side_effect=Exception()) + with pytest.raises(OperationalException, + match=r"Unexpected error.*loading backtest metadata\."): + load_backtest_metadata(testdatadir / 'nonexistant.file.json') + + def test_load_backtest_data_old_format(testdatadir, mocker): filename = testdatadir / "backtest-result_test222.json"