From fbd142844fff0065bdac6a9e8770990faf289ab3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Apr 2022 19:37:13 +0200 Subject: [PATCH] Refactor bt-caching stuff to it's own module --- freqtrade/data/btanalysis.py | 3 +- freqtrade/misc.py | 35 +----------------------- freqtrade/optimize/backtest_caching.py | 38 ++++++++++++++++++++++++++ freqtrade/optimize/backtesting.py | 2 +- freqtrade/optimize/optimize_reports.py | 4 +-- tests/optimize/test_backtesting.py | 2 +- 6 files changed, 45 insertions(+), 39 deletions(-) create mode 100644 freqtrade/optimize/backtest_caching.py diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 206a6f5f3..0c8e721c0 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -12,7 +12,8 @@ import pandas as pd from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.exceptions import OperationalException -from freqtrade.misc import get_backtest_metadata_filename, json_load +from freqtrade.misc import json_load +from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename from freqtrade.persistence import LocalTrade, Trade, init_db diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 55a533725..c3968e61c 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -2,13 +2,11 @@ Various tool function for Freqtrade and scripts """ import gzip -import hashlib import logging import re -from copy import deepcopy from datetime import datetime from pathlib import Path -from typing import Any, Iterator, List, Union +from typing import Any, Iterator, List from typing.io import IO from urllib.parse import urlparse @@ -251,34 +249,3 @@ def parse_db_uri_for_logging(uri: str): return uri pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0] return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@') - - -def get_strategy_run_id(strategy) -> str: - """ - Generate unique identification hash for a backtest run. Identical config and strategy file will - always return an identical hash. - :param strategy: strategy object. - :return: hex string id. - """ - digest = hashlib.sha1() - config = deepcopy(strategy.config) - - # Options that have no impact on results of individual backtest. - not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server') - for k in not_important_keys: - if k in config: - del config[k] - - # Explicitly allow NaN values (e.g. max_open_trades). - # as it does not matter for getting the hash. - digest.update(rapidjson.dumps(config, default=str, - number_mode=rapidjson.NM_NAN).encode('utf-8')) - with open(strategy.__file__, 'rb') as fp: - digest.update(fp.read()) - return digest.hexdigest().lower() - - -def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path: - """Return metadata filename for specified backtest results file.""" - filename = Path(filename) - return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}') diff --git a/freqtrade/optimize/backtest_caching.py b/freqtrade/optimize/backtest_caching.py new file mode 100644 index 000000000..c2a9903fa --- /dev/null +++ b/freqtrade/optimize/backtest_caching.py @@ -0,0 +1,38 @@ +import hashlib +from copy import deepcopy +from pathlib import Path +from typing import Union + +import rapidjson + + +def get_strategy_run_id(strategy) -> str: + """ + Generate unique identification hash for a backtest run. Identical config and strategy file will + always return an identical hash. + :param strategy: strategy object. + :return: hex string id. + """ + digest = hashlib.sha1() + config = deepcopy(strategy.config) + + # Options that have no impact on results of individual backtest. + not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server') + for k in not_important_keys: + if k in config: + del config[k] + + # Explicitly allow NaN values (e.g. max_open_trades). + # as it does not matter for getting the hash. + digest.update(rapidjson.dumps(config, default=str, + number_mode=rapidjson.NM_NAN).encode('utf-8')) + + with open(strategy.__file__, 'rb') as fp: + digest.update(fp.read()) + return digest.hexdigest().lower() + + +def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path: + """Return metadata filename for specified backtest results file.""" + filename = Path(filename) + return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0f816f295..db4496f0f 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -24,8 +24,8 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType TradingMode) from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds -from freqtrade.misc import get_strategy_run_id from freqtrade.mixins import LoggingMixin +from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, store_backtest_signal_candles, diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index dd058aff4..1d58dc339 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -11,8 +11,8 @@ from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT from freqtrade.data.btanalysis import (calculate_cagr, calculate_csum, calculate_market_change, calculate_max_drawdown) -from freqtrade.misc import (decimals_per_coin, file_dump_joblib, file_dump_json, - get_backtest_metadata_filename, round_coin_value) +from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value +from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename logger = logging.getLogger(__name__) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index a51e1b654..7494155b4 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -22,7 +22,7 @@ from freqtrade.data.history import get_timerange from freqtrade.enums import ExitType, RunMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange.exchange import timeframe_to_next_date -from freqtrade.misc import get_strategy_run_id +from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import LocalTrade from freqtrade.resolvers import StrategyResolver