From 8405ccc15eff2de0eb9183dd8d5243e60552559c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 Aug 2021 15:34:43 +0200 Subject: [PATCH] Seperate detail data loading from regular backest-data loading --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 2 +- freqtrade/configuration/configuration.py | 4 +-- freqtrade/optimize/backtesting.py | 39 ++++++++++++++---------- freqtrade/optimize/optimize_reports.py | 1 + freqtrade/rpc/api_server/api_backtest.py | 3 ++ freqtrade/rpc/api_server/api_schemas.py | 1 + 7 files changed, 32 insertions(+), 20 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index a10ea5568..899998310 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -22,7 +22,7 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", "max_open_trades", "stake_amount", "fee", "pairs"] ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", - "enable_protections", "dry_run_wallet", "detail_timeframe", + "enable_protections", "dry_run_wallet", "timeframe_detail", "strategy_list", "export", "exportfilename"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index a168a44cf..8f10bbd0a 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -135,7 +135,7 @@ AVAILABLE_CLI_OPTIONS = { help='Override the value of the `stake_amount` configuration setting.', ), # Backtesting - "detail_timeframe": Arg( + "timeframe_detail": Arg( '--timeframe-detail', help='Specify detail timeframe for backtesting (`1m`, `5m`, `30m`, `1h`, `1d`).', ), diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 1d95dfb03..b3dfebe86 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -242,9 +242,9 @@ class Configuration: except ValueError: pass - self._args_to_config(config, argname='detail_timeframe', + self._args_to_config(config, argname='timeframe_detail', logstring='Parameter --detail-timeframe detected, ' - 'using {} for intra-candle backtesting') + 'using {} for intra-candle backtesting ...') self._args_to_config(config, argname='stake_amount', logstring='Parameter --stake-amount detected, ' 'overriding stake_amount to: {} ...') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2106f76c6..156ff48be 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -87,7 +87,7 @@ class Backtesting: self.timeframe = str(self.config.get('timeframe')) self.timeframe_min = timeframe_to_minutes(self.timeframe) # Load detail timeframe if specified - self.timeframe_detail = str(self.config.get('detail_timeframe', '')) + self.timeframe_detail = str(self.config.get('timeframe_detail', '')) if self.timeframe_detail: self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail) if self.timeframe_min <= self.timeframe_detail_min: @@ -96,6 +96,7 @@ class Backtesting: else: self.timeframe_detail_min = 0 + self.detail_data: Dict[str, DataFrame] = {} self.pairlists = PairListManager(self.exchange, self.config) if 'VolumePairList' in self.pairlists.name_list: @@ -168,7 +169,7 @@ class Backtesting: conf['protections'] = strategy.protections self.protections = ProtectionManager(self.config, strategy.protections) - def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange, Dict[str, DataFrame]]: + def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]: """ Loads backtest data and returns the data combined with the timerange as tuple. @@ -184,18 +185,6 @@ class Backtesting: fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), ) - if self.timeframe_detail: - detail_data = history.load_data( - datadir=self.config['datadir'], - pairs=self.pairlists.whitelist, - timeframe=self.timeframe_detail, - timerange=self.timerange, - startup_candles=0, - fail_without_data=True, - data_format=self.config.get('dataformat_ohlcv', 'json'), - ) - else: - detail_data = None min_date, max_date = history.get_timerange(data) @@ -208,7 +197,24 @@ class Backtesting: self.required_startup, min_date) self.progress.set_new_value(1) - return data, self.timerange, detail_data + return data, self.timerange + + def load_bt_data_detail(self) -> None: + """ + Loads backtest detail data (smaller timeframe) if necessary. + """ + if self.timeframe_detail: + self.detail_data = history.load_data( + datadir=self.config['datadir'], + pairs=self.pairlists.whitelist, + timeframe=self.timeframe_detail, + timerange=self.timerange, + startup_candles=0, + fail_without_data=True, + data_format=self.config.get('dataformat_ohlcv', 'json'), + ) + else: + self.detail_data = {} def prepare_backtest(self, enable_protections): """ @@ -637,7 +643,8 @@ class Backtesting: """ data: Dict[str, Any] = {} - data, timerange, self.detail_data = self.load_bt_data() + data, timerange = self.load_bt_data() + self.load_bt_data_detail() logger.info("Dataload complete. Calculating indicators") for strat in self.strategylist: diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 7bb60228a..8bde48670 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -368,6 +368,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame], 'max_open_trades_setting': (config['max_open_trades'] if config['max_open_trades'] != float('inf') else -1), 'timeframe': config['timeframe'], + 'timeframe_detail': config.get('timeframe_detail', ''), 'timerange': config.get('timerange', ''), 'enable_protections': config.get('enable_protections', False), 'strategy_name': strategy, diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 2fa66645b..4623c187e 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -46,11 +46,14 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac if ( not ApiServer._bt or lastconfig.get('timeframe') != strat.timeframe + or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail') or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0) or lastconfig.get('timerange') != btconfig['timerange'] ): from freqtrade.optimize.backtesting import Backtesting ApiServer._bt = Backtesting(btconfig) + if ApiServer._bt.timeframe_detail: + ApiServer._bt.load_bt_data_detail() # Only reload data if timeframe changed. if ( diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 318762136..3adbebc16 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -324,6 +324,7 @@ class PairHistory(BaseModel): class BacktestRequest(BaseModel): strategy: str timeframe: Optional[str] + timeframe_detail: Optional[str] timerange: Optional[str] max_open_trades: Optional[int] stake_amount: Optional[Union[float, str]]