diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 529a12690..a21114901 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -91,9 +91,9 @@ class DataProvider: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) # Move informative start time respecting startup_candle_count - timerange.subtract_start( - self.get_required_startup_seconds(str(timeframe)) - ) + startup_candles = self.get_required_startup(str(timeframe)) + tf_seconds = timeframe_to_seconds(str(timeframe)) + timerange.subtract_start(tf_seconds * startup_candles) self.__cached_pairs_backtesting[saved_pair] = load_pair_history( pair=pair, timeframe=timeframe or self._config['timeframe'], @@ -105,16 +105,18 @@ class DataProvider: ) return self.__cached_pairs_backtesting[saved_pair].copy() - def get_required_startup_seconds(self, timeframe: str) -> int: - tf_seconds = timeframe_to_seconds(timeframe) - base_seconds = tf_seconds * self._config.get('startup_candle_count', 0) - if not self._config['freqai']['enabled']: - return base_seconds + def get_required_startup(self, timeframe: str) -> int: + if not self._config.get('freqai', {}).get('enabled', False): + return self._config.get('startup_candle_count', 0) else: - train_seconds = self._config['freqai']['train_period_days'] * 86400 - # multiplied by safety factor of 2 because FreqAI users - # typically do not know the correct window. - return base_seconds * 2 + int(train_seconds) + if not self._config['startup_candle_count']: + raise OperationalException('FreqAI backtesting module requires strategy ' + 'set startup_candle_count.') + tf_seconds = timeframe_to_seconds(timeframe) + train_candles = self._config['freqai']['train_period_days'] * 86400 / tf_seconds + total_candles = int(self._config.get('startup_candle_count', 0) + train_candles) + logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') + return total_candles def get_pair_dataframe( self, diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index c768fc30e..1a8063add 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1006,8 +1006,7 @@ class FreqaiDataKitchen: # Methods called by interface.py (load_freqai_model()) -def download_all_data_for_training(timerange: TimeRange, - dp: DataProvider, config: dict) -> None: +def download_all_data_for_training(dp: DataProvider, config: dict) -> None: """ Called only once upon start of bot to download the necessary data for populating indicators and training the model. @@ -1025,51 +1024,31 @@ def download_all_data_for_training(timerange: TimeRange, all_pairs = dynamic_expand_pairlist(config, markets) - new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) if not dp._exchange: # Not realistic - this is only called in live mode. raise OperationalException("Dataprovider did not have an exchange attached.") - refresh_backtest_ohlcv_data( - dp._exchange, - pairs=all_pairs, - timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"), - datadir=config["datadir"], - timerange=timerange, - new_pairs_days=new_pairs_days, - erase=False, - data_format=config.get("dataformat_ohlcv", "json"), - trading_mode=config.get("trading_mode", "spot"), - prepend=config.get("prepend_data", False), - ) - -def get_required_data_timerange( - config: dict -) -> TimeRange: - """ - Used by interface.py to pre-download necessary data for FreqAI - user. - """ time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() - data_load_timerange = TimeRange() - timeframes = config["freqai"]["feature_parameters"].get("include_timeframes") - - max_tf_seconds = 0 - for tf in timeframes: - secs = timeframe_to_seconds(tf) - if secs > max_tf_seconds: - max_tf_seconds = secs - - max_period = config.get('startup_candle_count', 20) * 2 - - additional_seconds = max_period * max_tf_seconds - - data_load_timerange.startts = int( - time - - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY - - additional_seconds - ) - data_load_timerange.stopts = int(time) - - return data_load_timerange + for tf in config["freqai"]["feature_parameters"].get("include_timeframes"): + timerange = TimeRange() + timerange.startts = int(time) + timerange.stopts = int(time) + startup_candles = dp.get_required_startup(str(tf)) + tf_seconds = timeframe_to_seconds(str(tf)) + timerange.subtract_start(tf_seconds * startup_candles) + new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) + # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function + # redownloads the funding rate for each pair. + refresh_backtest_ohlcv_data( + dp._exchange, + pairs=all_pairs, + timeframes=[tf], + datadir=config["datadir"], + timerange=timerange, + new_pairs_days=new_pairs_days, + erase=False, + data_format=config.get("dataformat_ohlcv", "json"), + trading_mode=config.get("trading_mode", "spot"), + prepend=config.get("prepend_data", False), + ) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8f0302ada..3d715c82d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -211,21 +211,12 @@ class Backtesting: """ self.progress.init_step(BacktestState.DATALOAD, 1) - # if self.config.get('freqai', {}).get('enabled', False): - # startup_candles = int(self.config.get('freqai', {}).get('startup_candles', 0)) - # if not startup_candles: - # raise OperationalException('FreqAI backtesting module requires user set ' - # 'startup_candles in config.') - # self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 0)) - # logger.info(f'Increasing startup_candle_count for freqai to {self.required_startup}') - # self.config['startup_candle_count'] = self.required_startup - data = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, timeframe=self.timeframe, timerange=self.timerange, - startup_candles=self.get_required_startup(self.timeframe), + startup_candles=self.dataprovider.get_required_startup(self.timeframe), fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), candle_type=self.config.get('candle_type_def', CandleType.SPOT) @@ -244,21 +235,6 @@ class Backtesting: self.progress.set_new_value(1) return data, self.timerange - def get_required_startup(self, timeframe: str) -> int: - if not self.config['freqai']['enabled']: - return self.required_startup - else: - if not self.config['startup_candle_count']: - raise OperationalException('FreqAI backtesting module requires strategy ' - 'set startup_candle_count.') - tf_seconds = timeframe_to_seconds(timeframe) - train_candles = self.config['freqai']['train_period_days'] * 86400 / tf_seconds - # multiplied by safety factor of 2 because FreqAI users - # typically do not know the correct window. - total_candles = self.required_startup * 2 + train_candles - logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') - return total_candles - def load_bt_data_detail(self) -> None: """ Loads backtest detail data (smaller timeframe) if necessary. diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 284727d2b..9124a0427 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -148,8 +148,7 @@ class IStrategy(ABC, HyperStrategyMixin): def load_freqAI_model(self) -> None: if self.config.get('freqai', {}).get('enabled', False): # Import here to avoid importing this if freqAI is disabled - from freqtrade.freqai.data_kitchen import (download_all_data_for_training, - get_required_data_timerange) + from freqtrade.freqai.data_kitchen import (download_all_data_for_training) from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) self.freqai_info = self.config["freqai"] @@ -161,8 +160,8 @@ class IStrategy(ABC, HyperStrategyMixin): "corr_pairlist, this may take a while if you do not have the " "data saved" ) - data_load_timerange = get_required_data_timerange(self.config) - download_all_data_for_training(data_load_timerange, self.dp, self.config) + # data_load_timerange = get_required_data_timerange(self.config) + download_all_data_for_training(self.dp, self.config) else: # Gracious failures if freqAI is disabled but "start" is called.