reduce code duplication, optimize auto data download per tf

freqtrade/data/dataprovider.py

@@ -91,9 +91,9 @@ class DataProvider:
             timerange = TimeRange.parse_timerange(None if self._config.get(
                 'timerange') is None else str(self._config.get('timerange')))
             # Move informative start time respecting startup_candle_count
-            timerange.subtract_start(
+            startup_candles = self.get_required_startup(str(timeframe))
-                self.get_required_startup_seconds(str(timeframe))
+            tf_seconds = timeframe_to_seconds(str(timeframe))
-            )
+            timerange.subtract_start(tf_seconds * startup_candles)
             self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
                 pair=pair,
                 timeframe=timeframe or self._config['timeframe'],
@@ -105,16 +105,18 @@ class DataProvider:
             )
         return self.__cached_pairs_backtesting[saved_pair].copy()
 
-    def get_required_startup_seconds(self, timeframe: str) -> int:
+    def get_required_startup(self, timeframe: str) -> int:
-        tf_seconds = timeframe_to_seconds(timeframe)
+        if not self._config.get('freqai', {}).get('enabled', False):
-        base_seconds = tf_seconds * self._config.get('startup_candle_count', 0)
+            return self._config.get('startup_candle_count', 0)
-        if not self._config['freqai']['enabled']:
-            return base_seconds
         else:
-            train_seconds = self._config['freqai']['train_period_days'] * 86400
+            if not self._config['startup_candle_count']:
-            # multiplied by safety factor of 2 because FreqAI users
+                raise OperationalException('FreqAI backtesting module requires strategy '
-            # typically do not know the correct window.
+                                           'set startup_candle_count.')
-            return base_seconds * 2 + int(train_seconds)
+            tf_seconds = timeframe_to_seconds(timeframe)
+            train_candles = self._config['freqai']['train_period_days'] * 86400 / tf_seconds
+            total_candles = int(self._config.get('startup_candle_count', 0) + train_candles)
+            logger.info(f'Increasing startup_candle_count for freqai to {total_candles}')
+            return total_candles
 
     def get_pair_dataframe(
         self,

freqtrade/freqai/data_kitchen.py

@@ -1006,8 +1006,7 @@ class FreqaiDataKitchen:
 # Methods called by interface.py (load_freqai_model())
 
 
-def download_all_data_for_training(timerange: TimeRange,
+def download_all_data_for_training(dp: DataProvider, config: dict) -> None:
-                                   dp: DataProvider, config: dict) -> None:
     """
     Called only once upon start of bot to download the necessary data for
     populating indicators and training the model.
@@ -1025,14 +1024,26 @@ def download_all_data_for_training(timerange: TimeRange,
 
     all_pairs = dynamic_expand_pairlist(config, markets)
 
-    new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY)
     if not dp._exchange:
         # Not realistic - this is only called in live mode.
         raise OperationalException("Dataprovider did not have an exchange attached.")
+
+    time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp()
+
+    for tf in config["freqai"]["feature_parameters"].get("include_timeframes"):
+        timerange = TimeRange()
+        timerange.startts = int(time)
+        timerange.stopts = int(time)
+        startup_candles = dp.get_required_startup(str(tf))
+        tf_seconds = timeframe_to_seconds(str(tf))
+        timerange.subtract_start(tf_seconds * startup_candles)
+        new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY)
+        # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function
+        # redownloads the funding rate for each pair.
         refresh_backtest_ohlcv_data(
             dp._exchange,
             pairs=all_pairs,
-        timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"),
+            timeframes=[tf],
             datadir=config["datadir"],
             timerange=timerange,
             new_pairs_days=new_pairs_days,
@@ -1041,35 +1052,3 @@ def download_all_data_for_training(timerange: TimeRange,
             trading_mode=config.get("trading_mode", "spot"),
             prepend=config.get("prepend_data", False),
         )
-
-
-def get_required_data_timerange(
-    config: dict
-) -> TimeRange:
-    """
-    Used by interface.py to pre-download necessary data for FreqAI
-    user.
-    """
-    time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp()
-    data_load_timerange = TimeRange()
-
-    timeframes = config["freqai"]["feature_parameters"].get("include_timeframes")
-
-    max_tf_seconds = 0
-    for tf in timeframes:
-        secs = timeframe_to_seconds(tf)
-        if secs > max_tf_seconds:
-            max_tf_seconds = secs
-
-    max_period = config.get('startup_candle_count', 20) * 2
-
-    additional_seconds = max_period * max_tf_seconds
-
-    data_load_timerange.startts = int(
-        time
-        - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY
-        - additional_seconds
-    )
-    data_load_timerange.stopts = int(time)
-
-    return data_load_timerange

freqtrade/optimize/backtesting.py

@@ -211,21 +211,12 @@ class Backtesting:
         """
         self.progress.init_step(BacktestState.DATALOAD, 1)
 
-        # if self.config.get('freqai', {}).get('enabled', False):
-        #     startup_candles = int(self.config.get('freqai', {}).get('startup_candles', 0))
-        #     if not startup_candles:
-        #         raise OperationalException('FreqAI backtesting module requires user set '
-        #                                    'startup_candles in config.')
-        #     self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 0))
-        #     logger.info(f'Increasing startup_candle_count for freqai to {self.required_startup}')
-        #     self.config['startup_candle_count'] = self.required_startup
-
         data = history.load_data(
             datadir=self.config['datadir'],
             pairs=self.pairlists.whitelist,
             timeframe=self.timeframe,
             timerange=self.timerange,
-            startup_candles=self.get_required_startup(self.timeframe),
+            startup_candles=self.dataprovider.get_required_startup(self.timeframe),
             fail_without_data=True,
             data_format=self.config.get('dataformat_ohlcv', 'json'),
             candle_type=self.config.get('candle_type_def', CandleType.SPOT)
@@ -244,21 +235,6 @@ class Backtesting:
         self.progress.set_new_value(1)
         return data, self.timerange
 
-    def get_required_startup(self, timeframe: str) -> int:
-        if not self.config['freqai']['enabled']:
-            return self.required_startup
-        else:
-            if not self.config['startup_candle_count']:
-                raise OperationalException('FreqAI backtesting module requires strategy '
-                                           'set startup_candle_count.')
-            tf_seconds = timeframe_to_seconds(timeframe)
-            train_candles = self.config['freqai']['train_period_days'] * 86400 / tf_seconds
-            # multiplied by safety factor of 2 because FreqAI users
-            # typically do not know the correct window.
-            total_candles = self.required_startup * 2 + train_candles
-            logger.info(f'Increasing startup_candle_count for freqai to {total_candles}')
-            return total_candles
-
     def load_bt_data_detail(self) -> None:
         """
         Loads backtest detail data (smaller timeframe) if necessary.

freqtrade/strategy/interface.py

@@ -148,8 +148,7 @@ class IStrategy(ABC, HyperStrategyMixin):
     def load_freqAI_model(self) -> None:
         if self.config.get('freqai', {}).get('enabled', False):
             # Import here to avoid importing this if freqAI is disabled
-            from freqtrade.freqai.data_kitchen import (download_all_data_for_training,
+            from freqtrade.freqai.data_kitchen import (download_all_data_for_training)
-                                                       get_required_data_timerange)
             from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
             self.freqai = FreqaiModelResolver.load_freqaimodel(self.config)
             self.freqai_info = self.config["freqai"]
@@ -161,8 +160,8 @@ class IStrategy(ABC, HyperStrategyMixin):
                     "corr_pairlist, this may take a while if you do not have the "
                     "data saved"
                 )
-                data_load_timerange = get_required_data_timerange(self.config)
+                # data_load_timerange = get_required_data_timerange(self.config)
-                download_all_data_for_training(data_load_timerange, self.dp, self.config)
+                download_all_data_for_training(self.dp, self.config)
 
         else:
             # Gracious failures if freqAI is disabled but "start" is called.