reduce code duplication, optimize auto data download per tf

freqtrade/freqai/data_kitchen.py

@@ -1006,8 +1006,7 @@ class FreqaiDataKitchen:
 # Methods called by interface.py (load_freqai_model())
 
 
-def download_all_data_for_training(timerange: TimeRange,
-                                   dp: DataProvider, config: dict) -> None:
+def download_all_data_for_training(dp: DataProvider, config: dict) -> None:
     """
     Called only once upon start of bot to download the necessary data for
     populating indicators and training the model.
@@ -1025,51 +1024,31 @@ def download_all_data_for_training(timerange: TimeRange,
 
     all_pairs = dynamic_expand_pairlist(config, markets)
 
-    new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY)
     if not dp._exchange:
         # Not realistic - this is only called in live mode.
         raise OperationalException("Dataprovider did not have an exchange attached.")
-    refresh_backtest_ohlcv_data(
-        dp._exchange,
-        pairs=all_pairs,
-        timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"),
-        datadir=config["datadir"],
-        timerange=timerange,
-        new_pairs_days=new_pairs_days,
-        erase=False,
-        data_format=config.get("dataformat_ohlcv", "json"),
-        trading_mode=config.get("trading_mode", "spot"),
-        prepend=config.get("prepend_data", False),
-    )
 
-
-def get_required_data_timerange(
-    config: dict
-) -> TimeRange:
-    """
-    Used by interface.py to pre-download necessary data for FreqAI
-    user.
-    """
     time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp()
-    data_load_timerange = TimeRange()
 
-    timeframes = config["freqai"]["feature_parameters"].get("include_timeframes")
-
-    max_tf_seconds = 0
-    for tf in timeframes:
-        secs = timeframe_to_seconds(tf)
-        if secs > max_tf_seconds:
-            max_tf_seconds = secs
-
-    max_period = config.get('startup_candle_count', 20) * 2
-
-    additional_seconds = max_period * max_tf_seconds
-
-    data_load_timerange.startts = int(
-        time
-        - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY
-        - additional_seconds
-    )
-    data_load_timerange.stopts = int(time)
-
-    return data_load_timerange
+    for tf in config["freqai"]["feature_parameters"].get("include_timeframes"):
+        timerange = TimeRange()
+        timerange.startts = int(time)
+        timerange.stopts = int(time)
+        startup_candles = dp.get_required_startup(str(tf))
+        tf_seconds = timeframe_to_seconds(str(tf))
+        timerange.subtract_start(tf_seconds * startup_candles)
+        new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY)
+        # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function
+        # redownloads the funding rate for each pair.
+        refresh_backtest_ohlcv_data(
+            dp._exchange,
+            pairs=all_pairs,
+            timeframes=[tf],
+            datadir=config["datadir"],
+            timerange=timerange,
+            new_pairs_days=new_pairs_days,
+            erase=False,
+            data_format=config.get("dataformat_ohlcv", "json"),
+            trading_mode=config.get("trading_mode", "spot"),
+            prepend=config.get("prepend_data", False),
+        )