make docs reflect reality, move download_all_data to new utils.py file, automatic startup_candle detection

freqtrade/freqai/data_kitchen.py

@@ -16,12 +16,8 @@ from sklearn.model_selection import train_test_split
 from sklearn.neighbors import NearestNeighbors
 
 from freqtrade.configuration import TimeRange
-from freqtrade.data.dataprovider import DataProvider
-from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_seconds
-from freqtrade.exchange.exchange import market_is_active
-from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.strategy.interface import IStrategy
 
 
@@ -1002,53 +998,3 @@ class FreqaiDataKitchen:
         if self.unique_classes:
             for label in self.unique_classes:
                 self.unique_class_list += list(self.unique_classes[label])
-
-# Methods called by interface.py (load_freqai_model())
-
-
-def download_all_data_for_training(dp: DataProvider, config: dict) -> None:
-    """
-    Called only once upon start of bot to download the necessary data for
-    populating indicators and training the model.
-    :param timerange: TimeRange = The full data timerange for populating the indicators
-                                    and training the model.
-    :param dp: DataProvider instance attached to the strategy
-    """
-
-    if dp._exchange is not None:
-        markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m)
-                   or config.get('include_inactive')]
-    else:
-        # This should not occur:
-        raise OperationalException('No exchange object found.')
-
-    all_pairs = dynamic_expand_pairlist(config, markets)
-
-    if not dp._exchange:
-        # Not realistic - this is only called in live mode.
-        raise OperationalException("Dataprovider did not have an exchange attached.")
-
-    time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp()
-
-    for tf in config["freqai"]["feature_parameters"].get("include_timeframes"):
-        timerange = TimeRange()
-        timerange.startts = int(time)
-        timerange.stopts = int(time)
-        startup_candles = dp.get_required_startup(str(tf))
-        tf_seconds = timeframe_to_seconds(str(tf))
-        timerange.subtract_start(tf_seconds * startup_candles)
-        new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY)
-        # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function
-        # redownloads the funding rate for each pair.
-        refresh_backtest_ohlcv_data(
-            dp._exchange,
-            pairs=all_pairs,
-            timeframes=[tf],
-            datadir=config["datadir"],
-            timerange=timerange,
-            new_pairs_days=new_pairs_days,
-            erase=False,
-            data_format=config.get("dataformat_ohlcv", "json"),
-            trading_mode=config.get("trading_mode", "spot"),
-            prepend=config.get("prepend_data", False),
-        )