download data homogeneously across timeframes

freqtrade/freqai/utils.py

@@ -1,17 +1,22 @@
-from freqtrade.data.dataprovider import DataProvider
+import logging
-from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
-from freqtrade.exchange.exchange import market_is_active
-from freqtrade.exchange import timeframe_to_seconds
-from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
 from datetime import datetime, timezone
-from freqtrade.exceptions import OperationalException
+
 from freqtrade.configuration import TimeRange
+from freqtrade.data.dataprovider import DataProvider
+from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
+from freqtrade.exceptions import OperationalException
+from freqtrade.exchange import timeframe_to_seconds
+from freqtrade.exchange.exchange import market_is_active
+from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
+
+
+logger = logging.getLogger(__name__)
 
 
 def download_all_data_for_training(dp: DataProvider, config: dict) -> None:
     """
     Called only once upon start of bot to download the necessary data for
-    populating indicators and training a FreqAI model.
+    populating indicators and training the model.
     :param timerange: TimeRange = The full data timerange for populating the indicators
                                     and training the model.
     :param dp: DataProvider instance attached to the strategy
@@ -26,31 +31,108 @@ def download_all_data_for_training(dp: DataProvider, config: dict) -> None:
 
     all_pairs = dynamic_expand_pairlist(config, markets)
 
+    timerange = get_required_data_timerange(config)
+
+    new_pairs_days = int((timerange.stopts - timerange.startts) / 86400)
     if not dp._exchange:
         # Not realistic - this is only called in live mode.
         raise OperationalException("Dataprovider did not have an exchange attached.")
+    refresh_backtest_ohlcv_data(
+        dp._exchange,
+        pairs=all_pairs,
+        timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"),
+        datadir=config["datadir"],
+        timerange=timerange,
+        new_pairs_days=new_pairs_days,
+        erase=False,
+        data_format=config.get("dataformat_ohlcv", "json"),
+        trading_mode=config.get("trading_mode", "spot"),
+        prepend=config.get("prepend_data", False),
+    )
 
+
+def get_required_data_timerange(
+    config: dict
+) -> TimeRange:
+    """
+    Used to compute the required data download time range
+    for auto data-download in FreqAI
+    """
     time = datetime.now(tz=timezone.utc).timestamp()
+    data_load_timerange = TimeRange()
 
-    for tf in config["freqai"]["feature_parameters"].get("include_timeframes"):
+    timeframes = config["freqai"]["feature_parameters"].get("include_timeframes")
-        timerange = TimeRange()
+
-        timerange.startts = int(time)
+    max_tf_seconds = 0
-        timerange.stopts = int(time)
+    for tf in timeframes:
-        startup_candles = dp.get_required_startup(str(tf))
+        secs = timeframe_to_seconds(tf)
-        tf_seconds = timeframe_to_seconds(str(tf))
+        if secs > max_tf_seconds:
-        timerange.subtract_start(tf_seconds * startup_candles)
+            max_tf_seconds = secs
-        new_pairs_days = int((timerange.stopts - timerange.startts) / 86400)
+
-        # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function
+    startup_candles = config.get('startup_candle_count', 0)
-        # redownloads the funding rate for each pair.
+    indicator_periods = config["freqai"]["feature_parameters"]["indicator_periods_candles"]
-        refresh_backtest_ohlcv_data(
+
-            dp._exchange,
+    # factor the max_period as a factor of safety.
-            pairs=all_pairs,
+    max_period = int(max(startup_candles, max(indicator_periods)) * 1.5)
-            timeframes=[tf],
+    config['startup_candle_count'] = max_period
-            datadir=config["datadir"],
+    logger.info(f'FreqAI auto-downloader using {max_period} startup candles.')
-            timerange=timerange,
+
-            new_pairs_days=new_pairs_days,
+    additional_seconds = max_period * max_tf_seconds
-            erase=False,
+
-            data_format=config.get("dataformat_ohlcv", "json"),
+    data_load_timerange.startts = int(
-            trading_mode=config.get("trading_mode", "spot"),
+        time
-            prepend=config.get("prepend_data", False),
+        - config["freqai"].get("train_period_days", 0) * 86400
-        )
+        - additional_seconds
+    )
+    data_load_timerange.stopts = int(time)
+
+    return data_load_timerange
+
+
+# Keep below for when we wish to download heterogeneously lengthed data for FreqAI.
+# def download_all_data_for_training(dp: DataProvider, config: dict) -> None:
+#     """
+#     Called only once upon start of bot to download the necessary data for
+#     populating indicators and training a FreqAI model.
+#     :param timerange: TimeRange = The full data timerange for populating the indicators
+#                                     and training the model.
+#     :param dp: DataProvider instance attached to the strategy
+#     """
+
+#     if dp._exchange is not None:
+#         markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m)
+#                    or config.get('include_inactive')]
+#     else:
+#         # This should not occur:
+#         raise OperationalException('No exchange object found.')
+
+#     all_pairs = dynamic_expand_pairlist(config, markets)
+
+#     if not dp._exchange:
+#         # Not realistic - this is only called in live mode.
+#         raise OperationalException("Dataprovider did not have an exchange attached.")
+
+#     time = datetime.now(tz=timezone.utc).timestamp()
+
+#     for tf in config["freqai"]["feature_parameters"].get("include_timeframes"):
+#         timerange = TimeRange()
+#         timerange.startts = int(time)
+#         timerange.stopts = int(time)
+#         startup_candles = dp.get_required_startup(str(tf))
+#         tf_seconds = timeframe_to_seconds(str(tf))
+#         timerange.subtract_start(tf_seconds * startup_candles)
+#         new_pairs_days = int((timerange.stopts - timerange.startts) / 86400)
+#         # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function
+#         # redownloads the funding rate for each pair.
+#         refresh_backtest_ohlcv_data(
+#             dp._exchange,
+#             pairs=all_pairs,
+#             timeframes=[tf],
+#             datadir=config["datadir"],
+#             timerange=timerange,
+#             new_pairs_days=new_pairs_days,
+#             erase=False,
+#             data_format=config.get("dataformat_ohlcv", "json"),
+#             trading_mode=config.get("trading_mode", "spot"),
+#             prepend=config.get("prepend_data", False),
+#         )

freqtrade/strategy/interface.py

@@ -157,12 +157,10 @@ class IStrategy(ABC, HyperStrategyMixin):
             if self.config.get('runmode') in (RunMode.DRY_RUN, RunMode.LIVE):
                 logger.info(
                     "Downloading all training data for all pairs in whitelist and "
-                    "corr_pairlist, this may take a while if you do not have the "
+                    "corr_pairlist, this may take a while if the data is not "
-                    "data saved"
+                    "already on disk."
                 )
-                # data_load_timerange = get_required_data_timerange(self.config)
                 download_all_data_for_training(self.dp, self.config)
-
         else:
             # Gracious failures if freqAI is disabled but "start" is called.
             class DummyClass():