From e7261cf51577dc30b530370f81df620c898f6a11 Mon Sep 17 00:00:00 2001
From: robcaulk <rob.caulk@gmail.com>
Date: Fri, 26 Aug 2022 15:30:28 +0200
Subject: [PATCH] add freqai utils.py file

---
 freqtrade/freqai/utils.py | 56 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)
 create mode 100644 freqtrade/freqai/utils.py

diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py
new file mode 100644
index 000000000..056458115
--- /dev/null
+++ b/freqtrade/freqai/utils.py
@@ -0,0 +1,56 @@
+from freqtrade.data.dataprovider import DataProvider
+from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
+from freqtrade.exchange.exchange import market_is_active
+from freqtrade.exchange import timeframe_to_seconds
+from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
+from datetime import datetime, timezone
+from freqtrade.exceptions import OperationalException
+from freqtrade.configuration import TimeRange
+
+
+def download_all_data_for_training(dp: DataProvider, config: dict) -> None:
+    """
+    Called only once upon start of bot to download the necessary data for
+    populating indicators and training a FreqAI model.
+    :param timerange: TimeRange = The full data timerange for populating the indicators
+                                    and training the model.
+    :param dp: DataProvider instance attached to the strategy
+    """
+
+    if dp._exchange is not None:
+        markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m)
+                   or config.get('include_inactive')]
+    else:
+        # This should not occur:
+        raise OperationalException('No exchange object found.')
+
+    all_pairs = dynamic_expand_pairlist(config, markets)
+
+    if not dp._exchange:
+        # Not realistic - this is only called in live mode.
+        raise OperationalException("Dataprovider did not have an exchange attached.")
+
+    time = datetime.now(tz=timezone.utc).timestamp()
+
+    for tf in config["freqai"]["feature_parameters"].get("include_timeframes"):
+        timerange = TimeRange()
+        timerange.startts = int(time)
+        timerange.stopts = int(time)
+        startup_candles = dp.get_required_startup(str(tf))
+        tf_seconds = timeframe_to_seconds(str(tf))
+        timerange.subtract_start(tf_seconds * startup_candles)
+        new_pairs_days = int((timerange.stopts - timerange.startts) / 86400)
+        # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function
+        # redownloads the funding rate for each pair.
+        refresh_backtest_ohlcv_data(
+            dp._exchange,
+            pairs=all_pairs,
+            timeframes=[tf],
+            datadir=config["datadir"],
+            timerange=timerange,
+            new_pairs_days=new_pairs_days,
+            erase=False,
+            data_format=config.get("dataformat_ohlcv", "json"),
+            trading_mode=config.get("trading_mode", "spot"),
+            prepend=config.get("prepend_data", False),
+        )