deprecate indicator_max_period_candles, automatically compute startup candles for FreqAI backtesting.

freqtrade/data/dataprovider.py

@@ -92,7 +92,7 @@ class DataProvider:
                 'timerange') is None else str(self._config.get('timerange')))
             # Move informative start time respecting startup_candle_count
             timerange.subtract_start(
-                timeframe_to_seconds(str(timeframe)) * self._config.get('startup_candle_count', 0)
+                self.get_required_startup_seconds(str(timeframe))
             )
             self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
                 pair=pair,
@@ -105,6 +105,17 @@ class DataProvider:
             )
         return self.__cached_pairs_backtesting[saved_pair].copy()
 
+    def get_required_startup_seconds(self, timeframe: str) -> int:
+        tf_seconds = timeframe_to_seconds(timeframe)
+        base_seconds = tf_seconds * self._config.get('startup_candle_count', 0)
+        if not self._config['freqai']['enabled']:
+            return base_seconds
+        else:
+            train_seconds = self._config['freqai']['train_period_days'] * 86400
+            # multiplied by safety factor of 2 because FreqAI users
+            # typically do not know the correct window.
+            return base_seconds * 2 + int(train_seconds)
+
     def get_pair_dataframe(
         self,
         pair: str,

freqtrade/freqai/data_kitchen.py

@@ -20,6 +20,8 @@ from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_seconds
+from freqtrade.exchange.exchange import market_is_active
+from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.strategy.interface import IStrategy
 
 
@@ -834,9 +836,7 @@ class FreqaiDataKitchen:
         # We notice that users like to use exotic indicators where
         # they do not know the required timeperiod. Here we include a factor
         # of safety by multiplying the user considered "max" by 2.
-        max_period = self.freqai_config["feature_parameters"].get(
-            "indicator_max_period_candles", 20
-        ) * 2
+        max_period = self.config.get('startup_candle_count', 20) * 2
         additional_seconds = max_period * max_tf_seconds
 
         if trained_timestamp != 0:
@@ -1015,12 +1015,15 @@ def download_all_data_for_training(timerange: TimeRange,
                                     and training the model.
     :param dp: DataProvider instance attached to the strategy
     """
-    all_pairs = copy.deepcopy(
-        config["freqai"]["feature_parameters"].get("include_corr_pairlist", [])
-    )
-    for pair in config.get("exchange", "").get("pair_whitelist"):
-        if pair not in all_pairs:
-            all_pairs.append(pair)
+
+    if dp._exchange is not None:
+        markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m)
+                   or config.get('include_inactive')]
+    else:
+        # This should not occur:
+        raise OperationalException('No exchange object found.')
+
+    all_pairs = dynamic_expand_pairlist(config, markets)
 
     new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY)
     if not dp._exchange:
@@ -1048,7 +1051,6 @@ def get_required_data_timerange(
     user.
     """
     time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp()
-    trained_timerange = TimeRange()
     data_load_timerange = TimeRange()
 
     timeframes = config["freqai"]["feature_parameters"].get("include_timeframes")
@@ -1059,15 +1061,9 @@ def get_required_data_timerange(
         if secs > max_tf_seconds:
             max_tf_seconds = secs
 
-    max_period = config["freqai"]["feature_parameters"].get(
-        "indicator_max_period_candles", 20
-    ) * 2
-    additional_seconds = max_period * max_tf_seconds
+    max_period = config.get('startup_candle_count', 20) * 2
 
-    trained_timerange.startts = int(
-        time - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY
-    )
-    trained_timerange.stopts = int(time)
+    additional_seconds = max_period * max_tf_seconds
 
     data_load_timerange.startts = int(
         time

freqtrade/optimize/backtesting.py

@@ -211,21 +211,21 @@ class Backtesting:
         """
         self.progress.init_step(BacktestState.DATALOAD, 1)
 
-        if self.config.get('freqai', {}).get('enabled', False):
-            startup_candles = int(self.config.get('freqai', {}).get('startup_candles', 0))
-            if not startup_candles:
-                raise OperationalException('FreqAI backtesting module requires user set '
-                                           'startup_candles in config.')
-            self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 0))
-            logger.info(f'Increasing startup_candle_count for freqai to {self.required_startup}')
-            self.config['startup_candle_count'] = self.required_startup
+        # if self.config.get('freqai', {}).get('enabled', False):
+        #     startup_candles = int(self.config.get('freqai', {}).get('startup_candles', 0))
+        #     if not startup_candles:
+        #         raise OperationalException('FreqAI backtesting module requires user set '
+        #                                    'startup_candles in config.')
+        #     self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 0))
+        #     logger.info(f'Increasing startup_candle_count for freqai to {self.required_startup}')
+        #     self.config['startup_candle_count'] = self.required_startup
 
         data = history.load_data(
             datadir=self.config['datadir'],
             pairs=self.pairlists.whitelist,
             timeframe=self.timeframe,
             timerange=self.timerange,
-            startup_candles=self.required_startup,
+            startup_candles=self.get_required_startup(self.timeframe),
             fail_without_data=True,
             data_format=self.config.get('dataformat_ohlcv', 'json'),
             candle_type=self.config.get('candle_type_def', CandleType.SPOT)
@@ -244,6 +244,21 @@ class Backtesting:
         self.progress.set_new_value(1)
         return data, self.timerange
 
+    def get_required_startup(self, timeframe: str) -> int:
+        if not self.config['freqai']['enabled']:
+            return self.required_startup
+        else:
+            if not self.config['startup_candle_count']:
+                raise OperationalException('FreqAI backtesting module requires strategy '
+                                           'set startup_candle_count.')
+            tf_seconds = timeframe_to_seconds(timeframe)
+            train_candles = self.config['freqai']['train_period_days'] * 86400 / tf_seconds
+            # multiplied by safety factor of 2 because FreqAI users
+            # typically do not know the correct window.
+            total_candles = self.required_startup * 2 + train_candles
+            logger.info(f'Increasing startup_candle_count for freqai to {total_candles}')
+            return total_candles
+
     def load_bt_data_detail(self) -> None:
         """
         Loads backtest detail data (smaller timeframe) if necessary.

freqtrade/strategy/interface.py

@@ -163,6 +163,7 @@ class IStrategy(ABC, HyperStrategyMixin):
                 )
                 data_load_timerange = get_required_data_timerange(self.config)
                 download_all_data_for_training(data_load_timerange, self.dp, self.config)
+
         else:
             # Gracious failures if freqAI is disabled but "start" is called.
             class DummyClass():

freqtrade/templates/FreqaiExampleStrategy.py

@@ -43,7 +43,8 @@ class FreqaiExampleStrategy(IStrategy):
     process_only_new_candles = True
     stoploss = -0.05
     use_exit_signal = True
-    startup_candle_count: int = 300
+    # this is the maximum period fed to talib (timeframe independent)
+    startup_candle_count: int = 20
     can_short = False
 
     linear_roi_offset = DecimalParameter(