ensure spice_rack is backtestable. Ensure download-data knows about the spice_rack informative pair requirements

freqtrade/commands/data_commands.py

@@ -12,6 +12,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_oh
 from freqtrade.enums import CandleType, RunMode, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import market_is_active, timeframe_to_minutes
+from freqtrade.freqai.utils import setup_freqai_spice_rack
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
 
@@ -48,6 +49,10 @@ def start_download_data(args: Dict[str, Any]) -> None:
 
     # Init exchange
     exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
+
+    if config.get('freqai_spice_rack', False):
+        config = setup_freqai_spice_rack(config, exchange)
+
     markets = [p for p, m in exchange.markets.items() if market_is_active(m)
                or config.get('include_inactive')]
 

freqtrade/freqai/utils.py

@@ -1,5 +1,6 @@
 import logging
 from datetime import datetime, timezone
+from typing import Any, Dict, Optional
 
 import numpy as np
 # for spice rack
@@ -12,7 +13,7 @@ from freqtrade.configuration import TimeRange
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
 from freqtrade.exceptions import OperationalException
-from freqtrade.exchange import timeframe_to_seconds
+from freqtrade.exchange import Exchange, timeframe_to_seconds
 from freqtrade.exchange.exchange import market_is_active
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.strategy import merge_informative_pair
@@ -170,6 +171,60 @@ def auto_populate_any_indicators(
 
     return df
 
+
+def setup_freqai_spice_rack(config: dict, exchange: Optional[Exchange]) -> Dict[str, Any]:
+    import difflib
+    import json
+    from pathlib import Path
+    auto_config = config.get('freqai_config', 'lightgbm_config.json')
+    with open(Path(__file__).parent / Path('spice_rack') / auto_config) as json_file:
+        freqai_config = json.load(json_file)
+        config['freqai'] = freqai_config['freqai']
+        config['freqai']['identifier'] = config['freqai_identifier']
+        corr_pairs = config['freqai']['feature_parameters']['include_corr_pairlist']
+        timeframes = config['freqai']['feature_parameters']['include_timeframes']
+        new_corr_pairs = []
+        new_tfs = []
+
+        if not exchange:
+            logger.warning('No dataprovider available.')
+            config['freqai']['enabled'] = False
+            return config
+        # find the closest pairs to what the default config wants
+        for pair in corr_pairs:
+            closest_pair = difflib.get_close_matches(
+                                        pair,
+                                        exchange.markets
+                                        )
+            if not closest_pair:
+                logger.warning(f'Could not find {pair} in markets, removing from '
+                               f'corr_pairlist.')
+            else:
+                closest_pair = closest_pair[0]
+
+            new_corr_pairs.append(closest_pair)
+            logger.info(f'Spice rack will use {closest_pair} as informative in FreqAI model.')
+
+        # find the closest matching timeframes to what the default config wants
+        if timeframe_to_seconds(config['timeframe']) > timeframe_to_seconds('15m'):
+            logger.warning('Default spice rack is designed for lower base timeframes (e.g. > '
+                           f'15m). But user passed {config["timeframe"]}.')
+        new_tfs.append(config['timeframe'])
+
+        list_tfs = [timeframe_to_seconds(tf) for tf
+                    in exchange.timeframes]
+        for tf in timeframes:
+            tf_secs = timeframe_to_seconds(tf)
+            closest_index = min(range(len(list_tfs)), key=lambda i: abs(list_tfs[i] - tf_secs))
+            closest_tf = exchange.timeframes[closest_index]
+            logger.info(f'Spice rack will use {closest_tf} as informative tf in FreqAI model.')
+            new_tfs.append(closest_tf)
+
+    config['freqai']['feature_parameters'].update({'include_timeframes': new_tfs})
+    config['freqai']['feature_parameters'].update({'include_corr_pairlist': new_corr_pairs})
+    config.update({"freqaimodel": 'LightGBMRegressorMultiTarget'})
+    return config
+
 # Keep below for when we wish to download heterogeneously lengthed data for FreqAI.
 # def download_all_data_for_training(dp: DataProvider, config: dict) -> None:
 #     """

freqtrade/optimize/backtesting.py

@@ -89,6 +89,10 @@ class Backtesting:
             self._exchange_name, self.config, load_leverage_tiers=True)
         self.dataprovider = DataProvider(self.config, self.exchange)
 
+        if config.get('freqai_spice_rack', False):
+            from freqtrade.freqai.utils import setup_freqai_spice_rack
+            self.config = setup_freqai_spice_rack(self.config, self.exchange)
+
         if self.config.get('strategy_list'):
             if self.config.get('freqai', {}).get('enabled', False):
                 raise OperationalException(

freqtrade/strategy/interface.py

@@ -5,7 +5,7 @@ This module defines the interface to apply for strategies
 import logging
 from abc import ABC, abstractmethod
 from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 import arrow
 from pandas import DataFrame
@@ -147,9 +147,9 @@ class IStrategy(ABC, HyperStrategyMixin):
     def load_freqAI_model(self) -> None:
         spice_rack = self.config.get('freqai_spice_rack', False)
         if self.config.get('freqai', {}).get('enabled', False) or spice_rack:
-            spice_rack = self.config.get('freqai_spice_rack', False)
             if spice_rack:
-                self.config = self.setup_freqai_spice_rack(self.config)
+                from freqtrade.freqai.utils import setup_freqai_spice_rack
+                self.config = setup_freqai_spice_rack(self.config, self.dp._exchange)
             # Import here to avoid importing this if freqAI is disabled
             from freqtrade.freqai.utils import download_all_data_for_training
             from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
@@ -190,60 +190,6 @@ class IStrategy(ABC, HyperStrategyMixin):
 
             self.freqai = DummyClass()  # type: ignore
 
-    def setup_freqai_spice_rack(self, config: dict) -> Dict[str, Any]:
-        import difflib
-        import json
-        from pathlib import Path
-        auto_config = config.get('freqai_config', 'lightgbm_config.json')
-        with open(Path('freqtrade') / 'freqai' / 'spice_rack'
-                  / auto_config) as json_file:
-            freqai_config = json.load(json_file)
-            config['freqai'] = freqai_config['freqai']
-            config['freqai']['identifier'] = config['freqai_identifier']
-            corr_pairs = config['freqai']['feature_parameters']['include_corr_pairlist']
-            timeframes = config['freqai']['feature_parameters']['include_timeframes']
-            new_corr_pairs = []
-            new_tfs = []
-
-            if not self.dp:
-                logger.warning('No dataprovider available.')
-                config['freqai']['enabled'] = False
-                return config
-            # find the closest pairs to what the default config wants
-            for pair in corr_pairs:
-                closest_pair = difflib.get_close_matches(
-                                            pair,
-                                            self.dp._exchange.markets  # type: ignore
-                                            )
-                if not closest_pair:
-                    logger.warning(f'Could not find {pair} in markets, removing from '
-                                   f'corr_pairlist.')
-                else:
-                    closest_pair = closest_pair[0]
-
-                new_corr_pairs.append(closest_pair)
-                logger.info(f'Spice rack will use {closest_pair} as informative in FreqAI model.')
-
-            # find the closest matching timeframes to what the default config wants
-            if timeframe_to_seconds(config['timeframe']) > timeframe_to_seconds('15m'):
-                logger.warning('Default spice rack is designed for lower base timeframes (e.g. > '
-                               f'15m). But user passed {config["timeframe"]}.')
-            new_tfs.append(config['timeframe'])
-
-            list_tfs = [timeframe_to_seconds(tf) for tf
-                        in self.dp._exchange.timeframes]  # type: ignore
-            for tf in timeframes:
-                tf_secs = timeframe_to_seconds(tf)
-                closest_index = min(range(len(list_tfs)), key=lambda i: abs(list_tfs[i] - tf_secs))
-                closest_tf = self.dp._exchange.timeframes[closest_index]  # type: ignore
-                logger.info(f'Spice rack will use {closest_tf} as informative tf in FreqAI model.')
-                new_tfs.append(closest_tf)
-
-        config['freqai']['feature_parameters'].update({'include_timeframes': new_tfs})
-        config['freqai']['feature_parameters'].update({'include_corr_pairlist': new_corr_pairs})
-        config.update({"freqaimodel": 'LightGBMRegressorMultiTarget'})
-        return config
-
     def ft_bot_start(self, **kwargs) -> None:
         """
         Strategy init - runs after dataprovider has been added.