Allow use of --strategy-list with freqai, with warning (#7455)

* Allow use of --strategy-list with freqai, with warning

* ensure populate_any_indicators is identical for resused identifiers

* use pair instead of metadata["pair"]

Co-authored-by: robcaulk <rob.caulk@gmail.com>

freqtrade/freqai/data_drawer.py

@@ -430,6 +430,16 @@ class FreqaiDataDrawer:
 
         return
 
+    def load_metadata(self, dk: FreqaiDataKitchen) -> None:
+        """
+        Load only metadata into datakitchen to increase performance during
+        presaved backtesting (prediction file loading).
+        """
+        with open(dk.data_path / f"{dk.model_filename}_metadata.json", "r") as fp:
+            dk.data = json.load(fp)
+            dk.training_features_list = dk.data["training_features_list"]
+            dk.label_list = dk.data["label_list"]
+
     def load_data(self, coin: str, dk: FreqaiDataKitchen) -> Any:
         """
         loads all data required to make a prediction on a sub-train time range

freqtrade/freqai/freqai_interface.py

@@ -65,7 +65,7 @@ class IFreqaiModel(ABC):
         self.first = True
         self.set_full_path()
         self.follow_mode: bool = self.freqai_info.get("follow_mode", False)
-        self.save_backtest_models: bool = self.freqai_info.get("save_backtest_models", False)
+        self.save_backtest_models: bool = self.freqai_info.get("save_backtest_models", True)
         if self.save_backtest_models:
             logger.info('Backtesting module configured to save all models.')
         self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode)
@@ -260,7 +260,7 @@ class IFreqaiModel(ABC):
                                                 tr_train.stopts,
                                                 tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT)
             logger.info(
-                f"Training {metadata['pair']}, {self.pair_it}/{self.total_pairs} pairs"
+                f"Training {pair}, {self.pair_it}/{self.total_pairs} pairs"
                 f" from {tr_train_startts_str} to {tr_train_stopts_str}, {train_it}/{total_trains} "
                 "trains"
             )
@@ -273,11 +273,13 @@ class IFreqaiModel(ABC):
             dk.set_new_model_names(pair, trained_timestamp)
 
             if dk.check_if_backtest_prediction_exists():
+                self.dd.load_metadata(dk)
+                self.check_if_feature_list_matches_strategy(dataframe_train, dk)
                 append_df = dk.get_backtesting_prediction()
                 dk.append_predictions(append_df)
             else:
                 if not self.model_exists(
-                    metadata["pair"], dk, trained_timestamp=trained_timestamp_int
+                    pair, dk, trained_timestamp=trained_timestamp_int
                 ):
                     dk.find_features(dataframe_train)
                     self.model = self.train(dataframe_train, pair, dk)
@@ -429,14 +431,16 @@ class IFreqaiModel(ABC):
         if "training_features_list_raw" in dk.data:
             feature_list = dk.data["training_features_list_raw"]
         else:
-            feature_list = dk.training_features_list
+            feature_list = dk.data['training_features_list']
         if dk.training_features_list != feature_list:
             raise OperationalException(
                 "Trying to access pretrained model with `identifier` "
                 "but found different features furnished by current strategy."
                 "Change `identifier` to train from scratch, or ensure the"
                 "strategy is furnishing the same features as the pretrained"
-                "model"
+                "model. In case of --strategy-list, please be aware that FreqAI "
+                "requires all strategies to maintain identical "
+                "populate_any_indicator() functions"
             )
 
     def data_cleaning_train(self, dk: FreqaiDataKitchen) -> None:

freqtrade/optimize/backtesting.py

@@ -91,8 +91,8 @@ class Backtesting:
 
         if self.config.get('strategy_list'):
             if self.config.get('freqai', {}).get('enabled', False):
-                raise OperationalException(
+                logger.warning("Using --strategy-list with FreqAI REQUIRES all strategies "
-                    "You can't use strategy_list and freqai at the same time.")
+                               "to have identical populate_any_indicators.")
             for strat in list(self.config['strategy_list']):
                 stratconf = deepcopy(self.config)
                 stratconf['strategy'] = strat

tests/freqai/test_freqai_backtesting.py

@@ -3,21 +3,21 @@ from datetime import datetime, timezone
 from pathlib import Path
 from unittest.mock import PropertyMock
 
-import pytest
+from freqtrade.commands.optimize_commands import setup_optimize_configuration
-
+from freqtrade.enums import RunMode
-from freqtrade.commands.optimize_commands import start_backtesting
-from freqtrade.exceptions import OperationalException
 from freqtrade.optimize.backtesting import Backtesting
 from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, log_has_re, patch_exchange,
                             patched_configuration_load_config_file)
 
 
-def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir):
+def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, caplog):
     patch_exchange(mocker)
 
+    now = datetime.now(timezone.utc)
     mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
                  PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT']))
-    # mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
+    mocker.patch('freqtrade.optimize.backtesting.history.load_data')
+    mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now))
 
     patched_configuration_load_config_file(mocker, freqai_conf)
 
@@ -30,9 +30,11 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir):
         '--strategy-list', CURRENT_TEST_STRATEGY
     ]
     args = get_args(args)
-    with pytest.raises(OperationalException,
+    bt_config = setup_optimize_configuration(args, RunMode.BACKTEST)
-                       match=r"You can't use strategy_list and freqai at the same time\."):
+    Backtesting(bt_config)
-        start_backtesting(args)
+    assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical '
+                      'populate_any_indicators.', caplog)
+    Backtesting.cleanup()
 
 
 def test_freqai_backtest_load_data(freqai_conf, mocker, caplog):