Merge pull request #8202 from freqtrade/remove-populate-any-indicators

remove populate_any_indicators

freqtrade/freqai/data_kitchen.py

@@ -1315,11 +1315,21 @@ class FreqaiDataKitchen:
         dataframe: DataFrame = dataframe containing populated indicators
         """
 
-        # this is a hack to check if the user is using the populate_any_indicators function
+        # check if the user is using the deprecated populate_any_indicators function
         new_version = inspect.getsource(strategy.populate_any_indicators) == (
             inspect.getsource(IStrategy.populate_any_indicators))
 
-        if new_version:
+        if not new_version:
+            raise OperationalException(
+                "You are using the `populate_any_indicators()` function"
+                " which was deprecated on March 1, 2023. Please refer "
+                "to the strategy migration guide to use the new "
+                "feature_engineering_* methods: \n"
+                "https://www.freqtrade.io/en/stable/strategy_migration/#freqai-strategy \n"
+                "And the feature_engineering_* documentation: \n"
+                "https://www.freqtrade.io/en/latest/freqai-feature-engineering/"
+                )
+
         tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes")
         pairs: List[str] = self.freqai_config["feature_parameters"].get(
             "include_corr_pairlist", [])
@@ -1363,85 +1373,6 @@ class FreqaiDataKitchen:
 
         return dataframe
 
-        else:
-            # the user is using the populate_any_indicators functions which is deprecated
-
-            df = self.use_strategy_to_populate_indicators_old_version(
-                strategy, corr_dataframes, base_dataframes, pair,
-                prediction_dataframe, do_corr_pairs)
-            return df
-
-    def use_strategy_to_populate_indicators_old_version(
-        self,
-        strategy: IStrategy,
-        corr_dataframes: dict = {},
-        base_dataframes: dict = {},
-        pair: str = "",
-        prediction_dataframe: DataFrame = pd.DataFrame(),
-        do_corr_pairs: bool = True,
-    ) -> DataFrame:
-        """
-        Use the user defined strategy for populating indicators during retrain
-        :param strategy: IStrategy = user defined strategy object
-        :param corr_dataframes: dict = dict containing the df pair dataframes
-                                (for user defined timeframes)
-        :param base_dataframes: dict = dict containing the current pair dataframes
-                                (for user defined timeframes)
-        :param metadata: dict = strategy furnished pair metadata
-        :return:
-        dataframe: DataFrame = dataframe containing populated indicators
-        """
-
-        # for prediction dataframe creation, we let dataprovider handle everything in the strategy
-        # so we create empty dictionaries, which allows us to pass None to
-        # `populate_any_indicators()`. Signaling we want the dp to give us the live dataframe.
-        tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes")
-        pairs: List[str] = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
-        if not prediction_dataframe.empty:
-            dataframe = prediction_dataframe.copy()
-            for tf in tfs:
-                base_dataframes[tf] = None
-                for p in pairs:
-                    if p not in corr_dataframes:
-                        corr_dataframes[p] = {}
-                    corr_dataframes[p][tf] = None
-        else:
-            dataframe = base_dataframes[self.config["timeframe"]].copy()
-
-        sgi = False
-        for tf in tfs:
-            if tf == tfs[-1]:
-                sgi = True  # doing this last allows user to use all tf raw prices in labels
-            dataframe = strategy.populate_any_indicators(
-                pair,
-                dataframe.copy(),
-                tf,
-                informative=base_dataframes[tf],
-                set_generalized_indicators=sgi
-            )
-
-        # ensure corr pairs are always last
-        for corr_pair in pairs:
-            if pair == corr_pair:
-                continue  # dont repeat anything from whitelist
-            for tf in tfs:
-                if pairs and do_corr_pairs:
-                    dataframe = strategy.populate_any_indicators(
-                        corr_pair,
-                        dataframe.copy(),
-                        tf,
-                        informative=corr_dataframes[corr_pair][tf]
-                    )
-
-        self.get_unique_classes_from_labels(dataframe)
-
-        dataframe = self.remove_special_chars_from_feature_names(dataframe)
-
-        if self.config.get('reduce_df_footprint', False):
-            dataframe = reduce_dataframe_footprint(dataframe)
-
-        return dataframe
-
     def fit_labels(self) -> None:
         """
         Fit the labels with a gaussian distribution

freqtrade/freqai/freqai_interface.py

@@ -1,4 +1,3 @@
-import inspect
 import logging
 import threading
 import time
@@ -106,8 +105,6 @@ class IFreqaiModel(ABC):
         self.max_system_threads = max(int(psutil.cpu_count() * 2 - 2), 1)
         self.can_short = True  # overridden in start() with strategy.can_short
 
-        self.warned_deprecated_populate_any_indicators = False
-
         record_params(config, self.full_path)
 
     def __getstate__(self):
@@ -138,9 +135,6 @@ class IFreqaiModel(ABC):
         self.data_provider = strategy.dp
         self.can_short = strategy.can_short
 
-        # check if the strategy has deprecated populate_any_indicators function
-        self.check_deprecated_populate_any_indicators(strategy)
-
         if self.live:
             self.inference_timer('start')
             self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
@@ -491,7 +485,7 @@ class IFreqaiModel(ABC):
                 "strategy is furnishing the same features as the pretrained"
                 "model. In case of --strategy-list, please be aware that FreqAI "
                 "requires all strategies to maintain identical "
-                "populate_any_indicator() functions"
+                "feature_engineering_* functions"
             )
 
     def data_cleaning_train(self, dk: FreqaiDataKitchen) -> None:
@@ -603,7 +597,7 @@ class IFreqaiModel(ABC):
         :param strategy: IStrategy = user defined strategy object
         :param dk: FreqaiDataKitchen = non-persistent data container for current coin/loop
         :param data_load_timerange: TimeRange = the amount of data to be loaded
-                                    for populate_any_indicators
+                                    for populating indicators
                                     (larger than new_trained_timerange so that
                                     new_trained_timerange does not contain any NaNs)
         """
@@ -809,7 +803,7 @@ class IFreqaiModel(ABC):
                 logger.warning("Couldn't cache corr_pair dataframes for improved performance. "
                                "Consider ensuring that the full coin/stake, e.g. XYZ/USD, "
                                "is included in the column names when you are creating features "
-                               "in `populate_any_indicators()`.")
+                               "in `feature_engineering_*` functions.")
             self.get_corr_dataframes = not bool(self.corr_dataframes)
         elif self.corr_dataframes:
             dataframe = dk.attach_corr_pair_columns(
@@ -936,26 +930,6 @@ class IFreqaiModel(ABC):
             dk.return_dataframe, saved_dataframe, how='left', left_on='date', right_on="date_pred")
         return dk
 
-    def check_deprecated_populate_any_indicators(self, strategy: IStrategy):
-        """
-        Check and warn if the deprecated populate_any_indicators function is used.
-        :param strategy: strategy object
-        """
-
-        if not self.warned_deprecated_populate_any_indicators:
-            self.warned_deprecated_populate_any_indicators = True
-            old_version = inspect.getsource(strategy.populate_any_indicators) != (
-                inspect.getsource(IStrategy.populate_any_indicators))
-
-            if old_version:
-                logger.warning("DEPRECATION WARNING: "
-                               "You are using the deprecated populate_any_indicators function. "
-                               "This function will raise an error on March 1 2023. "
-                               "Please update your strategy by using "
-                               "the new feature_engineering functions. See \n"
-                               "https://www.freqtrade.io/en/latest/freqai-feature-engineering/"
-                               "for details.")
-
     # Following methods which are overridden by user made prediction models.
     # See freqai/prediction_models/CatboostPredictionModel.py for an example.
 

freqtrade/optimize/backtesting.py

@@ -93,7 +93,7 @@ class Backtesting:
         if self.config.get('strategy_list'):
             if self.config.get('freqai', {}).get('enabled', False):
                 logger.warning("Using --strategy-list with FreqAI REQUIRES all strategies "
-                               "to have identical populate_any_indicators.")
+                               "to have identical feature_engineering_* functions.")
             for strat in list(self.config['strategy_list']):
                 stratconf = deepcopy(self.config)
                 stratconf['strategy'] = strat

tests/freqai/test_freqai_backtesting.py

@@ -35,8 +35,8 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, c
     args = get_args(args)
     bt_config = setup_optimize_configuration(args, RunMode.BACKTEST)
     Backtesting(bt_config)
-    assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical '
+    assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical',
-                      'populate_any_indicators.', caplog)
+                      caplog)
     Backtesting.cleanup()
 
 

tests/strategy/test_interface.py

@@ -291,18 +291,6 @@ def test_advise_all_indicators(default_conf, testdatadir) -> None:
     assert len(processed['UNITTEST/BTC']) == 103
 
 
-def test_populate_any_indicators(default_conf, testdatadir) -> None:
-    strategy = StrategyResolver.load_strategy(default_conf)
-
-    timerange = TimeRange.parse_timerange('1510694220-1510700340')
-    data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange,
-                     fill_up_missing=True)
-    processed = strategy.populate_any_indicators('UNITTEST/BTC', data, '5m')
-    assert processed == data
-    assert id(processed) == id(data)
-    assert len(processed['UNITTEST/BTC']) == 103
-
-
 def test_freqai_not_initialized(default_conf) -> None:
     strategy = StrategyResolver.load_strategy(default_conf)
     strategy.ft_bot_start()