From fd4e27d889c4b4d192373e86b4b4a591dab8783e Mon Sep 17 00:00:00 2001 From: robcaulk Date: Tue, 21 Feb 2023 14:22:40 +0100 Subject: [PATCH] remove populate_any_indicators --- freqtrade/freqai/data_kitchen.py | 139 ++++++------------------ freqtrade/freqai/freqai_interface.py | 32 +----- freqtrade/optimize/backtesting.py | 2 +- tests/freqai/test_freqai_backtesting.py | 4 +- tests/strategy/test_interface.py | 12 -- 5 files changed, 41 insertions(+), 148 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 30d2509b5..ba304aca3 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1315,123 +1315,54 @@ class FreqaiDataKitchen: dataframe: DataFrame = dataframe containing populated indicators """ - # this is a hack to check if the user is using the populate_any_indicators function + # check if the user is using the deprecated populate_any_indicators function new_version = inspect.getsource(strategy.populate_any_indicators) == ( inspect.getsource(IStrategy.populate_any_indicators)) - if new_version: - tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes") - pairs: List[str] = self.freqai_config["feature_parameters"].get( - "include_corr_pairlist", []) + if not new_version: + raise OperationalException( + "You are using the `populate_any_indicators()` function" + " which was deprecated on March 1, 2023. Please refer " + "to the strategy migration guide to use the new " + "feature_engineering_* methods: \n" + "https://www.freqtrade.io/en/stable/strategy_migration/#freqai-strategy \n" + "And the feature_engineering_* documentation: \n" + "https://www.freqtrade.io/en/latest/freqai-feature-engineering/" + ) - for tf in tfs: - if tf not in base_dataframes: - base_dataframes[tf] = pd.DataFrame() - for p in pairs: - if p not in corr_dataframes: - corr_dataframes[p] = {} - if tf not in corr_dataframes[p]: - corr_dataframes[p][tf] = pd.DataFrame() - - if not prediction_dataframe.empty: - dataframe = prediction_dataframe.copy() - else: - dataframe = base_dataframes[self.config["timeframe"]].copy() - - corr_pairs: List[str] = self.freqai_config["feature_parameters"].get( - "include_corr_pairlist", []) - dataframe = self.populate_features(dataframe.copy(), pair, strategy, - corr_dataframes, base_dataframes) - metadata = {"pair": pair} - dataframe = strategy.feature_engineering_standard(dataframe.copy(), metadata=metadata) - # ensure corr pairs are always last - for corr_pair in corr_pairs: - if pair == corr_pair: - continue # dont repeat anything from whitelist - if corr_pairs and do_corr_pairs: - dataframe = self.populate_features(dataframe.copy(), corr_pair, strategy, - corr_dataframes, base_dataframes, True) - - dataframe = strategy.set_freqai_targets(dataframe.copy(), metadata=metadata) - - self.get_unique_classes_from_labels(dataframe) - - dataframe = self.remove_special_chars_from_feature_names(dataframe) - - if self.config.get('reduce_df_footprint', False): - dataframe = reduce_dataframe_footprint(dataframe) - - return dataframe - - else: - # the user is using the populate_any_indicators functions which is deprecated - - df = self.use_strategy_to_populate_indicators_old_version( - strategy, corr_dataframes, base_dataframes, pair, - prediction_dataframe, do_corr_pairs) - return df - - def use_strategy_to_populate_indicators_old_version( - self, - strategy: IStrategy, - corr_dataframes: dict = {}, - base_dataframes: dict = {}, - pair: str = "", - prediction_dataframe: DataFrame = pd.DataFrame(), - do_corr_pairs: bool = True, - ) -> DataFrame: - """ - Use the user defined strategy for populating indicators during retrain - :param strategy: IStrategy = user defined strategy object - :param corr_dataframes: dict = dict containing the df pair dataframes - (for user defined timeframes) - :param base_dataframes: dict = dict containing the current pair dataframes - (for user defined timeframes) - :param metadata: dict = strategy furnished pair metadata - :return: - dataframe: DataFrame = dataframe containing populated indicators - """ - - # for prediction dataframe creation, we let dataprovider handle everything in the strategy - # so we create empty dictionaries, which allows us to pass None to - # `populate_any_indicators()`. Signaling we want the dp to give us the live dataframe. tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes") - pairs: List[str] = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) + pairs: List[str] = self.freqai_config["feature_parameters"].get( + "include_corr_pairlist", []) + + for tf in tfs: + if tf not in base_dataframes: + base_dataframes[tf] = pd.DataFrame() + for p in pairs: + if p not in corr_dataframes: + corr_dataframes[p] = {} + if tf not in corr_dataframes[p]: + corr_dataframes[p][tf] = pd.DataFrame() + if not prediction_dataframe.empty: dataframe = prediction_dataframe.copy() - for tf in tfs: - base_dataframes[tf] = None - for p in pairs: - if p not in corr_dataframes: - corr_dataframes[p] = {} - corr_dataframes[p][tf] = None else: dataframe = base_dataframes[self.config["timeframe"]].copy() - sgi = False - for tf in tfs: - if tf == tfs[-1]: - sgi = True # doing this last allows user to use all tf raw prices in labels - dataframe = strategy.populate_any_indicators( - pair, - dataframe.copy(), - tf, - informative=base_dataframes[tf], - set_generalized_indicators=sgi - ) - + corr_pairs: List[str] = self.freqai_config["feature_parameters"].get( + "include_corr_pairlist", []) + dataframe = self.populate_features(dataframe.copy(), pair, strategy, + corr_dataframes, base_dataframes) + metadata = {"pair": pair} + dataframe = strategy.feature_engineering_standard(dataframe.copy(), metadata=metadata) # ensure corr pairs are always last - for corr_pair in pairs: + for corr_pair in corr_pairs: if pair == corr_pair: continue # dont repeat anything from whitelist - for tf in tfs: - if pairs and do_corr_pairs: - dataframe = strategy.populate_any_indicators( - corr_pair, - dataframe.copy(), - tf, - informative=corr_dataframes[corr_pair][tf] - ) + if corr_pairs and do_corr_pairs: + dataframe = self.populate_features(dataframe.copy(), corr_pair, strategy, + corr_dataframes, base_dataframes, True) + + dataframe = strategy.set_freqai_targets(dataframe.copy(), metadata=metadata) self.get_unique_classes_from_labels(dataframe) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index c265e42f9..c7b39b4e8 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -1,4 +1,3 @@ -import inspect import logging import threading import time @@ -106,8 +105,6 @@ class IFreqaiModel(ABC): self.max_system_threads = max(int(psutil.cpu_count() * 2 - 2), 1) self.can_short = True # overridden in start() with strategy.can_short - self.warned_deprecated_populate_any_indicators = False - record_params(config, self.full_path) def __getstate__(self): @@ -138,9 +135,6 @@ class IFreqaiModel(ABC): self.data_provider = strategy.dp self.can_short = strategy.can_short - # check if the strategy has deprecated populate_any_indicators function - self.check_deprecated_populate_any_indicators(strategy) - if self.live: self.inference_timer('start') self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"]) @@ -489,7 +483,7 @@ class IFreqaiModel(ABC): "strategy is furnishing the same features as the pretrained" "model. In case of --strategy-list, please be aware that FreqAI " "requires all strategies to maintain identical " - "populate_any_indicator() functions" + "feature_engineering_* functions" ) def data_cleaning_train(self, dk: FreqaiDataKitchen) -> None: @@ -601,7 +595,7 @@ class IFreqaiModel(ABC): :param strategy: IStrategy = user defined strategy object :param dk: FreqaiDataKitchen = non-persistent data container for current coin/loop :param data_load_timerange: TimeRange = the amount of data to be loaded - for populate_any_indicators + for populating indicators (larger than new_trained_timerange so that new_trained_timerange does not contain any NaNs) """ @@ -806,7 +800,7 @@ class IFreqaiModel(ABC): logger.warning("Couldn't cache corr_pair dataframes for improved performance. " "Consider ensuring that the full coin/stake, e.g. XYZ/USD, " "is included in the column names when you are creating features " - "in `populate_any_indicators()`.") + "in `feature_engineering_*` functions.") self.get_corr_dataframes = not bool(self.corr_dataframes) elif self.corr_dataframes: dataframe = dk.attach_corr_pair_columns( @@ -933,26 +927,6 @@ class IFreqaiModel(ABC): dk.return_dataframe, saved_dataframe, how='left', left_on='date', right_on="date_pred") return dk - def check_deprecated_populate_any_indicators(self, strategy: IStrategy): - """ - Check and warn if the deprecated populate_any_indicators function is used. - :param strategy: strategy object - """ - - if not self.warned_deprecated_populate_any_indicators: - self.warned_deprecated_populate_any_indicators = True - old_version = inspect.getsource(strategy.populate_any_indicators) != ( - inspect.getsource(IStrategy.populate_any_indicators)) - - if old_version: - logger.warning("DEPRECATION WARNING: " - "You are using the deprecated populate_any_indicators function. " - "This function will raise an error on March 1 2023. " - "Please update your strategy by using " - "the new feature_engineering functions. See \n" - "https://www.freqtrade.io/en/latest/freqai-feature-engineering/" - "for details.") - # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModel.py for an example. diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 065a88f40..023be9a1a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -93,7 +93,7 @@ class Backtesting: if self.config.get('strategy_list'): if self.config.get('freqai', {}).get('enabled', False): logger.warning("Using --strategy-list with FreqAI REQUIRES all strategies " - "to have identical populate_any_indicators.") + "to have identical feature_engineering_* functions.") for strat in list(self.config['strategy_list']): stratconf = deepcopy(self.config) stratconf['strategy'] = strat diff --git a/tests/freqai/test_freqai_backtesting.py b/tests/freqai/test_freqai_backtesting.py index 60963e762..0a8059966 100644 --- a/tests/freqai/test_freqai_backtesting.py +++ b/tests/freqai/test_freqai_backtesting.py @@ -35,8 +35,8 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, c args = get_args(args) bt_config = setup_optimize_configuration(args, RunMode.BACKTEST) Backtesting(bt_config) - assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical ' - 'populate_any_indicators.', caplog) + assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical', + caplog) Backtesting.cleanup() diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index fe562907a..87075d56d 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -291,18 +291,6 @@ def test_advise_all_indicators(default_conf, testdatadir) -> None: assert len(processed['UNITTEST/BTC']) == 103 -def test_populate_any_indicators(default_conf, testdatadir) -> None: - strategy = StrategyResolver.load_strategy(default_conf) - - timerange = TimeRange.parse_timerange('1510694220-1510700340') - data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, - fill_up_missing=True) - processed = strategy.populate_any_indicators('UNITTEST/BTC', data, '5m') - assert processed == data - assert id(processed) == id(data) - assert len(processed['UNITTEST/BTC']) == 103 - - def test_freqai_not_initialized(default_conf) -> None: strategy = StrategyResolver.load_strategy(default_conf) strategy.ft_bot_start()