Merge pull request #8202 from freqtrade/remove-populate-any-indicators

remove populate_any_indicators
This commit is contained in:
Matthias 2023-03-03 06:33:25 +01:00 committed by GitHub
commit 6e9ff5fdd8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 41 additions and 148 deletions

View File

@ -1315,11 +1315,21 @@ class FreqaiDataKitchen:
dataframe: DataFrame = dataframe containing populated indicators dataframe: DataFrame = dataframe containing populated indicators
""" """
# this is a hack to check if the user is using the populate_any_indicators function # check if the user is using the deprecated populate_any_indicators function
new_version = inspect.getsource(strategy.populate_any_indicators) == ( new_version = inspect.getsource(strategy.populate_any_indicators) == (
inspect.getsource(IStrategy.populate_any_indicators)) inspect.getsource(IStrategy.populate_any_indicators))
if new_version: if not new_version:
raise OperationalException(
"You are using the `populate_any_indicators()` function"
" which was deprecated on March 1, 2023. Please refer "
"to the strategy migration guide to use the new "
"feature_engineering_* methods: \n"
"https://www.freqtrade.io/en/stable/strategy_migration/#freqai-strategy \n"
"And the feature_engineering_* documentation: \n"
"https://www.freqtrade.io/en/latest/freqai-feature-engineering/"
)
tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes") tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes")
pairs: List[str] = self.freqai_config["feature_parameters"].get( pairs: List[str] = self.freqai_config["feature_parameters"].get(
"include_corr_pairlist", []) "include_corr_pairlist", [])
@ -1363,85 +1373,6 @@ class FreqaiDataKitchen:
return dataframe return dataframe
else:
# the user is using the populate_any_indicators functions which is deprecated
df = self.use_strategy_to_populate_indicators_old_version(
strategy, corr_dataframes, base_dataframes, pair,
prediction_dataframe, do_corr_pairs)
return df
def use_strategy_to_populate_indicators_old_version(
self,
strategy: IStrategy,
corr_dataframes: dict = {},
base_dataframes: dict = {},
pair: str = "",
prediction_dataframe: DataFrame = pd.DataFrame(),
do_corr_pairs: bool = True,
) -> DataFrame:
"""
Use the user defined strategy for populating indicators during retrain
:param strategy: IStrategy = user defined strategy object
:param corr_dataframes: dict = dict containing the df pair dataframes
(for user defined timeframes)
:param base_dataframes: dict = dict containing the current pair dataframes
(for user defined timeframes)
:param metadata: dict = strategy furnished pair metadata
:return:
dataframe: DataFrame = dataframe containing populated indicators
"""
# for prediction dataframe creation, we let dataprovider handle everything in the strategy
# so we create empty dictionaries, which allows us to pass None to
# `populate_any_indicators()`. Signaling we want the dp to give us the live dataframe.
tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes")
pairs: List[str] = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
if not prediction_dataframe.empty:
dataframe = prediction_dataframe.copy()
for tf in tfs:
base_dataframes[tf] = None
for p in pairs:
if p not in corr_dataframes:
corr_dataframes[p] = {}
corr_dataframes[p][tf] = None
else:
dataframe = base_dataframes[self.config["timeframe"]].copy()
sgi = False
for tf in tfs:
if tf == tfs[-1]:
sgi = True # doing this last allows user to use all tf raw prices in labels
dataframe = strategy.populate_any_indicators(
pair,
dataframe.copy(),
tf,
informative=base_dataframes[tf],
set_generalized_indicators=sgi
)
# ensure corr pairs are always last
for corr_pair in pairs:
if pair == corr_pair:
continue # dont repeat anything from whitelist
for tf in tfs:
if pairs and do_corr_pairs:
dataframe = strategy.populate_any_indicators(
corr_pair,
dataframe.copy(),
tf,
informative=corr_dataframes[corr_pair][tf]
)
self.get_unique_classes_from_labels(dataframe)
dataframe = self.remove_special_chars_from_feature_names(dataframe)
if self.config.get('reduce_df_footprint', False):
dataframe = reduce_dataframe_footprint(dataframe)
return dataframe
def fit_labels(self) -> None: def fit_labels(self) -> None:
""" """
Fit the labels with a gaussian distribution Fit the labels with a gaussian distribution

View File

@ -1,4 +1,3 @@
import inspect
import logging import logging
import threading import threading
import time import time
@ -106,8 +105,6 @@ class IFreqaiModel(ABC):
self.max_system_threads = max(int(psutil.cpu_count() * 2 - 2), 1) self.max_system_threads = max(int(psutil.cpu_count() * 2 - 2), 1)
self.can_short = True # overridden in start() with strategy.can_short self.can_short = True # overridden in start() with strategy.can_short
self.warned_deprecated_populate_any_indicators = False
record_params(config, self.full_path) record_params(config, self.full_path)
def __getstate__(self): def __getstate__(self):
@ -138,9 +135,6 @@ class IFreqaiModel(ABC):
self.data_provider = strategy.dp self.data_provider = strategy.dp
self.can_short = strategy.can_short self.can_short = strategy.can_short
# check if the strategy has deprecated populate_any_indicators function
self.check_deprecated_populate_any_indicators(strategy)
if self.live: if self.live:
self.inference_timer('start') self.inference_timer('start')
self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"]) self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
@ -491,7 +485,7 @@ class IFreqaiModel(ABC):
"strategy is furnishing the same features as the pretrained" "strategy is furnishing the same features as the pretrained"
"model. In case of --strategy-list, please be aware that FreqAI " "model. In case of --strategy-list, please be aware that FreqAI "
"requires all strategies to maintain identical " "requires all strategies to maintain identical "
"populate_any_indicator() functions" "feature_engineering_* functions"
) )
def data_cleaning_train(self, dk: FreqaiDataKitchen) -> None: def data_cleaning_train(self, dk: FreqaiDataKitchen) -> None:
@ -603,7 +597,7 @@ class IFreqaiModel(ABC):
:param strategy: IStrategy = user defined strategy object :param strategy: IStrategy = user defined strategy object
:param dk: FreqaiDataKitchen = non-persistent data container for current coin/loop :param dk: FreqaiDataKitchen = non-persistent data container for current coin/loop
:param data_load_timerange: TimeRange = the amount of data to be loaded :param data_load_timerange: TimeRange = the amount of data to be loaded
for populate_any_indicators for populating indicators
(larger than new_trained_timerange so that (larger than new_trained_timerange so that
new_trained_timerange does not contain any NaNs) new_trained_timerange does not contain any NaNs)
""" """
@ -809,7 +803,7 @@ class IFreqaiModel(ABC):
logger.warning("Couldn't cache corr_pair dataframes for improved performance. " logger.warning("Couldn't cache corr_pair dataframes for improved performance. "
"Consider ensuring that the full coin/stake, e.g. XYZ/USD, " "Consider ensuring that the full coin/stake, e.g. XYZ/USD, "
"is included in the column names when you are creating features " "is included in the column names when you are creating features "
"in `populate_any_indicators()`.") "in `feature_engineering_*` functions.")
self.get_corr_dataframes = not bool(self.corr_dataframes) self.get_corr_dataframes = not bool(self.corr_dataframes)
elif self.corr_dataframes: elif self.corr_dataframes:
dataframe = dk.attach_corr_pair_columns( dataframe = dk.attach_corr_pair_columns(
@ -936,26 +930,6 @@ class IFreqaiModel(ABC):
dk.return_dataframe, saved_dataframe, how='left', left_on='date', right_on="date_pred") dk.return_dataframe, saved_dataframe, how='left', left_on='date', right_on="date_pred")
return dk return dk
def check_deprecated_populate_any_indicators(self, strategy: IStrategy):
"""
Check and warn if the deprecated populate_any_indicators function is used.
:param strategy: strategy object
"""
if not self.warned_deprecated_populate_any_indicators:
self.warned_deprecated_populate_any_indicators = True
old_version = inspect.getsource(strategy.populate_any_indicators) != (
inspect.getsource(IStrategy.populate_any_indicators))
if old_version:
logger.warning("DEPRECATION WARNING: "
"You are using the deprecated populate_any_indicators function. "
"This function will raise an error on March 1 2023. "
"Please update your strategy by using "
"the new feature_engineering functions. See \n"
"https://www.freqtrade.io/en/latest/freqai-feature-engineering/"
"for details.")
# Following methods which are overridden by user made prediction models. # Following methods which are overridden by user made prediction models.
# See freqai/prediction_models/CatboostPredictionModel.py for an example. # See freqai/prediction_models/CatboostPredictionModel.py for an example.

View File

@ -93,7 +93,7 @@ class Backtesting:
if self.config.get('strategy_list'): if self.config.get('strategy_list'):
if self.config.get('freqai', {}).get('enabled', False): if self.config.get('freqai', {}).get('enabled', False):
logger.warning("Using --strategy-list with FreqAI REQUIRES all strategies " logger.warning("Using --strategy-list with FreqAI REQUIRES all strategies "
"to have identical populate_any_indicators.") "to have identical feature_engineering_* functions.")
for strat in list(self.config['strategy_list']): for strat in list(self.config['strategy_list']):
stratconf = deepcopy(self.config) stratconf = deepcopy(self.config)
stratconf['strategy'] = strat stratconf['strategy'] = strat

View File

@ -35,8 +35,8 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, c
args = get_args(args) args = get_args(args)
bt_config = setup_optimize_configuration(args, RunMode.BACKTEST) bt_config = setup_optimize_configuration(args, RunMode.BACKTEST)
Backtesting(bt_config) Backtesting(bt_config)
assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical ' assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical',
'populate_any_indicators.', caplog) caplog)
Backtesting.cleanup() Backtesting.cleanup()

View File

@ -291,18 +291,6 @@ def test_advise_all_indicators(default_conf, testdatadir) -> None:
assert len(processed['UNITTEST/BTC']) == 103 assert len(processed['UNITTEST/BTC']) == 103
def test_populate_any_indicators(default_conf, testdatadir) -> None:
strategy = StrategyResolver.load_strategy(default_conf)
timerange = TimeRange.parse_timerange('1510694220-1510700340')
data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange,
fill_up_missing=True)
processed = strategy.populate_any_indicators('UNITTEST/BTC', data, '5m')
assert processed == data
assert id(processed) == id(data)
assert len(processed['UNITTEST/BTC']) == 103
def test_freqai_not_initialized(default_conf) -> None: def test_freqai_not_initialized(default_conf) -> None:
strategy = StrategyResolver.load_strategy(default_conf) strategy = StrategyResolver.load_strategy(default_conf)
strategy.ft_bot_start() strategy.ft_bot_start()