Merge pull request #8202 from freqtrade/remove-populate-any-indicators
remove populate_any_indicators
This commit is contained in:
Matthias
GitHub
commit
6e9ff5fdd8
@ -1315,123 +1315,54 @@ class FreqaiDataKitchen:
|
||||
dataframe: DataFrame = dataframe containing populated indicators
|
||||
"""
|
||||
|
||||
# this is a hack to check if the user is using the populate_any_indicators function
|
||||
# check if the user is using the deprecated populate_any_indicators function
|
||||
new_version = inspect.getsource(strategy.populate_any_indicators) == (
|
||||
inspect.getsource(IStrategy.populate_any_indicators))
|
||||
|
||||
if new_version:
|
||||
tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes")
|
||||
pairs: List[str] = self.freqai_config["feature_parameters"].get(
|
||||
"include_corr_pairlist", [])
|
||||
if not new_version:
|
||||
raise OperationalException(
|
||||
"You are using the `populate_any_indicators()` function"
|
||||
" which was deprecated on March 1, 2023. Please refer "
|
||||
"to the strategy migration guide to use the new "
|
||||
"feature_engineering_* methods: \n"
|
||||
"https://www.freqtrade.io/en/stable/strategy_migration/#freqai-strategy \n"
|
||||
"And the feature_engineering_* documentation: \n"
|
||||
"https://www.freqtrade.io/en/latest/freqai-feature-engineering/"
|
||||
)
|
||||
|
||||
for tf in tfs:
|
||||
if tf not in base_dataframes:
|
||||
base_dataframes[tf] = pd.DataFrame()
|
||||
for p in pairs:
|
||||
if p not in corr_dataframes:
|
||||
corr_dataframes[p] = {}
|
||||
if tf not in corr_dataframes[p]:
|
||||
corr_dataframes[p][tf] = pd.DataFrame()
|
||||
|
||||
if not prediction_dataframe.empty:
|
||||
dataframe = prediction_dataframe.copy()
|
||||
else:
|
||||
dataframe = base_dataframes[self.config["timeframe"]].copy()
|
||||
|
||||
corr_pairs: List[str] = self.freqai_config["feature_parameters"].get(
|
||||
"include_corr_pairlist", [])
|
||||
dataframe = self.populate_features(dataframe.copy(), pair, strategy,
|
||||
corr_dataframes, base_dataframes)
|
||||
metadata = {"pair": pair}
|
||||
dataframe = strategy.feature_engineering_standard(dataframe.copy(), metadata=metadata)
|
||||
# ensure corr pairs are always last
|
||||
for corr_pair in corr_pairs:
|
||||
if pair == corr_pair:
|
||||
continue # dont repeat anything from whitelist
|
||||
if corr_pairs and do_corr_pairs:
|
||||
dataframe = self.populate_features(dataframe.copy(), corr_pair, strategy,
|
||||
corr_dataframes, base_dataframes, True)
|
||||
|
||||
dataframe = strategy.set_freqai_targets(dataframe.copy(), metadata=metadata)
|
||||
|
||||
self.get_unique_classes_from_labels(dataframe)
|
||||
|
||||
dataframe = self.remove_special_chars_from_feature_names(dataframe)
|
||||
|
||||
if self.config.get('reduce_df_footprint', False):
|
||||
dataframe = reduce_dataframe_footprint(dataframe)
|
||||
|
||||
return dataframe
|
||||
|
||||
else:
|
||||
# the user is using the populate_any_indicators functions which is deprecated
|
||||
|
||||
df = self.use_strategy_to_populate_indicators_old_version(
|
||||
strategy, corr_dataframes, base_dataframes, pair,
|
||||
prediction_dataframe, do_corr_pairs)
|
||||
return df
|
||||
|
||||
def use_strategy_to_populate_indicators_old_version(
|
||||
self,
|
||||
strategy: IStrategy,
|
||||
corr_dataframes: dict = {},
|
||||
base_dataframes: dict = {},
|
||||
pair: str = "",
|
||||
prediction_dataframe: DataFrame = pd.DataFrame(),
|
||||
do_corr_pairs: bool = True,
|
||||
) -> DataFrame:
|
||||
"""
|
||||
Use the user defined strategy for populating indicators during retrain
|
||||
:param strategy: IStrategy = user defined strategy object
|
||||
:param corr_dataframes: dict = dict containing the df pair dataframes
|
||||
(for user defined timeframes)
|
||||
:param base_dataframes: dict = dict containing the current pair dataframes
|
||||
(for user defined timeframes)
|
||||
:param metadata: dict = strategy furnished pair metadata
|
||||
:return:
|
||||
dataframe: DataFrame = dataframe containing populated indicators
|
||||
"""
|
||||
|
||||
# for prediction dataframe creation, we let dataprovider handle everything in the strategy
|
||||
# so we create empty dictionaries, which allows us to pass None to
|
||||
# `populate_any_indicators()`. Signaling we want the dp to give us the live dataframe.
|
||||
tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes")
|
||||
pairs: List[str] = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
|
||||
pairs: List[str] = self.freqai_config["feature_parameters"].get(
|
||||
"include_corr_pairlist", [])
|
||||
|
||||
for tf in tfs:
|
||||
if tf not in base_dataframes:
|
||||
base_dataframes[tf] = pd.DataFrame()
|
||||
for p in pairs:
|
||||
if p not in corr_dataframes:
|
||||
corr_dataframes[p] = {}
|
||||
if tf not in corr_dataframes[p]:
|
||||
corr_dataframes[p][tf] = pd.DataFrame()
|
||||
|
||||
if not prediction_dataframe.empty:
|
||||
dataframe = prediction_dataframe.copy()
|
||||
for tf in tfs:
|
||||
base_dataframes[tf] = None
|
||||
for p in pairs:
|
||||
if p not in corr_dataframes:
|
||||
corr_dataframes[p] = {}
|
||||
corr_dataframes[p][tf] = None
|
||||
else:
|
||||
dataframe = base_dataframes[self.config["timeframe"]].copy()
|
||||
|
||||
sgi = False
|
||||
for tf in tfs:
|
||||
if tf == tfs[-1]:
|
||||
sgi = True # doing this last allows user to use all tf raw prices in labels
|
||||
dataframe = strategy.populate_any_indicators(
|
||||
pair,
|
||||
dataframe.copy(),
|
||||
tf,
|
||||
informative=base_dataframes[tf],
|
||||
set_generalized_indicators=sgi
|
||||
)
|
||||
|
||||
corr_pairs: List[str] = self.freqai_config["feature_parameters"].get(
|
||||
"include_corr_pairlist", [])
|
||||
dataframe = self.populate_features(dataframe.copy(), pair, strategy,
|
||||
corr_dataframes, base_dataframes)
|
||||
metadata = {"pair": pair}
|
||||
dataframe = strategy.feature_engineering_standard(dataframe.copy(), metadata=metadata)
|
||||
# ensure corr pairs are always last
|
||||
for corr_pair in pairs:
|
||||
for corr_pair in corr_pairs:
|
||||
if pair == corr_pair:
|
||||
continue # dont repeat anything from whitelist
|
||||
for tf in tfs:
|
||||
if pairs and do_corr_pairs:
|
||||
dataframe = strategy.populate_any_indicators(
|
||||
corr_pair,
|
||||
dataframe.copy(),
|
||||
tf,
|
||||
informative=corr_dataframes[corr_pair][tf]
|
||||
)
|
||||
if corr_pairs and do_corr_pairs:
|
||||
dataframe = self.populate_features(dataframe.copy(), corr_pair, strategy,
|
||||
corr_dataframes, base_dataframes, True)
|
||||
|
||||
dataframe = strategy.set_freqai_targets(dataframe.copy(), metadata=metadata)
|
||||
|
||||
self.get_unique_classes_from_labels(dataframe)
|
||||
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
import inspect
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
@ -106,8 +105,6 @@ class IFreqaiModel(ABC):
|
||||
self.max_system_threads = max(int(psutil.cpu_count() * 2 - 2), 1)
|
||||
self.can_short = True # overridden in start() with strategy.can_short
|
||||
|
||||
self.warned_deprecated_populate_any_indicators = False
|
||||
|
||||
record_params(config, self.full_path)
|
||||
|
||||
def __getstate__(self):
|
||||
@ -138,9 +135,6 @@ class IFreqaiModel(ABC):
|
||||
self.data_provider = strategy.dp
|
||||
self.can_short = strategy.can_short
|
||||
|
||||
# check if the strategy has deprecated populate_any_indicators function
|
||||
self.check_deprecated_populate_any_indicators(strategy)
|
||||
|
||||
if self.live:
|
||||
self.inference_timer('start')
|
||||
self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
|
||||
@ -491,7 +485,7 @@ class IFreqaiModel(ABC):
|
||||
"strategy is furnishing the same features as the pretrained"
|
||||
"model. In case of --strategy-list, please be aware that FreqAI "
|
||||
"requires all strategies to maintain identical "
|
||||
"populate_any_indicator() functions"
|
||||
"feature_engineering_* functions"
|
||||
)
|
||||
|
||||
def data_cleaning_train(self, dk: FreqaiDataKitchen) -> None:
|
||||
@ -603,7 +597,7 @@ class IFreqaiModel(ABC):
|
||||
:param strategy: IStrategy = user defined strategy object
|
||||
:param dk: FreqaiDataKitchen = non-persistent data container for current coin/loop
|
||||
:param data_load_timerange: TimeRange = the amount of data to be loaded
|
||||
for populate_any_indicators
|
||||
for populating indicators
|
||||
(larger than new_trained_timerange so that
|
||||
new_trained_timerange does not contain any NaNs)
|
||||
"""
|
||||
@ -809,7 +803,7 @@ class IFreqaiModel(ABC):
|
||||
logger.warning("Couldn't cache corr_pair dataframes for improved performance. "
|
||||
"Consider ensuring that the full coin/stake, e.g. XYZ/USD, "
|
||||
"is included in the column names when you are creating features "
|
||||
"in `populate_any_indicators()`.")
|
||||
"in `feature_engineering_*` functions.")
|
||||
self.get_corr_dataframes = not bool(self.corr_dataframes)
|
||||
elif self.corr_dataframes:
|
||||
dataframe = dk.attach_corr_pair_columns(
|
||||
@ -936,26 +930,6 @@ class IFreqaiModel(ABC):
|
||||
dk.return_dataframe, saved_dataframe, how='left', left_on='date', right_on="date_pred")
|
||||
return dk
|
||||
|
||||
def check_deprecated_populate_any_indicators(self, strategy: IStrategy):
|
||||
"""
|
||||
Check and warn if the deprecated populate_any_indicators function is used.
|
||||
:param strategy: strategy object
|
||||
"""
|
||||
|
||||
if not self.warned_deprecated_populate_any_indicators:
|
||||
self.warned_deprecated_populate_any_indicators = True
|
||||
old_version = inspect.getsource(strategy.populate_any_indicators) != (
|
||||
inspect.getsource(IStrategy.populate_any_indicators))
|
||||
|
||||
if old_version:
|
||||
logger.warning("DEPRECATION WARNING: "
|
||||
"You are using the deprecated populate_any_indicators function. "
|
||||
"This function will raise an error on March 1 2023. "
|
||||
"Please update your strategy by using "
|
||||
"the new feature_engineering functions. See \n"
|
||||
"https://www.freqtrade.io/en/latest/freqai-feature-engineering/"
|
||||
"for details.")
|
||||
|
||||
# Following methods which are overridden by user made prediction models.
|
||||
# See freqai/prediction_models/CatboostPredictionModel.py for an example.
|
||||
|
||||
|
||||
@ -93,7 +93,7 @@ class Backtesting:
|
||||
if self.config.get('strategy_list'):
|
||||
if self.config.get('freqai', {}).get('enabled', False):
|
||||
logger.warning("Using --strategy-list with FreqAI REQUIRES all strategies "
|
||||
"to have identical populate_any_indicators.")
|
||||
"to have identical feature_engineering_* functions.")
|
||||
for strat in list(self.config['strategy_list']):
|
||||
stratconf = deepcopy(self.config)
|
||||
stratconf['strategy'] = strat
|
||||
|
||||
@ -35,8 +35,8 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, c
|
||||
args = get_args(args)
|
||||
bt_config = setup_optimize_configuration(args, RunMode.BACKTEST)
|
||||
Backtesting(bt_config)
|
||||
assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical '
|
||||
'populate_any_indicators.', caplog)
|
||||
assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical',
|
||||
caplog)
|
||||
Backtesting.cleanup()
|
||||
|
||||
|
||||
|
||||
@ -291,18 +291,6 @@ def test_advise_all_indicators(default_conf, testdatadir) -> None:
|
||||
assert len(processed['UNITTEST/BTC']) == 103
|
||||
|
||||
|
||||
def test_populate_any_indicators(default_conf, testdatadir) -> None:
|
||||
strategy = StrategyResolver.load_strategy(default_conf)
|
||||
|
||||
timerange = TimeRange.parse_timerange('1510694220-1510700340')
|
||||
data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange,
|
||||
fill_up_missing=True)
|
||||
processed = strategy.populate_any_indicators('UNITTEST/BTC', data, '5m')
|
||||
assert processed == data
|
||||
assert id(processed) == id(data)
|
||||
assert len(processed['UNITTEST/BTC']) == 103
|
||||
|
||||
|
||||
def test_freqai_not_initialized(default_conf) -> None:
|
||||
strategy = StrategyResolver.load_strategy(default_conf)
|
||||
strategy.ft_bot_start()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user