freqAI Strategy - improve user experience
This commit is contained in:
Wagner Costa
parent
4601705814
commit
8227b4aafe
@ -1,4 +1,5 @@
|
||||
import copy
|
||||
import inspect
|
||||
import logging
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
@ -23,6 +24,7 @@ from freqtrade.constants import Config
|
||||
from freqtrade.data.converter import reduce_dataframe_footprint
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange import timeframe_to_seconds
|
||||
from freqtrade.strategy import merge_informative_pair
|
||||
from freqtrade.strategy.interface import IStrategy
|
||||
|
||||
|
||||
@ -1176,6 +1178,103 @@ class FreqaiDataKitchen:
|
||||
|
||||
return dataframe
|
||||
|
||||
def get_pair_data_for_features(self,
|
||||
pair: str,
|
||||
tf: str,
|
||||
strategy: IStrategy,
|
||||
corr_dataframes: dict = {},
|
||||
base_dataframes: dict = {},
|
||||
is_corr_pairs: bool = False) -> DataFrame:
|
||||
"""
|
||||
Get the data for the pair. If it's not in the dictionary, get it from the data provider
|
||||
:param pair: str = pair to get data for
|
||||
:param tf: str = timeframe to get data for
|
||||
:param strategy: IStrategy = user defined strategy object
|
||||
:param corr_dataframes: dict = dict containing the df pair dataframes
|
||||
(for user defined timeframes)
|
||||
:param base_dataframes: dict = dict containing the current pair dataframes
|
||||
(for user defined timeframes)
|
||||
:param is_corr_pairs: bool = whether the pair is a corr pair or not
|
||||
:return: dataframe = dataframe containing the pair data
|
||||
"""
|
||||
if is_corr_pairs:
|
||||
dataframe = corr_dataframes[pair][tf]
|
||||
if not dataframe.empty:
|
||||
return dataframe
|
||||
else:
|
||||
dataframe = strategy.dp.get_pair_dataframe(pair=pair, timeframe=tf)
|
||||
return dataframe
|
||||
else:
|
||||
dataframe = base_dataframes[tf]
|
||||
if not dataframe.empty:
|
||||
return dataframe
|
||||
else:
|
||||
dataframe = strategy.dp.get_pair_dataframe(pair=pair, timeframe=tf)
|
||||
return dataframe
|
||||
|
||||
def merge_features(self, df_main: DataFrame, df_to_merge: DataFrame,
|
||||
tf: str, timeframe_inf: str, suffix: str) -> DataFrame:
|
||||
"""
|
||||
Merge the features of the dataframe and remove HLCV and date added columns
|
||||
:param df_main: DataFrame = main dataframe
|
||||
:param df_to_merge: DataFrame = dataframe to merge
|
||||
:param tf: str = timeframe of the main dataframe
|
||||
:param timeframe_inf: str = timeframe of the dataframe to merge
|
||||
:param suffix: str = suffix to add to the columns of the dataframe to merge
|
||||
:return: dataframe = merged dataframe
|
||||
"""
|
||||
dataframe = merge_informative_pair(df_main, df_to_merge, tf, timeframe_inf=timeframe_inf,
|
||||
append_timeframe=False, suffix=suffix, ffill=True)
|
||||
skip_columns = [
|
||||
(f"{s}_{suffix}") for s in ["date", "open", "high", "low", "close", "volume"]
|
||||
]
|
||||
dataframe = dataframe.drop(columns=skip_columns)
|
||||
return dataframe
|
||||
|
||||
def populate_features(self, dataframe: DataFrame, pair: str, strategy: IStrategy,
|
||||
corr_dataframes: dict, base_dataframes: dict,
|
||||
is_corr_pairs: bool = False) -> DataFrame:
|
||||
"""
|
||||
Use the user defined strategy functions for populating features
|
||||
:param dataframe: DataFrame = dataframe to populate
|
||||
:param pair: str = pair to populate
|
||||
:param strategy: IStrategy = user defined strategy object
|
||||
:param corr_dataframes: dict = dict containing the df pair dataframes
|
||||
:param base_dataframes: dict = dict containing the current pair dataframes
|
||||
:param is_corr_pairs: bool = whether the pair is a corr pair or not
|
||||
:return: dataframe = populated dataframe
|
||||
"""
|
||||
tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes")
|
||||
|
||||
for tf in tfs:
|
||||
informative_df = self.get_pair_data_for_features(
|
||||
pair, tf, strategy, corr_dataframes, base_dataframes, is_corr_pairs)
|
||||
informative_copy = informative_df.copy()
|
||||
|
||||
for t in self.freqai_config["feature_parameters"]["indicator_periods_candles"]:
|
||||
df_features = strategy.freqai_feature_engineering_indicator_periods(
|
||||
informative_copy.copy(), t)
|
||||
suffix = f"{t}"
|
||||
informative_df = self.merge_features(informative_df, df_features, tf, tf, suffix)
|
||||
|
||||
generic_df = strategy.freqai_feature_engineering_generic(informative_copy.copy())
|
||||
suffix = "gen"
|
||||
|
||||
informative_df = self.merge_features(informative_df, generic_df, tf, tf, suffix)
|
||||
|
||||
indicators = [col for col in informative_df if col.startswith("%")]
|
||||
for n in range(self.freqai_config["feature_parameters"]["include_shifted_candles"] + 1):
|
||||
if n == 0:
|
||||
continue
|
||||
df_shift = informative_df[indicators].shift(n)
|
||||
df_shift = df_shift.add_suffix("_shift-" + str(n))
|
||||
informative_df = pd.concat((informative_df, df_shift), axis=1)
|
||||
|
||||
dataframe = self.merge_features(dataframe.copy(), informative_df,
|
||||
self.config["timeframe"], tf, f'{pair}_{tf}')
|
||||
|
||||
return dataframe
|
||||
|
||||
def use_strategy_to_populate_indicators(
|
||||
self,
|
||||
strategy: IStrategy,
|
||||
@ -1188,7 +1287,88 @@ class FreqaiDataKitchen:
|
||||
"""
|
||||
Use the user defined strategy for populating indicators during retrain
|
||||
:param strategy: IStrategy = user defined strategy object
|
||||
:param corr_dataframes: dict = dict containing the informative pair dataframes
|
||||
:param corr_dataframes: dict = dict containing the df pair dataframes
|
||||
(for user defined timeframes)
|
||||
:param base_dataframes: dict = dict containing the current pair dataframes
|
||||
(for user defined timeframes)
|
||||
:param pair: str = pair to populate
|
||||
:param prediction_dataframe: DataFrame = dataframe containing the pair data
|
||||
used for prediction
|
||||
:param do_corr_pairs: bool = whether to populate corr pairs or not
|
||||
:return:
|
||||
dataframe: DataFrame = dataframe containing populated indicators
|
||||
"""
|
||||
|
||||
# this is a hack to check if the user is using the populate_any_indicators function
|
||||
new_version = inspect.getsource(strategy.populate_any_indicators) == (
|
||||
inspect.getsource(IStrategy.populate_any_indicators))
|
||||
|
||||
if new_version:
|
||||
tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes")
|
||||
pairs: List[str] = self.freqai_config["feature_parameters"].get(
|
||||
"include_corr_pairlist", [])
|
||||
|
||||
if not prediction_dataframe.empty:
|
||||
dataframe = prediction_dataframe.copy()
|
||||
for tf in tfs:
|
||||
base_dataframes[tf] = pd.DataFrame()
|
||||
for p in pairs:
|
||||
if p not in corr_dataframes:
|
||||
corr_dataframes[p] = {}
|
||||
corr_dataframes[p][tf] = pd.DataFrame()
|
||||
else:
|
||||
dataframe = base_dataframes[self.config["timeframe"]].copy()
|
||||
|
||||
corr_pairs: List[str] = self.freqai_config["feature_parameters"].get(
|
||||
"include_corr_pairlist", [])
|
||||
dataframe = self.populate_features(dataframe.copy(), pair, strategy,
|
||||
corr_dataframes, base_dataframes)
|
||||
|
||||
# ensure corr pairs are always last
|
||||
for corr_pair in corr_pairs:
|
||||
if pair == corr_pair:
|
||||
continue # dont repeat anything from whitelist
|
||||
if corr_pairs and do_corr_pairs:
|
||||
dataframe = self.populate_features(dataframe.copy(), corr_pair, strategy,
|
||||
corr_dataframes, base_dataframes, True)
|
||||
|
||||
dataframe = strategy.freqai_feature_engineering_generalized_indicators(dataframe.copy())
|
||||
dataframe = strategy.freqai_set_targets(dataframe.copy())
|
||||
|
||||
self.get_unique_classes_from_labels(dataframe)
|
||||
|
||||
dataframe = self.remove_special_chars_from_feature_names(dataframe)
|
||||
|
||||
if self.config.get('reduce_df_footprint', False):
|
||||
dataframe = reduce_dataframe_footprint(dataframe)
|
||||
|
||||
return dataframe
|
||||
|
||||
else:
|
||||
# the user is using the populate_any_indicators functions which is deprecated
|
||||
logger.warning("DEPRECATION WARNING: "
|
||||
"You are using the deprecated populate_any_indicators function. "
|
||||
"Please update your strategy to use "
|
||||
"the new feature_engineering functions.")
|
||||
|
||||
df = self.use_strategy_to_populate_indicators_old_version(
|
||||
strategy, corr_dataframes, base_dataframes, pair,
|
||||
prediction_dataframe, do_corr_pairs)
|
||||
return df
|
||||
|
||||
def use_strategy_to_populate_indicators_old_version(
|
||||
self,
|
||||
strategy: IStrategy,
|
||||
corr_dataframes: dict = {},
|
||||
base_dataframes: dict = {},
|
||||
pair: str = "",
|
||||
prediction_dataframe: DataFrame = pd.DataFrame(),
|
||||
do_corr_pairs: bool = True,
|
||||
) -> DataFrame:
|
||||
"""
|
||||
Use the user defined strategy for populating indicators during retrain
|
||||
:param strategy: IStrategy = user defined strategy object
|
||||
:param corr_dataframes: dict = dict containing the df pair dataframes
|
||||
(for user defined timeframes)
|
||||
:param base_dataframes: dict = dict containing the current pair dataframes
|
||||
(for user defined timeframes)
|
||||
@ -1212,6 +1392,7 @@ class FreqaiDataKitchen:
|
||||
corr_dataframes[p][tf] = None
|
||||
else:
|
||||
dataframe = base_dataframes[self.config["timeframe"]].copy()
|
||||
# dataframe = strategy.dp.get_pair_dataframe(pair, self.config["timeframe"])
|
||||
|
||||
sgi = False
|
||||
for tf in tfs:
|
||||
|
||||
@ -598,6 +598,7 @@ class IStrategy(ABC, HyperStrategyMixin):
|
||||
informative: DataFrame = None,
|
||||
set_generalized_indicators: bool = False) -> DataFrame:
|
||||
"""
|
||||
DEPRECATED - USE FEATURE ENGINEERING FUNCTIONS INSTEAD
|
||||
Function designed to automatically generate, name and merge features
|
||||
from user indicated timeframes in the configuration file. User can add
|
||||
additional features here, but must follow the naming convention.
|
||||
@ -610,6 +611,45 @@ class IStrategy(ABC, HyperStrategyMixin):
|
||||
"""
|
||||
return df
|
||||
|
||||
def freqai_feature_engineering_indicator_periods(self, dataframe: DataFrame,
|
||||
period: int, **kwargs):
|
||||
"""
|
||||
This function will be called for all include_timeframes in each indicator_periods_candles
|
||||
(including corr_pairs).
|
||||
After that, the features will be shifted by the number of candles in the
|
||||
include_shifted_candles.
|
||||
:param df: strategy dataframe which will receive the features
|
||||
:param period: period of the indicator - usage example:
|
||||
dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
|
||||
"""
|
||||
return dataframe
|
||||
|
||||
def freqai_feature_engineering_generic(self, dataframe: DataFrame, **kwargs):
|
||||
"""
|
||||
This optional function will be called for all include_timeframes (including corr_pairs).
|
||||
After that, the features will be shifted by the number of candles in the
|
||||
include_shifted_candles.
|
||||
:param df: strategy dataframe which will receive the features
|
||||
dataframe["%-pct-change"] = dataframe["close"].pct_change()
|
||||
"""
|
||||
return dataframe
|
||||
|
||||
def freqai_feature_engineering_generalized_indicators(self, dataframe: DataFrame, **kwargs):
|
||||
"""
|
||||
This optional function will be called once with the dataframe of the main timeframe.
|
||||
:param df: strategy dataframe which will receive the features
|
||||
usage example: dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7
|
||||
"""
|
||||
return dataframe
|
||||
|
||||
def freqai_set_targets(self, dataframe, **kwargs):
|
||||
"""
|
||||
Required function to set the targets for the model.
|
||||
:param df: strategy dataframe which will receive the targets
|
||||
usage example: dataframe["&-target"] = dataframe["close"].shift(-1) / dataframe["close"]
|
||||
"""
|
||||
return dataframe
|
||||
|
||||
###
|
||||
# END - Intended to be overridden by strategy
|
||||
###
|
||||
|
||||
@ -47,16 +47,94 @@ class FreqaiExampleStrategy(IStrategy):
|
||||
std_dev_multiplier_sell = CategoricalParameter(
|
||||
[0.75, 1, 1.25, 1.5, 1.75], space="sell", default=1.25, optimize=True)
|
||||
|
||||
def populate_any_indicators(
|
||||
def freqai_feature_engineering_indicator_periods(self, dataframe, period, **kwargs):
|
||||
"""
|
||||
This function will be called for all include_timeframes in each indicator_periods_candles
|
||||
(including corr_pairs).
|
||||
After that, the features will be shifted by the number of candles in the
|
||||
include_shifted_candles.
|
||||
:param df: strategy dataframe which will receive the features
|
||||
:param period: period of the indicator - usage example:
|
||||
dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
|
||||
"""
|
||||
dataframe["%-rsi-period"] = ta.RSI(dataframe, timeperiod=period)
|
||||
dataframe["%-mfi-period"] = ta.MFI(dataframe, timeperiod=period)
|
||||
dataframe["%-adx-period"] = ta.ADX(dataframe, timeperiod=period)
|
||||
dataframe["%-sma-period"] = ta.SMA(dataframe, timeperiod=period)
|
||||
dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
|
||||
|
||||
bollinger = qtpylib.bollinger_bands(
|
||||
qtpylib.typical_price(dataframe), window=period, stds=2.2
|
||||
)
|
||||
dataframe["bb_lowerband-period"] = bollinger["lower"]
|
||||
dataframe["bb_middleband-period"] = bollinger["mid"]
|
||||
dataframe["bb_upperband-period"] = bollinger["upper"]
|
||||
|
||||
dataframe["%-bb_width-period"] = (
|
||||
dataframe["bb_upperband-period"]
|
||||
- dataframe["bb_lowerband-period"]
|
||||
) / dataframe["bb_middleband-period"]
|
||||
dataframe["%-close-bb_lower-period"] = (
|
||||
dataframe["close"] / dataframe["bb_lowerband-period"]
|
||||
)
|
||||
|
||||
dataframe["%-roc-period"] = ta.ROC(dataframe, timeperiod=period)
|
||||
|
||||
dataframe["%-relative_volume-period"] = (
|
||||
dataframe["volume"] / dataframe["volume"].rolling(period).mean()
|
||||
)
|
||||
|
||||
return dataframe
|
||||
|
||||
def freqai_feature_engineering_generic(self, dataframe, **kwargs):
|
||||
"""
|
||||
This optional function will be called for all include_timeframes (including corr_pairs).
|
||||
After that, the features will be shifted by the number of candles in the
|
||||
include_shifted_candles.
|
||||
:param df: strategy dataframe which will receive the features
|
||||
dataframe["%-pct-change"] = dataframe["close"].pct_change()
|
||||
"""
|
||||
dataframe["%-pct-change"] = dataframe["close"].pct_change()
|
||||
dataframe["%-raw_volume"] = dataframe["volume"]
|
||||
dataframe["%-raw_price"] = dataframe["close"]
|
||||
return dataframe
|
||||
|
||||
def freqai_feature_engineering_generalized_indicators(self, dataframe, **kwargs):
|
||||
"""
|
||||
This optional function will be called once with the dataframe of the main timeframe.
|
||||
:param df: strategy dataframe which will receive the features
|
||||
usage example: dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7
|
||||
"""
|
||||
dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7
|
||||
dataframe["%-hour_of_day"] = (dataframe["date"].dt.hour + 1) / 25
|
||||
return dataframe
|
||||
|
||||
def freqai_set_targets(self, dataframe, **kwargs):
|
||||
"""
|
||||
Required function to set the targets for the model.
|
||||
:param df: strategy dataframe which will receive the targets
|
||||
usage example: dataframe["&-target"] = dataframe["close"].shift(-1) / dataframe["close"]
|
||||
"""
|
||||
dataframe["&-s_close"] = (
|
||||
dataframe["close"]
|
||||
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
|
||||
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
|
||||
.mean()
|
||||
/ dataframe["close"]
|
||||
- 1
|
||||
)
|
||||
return dataframe
|
||||
|
||||
def populate_any_indicators_old(
|
||||
self, pair, df, tf, informative=None, set_generalized_indicators=False
|
||||
):
|
||||
"""
|
||||
DEPRECATED - USE FEATURE ENGINEERING FUNCTIONS INSTEAD
|
||||
Function designed to automatically generate, name and merge features
|
||||
from user indicated timeframes in the configuration file. User controls the indicators
|
||||
passed to the training/prediction by prepending indicators with `f'%-{pair}`
|
||||
(see convention below). I.e. user should not prepend any supporting metrics
|
||||
(e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the
|
||||
model.
|
||||
from user indicated timeframes in the configuration file. User can add
|
||||
additional features here, but must follow the naming convention.
|
||||
This method is *only* used in FreqaiDataKitchen class and therefore
|
||||
it is only called if FreqAI is active.
|
||||
:param pair: pair to be used as informative
|
||||
:param df: strategy dataframe which will receive merges from informatives
|
||||
:param tf: timeframe of the dataframe which will modify the feature names
|
||||
|
||||
Loading…
Reference in New Issue
Block a user