From 8227b4aafe51b30e5942d293e8d0052c968442dd Mon Sep 17 00:00:00 2001 From: Wagner Costa Date: Tue, 27 Dec 2022 11:37:01 -0300 Subject: [PATCH] freqAI Strategy - improve user experience --- freqtrade/freqai/data_kitchen.py | 183 ++++++++++++++++++- freqtrade/strategy/interface.py | 40 ++++ freqtrade/templates/FreqaiExampleStrategy.py | 90 ++++++++- 3 files changed, 306 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 9c8158c8a..c3e5929de 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1,4 +1,5 @@ import copy +import inspect import logging import shutil from datetime import datetime, timezone @@ -23,6 +24,7 @@ from freqtrade.constants import Config from freqtrade.data.converter import reduce_dataframe_footprint from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds +from freqtrade.strategy import merge_informative_pair from freqtrade.strategy.interface import IStrategy @@ -1176,6 +1178,103 @@ class FreqaiDataKitchen: return dataframe + def get_pair_data_for_features(self, + pair: str, + tf: str, + strategy: IStrategy, + corr_dataframes: dict = {}, + base_dataframes: dict = {}, + is_corr_pairs: bool = False) -> DataFrame: + """ + Get the data for the pair. If it's not in the dictionary, get it from the data provider + :param pair: str = pair to get data for + :param tf: str = timeframe to get data for + :param strategy: IStrategy = user defined strategy object + :param corr_dataframes: dict = dict containing the df pair dataframes + (for user defined timeframes) + :param base_dataframes: dict = dict containing the current pair dataframes + (for user defined timeframes) + :param is_corr_pairs: bool = whether the pair is a corr pair or not + :return: dataframe = dataframe containing the pair data + """ + if is_corr_pairs: + dataframe = corr_dataframes[pair][tf] + if not dataframe.empty: + return dataframe + else: + dataframe = strategy.dp.get_pair_dataframe(pair=pair, timeframe=tf) + return dataframe + else: + dataframe = base_dataframes[tf] + if not dataframe.empty: + return dataframe + else: + dataframe = strategy.dp.get_pair_dataframe(pair=pair, timeframe=tf) + return dataframe + + def merge_features(self, df_main: DataFrame, df_to_merge: DataFrame, + tf: str, timeframe_inf: str, suffix: str) -> DataFrame: + """ + Merge the features of the dataframe and remove HLCV and date added columns + :param df_main: DataFrame = main dataframe + :param df_to_merge: DataFrame = dataframe to merge + :param tf: str = timeframe of the main dataframe + :param timeframe_inf: str = timeframe of the dataframe to merge + :param suffix: str = suffix to add to the columns of the dataframe to merge + :return: dataframe = merged dataframe + """ + dataframe = merge_informative_pair(df_main, df_to_merge, tf, timeframe_inf=timeframe_inf, + append_timeframe=False, suffix=suffix, ffill=True) + skip_columns = [ + (f"{s}_{suffix}") for s in ["date", "open", "high", "low", "close", "volume"] + ] + dataframe = dataframe.drop(columns=skip_columns) + return dataframe + + def populate_features(self, dataframe: DataFrame, pair: str, strategy: IStrategy, + corr_dataframes: dict, base_dataframes: dict, + is_corr_pairs: bool = False) -> DataFrame: + """ + Use the user defined strategy functions for populating features + :param dataframe: DataFrame = dataframe to populate + :param pair: str = pair to populate + :param strategy: IStrategy = user defined strategy object + :param corr_dataframes: dict = dict containing the df pair dataframes + :param base_dataframes: dict = dict containing the current pair dataframes + :param is_corr_pairs: bool = whether the pair is a corr pair or not + :return: dataframe = populated dataframe + """ + tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes") + + for tf in tfs: + informative_df = self.get_pair_data_for_features( + pair, tf, strategy, corr_dataframes, base_dataframes, is_corr_pairs) + informative_copy = informative_df.copy() + + for t in self.freqai_config["feature_parameters"]["indicator_periods_candles"]: + df_features = strategy.freqai_feature_engineering_indicator_periods( + informative_copy.copy(), t) + suffix = f"{t}" + informative_df = self.merge_features(informative_df, df_features, tf, tf, suffix) + + generic_df = strategy.freqai_feature_engineering_generic(informative_copy.copy()) + suffix = "gen" + + informative_df = self.merge_features(informative_df, generic_df, tf, tf, suffix) + + indicators = [col for col in informative_df if col.startswith("%")] + for n in range(self.freqai_config["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + df_shift = informative_df[indicators].shift(n) + df_shift = df_shift.add_suffix("_shift-" + str(n)) + informative_df = pd.concat((informative_df, df_shift), axis=1) + + dataframe = self.merge_features(dataframe.copy(), informative_df, + self.config["timeframe"], tf, f'{pair}_{tf}') + + return dataframe + def use_strategy_to_populate_indicators( self, strategy: IStrategy, @@ -1188,7 +1287,88 @@ class FreqaiDataKitchen: """ Use the user defined strategy for populating indicators during retrain :param strategy: IStrategy = user defined strategy object - :param corr_dataframes: dict = dict containing the informative pair dataframes + :param corr_dataframes: dict = dict containing the df pair dataframes + (for user defined timeframes) + :param base_dataframes: dict = dict containing the current pair dataframes + (for user defined timeframes) + :param pair: str = pair to populate + :param prediction_dataframe: DataFrame = dataframe containing the pair data + used for prediction + :param do_corr_pairs: bool = whether to populate corr pairs or not + :return: + dataframe: DataFrame = dataframe containing populated indicators + """ + + # this is a hack to check if the user is using the populate_any_indicators function + new_version = inspect.getsource(strategy.populate_any_indicators) == ( + inspect.getsource(IStrategy.populate_any_indicators)) + + if new_version: + tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes") + pairs: List[str] = self.freqai_config["feature_parameters"].get( + "include_corr_pairlist", []) + + if not prediction_dataframe.empty: + dataframe = prediction_dataframe.copy() + for tf in tfs: + base_dataframes[tf] = pd.DataFrame() + for p in pairs: + if p not in corr_dataframes: + corr_dataframes[p] = {} + corr_dataframes[p][tf] = pd.DataFrame() + else: + dataframe = base_dataframes[self.config["timeframe"]].copy() + + corr_pairs: List[str] = self.freqai_config["feature_parameters"].get( + "include_corr_pairlist", []) + dataframe = self.populate_features(dataframe.copy(), pair, strategy, + corr_dataframes, base_dataframes) + + # ensure corr pairs are always last + for corr_pair in corr_pairs: + if pair == corr_pair: + continue # dont repeat anything from whitelist + if corr_pairs and do_corr_pairs: + dataframe = self.populate_features(dataframe.copy(), corr_pair, strategy, + corr_dataframes, base_dataframes, True) + + dataframe = strategy.freqai_feature_engineering_generalized_indicators(dataframe.copy()) + dataframe = strategy.freqai_set_targets(dataframe.copy()) + + self.get_unique_classes_from_labels(dataframe) + + dataframe = self.remove_special_chars_from_feature_names(dataframe) + + if self.config.get('reduce_df_footprint', False): + dataframe = reduce_dataframe_footprint(dataframe) + + return dataframe + + else: + # the user is using the populate_any_indicators functions which is deprecated + logger.warning("DEPRECATION WARNING: " + "You are using the deprecated populate_any_indicators function. " + "Please update your strategy to use " + "the new feature_engineering functions.") + + df = self.use_strategy_to_populate_indicators_old_version( + strategy, corr_dataframes, base_dataframes, pair, + prediction_dataframe, do_corr_pairs) + return df + + def use_strategy_to_populate_indicators_old_version( + self, + strategy: IStrategy, + corr_dataframes: dict = {}, + base_dataframes: dict = {}, + pair: str = "", + prediction_dataframe: DataFrame = pd.DataFrame(), + do_corr_pairs: bool = True, + ) -> DataFrame: + """ + Use the user defined strategy for populating indicators during retrain + :param strategy: IStrategy = user defined strategy object + :param corr_dataframes: dict = dict containing the df pair dataframes (for user defined timeframes) :param base_dataframes: dict = dict containing the current pair dataframes (for user defined timeframes) @@ -1212,6 +1392,7 @@ class FreqaiDataKitchen: corr_dataframes[p][tf] = None else: dataframe = base_dataframes[self.config["timeframe"]].copy() + # dataframe = strategy.dp.get_pair_dataframe(pair, self.config["timeframe"]) sgi = False for tf in tfs: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 781ae6c5c..6bcc2a23f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -598,6 +598,7 @@ class IStrategy(ABC, HyperStrategyMixin): informative: DataFrame = None, set_generalized_indicators: bool = False) -> DataFrame: """ + DEPRECATED - USE FEATURE ENGINEERING FUNCTIONS INSTEAD Function designed to automatically generate, name and merge features from user indicated timeframes in the configuration file. User can add additional features here, but must follow the naming convention. @@ -610,6 +611,45 @@ class IStrategy(ABC, HyperStrategyMixin): """ return df + def freqai_feature_engineering_indicator_periods(self, dataframe: DataFrame, + period: int, **kwargs): + """ + This function will be called for all include_timeframes in each indicator_periods_candles + (including corr_pairs). + After that, the features will be shifted by the number of candles in the + include_shifted_candles. + :param df: strategy dataframe which will receive the features + :param period: period of the indicator - usage example: + dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period) + """ + return dataframe + + def freqai_feature_engineering_generic(self, dataframe: DataFrame, **kwargs): + """ + This optional function will be called for all include_timeframes (including corr_pairs). + After that, the features will be shifted by the number of candles in the + include_shifted_candles. + :param df: strategy dataframe which will receive the features + dataframe["%-pct-change"] = dataframe["close"].pct_change() + """ + return dataframe + + def freqai_feature_engineering_generalized_indicators(self, dataframe: DataFrame, **kwargs): + """ + This optional function will be called once with the dataframe of the main timeframe. + :param df: strategy dataframe which will receive the features + usage example: dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7 + """ + return dataframe + + def freqai_set_targets(self, dataframe, **kwargs): + """ + Required function to set the targets for the model. + :param df: strategy dataframe which will receive the targets + usage example: dataframe["&-target"] = dataframe["close"].shift(-1) / dataframe["close"] + """ + return dataframe + ### # END - Intended to be overridden by strategy ### diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index fc39b0ab4..323919a47 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -47,16 +47,94 @@ class FreqaiExampleStrategy(IStrategy): std_dev_multiplier_sell = CategoricalParameter( [0.75, 1, 1.25, 1.5, 1.75], space="sell", default=1.25, optimize=True) - def populate_any_indicators( + def freqai_feature_engineering_indicator_periods(self, dataframe, period, **kwargs): + """ + This function will be called for all include_timeframes in each indicator_periods_candles + (including corr_pairs). + After that, the features will be shifted by the number of candles in the + include_shifted_candles. + :param df: strategy dataframe which will receive the features + :param period: period of the indicator - usage example: + dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period) + """ + dataframe["%-rsi-period"] = ta.RSI(dataframe, timeperiod=period) + dataframe["%-mfi-period"] = ta.MFI(dataframe, timeperiod=period) + dataframe["%-adx-period"] = ta.ADX(dataframe, timeperiod=period) + dataframe["%-sma-period"] = ta.SMA(dataframe, timeperiod=period) + dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period) + + bollinger = qtpylib.bollinger_bands( + qtpylib.typical_price(dataframe), window=period, stds=2.2 + ) + dataframe["bb_lowerband-period"] = bollinger["lower"] + dataframe["bb_middleband-period"] = bollinger["mid"] + dataframe["bb_upperband-period"] = bollinger["upper"] + + dataframe["%-bb_width-period"] = ( + dataframe["bb_upperband-period"] + - dataframe["bb_lowerband-period"] + ) / dataframe["bb_middleband-period"] + dataframe["%-close-bb_lower-period"] = ( + dataframe["close"] / dataframe["bb_lowerband-period"] + ) + + dataframe["%-roc-period"] = ta.ROC(dataframe, timeperiod=period) + + dataframe["%-relative_volume-period"] = ( + dataframe["volume"] / dataframe["volume"].rolling(period).mean() + ) + + return dataframe + + def freqai_feature_engineering_generic(self, dataframe, **kwargs): + """ + This optional function will be called for all include_timeframes (including corr_pairs). + After that, the features will be shifted by the number of candles in the + include_shifted_candles. + :param df: strategy dataframe which will receive the features + dataframe["%-pct-change"] = dataframe["close"].pct_change() + """ + dataframe["%-pct-change"] = dataframe["close"].pct_change() + dataframe["%-raw_volume"] = dataframe["volume"] + dataframe["%-raw_price"] = dataframe["close"] + return dataframe + + def freqai_feature_engineering_generalized_indicators(self, dataframe, **kwargs): + """ + This optional function will be called once with the dataframe of the main timeframe. + :param df: strategy dataframe which will receive the features + usage example: dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7 + """ + dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7 + dataframe["%-hour_of_day"] = (dataframe["date"].dt.hour + 1) / 25 + return dataframe + + def freqai_set_targets(self, dataframe, **kwargs): + """ + Required function to set the targets for the model. + :param df: strategy dataframe which will receive the targets + usage example: dataframe["&-target"] = dataframe["close"].shift(-1) / dataframe["close"] + """ + dataframe["&-s_close"] = ( + dataframe["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .mean() + / dataframe["close"] + - 1 + ) + return dataframe + + def populate_any_indicators_old( self, pair, df, tf, informative=None, set_generalized_indicators=False ): """ + DEPRECATED - USE FEATURE ENGINEERING FUNCTIONS INSTEAD Function designed to automatically generate, name and merge features - from user indicated timeframes in the configuration file. User controls the indicators - passed to the training/prediction by prepending indicators with `f'%-{pair}` - (see convention below). I.e. user should not prepend any supporting metrics - (e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the - model. + from user indicated timeframes in the configuration file. User can add + additional features here, but must follow the naming convention. + This method is *only* used in FreqaiDataKitchen class and therefore + it is only called if FreqAI is active. :param pair: pair to be used as informative :param df: strategy dataframe which will receive merges from informatives :param tf: timeframe of the dataframe which will modify the feature names