diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 9c8158c8a..075458a69 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1184,6 +1184,7 @@ class FreqaiDataKitchen: pair: str = "", prediction_dataframe: DataFrame = pd.DataFrame(), do_corr_pairs: bool = True, + set_only_targets: bool = False ) -> DataFrame: """ Use the user defined strategy for populating indicators during retrain @@ -1222,7 +1223,8 @@ class FreqaiDataKitchen: dataframe.copy(), tf, informative=base_dataframes[tf], - set_generalized_indicators=sgi + set_generalized_indicators=sgi, + set_only_targets=set_only_targets ) # ensure corr pairs are always last @@ -1235,7 +1237,8 @@ class FreqaiDataKitchen: corr_pair, dataframe.copy(), tf, - informative=corr_dataframes[corr_pair][tf] + informative=corr_dataframes[corr_pair][tf], + set_only_targets=set_only_targets ) self.get_unique_classes_from_labels(dataframe) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 9025f358a..13009d31d 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -149,12 +149,13 @@ class IFreqaiModel(ABC): # the concatenated results for the full backtesting period back to the strategy. elif not self.follow_mode: self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"]) - dataframe = self.dk.use_strategy_to_populate_indicators( - strategy, prediction_dataframe=dataframe, pair=metadata["pair"] - ) + + # dataframe = self.dk.use_strategy_to_populate_indicators( + # strategy, prediction_dataframe=dataframe, pair=metadata["pair"] + # ) if not self.config.get("freqai_backtest_live_models", False): logger.info(f"Training {len(self.dk.training_timeranges)} timeranges") - dk = self.start_backtesting(dataframe, metadata, self.dk) + dk = self.start_backtesting(dataframe, metadata, self.dk, strategy) dataframe = dk.remove_features_from_df(dk.return_dataframe) else: logger.info( @@ -255,7 +256,7 @@ class IFreqaiModel(ABC): self.dd.save_metric_tracker_to_disk() def start_backtesting( - self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen + self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen, strategy: IStrategy ) -> FreqaiDataKitchen: """ The main broad execution for backtesting. For backtesting, each pair enters and then gets @@ -267,12 +268,14 @@ class IFreqaiModel(ABC): :param dataframe: DataFrame = strategy passed dataframe :param metadata: Dict = pair metadata :param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only + :param strategy: Strategy to train on :return: FreqaiDataKitchen = Data management/analysis tool associated to present pair only """ self.pair_it += 1 train_it = 0 + set_only_targets = False # Loop enforcing the sliding window training/backtesting paradigm # tr_train is the training time range e.g. 1 historical month # tr_backtest is the backtesting time range e.g. the week directly @@ -301,14 +304,39 @@ class IFreqaiModel(ABC): dk.set_new_model_names(pair, timestamp_model_id) if dk.check_if_backtest_prediction_is_valid(len_backtest_df): - self.dd.load_metadata(dk) - dk.find_features(dataframe) - self.check_if_feature_list_matches_strategy(dk) + # self.dd.load_metadata(dk) + # dk.find_features(dataframe) + # self.check_if_feature_list_matches_strategy(dk) append_df = dk.get_backtesting_prediction() dk.append_predictions(append_df) else: - dataframe_train = dk.slice_dataframe(tr_train, dataframe) - dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe) + if set_only_targets is False: + dataframe = self.dk.use_strategy_to_populate_indicators( + strategy, prediction_dataframe=dataframe, + pair=metadata["pair"], set_only_targets=set_only_targets + ) + set_only_targets = True + + dataframe_base_train = dataframe.loc[dataframe["date"] < tr_train.stopdt, :] + dataframe_base_train = self.dk.use_strategy_to_populate_indicators( + strategy, prediction_dataframe=dataframe_base_train, + pair=metadata["pair"], set_only_targets=set_only_targets + ) + + # region need to verify if this block is needed + # (or slice dataframe_backtest from dataframe) + dataframe_base_backtest = dataframe.loc[dataframe["date"] < tr_backtest.stopdt, :] + dataframe_base_backtest = self.dk.use_strategy_to_populate_indicators( + strategy, prediction_dataframe=dataframe_base_backtest, + pair=metadata["pair"], set_only_targets=set_only_targets + ) + # endregion + + set_only_targets = True + + dataframe_train = dk.slice_dataframe(tr_train, dataframe_base_train) + dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe_base_backtest) + if not self.model_exists(dk): dk.find_features(dataframe_train) dk.find_labels(dataframe_train) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 781ae6c5c..4f1c08474 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -596,7 +596,8 @@ class IStrategy(ABC, HyperStrategyMixin): def populate_any_indicators(self, pair: str, df: DataFrame, tf: str, informative: DataFrame = None, - set_generalized_indicators: bool = False) -> DataFrame: + set_generalized_indicators: bool = False, + set_only_targets: bool = False) -> DataFrame: """ Function designed to automatically generate, name and merge features from user indicated timeframes in the configuration file. User can add @@ -607,6 +608,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param df: strategy dataframe which will receive merges from informatives :param tf: timeframe of the dataframe which will modify the feature names :param informative: the dataframe associated with the informative pair + :param set_only_targets: if True, only the target features will be set """ return df diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index fc39b0ab4..ff77a1dff 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -48,7 +48,13 @@ class FreqaiExampleStrategy(IStrategy): [0.75, 1, 1.25, 1.5, 1.75], space="sell", default=1.25, optimize=True) def populate_any_indicators( - self, pair, df, tf, informative=None, set_generalized_indicators=False + self, + pair, + df, + tf, + informative=None, + set_generalized_indicators=False, + set_only_targets=False ): """ Function designed to automatically generate, name and merge features @@ -63,66 +69,68 @@ class FreqaiExampleStrategy(IStrategy): :param informative: the dataframe associated with the informative pair """ - if informative is None: - informative = self.dp.get_pair_dataframe(pair, tf) + if not set_only_targets: + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) - # first loop is automatically duplicating indicators for time periods - for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: - t = int(t) - informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) - informative[f"%-{pair}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) - informative[f"%-{pair}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) + t = int(t) + informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) + informative[f"%-{pair}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) + informative[f"%-{pair}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) - bollinger = qtpylib.bollinger_bands( - qtpylib.typical_price(informative), window=t, stds=2.2 - ) - informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"] - informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"] - informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"] + bollinger = qtpylib.bollinger_bands( + qtpylib.typical_price(informative), window=t, stds=2.2 + ) + informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"] + informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"] + informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"] - informative[f"%-{pair}bb_width-period_{t}"] = ( - informative[f"{pair}bb_upperband-period_{t}"] - - informative[f"{pair}bb_lowerband-period_{t}"] - ) / informative[f"{pair}bb_middleband-period_{t}"] - informative[f"%-{pair}close-bb_lower-period_{t}"] = ( - informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"] - ) + informative[f"%-{pair}bb_width-period_{t}"] = ( + informative[f"{pair}bb_upperband-period_{t}"] + - informative[f"{pair}bb_lowerband-period_{t}"] + ) / informative[f"{pair}bb_middleband-period_{t}"] + informative[f"%-{pair}close-bb_lower-period_{t}"] = ( + informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"] + ) - informative[f"%-{pair}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) + informative[f"%-{pair}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) - informative[f"%-{pair}relative_volume-period_{t}"] = ( - informative["volume"] / informative["volume"].rolling(t).mean() - ) + informative[f"%-{pair}relative_volume-period_{t}"] = ( + informative["volume"] / informative["volume"].rolling(t).mean() + ) - informative[f"%-{pair}pct-change"] = informative["close"].pct_change() - informative[f"%-{pair}raw_volume"] = informative["volume"] - informative[f"%-{pair}raw_price"] = informative["close"] + informative[f"%-{pair}pct-change"] = informative["close"].pct_change() + informative[f"%-{pair}raw_volume"] = informative["volume"] + informative[f"%-{pair}raw_price"] = informative["close"] - indicators = [col for col in informative if col.startswith("%")] - # This loop duplicates and shifts all indicators to add a sense of recency to data - for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): - if n == 0: - continue - informative_shift = informative[indicators].shift(n) - informative_shift = informative_shift.add_suffix("_shift-" + str(n)) - informative = pd.concat((informative, informative_shift), axis=1) + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) - df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) - skip_columns = [ - (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] - ] - df = df.drop(columns=skip_columns) + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) + + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: + df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 + df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 - # Add generalized indicators here (because in live, it will call this - # function to populate indicators during training). Notice how we ensure not to - # add them multiple times if set_generalized_indicators: - df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 - df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 - # user adds targets here by prepending them with &- (see convention below) df["&-s_close"] = ( df["close"]