adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

freqai bt fix populate any indicators call - first commit

Browse Source
This commit is contained in:
Wagner Costa 2022-12-20 20:26:27 -03:00
parent 3012c55ec5
commit cc796cc594
4 changed files with 104 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
freqtrade/freqai/data_kitchen.py
View File

@ -1184,6 +1184,7 @@ class FreqaiDataKitchen:
pair: str = "", pair: str = "",
prediction_dataframe: DataFrame = pd.DataFrame(), prediction_dataframe: DataFrame = pd.DataFrame(),
do_corr_pairs: bool = True, do_corr_pairs: bool = True,
set_only_targets: bool = False
) -> DataFrame: ) -> DataFrame:
""" """
Use the user defined strategy for populating indicators during retrain Use the user defined strategy for populating indicators during retrain
@ -1222,7 +1223,8 @@ class FreqaiDataKitchen:
dataframe.copy(), dataframe.copy(),
tf, tf,
informative=base_dataframes[tf], informative=base_dataframes[tf],
set_generalized_indicators=sgi set_generalized_indicators=sgi,
set_only_targets=set_only_targets
) )
# ensure corr pairs are always last # ensure corr pairs are always last
@ -1235,7 +1237,8 @@ class FreqaiDataKitchen:
corr_pair, corr_pair,
dataframe.copy(), dataframe.copy(),
tf, tf,
informative=corr_dataframes[corr_pair][tf] informative=corr_dataframes[corr_pair][tf],
set_only_targets=set_only_targets
) )
self.get_unique_classes_from_labels(dataframe) self.get_unique_classes_from_labels(dataframe)

48
freqtrade/freqai/freqai_interface.py
View File

@ -149,12 +149,13 @@ class IFreqaiModel(ABC):
# the concatenated results for the full backtesting period back to the strategy. # the concatenated results for the full backtesting period back to the strategy.
elif not self.follow_mode: elif not self.follow_mode:
self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"]) self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
dataframe = self.dk.use_strategy_to_populate_indicators(
strategy, prediction_dataframe=dataframe, pair=metadata["pair"] # dataframe = self.dk.use_strategy_to_populate_indicators(
) # strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
# )
if not self.config.get("freqai_backtest_live_models", False): if not self.config.get("freqai_backtest_live_models", False):
logger.info(f"Training {len(self.dk.training_timeranges)} timeranges") logger.info(f"Training {len(self.dk.training_timeranges)} timeranges")
dk = self.start_backtesting(dataframe, metadata, self.dk) dk = self.start_backtesting(dataframe, metadata, self.dk, strategy)
dataframe = dk.remove_features_from_df(dk.return_dataframe) dataframe = dk.remove_features_from_df(dk.return_dataframe)
else: else:
logger.info( logger.info(
@ -255,7 +256,7 @@ class IFreqaiModel(ABC):
self.dd.save_metric_tracker_to_disk() self.dd.save_metric_tracker_to_disk()
def start_backtesting( def start_backtesting(
self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen, strategy: IStrategy
) -> FreqaiDataKitchen: ) -> FreqaiDataKitchen:
""" """
The main broad execution for backtesting. For backtesting, each pair enters and then gets The main broad execution for backtesting. For backtesting, each pair enters and then gets
@ -267,12 +268,14 @@ class IFreqaiModel(ABC):
:param dataframe: DataFrame = strategy passed dataframe :param dataframe: DataFrame = strategy passed dataframe
:param metadata: Dict = pair metadata :param metadata: Dict = pair metadata
:param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only :param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only
:param strategy: Strategy to train on
:return: :return:
FreqaiDataKitchen = Data management/analysis tool associated to present pair only FreqaiDataKitchen = Data management/analysis tool associated to present pair only
""" """
self.pair_it += 1 self.pair_it += 1
train_it = 0 train_it = 0
set_only_targets = False
# Loop enforcing the sliding window training/backtesting paradigm # Loop enforcing the sliding window training/backtesting paradigm
# tr_train is the training time range e.g. 1 historical month # tr_train is the training time range e.g. 1 historical month
# tr_backtest is the backtesting time range e.g. the week directly # tr_backtest is the backtesting time range e.g. the week directly
@ -301,14 +304,39 @@ class IFreqaiModel(ABC):
dk.set_new_model_names(pair, timestamp_model_id) dk.set_new_model_names(pair, timestamp_model_id)
if dk.check_if_backtest_prediction_is_valid(len_backtest_df): if dk.check_if_backtest_prediction_is_valid(len_backtest_df):
self.dd.load_metadata(dk) # self.dd.load_metadata(dk)
dk.find_features(dataframe) # dk.find_features(dataframe)
self.check_if_feature_list_matches_strategy(dk) # self.check_if_feature_list_matches_strategy(dk)
append_df = dk.get_backtesting_prediction() append_df = dk.get_backtesting_prediction()
dk.append_predictions(append_df) dk.append_predictions(append_df)
else: else:
dataframe_train = dk.slice_dataframe(tr_train, dataframe) if set_only_targets is False:
dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe) dataframe = self.dk.use_strategy_to_populate_indicators(
strategy, prediction_dataframe=dataframe,
pair=metadata["pair"], set_only_targets=set_only_targets
)
set_only_targets = True
dataframe_base_train = dataframe.loc[dataframe["date"] < tr_train.stopdt, :]
dataframe_base_train = self.dk.use_strategy_to_populate_indicators(
strategy, prediction_dataframe=dataframe_base_train,
pair=metadata["pair"], set_only_targets=set_only_targets
)
# region need to verify if this block is needed
# (or slice dataframe_backtest from dataframe)
dataframe_base_backtest = dataframe.loc[dataframe["date"] < tr_backtest.stopdt, :]
dataframe_base_backtest = self.dk.use_strategy_to_populate_indicators(
strategy, prediction_dataframe=dataframe_base_backtest,
pair=metadata["pair"], set_only_targets=set_only_targets
)
# endregion
set_only_targets = True
dataframe_train = dk.slice_dataframe(tr_train, dataframe_base_train)
dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe_base_backtest)
if not self.model_exists(dk): if not self.model_exists(dk):
dk.find_features(dataframe_train) dk.find_features(dataframe_train)
dk.find_labels(dataframe_train) dk.find_labels(dataframe_train)

4
freqtrade/strategy/interface.py
View File

@ -596,7 +596,8 @@ class IStrategy(ABC, HyperStrategyMixin):
def populate_any_indicators(self, pair: str, df: DataFrame, tf: str, def populate_any_indicators(self, pair: str, df: DataFrame, tf: str,
informative: DataFrame = None, informative: DataFrame = None,
set_generalized_indicators: bool = False) -> DataFrame: set_generalized_indicators: bool = False,
set_only_targets: bool = False) -> DataFrame:
""" """
Function designed to automatically generate, name and merge features Function designed to automatically generate, name and merge features
from user indicated timeframes in the configuration file. User can add from user indicated timeframes in the configuration file. User can add
@ -607,6 +608,7 @@ class IStrategy(ABC, HyperStrategyMixin):
:param df: strategy dataframe which will receive merges from informatives :param df: strategy dataframe which will receive merges from informatives
:param tf: timeframe of the dataframe which will modify the feature names :param tf: timeframe of the dataframe which will modify the feature names
:param informative: the dataframe associated with the informative pair :param informative: the dataframe associated with the informative pair
:param set_only_targets: if True, only the target features will be set
""" """
return df return df

108
freqtrade/templates/FreqaiExampleStrategy.py
View File

@ -48,7 +48,13 @@ class FreqaiExampleStrategy(IStrategy):
[0.75, 1, 1.25, 1.5, 1.75], space="sell", default=1.25, optimize=True) [0.75, 1, 1.25, 1.5, 1.75], space="sell", default=1.25, optimize=True)
def populate_any_indicators( def populate_any_indicators(
self, pair, df, tf, informative=None, set_generalized_indicators=False self,
pair,
df,
tf,
informative=None,
set_generalized_indicators=False,
set_only_targets=False
): ):
""" """
Function designed to automatically generate, name and merge features Function designed to automatically generate, name and merge features
@ -63,66 +69,68 @@ class FreqaiExampleStrategy(IStrategy):
:param informative: the dataframe associated with the informative pair :param informative: the dataframe associated with the informative pair
""" """
if informative is None: if not set_only_targets:
informative = self.dp.get_pair_dataframe(pair, tf) if informative is None:
informative = self.dp.get_pair_dataframe(pair, tf)
# first loop is automatically duplicating indicators for time periods # first loop is automatically duplicating indicators for time periods
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
t = int(t) t = int(t)
informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, timeperiod=t)
informative[f"%-{pair}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) informative[f"%-{pair}sma-period_{t}"] = ta.SMA(informative, timeperiod=t)
informative[f"%-{pair}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) informative[f"%-{pair}ema-period_{t}"] = ta.EMA(informative, timeperiod=t)
bollinger = qtpylib.bollinger_bands( bollinger = qtpylib.bollinger_bands(
qtpylib.typical_price(informative), window=t, stds=2.2 qtpylib.typical_price(informative), window=t, stds=2.2
) )
informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"] informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"]
informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"] informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"]
informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"] informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"]
informative[f"%-{pair}bb_width-period_{t}"] = ( informative[f"%-{pair}bb_width-period_{t}"] = (
informative[f"{pair}bb_upperband-period_{t}"] informative[f"{pair}bb_upperband-period_{t}"]
- informative[f"{pair}bb_lowerband-period_{t}"] - informative[f"{pair}bb_lowerband-period_{t}"]
) / informative[f"{pair}bb_middleband-period_{t}"] ) / informative[f"{pair}bb_middleband-period_{t}"]
informative[f"%-{pair}close-bb_lower-period_{t}"] = ( informative[f"%-{pair}close-bb_lower-period_{t}"] = (
informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"] informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"]
) )
informative[f"%-{pair}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) informative[f"%-{pair}roc-period_{t}"] = ta.ROC(informative, timeperiod=t)
informative[f"%-{pair}relative_volume-period_{t}"] = ( informative[f"%-{pair}relative_volume-period_{t}"] = (
informative["volume"] / informative["volume"].rolling(t).mean() informative["volume"] / informative["volume"].rolling(t).mean()
) )
informative[f"%-{pair}pct-change"] = informative["close"].pct_change() informative[f"%-{pair}pct-change"] = informative["close"].pct_change()
informative[f"%-{pair}raw_volume"] = informative["volume"] informative[f"%-{pair}raw_volume"] = informative["volume"]
informative[f"%-{pair}raw_price"] = informative["close"] informative[f"%-{pair}raw_price"] = informative["close"]
indicators = [col for col in informative if col.startswith("%")] indicators = [col for col in informative if col.startswith("%")]
# This loop duplicates and shifts all indicators to add a sense of recency to data # This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0: if n == 0:
continue continue
informative_shift = informative[indicators].shift(n) informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n)) informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1) informative = pd.concat((informative, informative_shift), axis=1)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True)
skip_columns = [ skip_columns = [
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
] ]
df = df.drop(columns=skip_columns) df = df.drop(columns=skip_columns)
# Add generalized indicators here (because in live, it will call this
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if set_generalized_indicators:
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
# Add generalized indicators here (because in live, it will call this
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if set_generalized_indicators: if set_generalized_indicators:
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
# user adds targets here by prepending them with &- (see convention below) # user adds targets here by prepending them with &- (see convention below)
df["&-s_close"] = ( df["&-s_close"] = (
df["close"] df["close"]