freqai bt fix populate any indicators call - first commit
This commit is contained in:
Wagner Costa
parent
3012c55ec5
commit
cc796cc594
@ -1184,6 +1184,7 @@ class FreqaiDataKitchen:
|
||||
pair: str = "",
|
||||
prediction_dataframe: DataFrame = pd.DataFrame(),
|
||||
do_corr_pairs: bool = True,
|
||||
set_only_targets: bool = False
|
||||
) -> DataFrame:
|
||||
"""
|
||||
Use the user defined strategy for populating indicators during retrain
|
||||
@ -1222,7 +1223,8 @@ class FreqaiDataKitchen:
|
||||
dataframe.copy(),
|
||||
tf,
|
||||
informative=base_dataframes[tf],
|
||||
set_generalized_indicators=sgi
|
||||
set_generalized_indicators=sgi,
|
||||
set_only_targets=set_only_targets
|
||||
)
|
||||
|
||||
# ensure corr pairs are always last
|
||||
@ -1235,7 +1237,8 @@ class FreqaiDataKitchen:
|
||||
corr_pair,
|
||||
dataframe.copy(),
|
||||
tf,
|
||||
informative=corr_dataframes[corr_pair][tf]
|
||||
informative=corr_dataframes[corr_pair][tf],
|
||||
set_only_targets=set_only_targets
|
||||
)
|
||||
|
||||
self.get_unique_classes_from_labels(dataframe)
|
||||
|
||||
@ -149,12 +149,13 @@ class IFreqaiModel(ABC):
|
||||
# the concatenated results for the full backtesting period back to the strategy.
|
||||
elif not self.follow_mode:
|
||||
self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
|
||||
dataframe = self.dk.use_strategy_to_populate_indicators(
|
||||
strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
|
||||
)
|
||||
|
||||
# dataframe = self.dk.use_strategy_to_populate_indicators(
|
||||
# strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
|
||||
# )
|
||||
if not self.config.get("freqai_backtest_live_models", False):
|
||||
logger.info(f"Training {len(self.dk.training_timeranges)} timeranges")
|
||||
dk = self.start_backtesting(dataframe, metadata, self.dk)
|
||||
dk = self.start_backtesting(dataframe, metadata, self.dk, strategy)
|
||||
dataframe = dk.remove_features_from_df(dk.return_dataframe)
|
||||
else:
|
||||
logger.info(
|
||||
@ -255,7 +256,7 @@ class IFreqaiModel(ABC):
|
||||
self.dd.save_metric_tracker_to_disk()
|
||||
|
||||
def start_backtesting(
|
||||
self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen
|
||||
self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen, strategy: IStrategy
|
||||
) -> FreqaiDataKitchen:
|
||||
"""
|
||||
The main broad execution for backtesting. For backtesting, each pair enters and then gets
|
||||
@ -267,12 +268,14 @@ class IFreqaiModel(ABC):
|
||||
:param dataframe: DataFrame = strategy passed dataframe
|
||||
:param metadata: Dict = pair metadata
|
||||
:param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only
|
||||
:param strategy: Strategy to train on
|
||||
:return:
|
||||
FreqaiDataKitchen = Data management/analysis tool associated to present pair only
|
||||
"""
|
||||
|
||||
self.pair_it += 1
|
||||
train_it = 0
|
||||
set_only_targets = False
|
||||
# Loop enforcing the sliding window training/backtesting paradigm
|
||||
# tr_train is the training time range e.g. 1 historical month
|
||||
# tr_backtest is the backtesting time range e.g. the week directly
|
||||
@ -301,14 +304,39 @@ class IFreqaiModel(ABC):
|
||||
dk.set_new_model_names(pair, timestamp_model_id)
|
||||
|
||||
if dk.check_if_backtest_prediction_is_valid(len_backtest_df):
|
||||
self.dd.load_metadata(dk)
|
||||
dk.find_features(dataframe)
|
||||
self.check_if_feature_list_matches_strategy(dk)
|
||||
# self.dd.load_metadata(dk)
|
||||
# dk.find_features(dataframe)
|
||||
# self.check_if_feature_list_matches_strategy(dk)
|
||||
append_df = dk.get_backtesting_prediction()
|
||||
dk.append_predictions(append_df)
|
||||
else:
|
||||
dataframe_train = dk.slice_dataframe(tr_train, dataframe)
|
||||
dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe)
|
||||
if set_only_targets is False:
|
||||
dataframe = self.dk.use_strategy_to_populate_indicators(
|
||||
strategy, prediction_dataframe=dataframe,
|
||||
pair=metadata["pair"], set_only_targets=set_only_targets
|
||||
)
|
||||
set_only_targets = True
|
||||
|
||||
dataframe_base_train = dataframe.loc[dataframe["date"] < tr_train.stopdt, :]
|
||||
dataframe_base_train = self.dk.use_strategy_to_populate_indicators(
|
||||
strategy, prediction_dataframe=dataframe_base_train,
|
||||
pair=metadata["pair"], set_only_targets=set_only_targets
|
||||
)
|
||||
|
||||
# region need to verify if this block is needed
|
||||
# (or slice dataframe_backtest from dataframe)
|
||||
dataframe_base_backtest = dataframe.loc[dataframe["date"] < tr_backtest.stopdt, :]
|
||||
dataframe_base_backtest = self.dk.use_strategy_to_populate_indicators(
|
||||
strategy, prediction_dataframe=dataframe_base_backtest,
|
||||
pair=metadata["pair"], set_only_targets=set_only_targets
|
||||
)
|
||||
# endregion
|
||||
|
||||
set_only_targets = True
|
||||
|
||||
dataframe_train = dk.slice_dataframe(tr_train, dataframe_base_train)
|
||||
dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe_base_backtest)
|
||||
|
||||
if not self.model_exists(dk):
|
||||
dk.find_features(dataframe_train)
|
||||
dk.find_labels(dataframe_train)
|
||||
|
||||
@ -596,7 +596,8 @@ class IStrategy(ABC, HyperStrategyMixin):
|
||||
|
||||
def populate_any_indicators(self, pair: str, df: DataFrame, tf: str,
|
||||
informative: DataFrame = None,
|
||||
set_generalized_indicators: bool = False) -> DataFrame:
|
||||
set_generalized_indicators: bool = False,
|
||||
set_only_targets: bool = False) -> DataFrame:
|
||||
"""
|
||||
Function designed to automatically generate, name and merge features
|
||||
from user indicated timeframes in the configuration file. User can add
|
||||
@ -607,6 +608,7 @@ class IStrategy(ABC, HyperStrategyMixin):
|
||||
:param df: strategy dataframe which will receive merges from informatives
|
||||
:param tf: timeframe of the dataframe which will modify the feature names
|
||||
:param informative: the dataframe associated with the informative pair
|
||||
:param set_only_targets: if True, only the target features will be set
|
||||
"""
|
||||
return df
|
||||
|
||||
|
||||
@ -48,7 +48,13 @@ class FreqaiExampleStrategy(IStrategy):
|
||||
[0.75, 1, 1.25, 1.5, 1.75], space="sell", default=1.25, optimize=True)
|
||||
|
||||
def populate_any_indicators(
|
||||
self, pair, df, tf, informative=None, set_generalized_indicators=False
|
||||
self,
|
||||
pair,
|
||||
df,
|
||||
tf,
|
||||
informative=None,
|
||||
set_generalized_indicators=False,
|
||||
set_only_targets=False
|
||||
):
|
||||
"""
|
||||
Function designed to automatically generate, name and merge features
|
||||
@ -63,66 +69,68 @@ class FreqaiExampleStrategy(IStrategy):
|
||||
:param informative: the dataframe associated with the informative pair
|
||||
"""
|
||||
|
||||
if informative is None:
|
||||
informative = self.dp.get_pair_dataframe(pair, tf)
|
||||
if not set_only_targets:
|
||||
if informative is None:
|
||||
informative = self.dp.get_pair_dataframe(pair, tf)
|
||||
|
||||
# first loop is automatically duplicating indicators for time periods
|
||||
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
|
||||
# first loop is automatically duplicating indicators for time periods
|
||||
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
|
||||
|
||||
t = int(t)
|
||||
informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
|
||||
informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
|
||||
informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, timeperiod=t)
|
||||
informative[f"%-{pair}sma-period_{t}"] = ta.SMA(informative, timeperiod=t)
|
||||
informative[f"%-{pair}ema-period_{t}"] = ta.EMA(informative, timeperiod=t)
|
||||
t = int(t)
|
||||
informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
|
||||
informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
|
||||
informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, timeperiod=t)
|
||||
informative[f"%-{pair}sma-period_{t}"] = ta.SMA(informative, timeperiod=t)
|
||||
informative[f"%-{pair}ema-period_{t}"] = ta.EMA(informative, timeperiod=t)
|
||||
|
||||
bollinger = qtpylib.bollinger_bands(
|
||||
qtpylib.typical_price(informative), window=t, stds=2.2
|
||||
)
|
||||
informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"]
|
||||
informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"]
|
||||
informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"]
|
||||
bollinger = qtpylib.bollinger_bands(
|
||||
qtpylib.typical_price(informative), window=t, stds=2.2
|
||||
)
|
||||
informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"]
|
||||
informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"]
|
||||
informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"]
|
||||
|
||||
informative[f"%-{pair}bb_width-period_{t}"] = (
|
||||
informative[f"{pair}bb_upperband-period_{t}"]
|
||||
- informative[f"{pair}bb_lowerband-period_{t}"]
|
||||
) / informative[f"{pair}bb_middleband-period_{t}"]
|
||||
informative[f"%-{pair}close-bb_lower-period_{t}"] = (
|
||||
informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"]
|
||||
)
|
||||
informative[f"%-{pair}bb_width-period_{t}"] = (
|
||||
informative[f"{pair}bb_upperband-period_{t}"]
|
||||
- informative[f"{pair}bb_lowerband-period_{t}"]
|
||||
) / informative[f"{pair}bb_middleband-period_{t}"]
|
||||
informative[f"%-{pair}close-bb_lower-period_{t}"] = (
|
||||
informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"]
|
||||
)
|
||||
|
||||
informative[f"%-{pair}roc-period_{t}"] = ta.ROC(informative, timeperiod=t)
|
||||
informative[f"%-{pair}roc-period_{t}"] = ta.ROC(informative, timeperiod=t)
|
||||
|
||||
informative[f"%-{pair}relative_volume-period_{t}"] = (
|
||||
informative["volume"] / informative["volume"].rolling(t).mean()
|
||||
)
|
||||
informative[f"%-{pair}relative_volume-period_{t}"] = (
|
||||
informative["volume"] / informative["volume"].rolling(t).mean()
|
||||
)
|
||||
|
||||
informative[f"%-{pair}pct-change"] = informative["close"].pct_change()
|
||||
informative[f"%-{pair}raw_volume"] = informative["volume"]
|
||||
informative[f"%-{pair}raw_price"] = informative["close"]
|
||||
informative[f"%-{pair}pct-change"] = informative["close"].pct_change()
|
||||
informative[f"%-{pair}raw_volume"] = informative["volume"]
|
||||
informative[f"%-{pair}raw_price"] = informative["close"]
|
||||
|
||||
indicators = [col for col in informative if col.startswith("%")]
|
||||
# This loop duplicates and shifts all indicators to add a sense of recency to data
|
||||
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
|
||||
if n == 0:
|
||||
continue
|
||||
informative_shift = informative[indicators].shift(n)
|
||||
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
|
||||
informative = pd.concat((informative, informative_shift), axis=1)
|
||||
indicators = [col for col in informative if col.startswith("%")]
|
||||
# This loop duplicates and shifts all indicators to add a sense of recency to data
|
||||
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
|
||||
if n == 0:
|
||||
continue
|
||||
informative_shift = informative[indicators].shift(n)
|
||||
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
|
||||
informative = pd.concat((informative, informative_shift), axis=1)
|
||||
|
||||
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True)
|
||||
skip_columns = [
|
||||
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
|
||||
]
|
||||
df = df.drop(columns=skip_columns)
|
||||
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True)
|
||||
skip_columns = [
|
||||
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
|
||||
]
|
||||
df = df.drop(columns=skip_columns)
|
||||
|
||||
# Add generalized indicators here (because in live, it will call this
|
||||
# function to populate indicators during training). Notice how we ensure not to
|
||||
# add them multiple times
|
||||
if set_generalized_indicators:
|
||||
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
|
||||
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
|
||||
|
||||
# Add generalized indicators here (because in live, it will call this
|
||||
# function to populate indicators during training). Notice how we ensure not to
|
||||
# add them multiple times
|
||||
if set_generalized_indicators:
|
||||
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
|
||||
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
|
||||
|
||||
# user adds targets here by prepending them with &- (see convention below)
|
||||
df["&-s_close"] = (
|
||||
df["close"]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user