adelorenzo
/
stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

take dynamic sized tail off historic_predictions as return dataframe to strategy.

This commit is contained in:
robcaulk 2022-08-12 13:13:08 +02:00
parent 7d448fd4ac
commit 3e38c1b0bd
2 changed files with 69 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
freqtrade/freqai/data_drawer.py
View File

@ -241,48 +241,48 @@ class FreqaiDataDrawer:
"""
# dynamic df returned to strategy and plotted in frequi
mrv_df = self.model_return_values[pair] = pd.DataFrame()
# mrv_df = self.model_return_values[pair] = pd.DataFrame()
# if user reused `identifier` in config and has historical predictions collected, load them
# so that frequi remains uninterrupted after a crash
hist_df = self.historic_predictions
if pair in hist_df:
len_diff = len(hist_df[pair].index) - len(pred_df.index)
if len_diff < 0:
df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]],
ignore_index=True, keys=hist_df[pair].keys())
else:
df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True)
df_concat = df_concat.fillna(0)
self.model_return_values[pair] = df_concat
logger.info(f'Setting initial FreqUI plots from historical data for {pair}.')
# if pair in hist_df:
len_diff = len(hist_df[pair].index) - len(pred_df.index)
if len_diff < 0:
df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]],
ignore_index=True, keys=hist_df[pair].keys())
else:
for label in pred_df.columns:
mrv_df[label] = pred_df[label]
if mrv_df[label].dtype == object:
continue
mrv_df[f"{label}_mean"] = dk.data["labels_mean"][label]
mrv_df[f"{label}_std"] = dk.data["labels_std"][label]
df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True)
df_concat = df_concat.fillna(0)
self.model_return_values[pair] = df_concat
logger.info(f'Setting initial FreqUI plots from historical data for {pair}.')
if self.freqai_info["feature_parameters"].get("DI_threshold", 0) > 0:
mrv_df["DI_values"] = dk.DI_values
# else:
# for label in pred_df.columns:
# mrv_df[label] = pred_df[label]
# if mrv_df[label].dtype == object:
# continue
# mrv_df[f"{label}_mean"] = dk.data["labels_mean"][label]
# mrv_df[f"{label}_std"] = dk.data["labels_std"][label]
mrv_df["do_predict"] = do_preds
# if self.freqai_info["feature_parameters"].get("DI_threshold", 0) > 0:
# mrv_df["DI_values"] = dk.DI_values
if dk.data['extra_returns_per_train']:
rets = dk.data['extra_returns_per_train']
for return_str in rets:
mrv_df[return_str] = rets[return_str]
# mrv_df["do_predict"] = do_preds
# for keras type models, the conv_window needs to be prepended so
# viewing is correct in frequi
if self.freqai_info.get('keras', False):
n_lost_points = self.freqai_info.get('conv_width', 2)
zeros_df = DataFrame(np.zeros((n_lost_points, len(mrv_df.columns))),
columns=mrv_df.columns)
self.model_return_values[pair] = pd.concat(
[zeros_df, mrv_df], axis=0, ignore_index=True)
# if dk.data['extra_returns_per_train']:
# rets = dk.data['extra_returns_per_train']
# for return_str in rets:
# mrv_df[return_str] = rets[return_str]
# # for keras type models, the conv_window needs to be prepended so
# # viewing is correct in frequi
# if self.freqai_info.get('keras', False):
# n_lost_points = self.freqai_info.get('conv_width', 2)
# zeros_df = DataFrame(np.zeros((n_lost_points, len(mrv_df.columns))),
# columns=mrv_df.columns)
# self.model_return_values[pair] = pd.concat(
# [zeros_df, mrv_df], axis=0, ignore_index=True)
def append_model_predictions(self, pair: str, predictions: DataFrame,
do_preds: NDArray[np.int_],
@ -292,23 +292,23 @@ class FreqaiDataDrawer:
# own return array in the same shape, we need to figure out how the size has changed
# and adapt our stored/returned info accordingly.
length_difference = len(self.model_return_values[pair]) - len_df
i = 0
# length_difference = len(self.model_return_values[pair]) - len_df
# i = 0
if length_difference == 0:
i = 1
elif length_difference > 0:
i = length_difference + 1
# if length_difference == 0:
# i = 1
# elif length_difference > 0:
# i = length_difference + 1
df = self.model_return_values[pair] = self.model_return_values[pair].shift(-i)
# df = self.model_return_values[pair] = self.model_return_values[pair].shift(-i)
if pair in self.historic_predictions:
hp_df = self.historic_predictions[pair]
# here are some pandas hula hoops to accommodate the possibility of a series
# or dataframe depending number of labels requested by user
nan_df = pd.DataFrame(np.nan, index=hp_df.index[-2:] + 2, columns=hp_df.columns)
hp_df = pd.concat([hp_df, nan_df], ignore_index=True, axis=0)
self.historic_predictions[pair] = hp_df[:-1]
# if pair in self.historic_predictions:
df = self.historic_predictions[pair]
# here are some pandas hula hoops to accommodate the possibility of a series
# or dataframe depending number of labels requested by user
nan_df = pd.DataFrame(np.nan, index=df.index[-2:] + 2, columns=df.columns)
df = pd.concat([df, nan_df], ignore_index=True, axis=0)
df = self.historic_predictions[pair] = df[:-1]
# incase user adds additional "predictions" e.g. predict_proba output:
for label in predictions.columns:
@ -328,16 +328,18 @@ class FreqaiDataDrawer:
for return_str in rets:
df[return_str].iloc[-1] = rets[return_str]
# append the new predictions to persistent storage
if pair in self.historic_predictions:
for key in df.keys():
self.historic_predictions[pair][key].iloc[-1] = df[key].iloc[-1]
self.model_return_values[pair] = df.tail(len_df).reset_index(drop=True)
if length_difference < 0:
prepend_df = pd.DataFrame(
np.zeros((abs(length_difference) - 1, len(df.columns))), columns=df.columns
)
df = pd.concat([prepend_df, df], axis=0)
# # append the new predictions to persistent storage
# if pair in self.historic_predictions:
# for key in df.keys():
# self.historic_predictions[pair][key].iloc[-1] = df[key].iloc[-1]
# if length_difference < 0:
# prepend_df = pd.DataFrame(
# np.zeros((abs(length_difference) - 1, len(df.columns))), columns=df.columns
# )
# df = pd.concat([prepend_df, df], axis=0)
def attach_return_values_to_return_dataframe(
self, pair: str, dataframe: DataFrame) -> DataFrame:

12
freqtrade/freqai/freqai_interface.py
View File

@ -319,9 +319,10 @@ class IFreqaiModel(ABC):
# first predictions are made on entire historical candle set coming from strategy. This
# allows FreqUI to show full return values.
pred_df, do_preds = self.predict(dataframe, dk)
self.dd.set_initial_return_values(pair, dk, pred_df, do_preds)
if pair not in self.dd.historic_predictions:
self.set_initial_historic_predictions(pred_df, dk, pair)
self.dd.set_initial_return_values(pair, dk, pred_df, do_preds)
dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
return
elif self.dk.check_if_model_expired(trained_timestamp):
@ -551,6 +552,15 @@ class IFreqaiModel(ABC):
for return_str in dk.data['extra_returns_per_train']:
hist_preds_df[return_str] = 0
# # for keras type models, the conv_window needs to be prepended so
# # viewing is correct in frequi
if self.freqai_info.get('keras', False):
n_lost_points = self.freqai_info.get('conv_width', 2)
zeros_df = DataFrame(np.zeros((n_lost_points, len(hist_preds_df.columns))),
columns=hist_preds_df.columns)
self.model_return_values[pair] = pd.concat(
[zeros_df, hist_preds_df], axis=0, ignore_index=True)
def fit_live_predictions(self, dk: FreqaiDataKitchen, pair: str) -> None:
"""
Fit the labels with a gaussian distribution