adelorenzo
/
stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

take dynamic sized tail off historic_predictions as return dataframe to strategy.

This commit is contained in:
robcaulk 2022-08-12 13:13:08 +02:00
parent 7d448fd4ac
commit 3e38c1b0bd
2 changed files with 69 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
freqtrade/freqai/data_drawer.py
View File

@ -241,48 +241,48 @@ class FreqaiDataDrawer:
""" """
# dynamic df returned to strategy and plotted in frequi # dynamic df returned to strategy and plotted in frequi
mrv_df = self.model_return_values[pair] = pd.DataFrame() # mrv_df = self.model_return_values[pair] = pd.DataFrame()
# if user reused `identifier` in config and has historical predictions collected, load them # if user reused `identifier` in config and has historical predictions collected, load them
# so that frequi remains uninterrupted after a crash # so that frequi remains uninterrupted after a crash
hist_df = self.historic_predictions hist_df = self.historic_predictions
if pair in hist_df: # if pair in hist_df:
len_diff = len(hist_df[pair].index) - len(pred_df.index) len_diff = len(hist_df[pair].index) - len(pred_df.index)
if len_diff < 0: if len_diff < 0:
df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]], df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]],
ignore_index=True, keys=hist_df[pair].keys()) ignore_index=True, keys=hist_df[pair].keys())
else:
df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True)
df_concat = df_concat.fillna(0)
self.model_return_values[pair] = df_concat
logger.info(f'Setting initial FreqUI plots from historical data for {pair}.')
else: else:
for label in pred_df.columns: df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True)
mrv_df[label] = pred_df[label] df_concat = df_concat.fillna(0)
if mrv_df[label].dtype == object: self.model_return_values[pair] = df_concat
continue logger.info(f'Setting initial FreqUI plots from historical data for {pair}.')
mrv_df[f"{label}_mean"] = dk.data["labels_mean"][label]
mrv_df[f"{label}_std"] = dk.data["labels_std"][label]
if self.freqai_info["feature_parameters"].get("DI_threshold", 0) > 0: # else:
mrv_df["DI_values"] = dk.DI_values # for label in pred_df.columns:
# mrv_df[label] = pred_df[label]
# if mrv_df[label].dtype == object:
# continue
# mrv_df[f"{label}_mean"] = dk.data["labels_mean"][label]
# mrv_df[f"{label}_std"] = dk.data["labels_std"][label]
mrv_df["do_predict"] = do_preds # if self.freqai_info["feature_parameters"].get("DI_threshold", 0) > 0:
# mrv_df["DI_values"] = dk.DI_values
if dk.data['extra_returns_per_train']: # mrv_df["do_predict"] = do_preds
rets = dk.data['extra_returns_per_train']
for return_str in rets:
mrv_df[return_str] = rets[return_str]
# for keras type models, the conv_window needs to be prepended so # if dk.data['extra_returns_per_train']:
# viewing is correct in frequi # rets = dk.data['extra_returns_per_train']
if self.freqai_info.get('keras', False): # for return_str in rets:
n_lost_points = self.freqai_info.get('conv_width', 2) # mrv_df[return_str] = rets[return_str]
zeros_df = DataFrame(np.zeros((n_lost_points, len(mrv_df.columns))),
columns=mrv_df.columns) # # for keras type models, the conv_window needs to be prepended so
self.model_return_values[pair] = pd.concat( # # viewing is correct in frequi
[zeros_df, mrv_df], axis=0, ignore_index=True) # if self.freqai_info.get('keras', False):
# n_lost_points = self.freqai_info.get('conv_width', 2)
# zeros_df = DataFrame(np.zeros((n_lost_points, len(mrv_df.columns))),
# columns=mrv_df.columns)
# self.model_return_values[pair] = pd.concat(
# [zeros_df, mrv_df], axis=0, ignore_index=True)
def append_model_predictions(self, pair: str, predictions: DataFrame, def append_model_predictions(self, pair: str, predictions: DataFrame,
do_preds: NDArray[np.int_], do_preds: NDArray[np.int_],
@ -292,23 +292,23 @@ class FreqaiDataDrawer:
# own return array in the same shape, we need to figure out how the size has changed # own return array in the same shape, we need to figure out how the size has changed
# and adapt our stored/returned info accordingly. # and adapt our stored/returned info accordingly.
length_difference = len(self.model_return_values[pair]) - len_df # length_difference = len(self.model_return_values[pair]) - len_df
i = 0 # i = 0
if length_difference == 0: # if length_difference == 0:
i = 1 # i = 1
elif length_difference > 0: # elif length_difference > 0:
i = length_difference + 1 # i = length_difference + 1
df = self.model_return_values[pair] = self.model_return_values[pair].shift(-i) # df = self.model_return_values[pair] = self.model_return_values[pair].shift(-i)
if pair in self.historic_predictions: # if pair in self.historic_predictions:
hp_df = self.historic_predictions[pair] df = self.historic_predictions[pair]
# here are some pandas hula hoops to accommodate the possibility of a series # here are some pandas hula hoops to accommodate the possibility of a series
# or dataframe depending number of labels requested by user # or dataframe depending number of labels requested by user
nan_df = pd.DataFrame(np.nan, index=hp_df.index[-2:] + 2, columns=hp_df.columns) nan_df = pd.DataFrame(np.nan, index=df.index[-2:] + 2, columns=df.columns)
hp_df = pd.concat([hp_df, nan_df], ignore_index=True, axis=0) df = pd.concat([df, nan_df], ignore_index=True, axis=0)
self.historic_predictions[pair] = hp_df[:-1] df = self.historic_predictions[pair] = df[:-1]
# incase user adds additional "predictions" e.g. predict_proba output: # incase user adds additional "predictions" e.g. predict_proba output:
for label in predictions.columns: for label in predictions.columns:
@ -328,16 +328,18 @@ class FreqaiDataDrawer:
for return_str in rets: for return_str in rets:
df[return_str].iloc[-1] = rets[return_str] df[return_str].iloc[-1] = rets[return_str]
# append the new predictions to persistent storage self.model_return_values[pair] = df.tail(len_df).reset_index(drop=True)
if pair in self.historic_predictions:
for key in df.keys():
self.historic_predictions[pair][key].iloc[-1] = df[key].iloc[-1]
if length_difference < 0: # # append the new predictions to persistent storage
prepend_df = pd.DataFrame( # if pair in self.historic_predictions:
np.zeros((abs(length_difference) - 1, len(df.columns))), columns=df.columns # for key in df.keys():
) # self.historic_predictions[pair][key].iloc[-1] = df[key].iloc[-1]
df = pd.concat([prepend_df, df], axis=0)
# if length_difference < 0:
# prepend_df = pd.DataFrame(
# np.zeros((abs(length_difference) - 1, len(df.columns))), columns=df.columns
# )
# df = pd.concat([prepend_df, df], axis=0)
def attach_return_values_to_return_dataframe( def attach_return_values_to_return_dataframe(
self, pair: str, dataframe: DataFrame) -> DataFrame: self, pair: str, dataframe: DataFrame) -> DataFrame:

12
freqtrade/freqai/freqai_interface.py
View File

@ -319,9 +319,10 @@ class IFreqaiModel(ABC):
# first predictions are made on entire historical candle set coming from strategy. This # first predictions are made on entire historical candle set coming from strategy. This
# allows FreqUI to show full return values. # allows FreqUI to show full return values.
pred_df, do_preds = self.predict(dataframe, dk) pred_df, do_preds = self.predict(dataframe, dk)
self.dd.set_initial_return_values(pair, dk, pred_df, do_preds)
if pair not in self.dd.historic_predictions: if pair not in self.dd.historic_predictions:
self.set_initial_historic_predictions(pred_df, dk, pair) self.set_initial_historic_predictions(pred_df, dk, pair)
self.dd.set_initial_return_values(pair, dk, pred_df, do_preds)
dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe) dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
return return
elif self.dk.check_if_model_expired(trained_timestamp): elif self.dk.check_if_model_expired(trained_timestamp):
@ -551,6 +552,15 @@ class IFreqaiModel(ABC):
for return_str in dk.data['extra_returns_per_train']: for return_str in dk.data['extra_returns_per_train']:
hist_preds_df[return_str] = 0 hist_preds_df[return_str] = 0
# # for keras type models, the conv_window needs to be prepended so
# # viewing is correct in frequi
if self.freqai_info.get('keras', False):
n_lost_points = self.freqai_info.get('conv_width', 2)
zeros_df = DataFrame(np.zeros((n_lost_points, len(hist_preds_df.columns))),
columns=hist_preds_df.columns)
self.model_return_values[pair] = pd.concat(
[zeros_df, hist_preds_df], axis=0, ignore_index=True)
def fit_live_predictions(self, dk: FreqaiDataKitchen, pair: str) -> None: def fit_live_predictions(self, dk: FreqaiDataKitchen, pair: str) -> None:
""" """
Fit the labels with a gaussian distribution Fit the labels with a gaussian distribution