From 3e38c1b0bd80498f0e6c7a03727653e24a148db6 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 12 Aug 2022 13:13:08 +0200 Subject: [PATCH] take dynamic sized tail off historic_predictions as return dataframe to strategy. --- freqtrade/freqai/data_drawer.py | 114 ++++++++++++++------------- freqtrade/freqai/freqai_interface.py | 12 ++- 2 files changed, 69 insertions(+), 57 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index db1206a39..0644ddbc9 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -241,48 +241,48 @@ class FreqaiDataDrawer: """ # dynamic df returned to strategy and plotted in frequi - mrv_df = self.model_return_values[pair] = pd.DataFrame() + # mrv_df = self.model_return_values[pair] = pd.DataFrame() # if user reused `identifier` in config and has historical predictions collected, load them # so that frequi remains uninterrupted after a crash hist_df = self.historic_predictions - if pair in hist_df: - len_diff = len(hist_df[pair].index) - len(pred_df.index) - if len_diff < 0: - df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]], - ignore_index=True, keys=hist_df[pair].keys()) - else: - df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True) - df_concat = df_concat.fillna(0) - self.model_return_values[pair] = df_concat - logger.info(f'Setting initial FreqUI plots from historical data for {pair}.') - + # if pair in hist_df: + len_diff = len(hist_df[pair].index) - len(pred_df.index) + if len_diff < 0: + df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]], + ignore_index=True, keys=hist_df[pair].keys()) else: - for label in pred_df.columns: - mrv_df[label] = pred_df[label] - if mrv_df[label].dtype == object: - continue - mrv_df[f"{label}_mean"] = dk.data["labels_mean"][label] - mrv_df[f"{label}_std"] = dk.data["labels_std"][label] + df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True) + df_concat = df_concat.fillna(0) + self.model_return_values[pair] = df_concat + logger.info(f'Setting initial FreqUI plots from historical data for {pair}.') - if self.freqai_info["feature_parameters"].get("DI_threshold", 0) > 0: - mrv_df["DI_values"] = dk.DI_values + # else: + # for label in pred_df.columns: + # mrv_df[label] = pred_df[label] + # if mrv_df[label].dtype == object: + # continue + # mrv_df[f"{label}_mean"] = dk.data["labels_mean"][label] + # mrv_df[f"{label}_std"] = dk.data["labels_std"][label] - mrv_df["do_predict"] = do_preds + # if self.freqai_info["feature_parameters"].get("DI_threshold", 0) > 0: + # mrv_df["DI_values"] = dk.DI_values - if dk.data['extra_returns_per_train']: - rets = dk.data['extra_returns_per_train'] - for return_str in rets: - mrv_df[return_str] = rets[return_str] + # mrv_df["do_predict"] = do_preds - # for keras type models, the conv_window needs to be prepended so - # viewing is correct in frequi - if self.freqai_info.get('keras', False): - n_lost_points = self.freqai_info.get('conv_width', 2) - zeros_df = DataFrame(np.zeros((n_lost_points, len(mrv_df.columns))), - columns=mrv_df.columns) - self.model_return_values[pair] = pd.concat( - [zeros_df, mrv_df], axis=0, ignore_index=True) + # if dk.data['extra_returns_per_train']: + # rets = dk.data['extra_returns_per_train'] + # for return_str in rets: + # mrv_df[return_str] = rets[return_str] + + # # for keras type models, the conv_window needs to be prepended so + # # viewing is correct in frequi + # if self.freqai_info.get('keras', False): + # n_lost_points = self.freqai_info.get('conv_width', 2) + # zeros_df = DataFrame(np.zeros((n_lost_points, len(mrv_df.columns))), + # columns=mrv_df.columns) + # self.model_return_values[pair] = pd.concat( + # [zeros_df, mrv_df], axis=0, ignore_index=True) def append_model_predictions(self, pair: str, predictions: DataFrame, do_preds: NDArray[np.int_], @@ -292,23 +292,23 @@ class FreqaiDataDrawer: # own return array in the same shape, we need to figure out how the size has changed # and adapt our stored/returned info accordingly. - length_difference = len(self.model_return_values[pair]) - len_df - i = 0 + # length_difference = len(self.model_return_values[pair]) - len_df + # i = 0 - if length_difference == 0: - i = 1 - elif length_difference > 0: - i = length_difference + 1 + # if length_difference == 0: + # i = 1 + # elif length_difference > 0: + # i = length_difference + 1 - df = self.model_return_values[pair] = self.model_return_values[pair].shift(-i) + # df = self.model_return_values[pair] = self.model_return_values[pair].shift(-i) - if pair in self.historic_predictions: - hp_df = self.historic_predictions[pair] - # here are some pandas hula hoops to accommodate the possibility of a series - # or dataframe depending number of labels requested by user - nan_df = pd.DataFrame(np.nan, index=hp_df.index[-2:] + 2, columns=hp_df.columns) - hp_df = pd.concat([hp_df, nan_df], ignore_index=True, axis=0) - self.historic_predictions[pair] = hp_df[:-1] + # if pair in self.historic_predictions: + df = self.historic_predictions[pair] + # here are some pandas hula hoops to accommodate the possibility of a series + # or dataframe depending number of labels requested by user + nan_df = pd.DataFrame(np.nan, index=df.index[-2:] + 2, columns=df.columns) + df = pd.concat([df, nan_df], ignore_index=True, axis=0) + df = self.historic_predictions[pair] = df[:-1] # incase user adds additional "predictions" e.g. predict_proba output: for label in predictions.columns: @@ -328,16 +328,18 @@ class FreqaiDataDrawer: for return_str in rets: df[return_str].iloc[-1] = rets[return_str] - # append the new predictions to persistent storage - if pair in self.historic_predictions: - for key in df.keys(): - self.historic_predictions[pair][key].iloc[-1] = df[key].iloc[-1] + self.model_return_values[pair] = df.tail(len_df).reset_index(drop=True) - if length_difference < 0: - prepend_df = pd.DataFrame( - np.zeros((abs(length_difference) - 1, len(df.columns))), columns=df.columns - ) - df = pd.concat([prepend_df, df], axis=0) + # # append the new predictions to persistent storage + # if pair in self.historic_predictions: + # for key in df.keys(): + # self.historic_predictions[pair][key].iloc[-1] = df[key].iloc[-1] + + # if length_difference < 0: + # prepend_df = pd.DataFrame( + # np.zeros((abs(length_difference) - 1, len(df.columns))), columns=df.columns + # ) + # df = pd.concat([prepend_df, df], axis=0) def attach_return_values_to_return_dataframe( self, pair: str, dataframe: DataFrame) -> DataFrame: diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 502d5ac34..4306b4879 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -319,9 +319,10 @@ class IFreqaiModel(ABC): # first predictions are made on entire historical candle set coming from strategy. This # allows FreqUI to show full return values. pred_df, do_preds = self.predict(dataframe, dk) - self.dd.set_initial_return_values(pair, dk, pred_df, do_preds) if pair not in self.dd.historic_predictions: self.set_initial_historic_predictions(pred_df, dk, pair) + self.dd.set_initial_return_values(pair, dk, pred_df, do_preds) + dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe) return elif self.dk.check_if_model_expired(trained_timestamp): @@ -551,6 +552,15 @@ class IFreqaiModel(ABC): for return_str in dk.data['extra_returns_per_train']: hist_preds_df[return_str] = 0 + # # for keras type models, the conv_window needs to be prepended so + # # viewing is correct in frequi + if self.freqai_info.get('keras', False): + n_lost_points = self.freqai_info.get('conv_width', 2) + zeros_df = DataFrame(np.zeros((n_lost_points, len(hist_preds_df.columns))), + columns=hist_preds_df.columns) + self.model_return_values[pair] = pd.concat( + [zeros_df, hist_preds_df], axis=0, ignore_index=True) + def fit_live_predictions(self, dk: FreqaiDataKitchen, pair: str) -> None: """ Fit the labels with a gaussian distribution