improve flexibility of user defined prediction dataframe

freqtrade/freqai/data_drawer.py

@@ -245,7 +245,7 @@ class FreqaiDataDrawer:
             logger.info(f'Setting initial FreqUI plots from historical data for {pair}.')
 
         else:
-            for label in dk.label_list:
+            for label in pred_df.columns:
                 mrv_df[label] = pred_df[label]
                 if mrv_df[label].dtype == object:
                     continue
@@ -278,15 +278,16 @@ class FreqaiDataDrawer:
         # strat seems to feed us variable sized dataframes - and since we are trying to build our
         # own return array in the same shape, we need to figure out how the size has changed
         # and adapt our stored/returned info accordingly.
-        length_difference = len(self.model_return_values[pair]) - len_df
-        i = 0
 
-        if length_difference == 0:
-            i = 1
-        elif length_difference > 0:
-            i = length_difference + 1
+        # length_difference = len(self.model_return_values[pair]) - len_df
+        # i = 0
 
-        df = self.model_return_values[pair] = self.model_return_values[pair].shift(-i)
+        # if length_difference == 0:
+        #     i = 1
+        # elif length_difference > 0:
+        #     i = length_difference + 1
+
+        df = self.model_return_values[pair] = self.model_return_values[pair].shift(-1)
 
         if pair in self.historic_predictions:
             hp_df = self.historic_predictions[pair]
@@ -296,7 +297,8 @@ class FreqaiDataDrawer:
             hp_df = pd.concat([hp_df, nan_df], ignore_index=True, axis=0)
             self.historic_predictions[pair] = hp_df[:-1]
 
-        for label in dk.label_list:
+        # incase user adds additional "predictions" e.g. predict_proba output:
+        for label in predictions.columns:
             df[label].iloc[-1] = predictions[label].iloc[-1]
             if df[label].dtype == object:
                 continue
@@ -318,11 +320,11 @@ class FreqaiDataDrawer:
             for key in df.keys():
                 self.historic_predictions[pair][key].iloc[-1] = df[key].iloc[-1]
 
-        if length_difference < 0:
-            prepend_df = pd.DataFrame(
-                np.zeros((abs(length_difference) - 1, len(df.columns))), columns=df.columns
-            )
-            df = pd.concat([prepend_df, df], axis=0)
+        # if length_difference < 0:
+        #     prepend_df = pd.DataFrame(
+        #         np.zeros((abs(length_difference) - 1, len(df.columns))), columns=df.columns
+        #     )
+        #     df = pd.concat([prepend_df, df], axis=0)
 
     def attach_return_values_to_return_dataframe(
             self, pair: str, dataframe: DataFrame) -> DataFrame:
@@ -343,7 +345,12 @@ class FreqaiDataDrawer:
 
         dk.find_features(dataframe)
 
-        for label in dk.label_list:
+        if self.freqai_info.get('predict_proba', []):
+            full_labels = dk.label_list + self.freqai_info['predict_proba']
+        else:
+            full_labels = dk.label_list
+
+        for label in full_labels:
             dataframe[label] = 0
             dataframe[f"{label}_mean"] = 0
             dataframe[f"{label}_std"] = 0

freqtrade/freqai/data_kitchen.py

@@ -342,7 +342,7 @@ class FreqaiDataKitchen:
         :df: Dataframe of predictions to be denormalized
         """
 
-        for label in self.label_list:
+        for label in df.columns:
             if df[label].dtype == object:
                 continue
             df[label] = (
@@ -716,14 +716,16 @@ class FreqaiDataKitchen:
         weights = np.exp(-np.arange(num_weights) / (wfactor * num_weights))[::-1]
         return weights
 
-    def append_predictions(self, predictions, do_predict, len_dataframe):
+    def append_predictions(self, predictions: DataFrame, do_predict: npt.ArrayLike) -> None:
         """
         Append backtest prediction from current backtest period to all previous periods
         """
 
         append_df = DataFrame()
-        for label in self.label_list:
+        for label in predictions.columns:
             append_df[label] = predictions[label]
+            if append_df[label].dtype == object:
+                continue
             append_df[f"{label}_mean"] = self.data["labels_mean"][label]
             append_df[f"{label}_std"] = self.data["labels_std"][label]
 
@@ -1009,7 +1011,7 @@ class FreqaiDataKitchen:
         import scipy as spy
 
         self.data["labels_mean"], self.data["labels_std"] = {}, {}
-        for label in self.label_list:
+        for label in self.data_dictionary["train_labels"].columns:
             if self.data_dictionary["train_labels"][label].dtype == object:
                 continue
             f = spy.stats.norm.fit(self.data_dictionary["train_labels"][label])

freqtrade/freqai/freqai_interface.py

@@ -221,7 +221,7 @@ class IFreqaiModel(ABC):
 
             pred_df, do_preds = self.predict(dataframe_backtest, dk)
 
-            dk.append_predictions(pred_df, do_preds, len(dataframe_backtest))
+            dk.append_predictions(pred_df, do_preds)
 
         dk.fill_predictions(dataframe)
 
@@ -543,15 +543,17 @@ class IFreqaiModel(ABC):
         self.dd.historic_predictions[pair] = pred_df
         hist_preds_df = self.dd.historic_predictions[pair]
 
+        for label in hist_preds_df.columns:
+            if hist_preds_df[label].dtype == object:
+                continue
+            hist_preds_df[f'{label}_mean'] = 0
+            hist_preds_df[f'{label}_std'] = 0
+
         hist_preds_df['do_predict'] = 0
 
         if self.freqai_info['feature_parameters'].get('DI_threshold', 0) > 0:
             hist_preds_df['DI_values'] = 0
 
-        for label in dk.data['labels_mean']:
-            hist_preds_df[f'{label}_mean'] = 0
-            hist_preds_df[f'{label}_std'] = 0
-
         for return_str in dk.data['extra_returns_per_train']:
             hist_preds_df[return_str] = 0