From 23cc21ce59093df511f6fcc89ae22ec0ab5829cb Mon Sep 17 00:00:00 2001
From: robcaulk <rob.caulk@gmail.com>
Date: Tue, 9 Aug 2022 17:31:38 +0200
Subject: [PATCH] add predict_proba to base classifier, improve historic
 predictions handling

---
 freqtrade/freqai/data_drawer.py               | 10 +++--
 freqtrade/freqai/data_kitchen.py              | 11 +++++
 .../prediction_models/CatboostClassifier.py   | 41 +++++++++++++++++--
 3 files changed, 54 insertions(+), 8 deletions(-)

diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py
index ec49c7050..0dacd4ee7 100644
--- a/freqtrade/freqai/data_drawer.py
+++ b/freqtrade/freqai/data_drawer.py
@@ -358,10 +358,12 @@ class FreqaiDataDrawer:
 
         dk.find_features(dataframe)
 
-        if self.freqai_info.get('predict_proba', []):
-            full_labels = dk.label_list + self.freqai_info['predict_proba']
-        else:
-            full_labels = dk.label_list
+        added_labels = []
+        if dk.unique_classes:
+            for label in dk.unique_classes:
+                added_labels += dk.unique_classes[label]
+   
+        full_labels = dk.label_list + added_labels
 
         for label in full_labels:
             dataframe[label] = 0
diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py
index 0cff9c90e..3eb89ce6d 100644
--- a/freqtrade/freqai/data_kitchen.py
+++ b/freqtrade/freqai/data_kitchen.py
@@ -90,6 +90,7 @@ class FreqaiDataKitchen:
         self.data['extra_returns_per_train'] = self.freqai_config.get('extra_returns_per_train', {})
         self.thread_count = self.freqai_config.get("data_kitchen_thread_count", -1)
         self.train_dates: DataFrame = pd.DataFrame()
+        self.unique_classes: Dict[str, list] = {}
 
     def set_paths(
         self,
@@ -977,6 +978,8 @@ class FreqaiDataKitchen:
                         informative=corr_dataframes[i][tf]
                     )
 
+        self.get_unique_classes_from_labels(dataframe)
+
         return dataframe
 
     def fit_labels(self) -> None:
@@ -1003,3 +1006,11 @@ class FreqaiDataKitchen:
             col for col in dataframe.columns if not col.startswith("%") or col.startswith("%%")
         ]
         return dataframe[to_keep]
+
+    def get_unique_classes_from_labels(self, dataframe: DataFrame) -> None:
+
+        self.find_features(dataframe)
+
+        for key in self.label_list:
+            if dataframe[key].dtype == object:
+                self.unique_classes[key] = dataframe[key].dropna().unique()
diff --git a/freqtrade/freqai/prediction_models/CatboostClassifier.py b/freqtrade/freqai/prediction_models/CatboostClassifier.py
index ac1386eee..7a4b06557 100644
--- a/freqtrade/freqai/prediction_models/CatboostClassifier.py
+++ b/freqtrade/freqai/prediction_models/CatboostClassifier.py
@@ -1,10 +1,12 @@
 import logging
-from typing import Any, Dict
-
+from typing import Any, Dict, Tuple
+import pandas as pd
+from pandas import DataFrame
 from catboost import CatBoostClassifier, Pool
-
+import numpy.typing as npt
+import numpy as np
 from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel
-
+from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
 
 logger = logging.getLogger(__name__)
 
@@ -39,3 +41,34 @@ class CatboostClassifier(BaseRegressionModel):
         cbr.fit(train_data)
 
         return cbr
+
+    def predict(
+        self, unfiltered_dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = False
+    ) -> Tuple[DataFrame, npt.NDArray[np.int_]]:
+        """
+        Filter the prediction features data and predict with it.
+        :param: unfiltered_dataframe: Full dataframe for the current backtest period.
+        :return:
+        :pred_df: dataframe containing the predictions
+        :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove
+        data (NaNs) or felt uncertain about data (PCA and DI index)
+        """
+
+        dk.find_features(unfiltered_dataframe)
+        filtered_dataframe, _ = dk.filter_features(
+            unfiltered_dataframe, dk.training_features_list, training_filter=False
+        )
+        filtered_dataframe = dk.normalize_data_from_metadata(filtered_dataframe)
+        dk.data_dictionary["prediction_features"] = filtered_dataframe
+
+        self.data_cleaning_predict(dk, filtered_dataframe)
+
+        predictions = self.model.predict(dk.data_dictionary["prediction_features"])
+        pred_df = DataFrame(predictions, columns=dk.label_list)
+
+        predictions_prob = self.model.predict_proba(dk.data_dictionary["prediction_features"])
+        pred_df_prob = DataFrame(predictions_prob, columns=self.model.classes_)
+
+        pred_df = pd.concat([pred_df, pred_df_prob], axis=1)
+
+        return (pred_df, dk.do_predict)