Merge branch 'develop' into backtest_live_models

freqtrade/freqai/data_drawer.py

@@ -636,6 +636,8 @@ class FreqaiDataDrawer:
                         axis=0,
                     )
 
+            self.current_candle = history_data[dk.pair][self.config['timeframe']].iloc[-1]['date']
+
     def load_all_pair_histories(self, timerange: TimeRange, dk: FreqaiDataKitchen) -> None:
         """
         Load pair histories for all whitelist and corr_pairlist pairs.

freqtrade/freqai/data_kitchen.py

@@ -1177,11 +1177,13 @@ class FreqaiDataKitchen:
         pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
 
         for pair in pairs:
+            pair = pair.replace(':', '')  # lightgbm doesnt like colons
             valid_strs = [f"%-{pair}", f"%{pair}", f"%_{pair}"]
             pair_cols = [col for col in dataframe.columns if
                          any(substr in col for substr in valid_strs)]
-            pair_cols.insert(0, 'date')
-            corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1)
+            if pair_cols:
+                pair_cols.insert(0, 'date')
+                corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1)
 
         return corr_dataframes
 
@@ -1199,8 +1201,9 @@ class FreqaiDataKitchen:
                     ready for training
         """
         pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
-
+        current_pair = current_pair.replace(':', '')
         for pair in pairs:
+            pair = pair.replace(':', '')  # lightgbm doesnt work with colons
             if current_pair != pair:
                 dataframe = dataframe.merge(corr_dataframes[pair], how='left', on='date')
 
@@ -1270,6 +1273,8 @@ class FreqaiDataKitchen:
 
         self.get_unique_classes_from_labels(dataframe)
 
+        dataframe = self.remove_special_chars_from_feature_names(dataframe)
+
         return dataframe
 
     def fit_labels(self) -> None:
@@ -1471,3 +1476,16 @@ class FreqaiDataKitchen:
                     assets_end_dates[asset].append(model_end_date)
 
         return assets_end_dates
+
+    def remove_special_chars_from_feature_names(self, dataframe: pd.DataFrame) -> pd.DataFrame:
+        """
+        Remove all special characters from feature strings (:)
+        :param dataframe: the dataframe that just finished indicator population. (unfiltered)
+        :return: dataframe with cleaned featrue names
+        """
+
+        spec_chars = [':']
+        for c in spec_chars:
+            dataframe.columns = dataframe.columns.str.replace(c, "")
+
+        return dataframe

freqtrade/freqai/freqai_interface.py

@@ -68,6 +68,9 @@ class IFreqaiModel(ABC):
         if self.save_backtest_models:
             logger.info('Backtesting module configured to save all models.')
         self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode)
+        # set current candle to arbitrary historical date
+        self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc)
+        self.dd.current_candle = self.current_candle
         self.scanning = False
         self.ft_params = self.freqai_info["feature_parameters"]
         self.corr_pairlist: List[str] = self.ft_params.get("include_corr_pairlist", [])
@@ -75,7 +78,7 @@ class IFreqaiModel(ABC):
         if self.keras and self.ft_params.get("DI_threshold", 0):
             self.ft_params["DI_threshold"] = 0
             logger.warning("DI threshold is not configured for Keras models yet. Deactivating.")
-        self.CONV_WIDTH = self.freqai_info.get("conv_width", 2)
+        self.CONV_WIDTH = self.freqai_info.get('conv_width', 1)
         if self.ft_params.get("inlier_metric_window", 0):
             self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2
         self.pair_it = 0
@@ -93,7 +96,6 @@ class IFreqaiModel(ABC):
         # get_corr_dataframes is controlling the caching of corr_dataframes
         # for improved performance. Careful with this boolean.
         self.get_corr_dataframes: bool = True
-
         self._threads: List[threading.Thread] = []
         self._stop_event = threading.Event()
 
@@ -338,6 +340,7 @@ class IFreqaiModel(ABC):
         if self.dd.historic_data:
             self.dd.update_historic_data(strategy, dk)
             logger.debug(f'Updating historic data on pair {metadata["pair"]}')
+            self.track_current_candle()
 
         if not self.follow_mode:
 
@@ -682,8 +685,6 @@ class IFreqaiModel(ABC):
                                    " avoid blinding open trades and degrading performance.")
                 self.pair_it = 0
                 self.inference_time = 0
-                if self.corr_pairlist:
-                    self.get_corr_dataframes = True
         return
 
     def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''):
@@ -759,12 +760,24 @@ class IFreqaiModel(ABC):
                                "is included in the column names when you are creating features "
                                "in `populate_any_indicators()`.")
             self.get_corr_dataframes = not bool(self.corr_dataframes)
-        else:
+        elif self.corr_dataframes:
             dataframe = dk.attach_corr_pair_columns(
                 dataframe, self.corr_dataframes, dk.pair)
 
         return dataframe
 
+    def track_current_candle(self):
+        """
+        Checks if the latest candle appended by the datadrawer is
+        equivalent to the latest candle seen by FreqAI. If not, it
+        asks to refresh the cached corr_dfs, and resets the pair
+        counter.
+        """
+        if self.dd.current_candle > self.current_candle:
+            self.get_corr_dataframes = True
+            self.pair_it = 1
+            self.current_candle = self.dd.current_candle
+
     def ensure_data_exists(self, dataframe_backtest: DataFrame,
                            tr_backtest: TimeRange, pair: str) -> bool:
         """