Version bump 2022.9.1

Fix feature list match for PCA

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2022.9'
+__version__ = '2022.9.1'
 
 if 'dev' in __version__:
     try:

freqtrade/freqai/base_models/BaseClassifierModel.py

@@ -92,7 +92,7 @@ class BaseClassifierModel(IFreqaiModel):
         filtered_df = dk.normalize_data_from_metadata(filtered_df)
         dk.data_dictionary["prediction_features"] = filtered_df
 
-        self.data_cleaning_predict(dk, filtered_df)
+        self.data_cleaning_predict(dk)
 
         predictions = self.model.predict(dk.data_dictionary["prediction_features"])
         pred_df = DataFrame(predictions, columns=dk.label_list)

freqtrade/freqai/base_models/BaseRegressionModel.py

@@ -92,7 +92,7 @@ class BaseRegressionModel(IFreqaiModel):
         dk.data_dictionary["prediction_features"] = filtered_df
 
         # optional additional data cleaning/analysis
-        self.data_cleaning_predict(dk, filtered_df)
+        self.data_cleaning_predict(dk)
 
         predictions = self.model.predict(dk.data_dictionary["prediction_features"])
         pred_df = DataFrame(predictions, columns=dk.label_list)

freqtrade/freqai/data_drawer.py

@@ -423,7 +423,7 @@ class FreqaiDataDrawer:
 
         dk.data["data_path"] = str(dk.data_path)
         dk.data["model_filename"] = str(dk.model_filename)
-        dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns)
+        dk.data["training_features_list"] = dk.training_features_list
         dk.data["label_list"] = dk.label_list
         # store the metadata
         with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp:

freqtrade/freqai/data_kitchen.py

@@ -210,7 +210,7 @@ class FreqaiDataKitchen:
         filtered_df = unfiltered_df.filter(training_feature_list, axis=1)
         filtered_df = filtered_df.replace([np.inf, -np.inf], np.nan)
 
-        drop_index = pd.isnull(filtered_df).any(1)  # get the rows that have NaNs,
+        drop_index = pd.isnull(filtered_df).any(axis=1)  # get the rows that have NaNs,
         drop_index = drop_index.replace(True, 1).replace(False, 0)  # pep8 requirement.
         if (training_filter):
             const_cols = list((filtered_df.nunique() == 1).loc[lambda x: x].index)
@@ -221,7 +221,7 @@ class FreqaiDataKitchen:
             # about removing any row with NaNs
             # if labels has multiple columns (user wants to train multiple modelEs), we detect here
             labels = unfiltered_df.filter(label_list, axis=1)
-            drop_index_labels = pd.isnull(labels).any(1)
+            drop_index_labels = pd.isnull(labels).any(axis=1)
             drop_index_labels = drop_index_labels.replace(True, 1).replace(False, 0)
             dates = unfiltered_df['date']
             filtered_df = filtered_df[
@@ -249,7 +249,7 @@ class FreqaiDataKitchen:
         else:
             # we are backtesting so we need to preserve row number to send back to strategy,
             # so now we use do_predict to avoid any prediction based on a NaN
-            drop_index = pd.isnull(filtered_df).any(1)
+            drop_index = pd.isnull(filtered_df).any(axis=1)
             self.data["filter_drop_index_prediction"] = drop_index
             filtered_df.fillna(0, inplace=True)
             # replacing all NaNs with zeros to avoid issues in 'prediction', but any prediction
@@ -808,7 +808,7 @@ class FreqaiDataKitchen:
                 :, :no_prev_pts
             ]
         distances = distances.replace([np.inf, -np.inf], np.nan)
-        drop_index = pd.isnull(distances).any(1)
+        drop_index = pd.isnull(distances).any(axis=1)
         distances = distances[drop_index == 0]
 
         inliers = pd.DataFrame(index=distances.index)
@@ -881,6 +881,7 @@ class FreqaiDataKitchen:
         """
         column_names = dataframe.columns
         features = [c for c in column_names if "%" in c]
+
         if not features:
             raise OperationalException("Could not find any features!")
 

freqtrade/freqai/freqai_interface.py

@@ -275,7 +275,8 @@ class IFreqaiModel(ABC):
 
             if dk.check_if_backtest_prediction_exists():
                 self.dd.load_metadata(dk)
-                self.check_if_feature_list_matches_strategy(dataframe_train, dk)
+                dk.find_features(dataframe_train)
+                self.check_if_feature_list_matches_strategy(dk)
                 append_df = dk.get_backtesting_prediction()
                 dk.append_predictions(append_df)
             else:
@@ -296,7 +297,6 @@ class IFreqaiModel(ABC):
                 else:
                     self.model = self.dd.load_data(pair, dk)
 
-                # self.check_if_feature_list_matches_strategy(dataframe_train, dk)
                 pred_df, do_preds = self.predict(dataframe_backtest, dk)
                 append_df = dk.get_predictions_to_append(pred_df, do_preds)
                 dk.append_predictions(append_df)
@@ -420,7 +420,7 @@ class IFreqaiModel(ABC):
         return
 
     def check_if_feature_list_matches_strategy(
-        self, dataframe: DataFrame, dk: FreqaiDataKitchen
+        self, dk: FreqaiDataKitchen
     ) -> None:
         """
         Ensure user is passing the proper feature set if they are reusing an `identifier` pointing
@@ -429,11 +429,12 @@ class IFreqaiModel(ABC):
         :param dk: FreqaiDataKitchen = non-persistent data container/analyzer for
                    current coin/bot loop
         """
-        dk.find_features(dataframe)
+
         if "training_features_list_raw" in dk.data:
             feature_list = dk.data["training_features_list_raw"]
         else:
             feature_list = dk.data['training_features_list']
+
         if dk.training_features_list != feature_list:
             raise OperationalException(
                 "Trying to access pretrained model with `identifier` "
@@ -481,13 +482,16 @@ class IFreqaiModel(ABC):
         if self.freqai_info["feature_parameters"].get('noise_standard_deviation', 0):
             dk.add_noise_to_training_features()
 
-    def data_cleaning_predict(self, dk: FreqaiDataKitchen, dataframe: DataFrame) -> None:
+    def data_cleaning_predict(self, dk: FreqaiDataKitchen) -> None:
         """
         Base data cleaning method for predict.
         Functions here are complementary to the functions of data_cleaning_train.
         """
         ft_params = self.freqai_info["feature_parameters"]
 
+        # ensure user is feeding the correct indicators to the model
+        self.check_if_feature_list_matches_strategy(dk)
+
         if ft_params.get('inlier_metric_window', 0):
             dk.compute_inlier_metric(set_='predict')
 
@@ -505,9 +509,6 @@ class IFreqaiModel(ABC):
         if ft_params.get("use_DBSCAN_to_remove_outliers", False):
             dk.use_DBSCAN_to_remove_outliers(predict=True)
 
-        # ensure user is feeding the correct indicators to the model
-        self.check_if_feature_list_matches_strategy(dk.data_dictionary['prediction_features'], dk)
-
     def model_exists(self, dk: FreqaiDataKitchen) -> bool:
         """
         Given a pair and path, check if a model already exists

freqtrade/rpc/api_server/webserver.py

@@ -198,8 +198,10 @@ class ApiServer(RPCHandler):
                 logger.debug(f"Found message of type: {message.get('type')}")
                 # Broadcast it
                 await self._ws_channel_manager.broadcast(message)
-                # Sleep, make this configurable?
-                await asyncio.sleep(0.1)
+                # Limit messages per sec.
+                # Could cause problems with queue size if too low, and
+                # problems with network traffik if too high.
+                await asyncio.sleep(0.001)
         except asyncio.CancelledError:
             pass
 

freqtrade/rpc/discord.py

@@ -30,9 +30,9 @@ class Discord(Webhook):
         pass
 
     def send_msg(self, msg) -> None:
-        logger.info(f"Sending discord message: {msg}")
 
         if msg['type'].value in self.config['discord']:
+            logger.info(f"Sending discord message: {msg}")
 
             msg['strategy'] = self.strategy
             msg['timeframe'] = self.timeframe

freqtrade/rpc/webhook.py

@@ -61,6 +61,14 @@ class Webhook(RPCHandler):
                                  RPCMessageType.STARTUP,
                                  RPCMessageType.WARNING):
                 valuedict = whconfig.get('webhookstatus')
+            elif msg['type'] in (
+                    RPCMessageType.PROTECTION_TRIGGER,
+                    RPCMessageType.PROTECTION_TRIGGER_GLOBAL,
+                    RPCMessageType.WHITELIST,
+                    RPCMessageType.ANALYZED_DF,
+                    RPCMessageType.STRATEGY_MSG):
+                # Don't fail for non-implemented types
+                return
             else:
                 raise NotImplementedError('Unknown message type: {}'.format(msg['type']))
             if not valuedict:

setup.py

@@ -72,7 +72,7 @@ setup(
         'pandas',
         'tables',
         'blosc',
-        'joblib',
+        'joblib>=1.2.0',
         'pyarrow; platform_machine != "armv7l"',
         'fastapi',
         'uvicorn',

tests/rpc/test_rpc_webhook.py

@@ -365,6 +365,14 @@ def test_exception_send_msg(default_conf, mocker, caplog):
     with pytest.raises(NotImplementedError):
         webhook.send_msg(msg)
 
+    # Test no failure for not implemented but known messagetypes
+    for e in RPCMessageType:
+        msg = {
+            'type': e,
+            'status': 'whatever'
+            }
+        webhook.send_msg(msg)
+
 
 def test__send_msg(default_conf, mocker, caplog):
     default_conf["webhook"] = get_webhook_dict()