From 2403a03fcbbae135b9919e635ef8d63257610c97 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Aug 2022 06:28:53 +0200 Subject: [PATCH 01/11] Version bump 2022.8 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 2572c03f1..6c5c52a04 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2022.8.dev' +__version__ = '2022.8' if 'dev' in __version__: try: From d0456b698cb2ed8f60d8a3b80165c36b657bf59c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 29 Sep 2022 07:22:41 +0200 Subject: [PATCH 02/11] Version bump 2022.9 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 6c5c52a04..634377e05 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2022.8' +__version__ = '2022.9' if 'dev' in __version__: try: From 8e101a9f1c3d0b76ef7be383644f62b2cf178d41 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Oct 2022 09:35:21 +0200 Subject: [PATCH 03/11] Disable log spam from analyze_df in webhook/discord --- freqtrade/rpc/discord.py | 2 +- freqtrade/rpc/webhook.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/discord.py b/freqtrade/rpc/discord.py index 9efe6f427..c48508300 100644 --- a/freqtrade/rpc/discord.py +++ b/freqtrade/rpc/discord.py @@ -30,9 +30,9 @@ class Discord(Webhook): pass def send_msg(self, msg) -> None: - logger.info(f"Sending discord message: {msg}") if msg['type'].value in self.config['discord']: + logger.info(f"Sending discord message: {msg}") msg['strategy'] = self.strategy msg['timeframe'] = self.timeframe diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 6109e80bc..bb3b3922f 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -61,6 +61,14 @@ class Webhook(RPCHandler): RPCMessageType.STARTUP, RPCMessageType.WARNING): valuedict = whconfig.get('webhookstatus') + elif msg['type'] in ( + RPCMessageType.PROTECTION_TRIGGER, + RPCMessageType.PROTECTION_TRIGGER_GLOBAL, + RPCMessageType.WHITELIST, + RPCMessageType.ANALYZED_DF, + RPCMessageType.STRATEGY_MSG): + # Don't fail for non-implemented types + return else: raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) if not valuedict: From 6841bdaa8193278d6863131f8866f4a59286da37 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Oct 2022 09:45:58 +0200 Subject: [PATCH 04/11] Update test to verify webhook won't log-spam on new messagetypes --- tests/rpc/test_rpc_webhook.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index 4d65b4966..3bbb85d54 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -365,6 +365,14 @@ def test_exception_send_msg(default_conf, mocker, caplog): with pytest.raises(NotImplementedError): webhook.send_msg(msg) + # Test no failure for not implemented but known messagetypes + for e in RPCMessageType: + msg = { + 'type': e, + 'status': 'whatever' + } + webhook.send_msg(msg) + def test__send_msg(default_conf, mocker, caplog): default_conf["webhook"] = get_webhook_dict() From 19b3669d971231654c3ff0ff5a52a6fe6a4e447e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Oct 2022 21:20:14 +0200 Subject: [PATCH 05/11] Decrease message throughput fixes memory leak by queue raising indefinitely --- freqtrade/rpc/api_server/webserver.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index df4324740..53af91477 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -198,8 +198,10 @@ class ApiServer(RPCHandler): logger.debug(f"Found message of type: {message.get('type')}") # Broadcast it await self._ws_channel_manager.broadcast(message) - # Sleep, make this configurable? - await asyncio.sleep(0.1) + # Limit messages per sec. + # Could cause problems with queue size if too low, and + # problems with network traffik if too high. + await asyncio.sleep(0.001) except asyncio.CancelledError: pass From 03256fc7768bfdebaa8ecbb9e547682ab36f0591 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Sat, 1 Oct 2022 16:50:29 +0200 Subject: [PATCH 06/11] Merge pull request #7508 from aemr3/fix-pca-errors Fix feature list match for PCA --- .../freqai/base_models/BaseClassifierModel.py | 2 +- .../freqai/base_models/BaseRegressionModel.py | 2 +- freqtrade/freqai/data_drawer.py | 2 +- freqtrade/freqai/data_kitchen.py | 1 + freqtrade/freqai/freqai_interface.py | 17 +++++++++-------- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/freqtrade/freqai/base_models/BaseClassifierModel.py b/freqtrade/freqai/base_models/BaseClassifierModel.py index 70f212d2a..09f1bf98c 100644 --- a/freqtrade/freqai/base_models/BaseClassifierModel.py +++ b/freqtrade/freqai/base_models/BaseClassifierModel.py @@ -92,7 +92,7 @@ class BaseClassifierModel(IFreqaiModel): filtered_df = dk.normalize_data_from_metadata(filtered_df) dk.data_dictionary["prediction_features"] = filtered_df - self.data_cleaning_predict(dk, filtered_df) + self.data_cleaning_predict(dk) predictions = self.model.predict(dk.data_dictionary["prediction_features"]) pred_df = DataFrame(predictions, columns=dk.label_list) diff --git a/freqtrade/freqai/base_models/BaseRegressionModel.py b/freqtrade/freqai/base_models/BaseRegressionModel.py index 2450bf305..5d89dd356 100644 --- a/freqtrade/freqai/base_models/BaseRegressionModel.py +++ b/freqtrade/freqai/base_models/BaseRegressionModel.py @@ -92,7 +92,7 @@ class BaseRegressionModel(IFreqaiModel): dk.data_dictionary["prediction_features"] = filtered_df # optional additional data cleaning/analysis - self.data_cleaning_predict(dk, filtered_df) + self.data_cleaning_predict(dk) predictions = self.model.predict(dk.data_dictionary["prediction_features"]) pred_df = DataFrame(predictions, columns=dk.label_list) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 1839724f8..471f6875c 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -423,7 +423,7 @@ class FreqaiDataDrawer: dk.data["data_path"] = str(dk.data_path) dk.data["model_filename"] = str(dk.model_filename) - dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns) + dk.data["training_features_list"] = dk.training_features_list dk.data["label_list"] = dk.label_list # store the metadata with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp: diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index f4fa4e5fd..697fd85cf 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -881,6 +881,7 @@ class FreqaiDataKitchen: """ column_names = dataframe.columns features = [c for c in column_names if "%" in c] + if not features: raise OperationalException("Could not find any features!") diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index d9f917338..78539bae5 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -275,7 +275,8 @@ class IFreqaiModel(ABC): if dk.check_if_backtest_prediction_exists(): self.dd.load_metadata(dk) - self.check_if_feature_list_matches_strategy(dataframe_train, dk) + dk.find_features(dataframe_train) + self.check_if_feature_list_matches_strategy(dk) append_df = dk.get_backtesting_prediction() dk.append_predictions(append_df) else: @@ -296,7 +297,6 @@ class IFreqaiModel(ABC): else: self.model = self.dd.load_data(pair, dk) - # self.check_if_feature_list_matches_strategy(dataframe_train, dk) pred_df, do_preds = self.predict(dataframe_backtest, dk) append_df = dk.get_predictions_to_append(pred_df, do_preds) dk.append_predictions(append_df) @@ -420,7 +420,7 @@ class IFreqaiModel(ABC): return def check_if_feature_list_matches_strategy( - self, dataframe: DataFrame, dk: FreqaiDataKitchen + self, dk: FreqaiDataKitchen ) -> None: """ Ensure user is passing the proper feature set if they are reusing an `identifier` pointing @@ -429,11 +429,12 @@ class IFreqaiModel(ABC): :param dk: FreqaiDataKitchen = non-persistent data container/analyzer for current coin/bot loop """ - dk.find_features(dataframe) + if "training_features_list_raw" in dk.data: feature_list = dk.data["training_features_list_raw"] else: feature_list = dk.data['training_features_list'] + if dk.training_features_list != feature_list: raise OperationalException( "Trying to access pretrained model with `identifier` " @@ -481,13 +482,16 @@ class IFreqaiModel(ABC): if self.freqai_info["feature_parameters"].get('noise_standard_deviation', 0): dk.add_noise_to_training_features() - def data_cleaning_predict(self, dk: FreqaiDataKitchen, dataframe: DataFrame) -> None: + def data_cleaning_predict(self, dk: FreqaiDataKitchen) -> None: """ Base data cleaning method for predict. Functions here are complementary to the functions of data_cleaning_train. """ ft_params = self.freqai_info["feature_parameters"] + # ensure user is feeding the correct indicators to the model + self.check_if_feature_list_matches_strategy(dk) + if ft_params.get('inlier_metric_window', 0): dk.compute_inlier_metric(set_='predict') @@ -505,9 +509,6 @@ class IFreqaiModel(ABC): if ft_params.get("use_DBSCAN_to_remove_outliers", False): dk.use_DBSCAN_to_remove_outliers(predict=True) - # ensure user is feeding the correct indicators to the model - self.check_if_feature_list_matches_strategy(dk.data_dictionary['prediction_features'], dk) - def model_exists(self, dk: FreqaiDataKitchen) -> bool: """ Given a pair and path, check if a model already exists From c53ff94b8e5de6a2cbbad69f386eb1c717211ce2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Sep 2022 13:47:26 +0200 Subject: [PATCH 07/11] Force joblib update via setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0581081fa..d3f9ea7c0 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ setup( 'pandas', 'tables', 'blosc', - 'joblib', + 'joblib>=1.2.0', 'pyarrow; platform_machine != "armv7l"', 'fastapi', 'uvicorn', From 59cfde3767f449c588500acb07c12503ac3d9032 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Sep 2022 15:43:05 +0200 Subject: [PATCH 08/11] Fix pandas deprecation warnings from freqAI --- freqtrade/freqai/data_kitchen.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 697fd85cf..766eb981f 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -210,7 +210,7 @@ class FreqaiDataKitchen: filtered_df = unfiltered_df.filter(training_feature_list, axis=1) filtered_df = filtered_df.replace([np.inf, -np.inf], np.nan) - drop_index = pd.isnull(filtered_df).any(1) # get the rows that have NaNs, + drop_index = pd.isnull(filtered_df).any(axis=1) # get the rows that have NaNs, drop_index = drop_index.replace(True, 1).replace(False, 0) # pep8 requirement. if (training_filter): const_cols = list((filtered_df.nunique() == 1).loc[lambda x: x].index) @@ -221,7 +221,7 @@ class FreqaiDataKitchen: # about removing any row with NaNs # if labels has multiple columns (user wants to train multiple modelEs), we detect here labels = unfiltered_df.filter(label_list, axis=1) - drop_index_labels = pd.isnull(labels).any(1) + drop_index_labels = pd.isnull(labels).any(axis=1) drop_index_labels = drop_index_labels.replace(True, 1).replace(False, 0) dates = unfiltered_df['date'] filtered_df = filtered_df[ @@ -249,7 +249,7 @@ class FreqaiDataKitchen: else: # we are backtesting so we need to preserve row number to send back to strategy, # so now we use do_predict to avoid any prediction based on a NaN - drop_index = pd.isnull(filtered_df).any(1) + drop_index = pd.isnull(filtered_df).any(axis=1) self.data["filter_drop_index_prediction"] = drop_index filtered_df.fillna(0, inplace=True) # replacing all NaNs with zeros to avoid issues in 'prediction', but any prediction @@ -808,7 +808,7 @@ class FreqaiDataKitchen: :, :no_prev_pts ] distances = distances.replace([np.inf, -np.inf], np.nan) - drop_index = pd.isnull(distances).any(1) + drop_index = pd.isnull(distances).any(axis=1) distances = distances[drop_index == 0] inliers = pd.DataFrame(index=distances.index) From 851d1e9da106aad2153ec38919873f61ce4b384f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Oct 2022 06:56:02 +0200 Subject: [PATCH 09/11] Version bump 2022.9.1 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 634377e05..2bd09a377 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2022.9' +__version__ = '2022.9.1' if 'dev' in __version__: try: From ec7d6634963cd9482f47e5db0c6e90e9ffd176ba Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 28 Oct 2022 19:34:30 +0200 Subject: [PATCH 10/11] Version bump 2022.10 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index ce5f420e4..74c558a37 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2022.9.1' +__version__ = '2022.10' if 'dev' in __version__: try: From 5c571f565ff639d10b5095db7f7429e35a9c0b73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Nov 2022 15:34:00 +0100 Subject: [PATCH 11/11] Version bump 2022.11 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 74c558a37..47d095f65 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2022.10' +__version__ = '2022.11' if 'dev' in __version__: try: