From 9633081c316fb21a716dddb7445ad77b88df4084 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 22 Feb 2023 22:01:41 +0100 Subject: [PATCH 1/3] remove remnants of follower, clean data-drawer, improve doc --- docs/freqai.md | 2 +- freqtrade/freqai/data_drawer.py | 26 ++------------------------ freqtrade/freqai/freqai_interface.py | 6 +++--- 3 files changed, 6 insertions(+), 28 deletions(-) diff --git a/docs/freqai.md b/docs/freqai.md index d84ec8d2b..b1cde2780 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -19,7 +19,7 @@ Features include: * **Automatic data download** - Compute timeranges for data downloads and update historic data (in live deployments) * **Cleaning of incoming data** - Handle NaNs safely before training and model inferencing * **Dimensionality reduction** - Reduce the size of the training data via [Principal Component Analysis](freqai-feature-engineering.md#data-dimensionality-reduction-with-principal-component-analysis) -* **Deploying bot fleets** - Set one bot to train models while a fleet of [follower bots](freqai-running.md#setting-up-a-follower) inference the models and handle trades +* **Deploying bot fleets** - Set one bot to train models while a fleet of [consumers](producer-consumer.md) use signals. ## Quick start diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index cb5b0f0fb..fc4c9f7b6 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -72,12 +72,7 @@ class FreqaiDataDrawer: self.model_return_values: Dict[str, DataFrame] = {} self.historic_data: Dict[str, Dict[str, DataFrame]] = {} self.historic_predictions: Dict[str, DataFrame] = {} - self.follower_dict: Dict[str, pair_info] = {} self.full_path = full_path - self.follower_name: str = self.config.get("bot_name", "follower1") - self.follower_dict_path = Path( - self.full_path / f"follower_dictionary-{self.follower_name}.json" - ) self.historic_predictions_path = Path(self.full_path / "historic_predictions.pkl") self.historic_predictions_bkp_path = Path( self.full_path / "historic_predictions.backup.pkl") @@ -218,14 +213,6 @@ class FreqaiDataDrawer: rapidjson.dump(self.pair_dict, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE) - def save_follower_dict_to_disk(self): - """ - Save follower dictionary to disk (used by strategy for persistent prediction targets) - """ - with open(self.follower_dict_path, "w") as fp: - rapidjson.dump(self.follower_dict, fp, default=self.np_encoder, - number_mode=rapidjson.NM_NATIVE) - def save_global_metadata_to_disk(self, metadata: Dict[str, Any]): """ Save global metadata json to disk @@ -239,7 +226,7 @@ class FreqaiDataDrawer: if isinstance(object, np.generic): return object.item() - def get_pair_dict_info(self, pair: str) -> Tuple[str, int, bool]: + def get_pair_dict_info(self, pair: str) -> Tuple[str, int]: """ Locate and load existing model metadata from persistent storage. If not located, create a new one and append the current pair to it and prepare it for its first @@ -248,12 +235,9 @@ class FreqaiDataDrawer: :return: model_filename: str = unique filename used for loading persistent objects from disk trained_timestamp: int = the last time the coin was trained - return_null_array: bool = Follower could not find pair metadata """ pair_dict = self.pair_dict.get(pair) - # data_path_set = self.pair_dict.get(pair, self.empty_pair_dict).get("data_path", "") - return_null_array = False if pair_dict: model_filename = pair_dict["model_filename"] @@ -263,7 +247,7 @@ class FreqaiDataDrawer: model_filename = "" trained_timestamp = 0 - return model_filename, trained_timestamp, return_null_array + return model_filename, trained_timestamp def set_pair_dict_info(self, metadata: dict) -> None: pair_in_dict = self.pair_dict.get(metadata["pair"]) @@ -417,12 +401,6 @@ class FreqaiDataDrawer: shutil.rmtree(v) deleted += 1 - def update_follower_metadata(self): - # follower needs to load from disk to get any changes made by leader to pair_dict - self.load_drawer_from_disk() - if self.config.get("freqai", {}).get("purge_old_models", False): - self.purge_old_models() - def save_metadata(self, dk: FreqaiDataKitchen) -> None: """ Saves only metadata for backtesting studies if user prefers diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index f3b4b0e54..fab5cbff8 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -227,7 +227,7 @@ class IFreqaiModel(ABC): logger.warning(f'{pair} not in current whitelist, removing from train queue.') continue - (_, trained_timestamp, _) = self.dd.get_pair_dict_info(pair) + (_, trained_timestamp) = self.dd.get_pair_dict_info(pair) dk = FreqaiDataKitchen(self.config, self.live, pair) ( @@ -285,7 +285,7 @@ class IFreqaiModel(ABC): # following tr_train. Both of these windows slide through the # entire backtest for tr_train, tr_backtest in zip(dk.training_timeranges, dk.backtesting_timeranges): - (_, _, _) = self.dd.get_pair_dict_info(pair) + (_, _) = self.dd.get_pair_dict_info(pair) train_it += 1 total_trains = len(dk.backtesting_timeranges) self.training_timerange = tr_train @@ -382,7 +382,7 @@ class IFreqaiModel(ABC): """ # get the model metadata associated with the current pair - (_, trained_timestamp, return_null_array) = self.dd.get_pair_dict_info(metadata["pair"]) + (_, trained_timestamp) = self.dd.get_pair_dict_info(metadata["pair"]) # append the historic data once per round if self.dd.historic_data: From b8f011a2ab70123f388d0902be887c801baa16f8 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 22 Feb 2023 22:27:56 +0100 Subject: [PATCH 2/3] give users ability to decide how many models to keep in dry/live --- config_examples/config_freqai.example.json | 2 +- docs/freqai-configuration.md | 2 +- docs/freqai-parameter-table.md | 2 +- freqtrade/constants.py | 2 +- freqtrade/freqai/data_drawer.py | 10 ++++++++-- freqtrade/freqai/freqai_interface.py | 3 +-- freqtrade/templates/FreqaiExampleHybridStrategy.py | 2 +- tests/freqai/conftest.py | 2 +- 8 files changed, 15 insertions(+), 10 deletions(-) diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json index 645c30227..65a93379e 100644 --- a/config_examples/config_freqai.example.json +++ b/config_examples/config_freqai.example.json @@ -48,7 +48,7 @@ ], "freqai": { "enabled": true, - "purge_old_models": true, + "purge_old_models": 2, "train_period_days": 15, "backtest_period_days": 7, "live_retrain_hours": 0, diff --git a/docs/freqai-configuration.md b/docs/freqai-configuration.md index 88415bf59..886dc2338 100644 --- a/docs/freqai-configuration.md +++ b/docs/freqai-configuration.md @@ -9,7 +9,7 @@ FreqAI is configured through the typical [Freqtrade config file](configuration.m ```json "freqai": { "enabled": true, - "purge_old_models": true, + "purge_old_models": 2, "train_period_days": 30, "backtest_period_days": 7, "identifier" : "unique-id", diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md index 23d2be8ef..7f0b0c213 100644 --- a/docs/freqai-parameter-table.md +++ b/docs/freqai-parameter-table.md @@ -15,7 +15,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `identifier` | **Required.**
A unique ID for the current model. If models are saved to disk, the `identifier` allows for reloading specific pre-trained models/data.
**Datatype:** String. | `live_retrain_hours` | Frequency of retraining during dry/live runs.
**Datatype:** Float > 0.
Default: `0` (models retrain as often as possible). | `expiration_hours` | Avoid making predictions if a model is more than `expiration_hours` old.
**Datatype:** Positive integer.
Default: `0` (models never expire). -| `purge_old_models` | Delete all unused models during live runs (not relevant to backtesting). If set to false (not default), dry/live runs will accumulate all unused models to disk. If
**Datatype:** Boolean.
Default: `True`. +| `purge_old_models` | Number of models to keep on disk (not relevant to backtesting). Default is 2, dry/live runs will keep 2 models on disk. Setting to 0 keeps all models. If
**Datatype:** Boolean.
Default: `2`. | `save_backtest_models` | Save models to disk when running backtesting. Backtesting operates most efficiently by saving the prediction data and reusing them directly for subsequent runs (when you wish to tune entry/exit parameters). Saving backtesting models to disk also allows to use the same model files for starting a dry/live instance with the same model `identifier`.
**Datatype:** Boolean.
Default: `False` (no models are saved). | `fit_live_predictions_candles` | Number of historical candles to use for computing target (label) statistics from prediction data, instead of from the training dataset (more information can be found [here](freqai-configuration.md#creating-a-dynamic-target-threshold)).
**Datatype:** Positive integer. | `continual_learning` | Use the final state of the most recently trained model as starting point for the new model, allowing for incremental learning (more information can be found [here](freqai-running.md#continual-learning)).
**Datatype:** Boolean.
Default: `False`. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index a724664a4..84c9b5cc9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -546,7 +546,7 @@ CONF_SCHEMA = { "enabled": {"type": "boolean", "default": False}, "keras": {"type": "boolean", "default": False}, "write_metrics_to_disk": {"type": "boolean", "default": False}, - "purge_old_models": {"type": "boolean", "default": True}, + "purge_old_models": {"type": ["boolean", "number"], "default": 2}, "conv_width": {"type": "integer", "default": 1}, "train_period_days": {"type": "integer", "default": 0}, "backtest_period_days": {"type": "number", "default": 7}, diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index fc4c9f7b6..c90bb23fc 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -366,6 +366,12 @@ class FreqaiDataDrawer: def purge_old_models(self) -> None: + num_keep = self.freqai_info["purge_old_models"] + if not num_keep: + return + elif type(num_keep) == bool: + num_keep = 2 + model_folders = [x for x in self.full_path.iterdir() if x.is_dir()] pattern = re.compile(r"sub-train-(\w+)_(\d{10})") @@ -388,11 +394,11 @@ class FreqaiDataDrawer: delete_dict[coin]["timestamps"][int(timestamp)] = dir for coin in delete_dict: - if delete_dict[coin]["num_folders"] > 2: + if delete_dict[coin]["num_folders"] > num_keep: sorted_dict = collections.OrderedDict( sorted(delete_dict[coin]["timestamps"].items()) ) - num_delete = len(sorted_dict) - 2 + num_delete = len(sorted_dict) - num_keep deleted = 0 for k, v in sorted_dict.items(): if deleted >= num_delete: diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index fab5cbff8..0a4648c8a 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -629,8 +629,7 @@ class IFreqaiModel(ABC): if self.plot_features: plot_feature_importance(model, pair, dk, self.plot_features) - if self.freqai_info.get("purge_old_models", False): - self.dd.purge_old_models() + self.dd.purge_old_models() def set_initial_historic_predictions( self, pred_df: DataFrame, dk: FreqaiDataKitchen, pair: str, strat_df: DataFrame diff --git a/freqtrade/templates/FreqaiExampleHybridStrategy.py b/freqtrade/templates/FreqaiExampleHybridStrategy.py index 8a99dabc1..0e7113f8c 100644 --- a/freqtrade/templates/FreqaiExampleHybridStrategy.py +++ b/freqtrade/templates/FreqaiExampleHybridStrategy.py @@ -27,7 +27,7 @@ class FreqaiExampleHybridStrategy(IStrategy): "freqai": { "enabled": true, - "purge_old_models": true, + "purge_old_models": 2, "train_period_days": 15, "identifier": "uniqe-id", "feature_parameters": { diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index 5e8945239..97f2c2246 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -27,7 +27,7 @@ def freqai_conf(default_conf, tmpdir): "timerange": "20180110-20180115", "freqai": { "enabled": True, - "purge_old_models": True, + "purge_old_models": 2, "train_period_days": 2, "backtest_period_days": 10, "live_retrain_hours": 0, From c283e223258d5be1e61627e0276445e1cfc9ff31 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 24 Feb 2023 10:54:43 +0100 Subject: [PATCH 3/3] fix purge_old_models description in parameter table --- docs/freqai-parameter-table.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md index 7f0b0c213..adcafbc0b 100644 --- a/docs/freqai-parameter-table.md +++ b/docs/freqai-parameter-table.md @@ -15,7 +15,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `identifier` | **Required.**
A unique ID for the current model. If models are saved to disk, the `identifier` allows for reloading specific pre-trained models/data.
**Datatype:** String. | `live_retrain_hours` | Frequency of retraining during dry/live runs.
**Datatype:** Float > 0.
Default: `0` (models retrain as often as possible). | `expiration_hours` | Avoid making predictions if a model is more than `expiration_hours` old.
**Datatype:** Positive integer.
Default: `0` (models never expire). -| `purge_old_models` | Number of models to keep on disk (not relevant to backtesting). Default is 2, dry/live runs will keep 2 models on disk. Setting to 0 keeps all models. If
**Datatype:** Boolean.
Default: `2`. +| `purge_old_models` | Number of models to keep on disk (not relevant to backtesting). Default is 2, which means that dry/live runs will keep the latest 2 models on disk. Setting to 0 keeps all models. This parameter also accepts a boolean to maintain backwards compatibility.
**Datatype:** Integer.
Default: `2`. | `save_backtest_models` | Save models to disk when running backtesting. Backtesting operates most efficiently by saving the prediction data and reusing them directly for subsequent runs (when you wish to tune entry/exit parameters). Saving backtesting models to disk also allows to use the same model files for starting a dry/live instance with the same model `identifier`.
**Datatype:** Boolean.
Default: `False` (no models are saved). | `fit_live_predictions_candles` | Number of historical candles to use for computing target (label) statistics from prediction data, instead of from the training dataset (more information can be found [here](freqai-configuration.md#creating-a-dynamic-target-threshold)).
**Datatype:** Positive integer. | `continual_learning` | Use the final state of the most recently trained model as starting point for the new model, allowing for incremental learning (more information can be found [here](freqai-running.md#continual-learning)).
**Datatype:** Boolean.
Default: `False`.