From 0306f5ca138449617438f8d8db17c34cbdaa688c Mon Sep 17 00:00:00 2001
From: robcaulk <rob.caulk@gmail.com>
Date: Tue, 31 May 2022 11:58:21 +0200
Subject: [PATCH] Add autopurge feature so that FreqAI cleans up after itself
 when it no longer needs old models on disk

---
 docs/freqai.md                       | 15 +++++++++++
 freqtrade/freqai/data_drawer.py      | 39 ++++++++++++++++++++++++++++
 freqtrade/freqai/data_kitchen.py     |  2 +-
 freqtrade/freqai/freqai_interface.py |  9 +++++++
 4 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/docs/freqai.md b/docs/freqai.md
index c4ac30415..fa2ca3724 100644
--- a/docs/freqai.md
+++ b/docs/freqai.md
@@ -423,6 +423,21 @@ by a leader with the same `identifier`. In this example, the user has a leader b
 `identifier: "example"` already running or launching simultaneously as the present follower. 
 The follower will load models created by the leader and inference them to obtain predictions.
 
+### Purging old model data
+
+FreqAI stores new model files each time it retrains. These files become obsolete as new models 
+are trained and FreqAI adapts to the new market conditions. Users planning to leave FreqAI running 
+for extended periods of time with high frequency retraining should set `purge_old_models` in their 
+config:
+
+```json
+    "freqai": {
+        "purge_old_models": true,
+    }
+```
+
+which will automatically purge all models older than the two most recently trained ones.
+
 <!-- ## Dynamic target expectation
 
 The labels used for model training have a unique statistical distribution for each separate model training. 
diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py
index e5ed529aa..346dbaaf1 100644
--- a/freqtrade/freqai/data_drawer.py
+++ b/freqtrade/freqai/data_drawer.py
@@ -1,7 +1,10 @@
 
+import collections
 import copy
 import json
 import logging
+import re
+import shutil
 from pathlib import Path
 from typing import Any, Dict, Tuple
 
@@ -165,6 +168,42 @@ class FreqaiDataDrawer:
         dh.full_target_mean = np.zeros(len_df)
         dh.full_target_std = np.zeros(len_df)
 
+    def purge_old_models(self) -> None:
+
+        model_folders = [x for x in self.full_path.iterdir() if x.is_dir()]
+
+        pattern = re.compile(r"sub-train-(\w+)(\d{10})")
+
+        delete_dict: Dict[str, Any] = {}
+
+        for dir in model_folders:
+            result = pattern.match(str(dir.name))
+            if result is None:
+                break
+            coin = result.group(1)
+            timestamp = result.group(2)
+
+            if coin not in delete_dict:
+                delete_dict[coin] = {}
+                delete_dict[coin]['num_folders'] = 1
+                delete_dict[coin]['timestamps'] = {int(timestamp): dir}
+            else:
+                delete_dict[coin]['num_folders'] += 1
+                delete_dict[coin]['timestamps'][int(timestamp)] = dir
+
+        for coin in delete_dict:
+            if delete_dict[coin]['num_folders'] > 2:
+                sorted_dict = collections.OrderedDict(
+                    sorted(delete_dict[coin]['timestamps'].items()))
+                num_delete = len(sorted_dict) - 2
+                deleted = 0
+                for k, v in sorted_dict.items():
+                    if deleted >= num_delete:
+                        break
+                    logger.info(f'Freqai purging old model file {v}')
+                    shutil.rmtree(v)
+                    deleted += 1
+
     # to be used if we want to send predictions directly to the follower instead of forcing
     # follower to load models and inference
     # def save_model_return_values_to_disk(self) -> None:
diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py
index bab8fe12c..37df54042 100644
--- a/freqtrade/freqai/data_kitchen.py
+++ b/freqtrade/freqai/data_kitchen.py
@@ -596,7 +596,7 @@ class FreqaiDataKitchen:
 
         do_predict = np.where(
             distance.min(axis=0) / self.data["avg_mean_dist"]
-            < self.config["freqai"]["feature_parameters"]["DI_threshold"],
+            < self.freqai_config.get("feature_parameters", {}).get("DI_threshold"),
             1,
             0,
         )
diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py
index 0e91333bd..ea48949b3 100644
--- a/freqtrade/freqai/freqai_interface.py
+++ b/freqtrade/freqai/freqai_interface.py
@@ -190,6 +190,10 @@ class IFreqaiModel(ABC):
         dh: FreqaiDataKitchen = Data management/analysis tool assoicated to present pair only
         """
 
+        if self.follow_mode:
+            # follower needs to load off disk to get any changes made by leader to pair_dict
+            self.data_drawer.load_drawer_from_disk()
+
         (model_filename,
          trained_timestamp,
          coin_first,
@@ -376,6 +380,11 @@ class IFreqaiModel(ABC):
             self.data_drawer.pair_to_end_of_training_queue(metadata['pair'])
         self.training_on_separate_thread = False
         self.retrain = False
+
+        # each time we finish a training, we check the directory to purge old models.
+        if self.freqai_info.get('purge_old_models', False):
+            self.data_drawer.purge_old_models()
+
         return
 
     def train_model_in_series(self, new_trained_timerange: TimeRange, metadata: dict,