adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #8210 from freqtrade/clean-data-drawer

Browse Source 
Allow user to control number of historical model files
This commit is contained in:
Matthias
2023-02-24 13:19:38 +01:00
committed by GitHub
parent 607d90ca5d c283e22325
commit ac2a2512ef
9 changed files with 21 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
freqtrade/constants.py
View File

@@ -546,7 +546,7 @@ CONF_SCHEMA = {
"enabled": {"type": "boolean", "default": False},
"keras": {"type": "boolean", "default": False},
"write_metrics_to_disk": {"type": "boolean", "default": False},
"purge_old_models": {"type": "boolean", "default": True},
"purge_old_models": {"type": ["boolean", "number"], "default": 2},
"conv_width": {"type": "integer", "default": 1},
"train_period_days": {"type": "integer", "default": 0},
"backtest_period_days": {"type": "number", "default": 7},

36
freqtrade/freqai/data_drawer.py
View File

@@ -72,12 +72,7 @@ class FreqaiDataDrawer:
self.model_return_values: Dict[str, DataFrame] = {}
self.historic_data: Dict[str, Dict[str, DataFrame]] = {}
self.historic_predictions: Dict[str, DataFrame] = {}
self.follower_dict: Dict[str, pair_info] = {}
self.full_path = full_path
self.follower_name: str = self.config.get("bot_name", "follower1")
self.follower_dict_path = Path(
self.full_path / f"follower_dictionary-{self.follower_name}.json"
)
self.historic_predictions_path = Path(self.full_path / "historic_predictions.pkl")
self.historic_predictions_bkp_path = Path(
self.full_path / "historic_predictions.backup.pkl")
@@ -218,14 +213,6 @@ class FreqaiDataDrawer:
rapidjson.dump(self.pair_dict, fp, default=self.np_encoder,
number_mode=rapidjson.NM_NATIVE)
def save_follower_dict_to_disk(self):
"""
Save follower dictionary to disk (used by strategy for persistent prediction targets)
"""
with open(self.follower_dict_path, "w") as fp:
rapidjson.dump(self.follower_dict, fp, default=self.np_encoder,
number_mode=rapidjson.NM_NATIVE)
def save_global_metadata_to_disk(self, metadata: Dict[str, Any]):
"""
Save global metadata json to disk
@@ -239,7 +226,7 @@ class FreqaiDataDrawer:
if isinstance(object, np.generic):
return object.item()
def get_pair_dict_info(self, pair: str) -> Tuple[str, int, bool]:
def get_pair_dict_info(self, pair: str) -> Tuple[str, int]:
"""
Locate and load existing model metadata from persistent storage. If not located,
create a new one and append the current pair to it and prepare it for its first
@@ -248,12 +235,9 @@ class FreqaiDataDrawer:
:return:
model_filename: str = unique filename used for loading persistent objects from disk
trained_timestamp: int = the last time the coin was trained
return_null_array: bool = Follower could not find pair metadata
"""
pair_dict = self.pair_dict.get(pair)
# data_path_set = self.pair_dict.get(pair, self.empty_pair_dict).get("data_path", "")
return_null_array = False
if pair_dict:
model_filename = pair_dict["model_filename"]
@@ -263,7 +247,7 @@ class FreqaiDataDrawer:
model_filename = ""
trained_timestamp = 0
return model_filename, trained_timestamp, return_null_array
return model_filename, trained_timestamp
def set_pair_dict_info(self, metadata: dict) -> None:
pair_in_dict = self.pair_dict.get(metadata["pair"])
@@ -382,6 +366,12 @@ class FreqaiDataDrawer:
def purge_old_models(self) -> None:
num_keep = self.freqai_info["purge_old_models"]
if not num_keep:
return
elif type(num_keep) == bool:
num_keep = 2
model_folders = [x for x in self.full_path.iterdir() if x.is_dir()]
pattern = re.compile(r"sub-train-(\w+)_(\d{10})")
@@ -404,11 +394,11 @@ class FreqaiDataDrawer:
delete_dict[coin]["timestamps"][int(timestamp)] = dir
for coin in delete_dict:
if delete_dict[coin]["num_folders"] > 2:
if delete_dict[coin]["num_folders"] > num_keep:
sorted_dict = collections.OrderedDict(
sorted(delete_dict[coin]["timestamps"].items())
)
num_delete = len(sorted_dict) - 2
num_delete = len(sorted_dict) - num_keep
deleted = 0
for k, v in sorted_dict.items():
if deleted >= num_delete:
@@ -417,12 +407,6 @@ class FreqaiDataDrawer:
shutil.rmtree(v)
deleted += 1
def update_follower_metadata(self):
# follower needs to load from disk to get any changes made by leader to pair_dict
self.load_drawer_from_disk()
if self.config.get("freqai", {}).get("purge_old_models", False):
self.purge_old_models()
def save_metadata(self, dk: FreqaiDataKitchen) -> None:
"""
Saves only metadata for backtesting studies if user prefers

9
freqtrade/freqai/freqai_interface.py
View File

@@ -227,7 +227,7 @@ class IFreqaiModel(ABC):
logger.warning(f'{pair} not in current whitelist, removing from train queue.')
continue
(_, trained_timestamp, _) = self.dd.get_pair_dict_info(pair)
(_, trained_timestamp) = self.dd.get_pair_dict_info(pair)
dk = FreqaiDataKitchen(self.config, self.live, pair)
(
@@ -285,7 +285,7 @@ class IFreqaiModel(ABC):
# following tr_train. Both of these windows slide through the
# entire backtest
for tr_train, tr_backtest in zip(dk.training_timeranges, dk.backtesting_timeranges):
(_, _, _) = self.dd.get_pair_dict_info(pair)
(_, _) = self.dd.get_pair_dict_info(pair)
train_it += 1
total_trains = len(dk.backtesting_timeranges)
self.training_timerange = tr_train
@@ -382,7 +382,7 @@ class IFreqaiModel(ABC):
"""
# get the model metadata associated with the current pair
(_, trained_timestamp, return_null_array) = self.dd.get_pair_dict_info(metadata["pair"])
(_, trained_timestamp) = self.dd.get_pair_dict_info(metadata["pair"])
# append the historic data once per round
if self.dd.historic_data:
@@ -629,8 +629,7 @@ class IFreqaiModel(ABC):
if self.plot_features:
plot_feature_importance(model, pair, dk, self.plot_features)
if self.freqai_info.get("purge_old_models", False):
self.dd.purge_old_models()
self.dd.purge_old_models()
def set_initial_historic_predictions(
self, pred_df: DataFrame, dk: FreqaiDataKitchen, pair: str, strat_df: DataFrame

2
freqtrade/templates/FreqaiExampleHybridStrategy.py
View File

@@ -27,7 +27,7 @@ class FreqaiExampleHybridStrategy(IStrategy):
"freqai": {
"enabled": true,
"purge_old_models": true,
"purge_old_models": 2,
"train_period_days": 15,
"identifier": "uniqe-id",
"feature_parameters": {