From 76b33359a939e8db89b55f268e9d650f78c51bf3 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 9 Oct 2022 20:22:42 +0200 Subject: [PATCH] add an optional metric tracker to collect train timings, inference timings, and cpu load data --- docs/freqai-parameter-table.md | 1 + freqtrade/constants.py | 2 + freqtrade/freqai/data_drawer.py | 55 ++++++++++++++++++++++++++-- freqtrade/freqai/freqai_interface.py | 21 ++++++++--- 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md index 38d7ece94..4ec92521d 100644 --- a/docs/freqai-parameter-table.md +++ b/docs/freqai-parameter-table.md @@ -37,6 +37,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `noise_standard_deviation` | If set, FreqAI adds noise to the training features with the aim of preventing overfitting. FreqAI generates random deviates from a gaussian distribution with a standard deviation of `noise_standard_deviation` and adds them to all data points. `noise_standard_deviation` should be kept relative to the normalized space, i.e., between -1 and 1. In other words, since data in FreqAI is always normalized to be between -1 and 1, `noise_standard_deviation: 0.05` would result in 32% of the data being randomly increased/decreased by more than 2.5% (i.e., the percent of data falling within the first standard deviation).
**Datatype:** Integer.
Default: `0`. | `outlier_protection_percentage` | Enable to prevent outlier detection methods from discarding too much data. If more than `outlier_protection_percentage` % of points are detected as outliers by the SVM or DBSCAN, FreqAI will log a warning message and ignore outlier detection, i.e., the original dataset will be kept intact. If the outlier protection is triggered, no predictions will be made based on the training dataset.
**Datatype:** Float.
Default: `30`. | `reverse_train_test_order` | Split the feature dataset (see below) and use the latest data split for training and test on historical split of the data. This allows the model to be trained up to the most recent data point, while avoiding overfitting. However, you should be careful to understand the unorthodox nature of this parameter before employing it.
**Datatype:** Boolean.
Default: `False` (no reversal). +| `write_metrics_to_disk` | Collect train timings, inference timings and cpu usage in json file.
**Datatype:** Boolean.
Default: `False` | | **Data split parameters** | `data_split_parameters` | Include any additional parameters available from Scikit-learn `test_train_split()`, which are shown [here](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html) (external website).
**Datatype:** Dictionary. | `test_size` | The fraction of data that should be used for testing instead of training.
**Datatype:** Positive float < 1. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e0eb5e288..fc218772f 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -537,6 +537,8 @@ CONF_SCHEMA = { "properties": { "enabled": {"type": "boolean", "default": False}, "keras": {"type": "boolean", "default": False}, + "write_metrics_to_disk": {"type": "boolean", "default": False}, + "purge_old_models": {"type": "boolean", "default": True}, "conv_width": {"type": "integer", "default": 2}, "train_period_days": {"type": "integer", "default": 0}, "backtest_period_days": {"type": "number", "default": 7}, diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 31c76a68e..61ad6e077 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -9,6 +9,7 @@ from typing import Any, Dict, Tuple, TypedDict import numpy as np import pandas as pd +import psutil import rapidjson from joblib import dump, load from joblib.externals import cloudpickle @@ -78,25 +79,53 @@ class FreqaiDataDrawer: self.historic_predictions_bkp_path = Path( self.full_path / "historic_predictions.backup.pkl") self.pair_dictionary_path = Path(self.full_path / "pair_dictionary.json") + self.metric_tracker_path = Path(self.full_path / "metric_tracker.json") self.follow_mode = follow_mode if follow_mode: self.create_follower_dict() self.load_drawer_from_disk() self.load_historic_predictions_from_disk() + self.load_metric_tracker_from_disk() self.training_queue: Dict[str, int] = {} self.history_lock = threading.Lock() self.save_lock = threading.Lock() self.pair_dict_lock = threading.Lock() + self.metric_tracker_lock = threading.Lock() self.old_DBSCAN_eps: Dict[str, float] = {} self.empty_pair_dict: pair_info = { "model_filename": "", "trained_timestamp": 0, "data_path": "", "extras": {}} + self.metric_tracker: Dict[str, Dict[str, list]] = {} + + def update_metric_tracker(self, metric: str, value: float, pair: str) -> None: + """ + General utility for adding and updating custom metrics. Typically used + for adding training performance, train timings, inferenc timings, cpu loads etc. + """ + with self.metric_tracker_lock: + if pair not in self.metric_tracker: + self.metric_tracker[pair] = {} + if metric not in self.metric_tracker[pair]: + self.metric_tracker[pair][metric] = [] + + self.metric_tracker[pair][metric].append(value) + + def collect_metrics(self, time_spent: float, pair: str): + """ + Add metrics to the metric tracker dictionary + """ + load1, load5, load15 = psutil.getloadavg() + cpus = psutil.cpu_count() + self.update_metric_tracker('train_time', time_spent, pair) + self.update_metric_tracker('cpu_load1min', load1 / cpus, pair) + self.update_metric_tracker('cpu_load5min', load5 / cpus, pair) + self.update_metric_tracker('cpu_load15min', load15 / cpus, pair) def load_drawer_from_disk(self): """ Locate and load a previously saved data drawer full of all pair model metadata in present model folder. - :return: bool - whether or not the drawer was located + Load any existing metric tracker that may be present. """ exists = self.pair_dictionary_path.is_file() if exists: @@ -110,7 +139,18 @@ class FreqaiDataDrawer: "sending null values back to strategy" ) - return exists + def load_metric_tracker_from_disk(self): + """ + Tries to load an existing metrics dictionary if the user + wants to collect metrics. + """ + if self.freqai_info.get('write_metrics_to_disk', False): + exists = self.metric_tracker_path.is_file() + if exists: + with open(self.metric_tracker_path, "r") as fp: + self.metric_tracker = json.load(fp) + else: + logger.info("Could not find existing metric tracker, starting from scratch") def load_historic_predictions_from_disk(self): """ @@ -146,7 +186,7 @@ class FreqaiDataDrawer: def save_historic_predictions_to_disk(self): """ - Save data drawer full of all pair model metadata in present model folder. + Save historic predictions pickle to disk """ with open(self.historic_predictions_path, "wb") as fp: cloudpickle.dump(self.historic_predictions, fp, protocol=cloudpickle.DEFAULT_PROTOCOL) @@ -154,6 +194,15 @@ class FreqaiDataDrawer: # create a backup shutil.copy(self.historic_predictions_path, self.historic_predictions_bkp_path) + def save_metric_tracker_to_disk(self): + """ + Save metric tracker of all pair metrics collected. + """ + with self.save_lock: + with open(self.metric_tracker_path, 'w') as fp: + rapidjson.dump(self.metric_tracker, fp, default=self.np_encoder, + number_mode=rapidjson.NM_NATIVE) + def save_drawer_to_disk(self): """ Save data drawer full of all pair model metadata in present model folder. diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 394b98e94..c6b79d1b5 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -144,7 +144,7 @@ class IFreqaiModel(ABC): dataframe = dk.remove_features_from_df(dk.return_dataframe) self.clean_up() if self.live: - self.inference_timer('stop') + self.inference_timer('stop', metadata["pair"]) return dataframe def clean_up(self): @@ -214,12 +214,14 @@ class IFreqaiModel(ABC): logger.warning(f"Training {pair} raised exception {msg.__class__.__name__}. " f"Message: {msg}, skipping.") - self.train_timer('stop') + self.train_timer('stop', pair) # only rotate the queue after the first has been trained. self.train_queue.rotate(-1) self.dd.save_historic_predictions_to_disk() + if self.freqai_info.get('write_metrics_to_disk', False): + self.dd.save_metric_tracker_to_disk() def start_backtesting( self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen @@ -658,7 +660,7 @@ class IFreqaiModel(ABC): return - def inference_timer(self, do='start'): + def inference_timer(self, do: str = 'start', pair: str = ''): """ Timer designed to track the cumulative time spent in FreqAI for one pass through the whitelist. This will check if the time spent is more than 1/4 the time @@ -669,7 +671,10 @@ class IFreqaiModel(ABC): self.begin_time = time.time() elif do == 'stop': end = time.time() - self.inference_time += (end - self.begin_time) + time_spent = (end - self.begin_time) + if self.freqai_info.get('write_metrics_to_disk', False): + self.dd.update_metric_tracker('inference_time', time_spent, pair) + self.inference_time += time_spent if self.pair_it == self.total_pairs: logger.info( f'Total time spent inferencing pairlist {self.inference_time:.2f} seconds') @@ -680,7 +685,7 @@ class IFreqaiModel(ABC): self.inference_time = 0 return - def train_timer(self, do='start'): + def train_timer(self, do: str = 'start', pair: str = ''): """ Timer designed to track the cumulative time spent training the full pairlist in FreqAI. @@ -690,7 +695,11 @@ class IFreqaiModel(ABC): self.begin_time_train = time.time() elif do == 'stop': end = time.time() - self.train_time += (end - self.begin_time_train) + time_spent = (end - self.begin_time_train) + if self.freqai_info.get('write_metrics_to_disk', False): + self.dd.collect_metrics(time_spent, pair) + + self.train_time += time_spent if self.pair_it_train == self.total_pairs: logger.info( f'Total time spent training pairlist {self.train_time:.2f} seconds')