From 9d184586f1173bb06ba78d4e4230d41271050cef Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Sat, 16 Jul 2022 21:16:59 +0200 Subject: [PATCH 1/4] fix bug in historic prediction saving --- freqtrade/freqai/data_drawer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 321295c53..4e8858349 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -93,14 +93,14 @@ class FreqaiDataDrawer: return exists - def save_drawer_to_disk(self): + def save_historic_predictions_to_disk(self): """ Save data drawer full of all pair model metadata in present model folder. """ with open(self.full_path / str("historic_predictions.pkl"), "wb") as fp: pickle.dump(self.historic_predictions, fp, protocol=pickle.HIGHEST_PROTOCOL) - def save_historic_predictions_to_disk(self): + def save_drawer_to_disk(self): """ Save data drawer full of all pair model metadata in present model folder. """ From 41eeb9917788abbbf3e88f96bebc609c3906aa3a Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Sun, 17 Jul 2022 10:05:21 +0200 Subject: [PATCH 2/4] load pickle file for writing --- freqtrade/freqai/data_drawer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 4e8858349..41736c154 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -78,9 +78,9 @@ class FreqaiDataDrawer: """ exists = Path(self.full_path / str("historic_predictions.pkl")).resolve().exists() if exists: - with open(self.full_path / str("historic_predictions.pkl"), "rb") as fp: + with open(self.full_path / str("historic_predictions.pkl"), "wb") as fp: self.historic_predictions = pickle.load(fp) - logger.info(f"Found existing historic predictions at {self.full_path}, but beware of " + logger.info(f"Found existing historic predictions at {self.full_path}, but beware " "that statistics may be inaccurate if the bot has been offline for " "an extended period of time.") elif not self.follow_mode: From 921f3899f062e042cd9fd26c48263405fb63871c Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Sun, 17 Jul 2022 16:06:36 +0200 Subject: [PATCH 3/4] revert pickle reading for historic predictions --- freqtrade/freqai/data_drawer.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 41736c154..5488a7e6b 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -78,7 +78,7 @@ class FreqaiDataDrawer: """ exists = Path(self.full_path / str("historic_predictions.pkl")).resolve().exists() if exists: - with open(self.full_path / str("historic_predictions.pkl"), "wb") as fp: + with open(self.full_path / str("historic_predictions.pkl"), "rb") as fp: self.historic_predictions = pickle.load(fp) logger.info(f"Found existing historic predictions at {self.full_path}, but beware " "that statistics may be inaccurate if the bot has been offline for " @@ -235,13 +235,14 @@ class FreqaiDataDrawer: i = length_difference + 1 df = self.model_return_values[pair] = self.model_return_values[pair].shift(-i) - hp_df = self.historic_predictions[pair] - # here are some pandas hula hoops to accommodate the possibility of a series - # or dataframe depending number of labels requested by user - nan_df = pd.DataFrame(np.nan, index=hp_df.index[-2:] + 2, columns=hp_df.columns) - hp_df = pd.concat([hp_df, nan_df], ignore_index=True, axis=0) - hp_df = pd.concat([hp_df, nan_df[-2:-1]], axis=0) + if pair in self.historic_predictions: + hp_df = self.historic_predictions[pair] + # here are some pandas hula hoops to accommodate the possibility of a series + # or dataframe depending number of labels requested by user + nan_df = pd.DataFrame(np.nan, index=hp_df.index[-2:] + 2, columns=hp_df.columns) + hp_df = pd.concat([hp_df, nan_df], ignore_index=True, axis=0) + self.historic_predictions[pair] = hp_df[:-1] for label in dk.label_list: df[label].iloc[-1] = predictions[label].iloc[-1] @@ -254,7 +255,8 @@ class FreqaiDataDrawer: df["DI_values"].iloc[-1] = dk.DI_values[-1] # append the new predictions to persistent storage - hp_df.iloc[-1] = df[label].iloc[-1] + if pair in self.historic_predictions: + self.historic_predictions[pair].iloc[-1] = df[label].iloc[-1] if length_difference < 0: prepend_df = pd.DataFrame( From ed0f8b118952f43e3d45a7e70e302e14fd220760 Mon Sep 17 00:00:00 2001 From: lolong Date: Mon, 18 Jul 2022 11:57:52 +0200 Subject: [PATCH 4/4] Improve FreqAI documentation (#7072) Improve doc + some other small fixes Co-authored-by: robcaulk --- docs/freqai.md | 13 ++++++------- freqtrade/freqai/data_kitchen.py | 2 +- .../CatboostPredictionMultiModel.py | 6 ++++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/freqai.md b/docs/freqai.md index a0a11ac35..a5d7458f5 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -408,12 +408,9 @@ It is common to want constant retraining, in whichcase, user should set `live_re ### Controlling the model learning process -The user can define model settings for the data split `data_split_parameters` and learning parameters -`model_training_parameters`. Users are encouraged to visit the Catboost documentation -for more information on how to select these values. `n_estimators` increases the -computational effort and the fit to the training data. If a user has a GPU -installed in their system, they may benefit from changing `task_type` to `GPU`. -The `weight_factor` allows the user to weight more recent data more strongly +Model training parameters are unqiue to the library employed by the user. FreqAI allows users to set any parameter for any library using the `model_training_parameters` dictionary in the user configuration file. The example configuration files show some of the example parameters associated with `Catboost` and `LightGBM`, but users can add any parameters available in those libraries. + +Data split parameters are defined in `data_split_parameters` which can be any parameters associated with `Sklearn`'s `train_test_split()` function. Meanwhile, FreqAI includes some additional parameters such `weight_factor` which allows the user to weight more recent data more strongly than past data via an exponential function: $$ W_i = \exp(\frac{-i}{\alpha*n}) $$ @@ -422,7 +419,9 @@ where $W_i$ is the weight of data point $i$ in a total set of $n$ data points._ ![weight-factor](assets/weights_factor.png) -Finally, `period` defines the offset used for the `labels`. In the present example, +`train_test_split()` has a parameters called `shuffle`, which users also have access to in FreqAI, that allows them to keep the data unshuffled. This is particularly useful to avoid biasing training with temporally autocorrelated data. + +Finally, `label_period_candles` defines the offset used for the `labels`. In the present example, the user is asking for `labels` that are 24 candles in the future. ### Removing outliers with the Dissimilarity Index diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 56c1a67ed..f320bdc2f 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -799,7 +799,7 @@ class FreqaiDataKitchen: max_timeframe_chars = self.freqai_config.get( "feature_parameters", {}).get("include_timeframes")[-1] max_period = self.freqai_config.get("feature_parameters", {}).get( - "indicator_max_period", 50 + "indicator_max_period_candles", 50 ) additional_seconds = 0 if max_timeframe_chars[-1] == "d": diff --git a/freqtrade/freqai/prediction_models/CatboostPredictionMultiModel.py b/freqtrade/freqai/prediction_models/CatboostPredictionMultiModel.py index f55563c83..becfb43eb 100644 --- a/freqtrade/freqai/prediction_models/CatboostPredictionMultiModel.py +++ b/freqtrade/freqai/prediction_models/CatboostPredictionMultiModel.py @@ -35,10 +35,12 @@ class CatboostPredictionMultiModel(BaseRegressionModel): X = data_dictionary["train_features"] y = data_dictionary["train_labels"] - # eval_set = (data_dictionary["test_features"], data_dictionary["test_labels"]) + eval_set = (data_dictionary["test_features"], data_dictionary["test_labels"]) sample_weight = data_dictionary["train_weights"] model = MultiOutputRegressor(estimator=cbr) model.fit(X=X, y=y, sample_weight=sample_weight) # , eval_set=eval_set) - + train_score = model.score(X, y) + test_score = model.score(*eval_set) + logger.info(f"Train score {train_score}, Test score {test_score}") return model