From 3fc92b1b213533e11e9844ab32bb12b3d2c62672 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 11 Jul 2022 11:33:59 +0200 Subject: [PATCH] Create BaseRegression model - designed to reduce code duplication across currently available models. --- .../prediction_models/BaseRegressionModel.py | 104 ++++++++++++++++++ .../CatboostPredictionModel.py | 94 +--------------- .../CatboostPredictionMultiModel.py | 95 +--------------- .../LightGBMPredictionModel.py | 95 +--------------- 4 files changed, 113 insertions(+), 275 deletions(-) create mode 100644 freqtrade/freqai/prediction_models/BaseRegressionModel.py diff --git a/freqtrade/freqai/prediction_models/BaseRegressionModel.py b/freqtrade/freqai/prediction_models/BaseRegressionModel.py new file mode 100644 index 000000000..7b8d606df --- /dev/null +++ b/freqtrade/freqai/prediction_models/BaseRegressionModel.py @@ -0,0 +1,104 @@ +import logging +from typing import Tuple + +from pandas import DataFrame + +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.freqai.freqai_interface import IFreqaiModel + + +logger = logging.getLogger(__name__) + + +class BaseRegressionModel(IFreqaiModel): + """ + User created prediction model. The class needs to override three necessary + functions, predict(), train(), fit(). The class inherits ModelHandler which + has its own DataHandler where data is held, saved, loaded, and managed. + """ + + def return_values(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> DataFrame: + """ + User uses this function to add any additional return values to the dataframe. + e.g. + dataframe['volatility'] = dk.volatility_values + """ + + return dataframe + + def train( + self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen + ) -> Tuple[DataFrame, DataFrame]: + """ + Filter the training data and train a model to it. Train makes heavy use of the datakitchen + for storing, saving, loading, and analyzing the data. + :params: + :unfiltered_dataframe: Full dataframe for the current training period + :metadata: pair metadata from strategy. + :returns: + :model: Trained model which can be used to inference (self.predict) + """ + + logger.info("--------------------Starting training " f"{pair} --------------------") + + # filter the features requested by user in the configuration file and elegantly handle NaNs + features_filtered, labels_filtered = dk.filter_features( + unfiltered_dataframe, + dk.training_features_list, + dk.label_list, + training_filter=True, + ) + + # split data into train/test data. + data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered) + dk.fit_labels() # fit labels to a cauchy distribution so we know what to expect in strategy + # normalize all data based on train_dataset only + data_dictionary = dk.normalize_data(data_dictionary) + + # optional additional data cleaning/analysis + self.data_cleaning_train(dk) + + logger.info( + f'Training model on {len(dk.data_dictionary["train_features"].columns)}' " features" + ) + logger.info(f'Training model on {len(data_dictionary["train_features"])} data points') + + model = self.fit(data_dictionary) + + logger.info(f"--------------------done training {pair}--------------------") + + return model + + def predict( + self, unfiltered_dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = False + ) -> Tuple[DataFrame, DataFrame]: + """ + Filter the prediction features data and predict with it. + :param: unfiltered_dataframe: Full dataframe for the current backtest period. + :return: + :pred_df: dataframe containing the predictions + :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove + data (NaNs) or felt uncertain about data (PCA and DI index) + """ + + dk.find_features(unfiltered_dataframe) + filtered_dataframe, _ = dk.filter_features( + unfiltered_dataframe, dk.training_features_list, training_filter=False + ) + filtered_dataframe = dk.normalize_data_from_metadata(filtered_dataframe) + dk.data_dictionary["prediction_features"] = filtered_dataframe + + # optional additional data cleaning/analysis + self.data_cleaning_predict(dk, filtered_dataframe) + + predictions = self.model.predict(dk.data_dictionary["prediction_features"]) + pred_df = DataFrame(predictions, columns=dk.label_list) + + for label in dk.label_list: + pred_df[label] = ( + (pred_df[label] + 1) + * (dk.data["labels_max"][label] - dk.data["labels_min"][label]) + / 2 + ) + dk.data["labels_min"][label] + + return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/prediction_models/CatboostPredictionModel.py b/freqtrade/freqai/prediction_models/CatboostPredictionModel.py index e6153cc27..fafb12abe 100644 --- a/freqtrade/freqai/prediction_models/CatboostPredictionModel.py +++ b/freqtrade/freqai/prediction_models/CatboostPredictionModel.py @@ -1,75 +1,21 @@ import logging -from typing import Any, Dict, Tuple +from typing import Any, Dict from catboost import CatBoostRegressor, Pool -from pandas import DataFrame -from freqtrade.freqai.data_kitchen import FreqaiDataKitchen -from freqtrade.freqai.freqai_interface import IFreqaiModel +from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel logger = logging.getLogger(__name__) -class CatboostPredictionModel(IFreqaiModel): +class CatboostPredictionModel(BaseRegressionModel): """ User created prediction model. The class needs to override three necessary functions, predict(), train(), fit(). The class inherits ModelHandler which has its own DataHandler where data is held, saved, loaded, and managed. """ - def return_values(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> DataFrame: - """ - User uses this function to add any additional return values to the dataframe. - e.g. - dataframe['volatility'] = dk.volatility_values - """ - - return dataframe - - def train( - self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen - ) -> Tuple[DataFrame, DataFrame]: - """ - Filter the training data and train a model to it. Train makes heavy use of the datakitchen - for storing, saving, loading, and analyzing the data. - :params: - :unfiltered_dataframe: Full dataframe for the current training period - :metadata: pair metadata from strategy. - :returns: - :model: Trained model which can be used to inference (self.predict) - """ - - logger.info("--------------------Starting training " f"{pair} --------------------") - - # filter the features requested by user in the configuration file and elegantly handle NaNs - features_filtered, labels_filtered = dk.filter_features( - unfiltered_dataframe, - dk.training_features_list, - dk.label_list, - training_filter=True, - ) - - # split data into train/test data. - data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered) - dk.fit_labels() # fit labels to a cauchy distribution so we know what to expect in strategy - # normalize all data based on train_dataset only - data_dictionary = dk.normalize_data(data_dictionary) - - # optional additional data cleaning/analysis - self.data_cleaning_train(dk) - - logger.info( - f'Training model on {len(dk.data_dictionary["train_features"].columns)}' " features" - ) - logger.info(f'Training model on {len(data_dictionary["train_features"])} data points') - - model = self.fit(data_dictionary) - - logger.info(f"--------------------done training {pair}--------------------") - - return model - def fit(self, data_dictionary: Dict) -> Any: """ User sets up the training and test data to fit their desired model here @@ -99,37 +45,3 @@ class CatboostPredictionModel(IFreqaiModel): model.fit(X=train_data, eval_set=test_data) return model - - def predict( - self, unfiltered_dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = False - ) -> Tuple[DataFrame, DataFrame]: - """ - Filter the prediction features data and predict with it. - :param: unfiltered_dataframe: Full dataframe for the current backtest period. - :return: - :pred_df: dataframe containing the predictions - :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove - data (NaNs) or felt uncertain about data (PCA and DI index) - """ - - dk.find_features(unfiltered_dataframe) - filtered_dataframe, _ = dk.filter_features( - unfiltered_dataframe, dk.training_features_list, training_filter=False - ) - filtered_dataframe = dk.normalize_data_from_metadata(filtered_dataframe) - dk.data_dictionary["prediction_features"] = filtered_dataframe - - # optional additional data cleaning/analysis - self.data_cleaning_predict(dk, filtered_dataframe) - - predictions = self.model.predict(dk.data_dictionary["prediction_features"]) - pred_df = DataFrame(predictions, columns=dk.label_list) - - for label in dk.label_list: - pred_df[label] = ( - (pred_df[label] + 1) - * (dk.data["labels_max"][label] - dk.data["labels_min"][label]) - / 2 - ) + dk.data["labels_min"][label] - - return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/prediction_models/CatboostPredictionMultiModel.py b/freqtrade/freqai/prediction_models/CatboostPredictionMultiModel.py index c4d92d7bb..f55563c83 100644 --- a/freqtrade/freqai/prediction_models/CatboostPredictionMultiModel.py +++ b/freqtrade/freqai/prediction_models/CatboostPredictionMultiModel.py @@ -1,77 +1,22 @@ import logging -from typing import Any, Dict, Tuple +from typing import Any, Dict from catboost import CatBoostRegressor # , Pool -from pandas import DataFrame from sklearn.multioutput import MultiOutputRegressor -from freqtrade.freqai.data_kitchen import FreqaiDataKitchen -from freqtrade.freqai.freqai_interface import IFreqaiModel +from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel logger = logging.getLogger(__name__) -class CatboostPredictionMultiModel(IFreqaiModel): +class CatboostPredictionMultiModel(BaseRegressionModel): """ User created prediction model. The class needs to override three necessary functions, predict(), train(), fit(). The class inherits ModelHandler which has its own DataHandler where data is held, saved, loaded, and managed. """ - def return_values(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> DataFrame: - """ - User uses this function to add any additional return values to the dataframe. - e.g. - dataframe['volatility'] = dk.volatility_values - """ - - return dataframe - - def train( - self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen - ) -> Tuple[DataFrame, DataFrame]: - """ - Filter the training data and train a model to it. Train makes heavy use of the datahkitchen - for storing, saving, loading, and analyzing the data. - :params: - :unfiltered_dataframe: Full dataframe for the current training period - :metadata: pair metadata from strategy. - :returns: - :model: Trained model which can be used to inference (self.predict) - """ - - logger.info("--------------------Starting training " f"{pair} --------------------") - - # unfiltered_labels = self.make_labels(unfiltered_dataframe, dk) - # filter the features requested by user in the configuration file and elegantly handle NaNs - features_filtered, labels_filtered = dk.filter_features( - unfiltered_dataframe, - dk.training_features_list, - dk.label_list, - training_filter=True, - ) - - # split data into train/test data. - data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered) - dk.fit_labels() # fit labels to a cauchy distribution so we know what to expect in strategy - # normalize all data based on train_dataset only - data_dictionary = dk.normalize_data(data_dictionary) - - # optional additional data cleaning/analysis - self.data_cleaning_train(dk) - - logger.info( - f'Training model on {len(dk.data_dictionary["train_features"].columns)}' " features" - ) - logger.info(f'Training model on {len(data_dictionary["train_features"])} data points') - - model = self.fit(data_dictionary) - - logger.info(f"--------------------done training {pair}--------------------") - - return model - def fit(self, data_dictionary: Dict) -> Any: """ User sets up the training and test data to fit their desired model here @@ -97,37 +42,3 @@ class CatboostPredictionMultiModel(IFreqaiModel): model.fit(X=X, y=y, sample_weight=sample_weight) # , eval_set=eval_set) return model - - def predict( - self, unfiltered_dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = False - ) -> Tuple[DataFrame, DataFrame]: - """ - Filter the prediction features data and predict with it. - :param: unfiltered_dataframe: Full dataframe for the current backtest period. - :return: - :pred_df: dataframe containing the predictions - :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove - data (NaNs) or felt uncertain about data (PCA and DI index) - """ - - dk.find_features(unfiltered_dataframe) - filtered_dataframe, _ = dk.filter_features( - unfiltered_dataframe, dk.training_features_list, training_filter=False - ) - filtered_dataframe = dk.normalize_data_from_metadata(filtered_dataframe) - dk.data_dictionary["prediction_features"] = filtered_dataframe - - # optional additional data cleaning/analysis - self.data_cleaning_predict(dk, filtered_dataframe) - - predictions = self.model.predict(dk.data_dictionary["prediction_features"]) - pred_df = DataFrame(predictions, columns=dk.label_list) - - for label in dk.label_list: - pred_df[label] = ( - (pred_df[label] + 1) - * (dk.data["labels_max"][label] - dk.data["labels_min"][label]) - / 2 - ) + dk.data["labels_min"][label] - - return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/prediction_models/LightGBMPredictionModel.py b/freqtrade/freqai/prediction_models/LightGBMPredictionModel.py index d59a0cc33..6a91837da 100644 --- a/freqtrade/freqai/prediction_models/LightGBMPredictionModel.py +++ b/freqtrade/freqai/prediction_models/LightGBMPredictionModel.py @@ -1,76 +1,21 @@ import logging -from typing import Any, Dict, Tuple +from typing import Any, Dict from lightgbm import LGBMRegressor -from pandas import DataFrame -from freqtrade.freqai.data_kitchen import FreqaiDataKitchen -from freqtrade.freqai.freqai_interface import IFreqaiModel +from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel logger = logging.getLogger(__name__) -class LightGBMPredictionModel(IFreqaiModel): +class LightGBMPredictionModel(BaseRegressionModel): """ User created prediction model. The class needs to override three necessary functions, predict(), train(), fit(). The class inherits ModelHandler which has its own DataHandler where data is held, saved, loaded, and managed. """ - def return_values(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> DataFrame: - """ - User uses this function to add any additional return values to the dataframe. - e.g. - dataframe['volatility'] = dk.volatility_values - """ - - return dataframe - - def train( - self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen - ) -> Tuple[DataFrame, DataFrame]: - """ - Filter the training data and train a model to it. Train makes heavy use of the datahkitchen - for storing, saving, loading, and analyzing the data. - :params: - :unfiltered_dataframe: Full dataframe for the current training period - :metadata: pair metadata from strategy. - :returns: - :model: Trained model which can be used to inference (self.predict) - """ - - logger.info("--------------------Starting training " f"{pair} --------------------") - - # unfiltered_labels = self.make_labels(unfiltered_dataframe, dk) - # filter the features requested by user in the configuration file and elegantly handle NaNs - features_filtered, labels_filtered = dk.filter_features( - unfiltered_dataframe, - dk.training_features_list, - dk.label_list, - training_filter=True, - ) - - # split data into train/test data. - data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered) - dk.fit_labels() # fit labels to a cauchy distribution so we know what to expect in strategy - # normalize all data based on train_dataset only - data_dictionary = dk.normalize_data(data_dictionary) - - # optional additional data cleaning/analysis - self.data_cleaning_train(dk) - - logger.info( - f'Training model on {len(dk.data_dictionary["train_features"].columns)}' " features" - ) - logger.info(f'Training model on {len(data_dictionary["train_features"])} data points') - - model = self.fit(data_dictionary) - - logger.info(f"--------------------done training {pair}--------------------") - - return model - def fit(self, data_dictionary: Dict) -> Any: """ Most regressors use the same function names and arguments e.g. user @@ -89,37 +34,3 @@ class LightGBMPredictionModel(IFreqaiModel): model.fit(X=X, y=y, eval_set=eval_set) return model - - def predict( - self, unfiltered_dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = False - ) -> Tuple[DataFrame, DataFrame]: - """ - Filter the prediction features data and predict with it. - :param: unfiltered_dataframe: Full dataframe for the current backtest period. - :return: - :pred_df: dataframe containing the predictions - :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove - data (NaNs) or felt uncertain about data (PCA and DI index) - """ - - dk.find_features(unfiltered_dataframe) - filtered_dataframe, _ = dk.filter_features( - unfiltered_dataframe, dk.training_features_list, training_filter=False - ) - filtered_dataframe = dk.normalize_data_from_metadata(filtered_dataframe) - dk.data_dictionary["prediction_features"] = filtered_dataframe - - # optional additional data cleaning/analysis - self.data_cleaning_predict(dk, filtered_dataframe) - - predictions = self.model.predict(dk.data_dictionary["prediction_features"]) - pred_df = DataFrame(predictions, columns=dk.label_list) - - for label in dk.label_list: - pred_df[label] = ( - (pred_df[label] + 1) - * (dk.data["labels_max"][label] - dk.data["labels_min"][label]) - / 2 - ) + dk.data["labels_min"][label] - - return (pred_df, dk.do_predict)