From 29c2d1d1891f7e804a133908702f435ff4fd8f32 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 4 May 2022 17:53:40 +0200 Subject: [PATCH] use logger in favor of print --- freqtrade/freqai/data_handler.py | 23 +++++++++++-------- freqtrade/freqai/freqai_interface.py | 15 ++++++++---- freqtrade/templates/ExamplePredictionModel.py | 18 +++++++++------ 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/freqtrade/freqai/data_handler.py b/freqtrade/freqai/data_handler.py index 7264c6fab..9ab47d223 100644 --- a/freqtrade/freqai/data_handler.py +++ b/freqtrade/freqai/data_handler.py @@ -1,6 +1,7 @@ import copy import datetime import json +import logging import pickle as pk from pathlib import Path from typing import Any, Dict, List, Tuple @@ -17,6 +18,8 @@ from freqtrade.configuration import TimeRange SECONDS_IN_DAY = 86400 +logger = logging.getLogger(__name__) + class DataHandler: """ @@ -175,7 +178,7 @@ class DataHandler: labels = labels[ (drop_index == 0) & (drop_index_labels == 0) ] # assuming the labels depend entirely on the dataframe here. - print( + logger.info( "dropped", len(unfiltered_dataframe) - len(filtered_dataframe), "training data points due to NaNs, ensure you have downloaded", @@ -193,7 +196,7 @@ class DataHandler: # that was based on a single NaN is ultimately protected from buys with do_predict drop_index = ~drop_index self.do_predict = np.array(drop_index.replace(True, 1).replace(False, 0)) - print( + logger.info( "dropped", len(self.do_predict) - self.do_predict.sum(), "of", @@ -350,8 +353,8 @@ class DataHandler: pca2 = PCA(n_components=n_keep_components) self.data["n_kept_components"] = n_keep_components pca2 = pca2.fit(self.data_dictionary["train_features"]) - print("reduced feature dimension by", n_components - n_keep_components) - print("explained variance", np.sum(pca2.explained_variance_ratio_)) + logger.info("reduced feature dimension by", n_components - n_keep_components) + logger.info("explained variance", np.sum(pca2.explained_variance_ratio_)) train_components = pca2.transform(self.data_dictionary["train_features"]) test_components = pca2.transform(self.data_dictionary["test_features"]) @@ -377,10 +380,10 @@ class DataHandler: return None def compute_distances(self) -> float: - print("computing average mean distance for all training points") + logger.info("computing average mean distance for all training points") pairwise = pairwise_distances(self.data_dictionary["train_features"], n_jobs=-1) avg_mean_dist = pairwise.mean(axis=1).mean() - print("avg_mean_dist", avg_mean_dist) + logger.info("avg_mean_dist", avg_mean_dist) return avg_mean_dist @@ -407,7 +410,7 @@ class DataHandler: drop_index = ~drop_index do_predict = np.array(drop_index.replace(True, 1).replace(False, 0)) - print( + logger.info( "remove_outliers() tossed", len(do_predict) - do_predict.sum(), "predictions because they were beyond 3 std deviations from training data.", @@ -472,7 +475,7 @@ class DataHandler: for p in config["freqai"]["corr_pairlist"]: features.append(p.split("/")[0] + "-" + ft + shift + "_" + tf) - print("number of features", len(features)) + logger.info("number of features", len(features)) return features def check_if_pred_in_training_spaces(self) -> None: @@ -483,7 +486,7 @@ class DataHandler: from the training data set. """ - print("checking if prediction features are in AOA") + logger.info("checking if prediction features are in AOA") distance = pairwise_distances( self.data_dictionary["train_features"], self.data_dictionary["prediction_features"], @@ -497,7 +500,7 @@ class DataHandler: 0, ) - print( + logger.info( "Distance checker tossed", len(do_predict) - do_predict.sum(), "predictions for being too far from training data", diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 9f04b09cd..05a0594f3 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -1,6 +1,7 @@ import gc +import logging import shutil -from abc import ABC +from abc import ABC, abstractmethod from pathlib import Path from typing import Any, Dict, Tuple @@ -12,6 +13,7 @@ from freqtrade.freqai.data_handler import DataHandler pd.options.mode.chained_assignment = None +logger = logging.getLogger(__name__) class IFreqaiModel(ABC): @@ -67,7 +69,7 @@ class IFreqaiModel(ABC): self.pair = metadata["pair"] self.dh = DataHandler(self.config, dataframe) - print( + logger.info( "going to train", len(self.dh.training_timeranges), "timeranges:", @@ -88,7 +90,7 @@ class IFreqaiModel(ABC): self.freqai_info["training_timerange"] = tr_train dataframe_train = self.dh.slice_dataframe(tr_train, dataframe) dataframe_backtest = self.dh.slice_dataframe(tr_backtest, dataframe) - print("training", self.pair, "for", tr_train) + logger.info("training", self.pair, "for", tr_train) # self.dh.model_path = self.full_path + "/" + "sub-train" + "-" + str(tr_train) + "/" self.dh.model_path = Path(self.full_path / str("sub-train" + "-" + str(tr_train))) if not self.model_exists(self.pair, training_timerange=tr_train): @@ -114,6 +116,7 @@ class IFreqaiModel(ABC): return dataframe + @abstractmethod def train(self, unfiltered_dataframe: DataFrame, metadata: dict) -> Any: """ Filter the training data and train a model to it. Train makes heavy use of the datahandler @@ -127,6 +130,7 @@ class IFreqaiModel(ABC): return Any + @abstractmethod def fit(self) -> Any: """ Most regressors use the same function names and arguments e.g. user @@ -139,6 +143,7 @@ class IFreqaiModel(ABC): return Any + @abstractmethod def predict(self, dataframe: DataFrame) -> Tuple[np.array, np.array]: """ Filter the prediction features data and predict with it. @@ -162,7 +167,7 @@ class IFreqaiModel(ABC): path_to_modelfile = Path(self.dh.model_path / str(self.dh.model_filename + "_model.joblib")) file_exists = path_to_modelfile.is_file() if file_exists: - print("Found model at", self.dh.model_path / self.dh.model_filename) + logger.info("Found model at", self.dh.model_path / self.dh.model_filename) else: - print("Could not find model at", self.dh.model_path / self.dh.model_filename) + logger.info("Could not find model at", self.dh.model_path / self.dh.model_filename) return file_exists diff --git a/freqtrade/templates/ExamplePredictionModel.py b/freqtrade/templates/ExamplePredictionModel.py index feeed11a9..4906b8c04 100644 --- a/freqtrade/templates/ExamplePredictionModel.py +++ b/freqtrade/templates/ExamplePredictionModel.py @@ -1,3 +1,4 @@ +import logging from typing import Any, Dict, Tuple import pandas as pd @@ -7,6 +8,9 @@ from pandas import DataFrame from freqtrade.freqai.freqai_interface import IFreqaiModel +logger = logging.getLogger(__name__) + + class ExamplePredictionModel(IFreqaiModel): """ User created prediction model. The class needs to override three necessary @@ -32,7 +36,7 @@ class ExamplePredictionModel(IFreqaiModel): self.dh.data["s_mean"] = dataframe["s"].mean() self.dh.data["s_std"] = dataframe["s"].std() - print("label mean", self.dh.data["s_mean"], "label std", self.dh.data["s_std"]) + logger.info("label mean", self.dh.data["s_mean"], "label std", self.dh.data["s_std"]) return dataframe["s"] @@ -46,7 +50,7 @@ class ExamplePredictionModel(IFreqaiModel): :returns: :model: Trained model which can be used to inference (self.predict) """ - print("--------------------Starting training--------------------") + logger.info("--------------------Starting training--------------------") # create the full feature list based on user config info self.dh.training_features_list = self.dh.build_feature_list(self.config) @@ -73,12 +77,12 @@ class ExamplePredictionModel(IFreqaiModel): if self.feature_parameters["DI_threshold"]: self.dh.data["avg_mean_dist"] = self.dh.compute_distances() - print("length of train data", len(data_dictionary["train_features"])) + logger.info("length of train data", len(data_dictionary["train_features"])) model = self.fit(data_dictionary) - print("Finished training") - print(f'--------------------done training {metadata["pair"]}--------------------') + logger.info("Finished training") + logger.info(f'--------------------done training {metadata["pair"]}--------------------') return model @@ -121,7 +125,7 @@ class ExamplePredictionModel(IFreqaiModel): data (NaNs) or felt uncertain about data (PCA and DI index) """ - print("--------------------Starting prediction--------------------") + logger.info("--------------------Starting prediction--------------------") original_feature_list = self.dh.build_feature_list(self.config) filtered_dataframe, _ = self.dh.filter_features( @@ -150,6 +154,6 @@ class ExamplePredictionModel(IFreqaiModel): # compute the non-standardized predictions predictions = predictions * self.dh.data["labels_std"] + self.dh.data["labels_mean"] - print("--------------------Finished prediction--------------------") + logger.info("--------------------Finished prediction--------------------") return (predictions, self.dh.do_predict)