adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use logger in favor of print

Browse Source
This commit is contained in:
robcaulk 2022-05-04 17:53:40 +02:00
parent 99f7e44c30
commit 29c2d1d189
3 changed files with 34 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
freqtrade/freqai/data_handler.py
View File

@ -1,6 +1,7 @@
import copy import copy
import datetime import datetime
import json import json
import logging
import pickle as pk import pickle as pk
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Tuple from typing import Any, Dict, List, Tuple
@ -17,6 +18,8 @@ from freqtrade.configuration import TimeRange
SECONDS_IN_DAY = 86400 SECONDS_IN_DAY = 86400
logger = logging.getLogger(__name__)
class DataHandler: class DataHandler:
""" """
@ -175,7 +178,7 @@ class DataHandler:
labels = labels[ labels = labels[
(drop_index == 0) & (drop_index_labels == 0) (drop_index == 0) & (drop_index_labels == 0)
] # assuming the labels depend entirely on the dataframe here. ] # assuming the labels depend entirely on the dataframe here.
print( logger.info(
"dropped", "dropped",
len(unfiltered_dataframe) - len(filtered_dataframe), len(unfiltered_dataframe) - len(filtered_dataframe),
"training data points due to NaNs, ensure you have downloaded", "training data points due to NaNs, ensure you have downloaded",
@ -193,7 +196,7 @@ class DataHandler:
# that was based on a single NaN is ultimately protected from buys with do_predict # that was based on a single NaN is ultimately protected from buys with do_predict
drop_index = ~drop_index drop_index = ~drop_index
self.do_predict = np.array(drop_index.replace(True, 1).replace(False, 0)) self.do_predict = np.array(drop_index.replace(True, 1).replace(False, 0))
print( logger.info(
"dropped", "dropped",
len(self.do_predict) - self.do_predict.sum(), len(self.do_predict) - self.do_predict.sum(),
"of", "of",
@ -350,8 +353,8 @@ class DataHandler:
pca2 = PCA(n_components=n_keep_components) pca2 = PCA(n_components=n_keep_components)
self.data["n_kept_components"] = n_keep_components self.data["n_kept_components"] = n_keep_components
pca2 = pca2.fit(self.data_dictionary["train_features"]) pca2 = pca2.fit(self.data_dictionary["train_features"])
print("reduced feature dimension by", n_components - n_keep_components) logger.info("reduced feature dimension by", n_components - n_keep_components)
print("explained variance", np.sum(pca2.explained_variance_ratio_)) logger.info("explained variance", np.sum(pca2.explained_variance_ratio_))
train_components = pca2.transform(self.data_dictionary["train_features"]) train_components = pca2.transform(self.data_dictionary["train_features"])
test_components = pca2.transform(self.data_dictionary["test_features"]) test_components = pca2.transform(self.data_dictionary["test_features"])
@ -377,10 +380,10 @@ class DataHandler:
return None return None
def compute_distances(self) -> float: def compute_distances(self) -> float:
print("computing average mean distance for all training points") logger.info("computing average mean distance for all training points")
pairwise = pairwise_distances(self.data_dictionary["train_features"], n_jobs=-1) pairwise = pairwise_distances(self.data_dictionary["train_features"], n_jobs=-1)
avg_mean_dist = pairwise.mean(axis=1).mean() avg_mean_dist = pairwise.mean(axis=1).mean()
print("avg_mean_dist", avg_mean_dist) logger.info("avg_mean_dist", avg_mean_dist)
return avg_mean_dist return avg_mean_dist
@ -407,7 +410,7 @@ class DataHandler:
drop_index = ~drop_index drop_index = ~drop_index
do_predict = np.array(drop_index.replace(True, 1).replace(False, 0)) do_predict = np.array(drop_index.replace(True, 1).replace(False, 0))
print( logger.info(
"remove_outliers() tossed", "remove_outliers() tossed",
len(do_predict) - do_predict.sum(), len(do_predict) - do_predict.sum(),
"predictions because they were beyond 3 std deviations from training data.", "predictions because they were beyond 3 std deviations from training data.",
@ -472,7 +475,7 @@ class DataHandler:
for p in config["freqai"]["corr_pairlist"]: for p in config["freqai"]["corr_pairlist"]:
features.append(p.split("/")[0] + "-" + ft + shift + "_" + tf) features.append(p.split("/")[0] + "-" + ft + shift + "_" + tf)
print("number of features", len(features)) logger.info("number of features", len(features))
return features return features
def check_if_pred_in_training_spaces(self) -> None: def check_if_pred_in_training_spaces(self) -> None:
@ -483,7 +486,7 @@ class DataHandler:
from the training data set. from the training data set.
""" """
print("checking if prediction features are in AOA") logger.info("checking if prediction features are in AOA")
distance = pairwise_distances( distance = pairwise_distances(
self.data_dictionary["train_features"], self.data_dictionary["train_features"],
self.data_dictionary["prediction_features"], self.data_dictionary["prediction_features"],
@ -497,7 +500,7 @@ class DataHandler:
0, 0,
) )
print( logger.info(
"Distance checker tossed", "Distance checker tossed",
len(do_predict) - do_predict.sum(), len(do_predict) - do_predict.sum(),
"predictions for being too far from training data", "predictions for being too far from training data",

15
freqtrade/freqai/freqai_interface.py
View File

@ -1,6 +1,7 @@
import gc import gc
import logging
import shutil import shutil
from abc import ABC from abc import ABC, abstractmethod
from pathlib import Path from pathlib import Path
from typing import Any, Dict, Tuple from typing import Any, Dict, Tuple
@ -12,6 +13,7 @@ from freqtrade.freqai.data_handler import DataHandler
pd.options.mode.chained_assignment = None pd.options.mode.chained_assignment = None
logger = logging.getLogger(__name__)
class IFreqaiModel(ABC): class IFreqaiModel(ABC):
@ -67,7 +69,7 @@ class IFreqaiModel(ABC):
self.pair = metadata["pair"] self.pair = metadata["pair"]
self.dh = DataHandler(self.config, dataframe) self.dh = DataHandler(self.config, dataframe)
print( logger.info(
"going to train", "going to train",
len(self.dh.training_timeranges), len(self.dh.training_timeranges),
"timeranges:", "timeranges:",
@ -88,7 +90,7 @@ class IFreqaiModel(ABC):
self.freqai_info["training_timerange"] = tr_train self.freqai_info["training_timerange"] = tr_train
dataframe_train = self.dh.slice_dataframe(tr_train, dataframe) dataframe_train = self.dh.slice_dataframe(tr_train, dataframe)
dataframe_backtest = self.dh.slice_dataframe(tr_backtest, dataframe) dataframe_backtest = self.dh.slice_dataframe(tr_backtest, dataframe)
print("training", self.pair, "for", tr_train) logger.info("training", self.pair, "for", tr_train)
# self.dh.model_path = self.full_path + "/" + "sub-train" + "-" + str(tr_train) + "/" # self.dh.model_path = self.full_path + "/" + "sub-train" + "-" + str(tr_train) + "/"
self.dh.model_path = Path(self.full_path / str("sub-train" + "-" + str(tr_train))) self.dh.model_path = Path(self.full_path / str("sub-train" + "-" + str(tr_train)))
if not self.model_exists(self.pair, training_timerange=tr_train): if not self.model_exists(self.pair, training_timerange=tr_train):
@ -114,6 +116,7 @@ class IFreqaiModel(ABC):
return dataframe return dataframe
@abstractmethod
def train(self, unfiltered_dataframe: DataFrame, metadata: dict) -> Any: def train(self, unfiltered_dataframe: DataFrame, metadata: dict) -> Any:
""" """
Filter the training data and train a model to it. Train makes heavy use of the datahandler Filter the training data and train a model to it. Train makes heavy use of the datahandler
@ -127,6 +130,7 @@ class IFreqaiModel(ABC):
return Any return Any
@abstractmethod
def fit(self) -> Any: def fit(self) -> Any:
""" """
Most regressors use the same function names and arguments e.g. user Most regressors use the same function names and arguments e.g. user
@ -139,6 +143,7 @@ class IFreqaiModel(ABC):
return Any return Any
@abstractmethod
def predict(self, dataframe: DataFrame) -> Tuple[np.array, np.array]: def predict(self, dataframe: DataFrame) -> Tuple[np.array, np.array]:
""" """
Filter the prediction features data and predict with it. Filter the prediction features data and predict with it.
@ -162,7 +167,7 @@ class IFreqaiModel(ABC):
path_to_modelfile = Path(self.dh.model_path / str(self.dh.model_filename + "_model.joblib")) path_to_modelfile = Path(self.dh.model_path / str(self.dh.model_filename + "_model.joblib"))
file_exists = path_to_modelfile.is_file() file_exists = path_to_modelfile.is_file()
if file_exists: if file_exists:
print("Found model at", self.dh.model_path / self.dh.model_filename) logger.info("Found model at", self.dh.model_path / self.dh.model_filename)
else: else:
print("Could not find model at", self.dh.model_path / self.dh.model_filename) logger.info("Could not find model at", self.dh.model_path / self.dh.model_filename)
return file_exists return file_exists

18
freqtrade/templates/ExamplePredictionModel.py
View File

@ -1,3 +1,4 @@
import logging
from typing import Any, Dict, Tuple from typing import Any, Dict, Tuple
import pandas as pd import pandas as pd
@ -7,6 +8,9 @@ from pandas import DataFrame
from freqtrade.freqai.freqai_interface import IFreqaiModel from freqtrade.freqai.freqai_interface import IFreqaiModel
logger = logging.getLogger(__name__)
class ExamplePredictionModel(IFreqaiModel): class ExamplePredictionModel(IFreqaiModel):
""" """
User created prediction model. The class needs to override three necessary User created prediction model. The class needs to override three necessary
@ -32,7 +36,7 @@ class ExamplePredictionModel(IFreqaiModel):
self.dh.data["s_mean"] = dataframe["s"].mean() self.dh.data["s_mean"] = dataframe["s"].mean()
self.dh.data["s_std"] = dataframe["s"].std() self.dh.data["s_std"] = dataframe["s"].std()
print("label mean", self.dh.data["s_mean"], "label std", self.dh.data["s_std"]) logger.info("label mean", self.dh.data["s_mean"], "label std", self.dh.data["s_std"])
return dataframe["s"] return dataframe["s"]
@ -46,7 +50,7 @@ class ExamplePredictionModel(IFreqaiModel):
:returns: :returns:
:model: Trained model which can be used to inference (self.predict) :model: Trained model which can be used to inference (self.predict)
""" """
print("--------------------Starting training--------------------") logger.info("--------------------Starting training--------------------")
# create the full feature list based on user config info # create the full feature list based on user config info
self.dh.training_features_list = self.dh.build_feature_list(self.config) self.dh.training_features_list = self.dh.build_feature_list(self.config)
@ -73,12 +77,12 @@ class ExamplePredictionModel(IFreqaiModel):
if self.feature_parameters["DI_threshold"]: if self.feature_parameters["DI_threshold"]:
self.dh.data["avg_mean_dist"] = self.dh.compute_distances() self.dh.data["avg_mean_dist"] = self.dh.compute_distances()
print("length of train data", len(data_dictionary["train_features"])) logger.info("length of train data", len(data_dictionary["train_features"]))
model = self.fit(data_dictionary) model = self.fit(data_dictionary)
print("Finished training") logger.info("Finished training")
print(f'--------------------done training {metadata["pair"]}--------------------') logger.info(f'--------------------done training {metadata["pair"]}--------------------')
return model return model
@ -121,7 +125,7 @@ class ExamplePredictionModel(IFreqaiModel):
data (NaNs) or felt uncertain about data (PCA and DI index) data (NaNs) or felt uncertain about data (PCA and DI index)
""" """
print("--------------------Starting prediction--------------------") logger.info("--------------------Starting prediction--------------------")
original_feature_list = self.dh.build_feature_list(self.config) original_feature_list = self.dh.build_feature_list(self.config)
filtered_dataframe, _ = self.dh.filter_features( filtered_dataframe, _ = self.dh.filter_features(
@ -150,6 +154,6 @@ class ExamplePredictionModel(IFreqaiModel):
# compute the non-standardized predictions # compute the non-standardized predictions
predictions = predictions * self.dh.data["labels_std"] + self.dh.data["labels_mean"] predictions = predictions * self.dh.data["labels_std"] + self.dh.data["labels_mean"]
print("--------------------Finished prediction--------------------") logger.info("--------------------Finished prediction--------------------")
return (predictions, self.dh.do_predict) return (predictions, self.dh.do_predict)