adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix bug for target_mean/std array merging in backtesting

Browse Source
This commit is contained in:
robcaulk
2022-05-26 21:07:50 +02:00
parent ff531c416f
commit 6193205012
6 changed files with 186 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
freqtrade/freqai/data_kitchen.py
View File

@@ -141,9 +141,9 @@ class FreqaiDataKitchen:
:model: User trained model which can be inferenced for new predictions
"""
# if self.live:
self.model_filename = self.data_drawer.pair_dict[coin]['model_filename']
self.data_path = Path(self.data_drawer.pair_dict[coin]['data_path'])
if self.live:
self.model_filename = self.data_drawer.pair_dict[coin]['model_filename']
self.data_path = Path(self.data_drawer.pair_dict[coin]['data_path'])
with open(self.data_path / str(self.model_filename + "_metadata.json"), "r") as fp:
self.data = json.load(fp)
@@ -329,42 +329,6 @@ class FreqaiDataKitchen:
:data_dictionary: updated dictionary with standardized values.
"""
# standardize the data by training stats
train_mean = data_dictionary["train_features"].mean()
train_std = data_dictionary["train_features"].std()
data_dictionary["train_features"] = (
data_dictionary["train_features"] - train_mean
) / train_std
data_dictionary["test_features"] = (
data_dictionary["test_features"] - train_mean
) / train_std
train_labels_std = data_dictionary["train_labels"].std()
train_labels_mean = data_dictionary["train_labels"].mean()
data_dictionary["train_labels"] = (
data_dictionary["train_labels"] - train_labels_mean
) / train_labels_std
data_dictionary["test_labels"] = (
data_dictionary["test_labels"] - train_labels_mean
) / train_labels_std
for item in train_std.keys():
self.data[item + "_std"] = train_std[item]
self.data[item + "_mean"] = train_mean[item]
self.data["labels_std"] = train_labels_std
self.data["labels_mean"] = train_labels_mean
return data_dictionary
def standardize_data(self, data_dictionary: Dict) -> Dict[Any, Any]:
"""
Standardize all data in the data_dictionary according to the training dataset
:params:
:data_dictionary: dictionary containing the cleaned and split training/test data/labels
:returns:
:data_dictionary: updated dictionary with standardized values.
"""
# standardize the data by training stats
train_max = data_dictionary["train_features"].max()
train_min = data_dictionary["train_features"].min()
data_dictionary["train_features"] = 2 * (
@@ -392,9 +356,9 @@ class FreqaiDataKitchen:
return data_dictionary
def standardize_data_from_metadata(self, df: DataFrame) -> DataFrame:
def normalize_data_from_metadata(self, df: DataFrame) -> DataFrame:
"""
Standardizes a set of data using the mean and standard deviation from
Normalize a set of data using the mean and standard deviation from
the associated training data.
:params:
:df: Dataframe to be standardized
@@ -406,19 +370,6 @@ class FreqaiDataKitchen:
return df
def normalize_data_from_metadata(self, df: DataFrame) -> DataFrame:
"""
Normalizes a set of data using the mean and standard deviation from
the associated training data.
:params:
:df: Dataframe to be standardized
"""
for item in df.keys():
df[item] = (df[item] - self.data[item + "_mean"]) / self.data[item + "_std"]
return df
def split_timerange(
self, tr: str, train_split: int = 28, bt_split: int = 7
) -> Tuple[list, list]:
@@ -657,12 +608,12 @@ class FreqaiDataKitchen:
"""
ones = np.ones(len_dataframe)
s_mean, s_std = ones * self.data["s_mean"], ones * self.data["s_std"]
target_mean, target_std = ones * self.data["target_mean"], ones * self.data["target_std"]
self.full_predictions = np.append(self.full_predictions, predictions)
self.full_do_predict = np.append(self.full_do_predict, do_predict)
self.full_target_mean = np.append(self.full_target_mean, s_mean)
self.full_target_std = np.append(self.full_target_std, s_std)
self.full_target_mean = np.append(self.full_target_mean, target_mean)
self.full_target_std = np.append(self.full_target_std, target_std)
return
@@ -827,6 +778,23 @@ class FreqaiDataKitchen:
return dataframe
def fit_labels(self) -> None:
import scipy as spy
f = spy.stats.norm.fit(self.data_dictionary["train_labels"])
# KEEPME incase we want to let user start to grab quantiles.
# upper_q = spy.stats.norm.ppf(self.freqai_config['feature_parameters'][
# 'target_quantile'], *f)
# lower_q = spy.stats.norm.ppf(1 - self.freqai_config['feature_parameters'][
# 'target_quantile'], *f)
self.data["target_mean"], self.data["target_std"] = f[0], f[1]
# self.data["upper_quantile"] = upper_q
# self.data["lower_quantile"] = lower_q
return
def np_encoder(self, object):
if isinstance(object, np.generic):
return object.item()
@@ -968,3 +936,52 @@ class FreqaiDataKitchen:
# )
# return
# def standardize_data(self, data_dictionary: Dict) -> Dict[Any, Any]:
# """
# standardize all data in the data_dictionary according to the training dataset
# :params:
# :data_dictionary: dictionary containing the cleaned and split training/test data/labels
# :returns:
# :data_dictionary: updated dictionary with standardized values.
# """
# # standardize the data by training stats
# train_mean = data_dictionary["train_features"].mean()
# train_std = data_dictionary["train_features"].std()
# data_dictionary["train_features"] = (
# data_dictionary["train_features"] - train_mean
# ) / train_std
# data_dictionary["test_features"] = (
# data_dictionary["test_features"] - train_mean
# ) / train_std
# train_labels_std = data_dictionary["train_labels"].std()
# train_labels_mean = data_dictionary["train_labels"].mean()
# data_dictionary["train_labels"] = (
# data_dictionary["train_labels"] - train_labels_mean
# ) / train_labels_std
# data_dictionary["test_labels"] = (
# data_dictionary["test_labels"] - train_labels_mean
# ) / train_labels_std
# for item in train_std.keys():
# self.data[item + "_std"] = train_std[item]
# self.data[item + "_mean"] = train_mean[item]
# self.data["labels_std"] = train_labels_std
# self.data["labels_mean"] = train_labels_mean
# return data_dictionary
# def standardize_data_from_metadata(self, df: DataFrame) -> DataFrame:
# """
# Normalizes a set of data using the mean and standard deviation from
# the associated training data.
# :params:
# :df: Dataframe to be standardized
# """
# for item in df.keys():
# df[item] = (df[item] - self.data[item + "_mean"]) / self.data[item + "_std"]
# return df

23
freqtrade/freqai/freqai_interface.py
View File

@@ -158,12 +158,7 @@ class IFreqaiModel(ABC):
else:
self.model = dh.load_data(metadata['pair'])
# strategy_provided_features = self.dh.find_features(dataframe_train)
# # FIXME doesnt work with PCA
# if strategy_provided_features != self.dh.training_features_list:
# logger.info("User changed input features, retraining model.")
# self.model = self.train(dataframe_train, metadata)
# self.dh.save_data(self.model)
self.check_if_feature_list_matches_strategy(dataframe_train, dh)
preds, do_preds = self.predict(dataframe_backtest, dh)
@@ -220,16 +215,23 @@ class IFreqaiModel(ABC):
self.model = dh.load_data(coin=metadata['pair'])
# FIXME
# strategy_provided_features = dh.find_features(dataframe)
# if strategy_provided_features != dh.training_features_list:
# self.train_model_in_series(new_trained_timerange, metadata, strategy)
self.check_if_feature_list_matches_strategy(dataframe, dh)
preds, do_preds = self.predict(dataframe, dh)
dh.append_predictions(preds, do_preds, len(dataframe))
return dh
def check_if_feature_list_matches_strategy(self, dataframe: DataFrame,
dh: FreqaiDataKitchen) -> None:
strategy_provided_features = dh.find_features(dataframe)
if strategy_provided_features != dh.training_features_list:
raise OperationalException("Trying to access pretrained model with `identifier` "
"but found different features furnished by current strategy."
"Change `identifer` to train from scratch, or ensure the"
"strategy is furnishing the same features as the pretrained"
"model")
def data_cleaning_train(self, dh: FreqaiDataKitchen) -> None:
"""
Base data cleaning method for train
@@ -237,6 +239,7 @@ class IFreqaiModel(ABC):
based on user decided logic. See FreqaiDataKitchen::remove_outliers() for an example
of how outlier data points are dropped from the dataframe used for training.
"""
if self.freqai_info.get('feature_parameters', {}).get('principal_component_analysis'):
dh.principal_component_analysis()

13
freqtrade/freqai/prediction_models/CatboostPredictionModel.py
View File

@@ -33,10 +33,6 @@ class CatboostPredictionModel(IFreqaiModel):
/ dataframe["close"]
- 1
)
dh.data["s_mean"] = dataframe["s"].mean()
dh.data["s_std"] = dataframe["s"].std()
# logger.info("label mean", dh.data["s_mean"], "label std", dh.data["s_std"])
return dataframe["s"]
@@ -68,8 +64,9 @@ class CatboostPredictionModel(IFreqaiModel):
# split data into train/test data.
data_dictionary = dh.make_train_test_datasets(features_filtered, labels_filtered)
# standardize all data based on train_dataset only
data_dictionary = dh.standardize_data(data_dictionary)
dh.fit_labels() # fit labels to a cauchy distribution so we know what to expect in strategy
# normalize all data based on train_dataset only
data_dictionary = dh.normalize_data(data_dictionary)
# optional additional data cleaning/analysis
self.data_cleaning_train(dh)
@@ -128,7 +125,7 @@ class CatboostPredictionModel(IFreqaiModel):
filtered_dataframe, _ = dh.filter_features(
unfiltered_dataframe, original_feature_list, training_filter=False
)
filtered_dataframe = dh.standardize_data_from_metadata(filtered_dataframe)
filtered_dataframe = dh.normalize_data_from_metadata(filtered_dataframe)
dh.data_dictionary["prediction_features"] = filtered_dataframe
# optional additional data cleaning/analysis
@@ -136,7 +133,7 @@ class CatboostPredictionModel(IFreqaiModel):
predictions = self.model.predict(dh.data_dictionary["prediction_features"])
# compute the non-standardized predictions
# compute the non-normalized predictions
dh.predictions = (predictions + 1) * (dh.data["labels_max"] -
dh.data["labels_min"]) / 2 + dh.data["labels_min"]