adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

isolate data_drawer functions from data_kitchen, accommodate tests, add new test

Browse Source
This commit is contained in:
robcaulk
2022-07-26 10:24:14 +02:00
parent 56b17e6f3c
commit e213d0ad55
12 changed files with 606 additions and 376 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
freqtrade/freqai/freqai_interface.py
View File

@@ -102,7 +102,7 @@ class IFreqaiModel(ABC):
self.dd.set_pair_dict_info(metadata)
if self.live:
self.dk = FreqaiDataKitchen(self.config, self.dd, self.live, metadata["pair"])
self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
dk = self.start_live(dataframe, metadata, strategy, self.dk)
# For backtesting, each pair enters and then gets trained for each window along the
@@ -111,7 +111,7 @@ class IFreqaiModel(ABC):
# FreqAI slides the window and sequentially builds the backtesting results before returning
# the concatenated results for the full backtesting period back to the strategy.
elif not self.follow_mode:
self.dk = FreqaiDataKitchen(self.config, self.dd, self.live, metadata["pair"])
self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
logger.info(f"Training {len(self.dk.training_timeranges)} timeranges")
dataframe = self.dk.use_strategy_to_populate_indicators(
@@ -138,7 +138,7 @@ class IFreqaiModel(ABC):
if self.dd.pair_dict[pair]["priority"] != 1:
continue
dk = FreqaiDataKitchen(self.config, self.dd, self.live, pair)
dk = FreqaiDataKitchen(self.config, self.live, pair)
dk.set_paths(pair, trained_timestamp)
(
retrain,
@@ -217,9 +217,9 @@ class IFreqaiModel(ABC):
self.dd.pair_dict[metadata["pair"]]["trained_timestamp"] = int(
trained_timestamp.stopts)
dk.set_new_model_names(metadata["pair"], trained_timestamp)
dk.save_data(self.model, metadata["pair"])
self.dd.save_data(self.model, metadata["pair"], dk)
else:
self.model = dk.load_data(metadata["pair"])
self.model = self.dd.load_data(metadata["pair"], dk)
self.check_if_feature_list_matches_strategy(dataframe_train, dk)
@@ -260,7 +260,7 @@ class IFreqaiModel(ABC):
# append the historic data once per round
if self.dd.historic_data:
dk.update_historic_data(strategy)
self.dd.update_historic_data(strategy, dk)
logger.debug(f'Updating historic data on pair {metadata["pair"]}')
if not self.follow_mode:
@@ -278,7 +278,7 @@ class IFreqaiModel(ABC):
"data saved"
)
dk.download_all_data_for_training(data_load_timerange)
dk.load_all_pair_histories(data_load_timerange)
self.dd.load_all_pair_histories(data_load_timerange, dk)
if not self.scanning:
self.scanning = True
@@ -292,7 +292,7 @@ class IFreqaiModel(ABC):
)
# load the model and associated data into the data kitchen
self.model = dk.load_data(coin=metadata["pair"])
self.model = self.dd.load_data(metadata["pair"], dk)
dataframe = self.dk.use_strategy_to_populate_indicators(
strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
@@ -468,8 +468,8 @@ class IFreqaiModel(ABC):
new_trained_timerange does not contain any NaNs)
"""
corr_dataframes, base_dataframes = dk.get_base_and_corr_dataframes(
data_load_timerange, pair
corr_dataframes, base_dataframes = self.dd.get_base_and_corr_dataframes(
data_load_timerange, pair, dk
)
unfiltered_dataframe = dk.use_strategy_to_populate_indicators(
@@ -489,7 +489,7 @@ class IFreqaiModel(ABC):
if self.dd.pair_dict[pair]["priority"] == 1 and self.scanning:
with self.lock:
self.dd.pair_to_end_of_training_queue(pair)
dk.save_data(model, coin=pair)
self.dd.save_data(model, pair, dk)
if self.freqai_info.get("purge_old_models", False):
self.dd.purge_old_models()
@@ -505,6 +505,20 @@ class IFreqaiModel(ABC):
self.dd.historic_predictions[pair] = pd.DataFrame()
self.dd.historic_predictions[pair] = copy.deepcopy(pred_df)
def fit_live_predictions(self, dk: FreqaiDataKitchen) -> None:
"""
Fit the labels with a gaussian distribution
"""
import scipy as spy
num_candles = self.freqai_info.get("fit_live_predictions_candles", 100)
dk.data["labels_mean"], dk.data["labels_std"] = {}, {}
for label in dk.label_list:
f = spy.stats.norm.fit(self.dd.historic_predictions[dk.pair][label].tail(num_candles))
dk.data["labels_mean"][label], dk.data["labels_std"][label] = f[0], f[1]
return
# Following methods which are overridden by user made prediction models.
# See freqai/prediction_models/CatboostPredictionModlel.py for an example.