diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 6a1d97470..2523cd561 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -1,5 +1,8 @@ +# import contextlib import gc import logging +# import sys +import threading from abc import ABC, abstractmethod from pathlib import Path from typing import Any, Dict, Tuple @@ -16,6 +19,24 @@ from freqtrade.strategy.interface import IStrategy pd.options.mode.chained_assignment = None logger = logging.getLogger(__name__) +# FIXME: suppress stdout for background training +# class DummyFile(object): +# def write(self, x): pass + + +# @contextlib.contextmanager +# def nostdout(): +# save_stdout = sys.stdout +# sys.stdout = DummyFile() +# yield +# sys.stdout = save_stdout + + +def threaded(fn): + def wrapper(*args, **kwargs): + threading.Thread(target=fn, args=args, kwargs=kwargs).start() + return wrapper + class IFreqaiModel(ABC): """ @@ -39,6 +60,8 @@ class IFreqaiModel(ABC): self.current_time = None self.model = None self.predictions = None + self.training_on_separate_thread = False + self.retrain = False def start(self, dataframe: DataFrame, metadata: dict, strategy: IStrategy) -> DataFrame: """ @@ -122,25 +145,26 @@ class IFreqaiModel(ABC): training_timerange=self.freqai_info[ 'live_trained_timerange']) - (retrain, - new_trained_timerange) = self.dh.check_if_new_training_required(self.freqai_info[ + if not self.training_on_separate_thread: + # this will also prevent other pairs from trying to train simultaneously. + (self.retrain, + new_trained_timerange) = self.dh.check_if_new_training_required(self.freqai_info[ 'live_trained_timerange'], - metadata) + metadata) + else: + logger.info("FreqAI training a new model on background thread.") + self.retrain = False - if retrain or not file_exists: - self.dh.download_new_data_for_retraining(new_trained_timerange, metadata) - corr_dataframes, base_dataframes = self.dh.load_pairs_histories(new_trained_timerange, - metadata) - - unfiltered_dataframe = self.dh.use_strategy_to_populate_indicators(strategy, - corr_dataframes, - base_dataframes, - metadata) - - self.model = self.train(unfiltered_dataframe, metadata) - self.dh.save_data(self.model) + if self.retrain or not file_exists: + self.training_on_separate_thread = True # acts like a lock + self.retrain_model_on_separate_thread(new_trained_timerange, metadata, strategy) self.model = self.dh.load_data() + + strategy_provided_features = self.dh.find_features(dataframe) + if strategy_provided_features != self.dh.training_features_list: + self.train_model_in_series(new_trained_timerange, metadata, strategy) + preds, do_preds = self.predict(dataframe, metadata) self.dh.append_predictions(preds, do_preds, len(dataframe)) @@ -206,3 +230,38 @@ class IFreqaiModel(ABC): else: logger.info("Could not find model at %s", self.dh.model_path / self.dh.model_filename) return file_exists + + @threaded + def retrain_model_on_separate_thread(self, new_trained_timerange: str, metadata: dict, + strategy: IStrategy): + + # with nostdout(): + self.dh.download_new_data_for_retraining(new_trained_timerange, metadata) + corr_dataframes, base_dataframes = self.dh.load_pairs_histories(new_trained_timerange, + metadata) + + unfiltered_dataframe = self.dh.use_strategy_to_populate_indicators(strategy, + corr_dataframes, + base_dataframes, + metadata) + + self.model = self.train(unfiltered_dataframe, metadata) + self.dh.save_data(self.model) + + self.training_on_separate_thread = False + self.retrain = False + + def train_model_in_series(self, new_trained_timerange: str, metadata: dict, + strategy: IStrategy): + + self.dh.download_new_data_for_retraining(new_trained_timerange, metadata) + corr_dataframes, base_dataframes = self.dh.load_pairs_histories(new_trained_timerange, + metadata) + + unfiltered_dataframe = self.dh.use_strategy_to_populate_indicators(strategy, + corr_dataframes, + base_dataframes, + metadata) + + self.model = self.train(unfiltered_dataframe, metadata) + self.dh.save_data(self.model) diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 6478ca167..c8befebcf 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -144,8 +144,6 @@ class FreqaiExampleStrategy(IStrategy): self.freqai_info = self.config["freqai"] self.pair = metadata['pair'] - print("Populating indicators...") - # the following loops are necessary for building the features # indicated by the user in the configuration file. for tf in self.freqai_info["timeframes"]: