From 4d472a0ea178b1619bb1bd42b0a4ad147d04ece8 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Tue, 7 Jun 2022 22:14:01 -0600 Subject: [PATCH 1/4] merging datarehaul into scanning branch --- freqtrade/freqai/data_kitchen.py | 27 +++--- freqtrade/freqai/freqai_interface.py | 139 +++++++++++++++------------ freqtrade/strategy/interface.py | 2 +- 3 files changed, 94 insertions(+), 74 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index e7ef69903..afc55a1a2 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -71,7 +71,7 @@ class FreqaiDataKitchen: self.data_drawer = data_drawer - def set_paths(self, metadata: dict, trained_timestamp: int = None,) -> None: + def set_paths(self, pair: str, trained_timestamp: int = None,) -> None: """ Set the paths to the data for the present coin/botloop :params: @@ -83,7 +83,7 @@ class FreqaiDataKitchen: str(self.freqai_config.get('identifier'))) self.data_path = Path(self.full_path / str("sub-train" + "-" + - metadata['pair'].split("/")[0] + + pair.split("/")[0] + str(trained_timestamp))) return @@ -796,12 +796,12 @@ class FreqaiDataKitchen: return retrain, trained_timerange, data_load_timerange - def set_new_model_names(self, metadata: dict, trained_timerange: TimeRange): + def set_new_model_names(self, pair: str, trained_timerange: TimeRange): - coin, _ = metadata['pair'].split("/") + coin, _ = pair.split("/") # set the new data_path self.data_path = Path(self.full_path / str("sub-train" + "-" + - metadata['pair'].split("/")[0] + + pair.split("/")[0] + str(int(trained_timerange.stopts)))) self.model_filename = "cb_" + coin.lower() + "_" + str(int(trained_timerange.stopts)) @@ -918,7 +918,7 @@ class FreqaiDataKitchen: 'trading_mode', 'spot')) def get_base_and_corr_dataframes(self, timerange: TimeRange, - metadata: dict) -> Tuple[Dict[Any, Any], Dict[Any, Any]]: + pair: str) -> Tuple[Dict[Any, Any], Dict[Any, Any]]: """ Searches through our historic_data in memory and returns the dataframes relevant to the present pair. @@ -927,6 +927,7 @@ class FreqaiDataKitchen: for training according to user defined train_period metadata: dict = strategy furnished pair metadata """ + with self.data_drawer.history_lock: corr_dataframes: Dict[Any, Any] = {} base_dataframes: Dict[Any, Any] = {} @@ -940,7 +941,7 @@ class FreqaiDataKitchen: ) if pairs: for p in pairs: - if metadata['pair'] in p: + if pair in p: continue # dont repeat anything from whitelist if p not in corr_dataframes: corr_dataframes[p] = {} @@ -984,7 +985,7 @@ class FreqaiDataKitchen: def use_strategy_to_populate_indicators(self, strategy: IStrategy, corr_dataframes: dict, base_dataframes: dict, - metadata: dict) -> DataFrame: + pair: str) -> DataFrame: """ Use the user defined strategy for populating indicators during retrain @@ -1003,19 +1004,19 @@ class FreqaiDataKitchen: for tf in self.freqai_config.get("timeframes"): dataframe = strategy.populate_any_indicators( - metadata, - metadata['pair'], + pair, + pair, dataframe.copy(), tf, base_dataframes[tf], - coin=metadata['pair'].split("/")[0] + "-" + coin=pair.split("/")[0] + "-" ) if pairs: for i in pairs: - if metadata['pair'] in i: + if pair in i: continue # dont repeat anything from whitelist dataframe = strategy.populate_any_indicators( - metadata, + pair, i, dataframe.copy(), tf, diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 1f194860d..dc7b3a750 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -63,6 +63,8 @@ class IFreqaiModel(ABC): self.lock = threading.Lock() self.follow_mode = self.freqai_info.get('follow_mode', False) self.identifier = self.freqai_info.get('identifier', 'no_id_provided') + self.scanning = False + self.ready_to_scan = False def assert_config(self, config: Dict[str, Any]) -> None: @@ -91,17 +93,9 @@ class IFreqaiModel(ABC): # and we keep the flag self.training_on_separate_threaad in the current object to help # determine what the current pair will do if self.live: - if (not self.training_on_separate_thread and - self.data_drawer.pair_dict[metadata['pair']]['priority'] == 1): - - self.dh = FreqaiDataKitchen(self.config, self.data_drawer, - self.live, metadata["pair"]) - dh = self.start_live(dataframe, metadata, strategy, self.dh, trainable=True) - else: - # we will have at max 2 separate instances of the kitchen at once. - self.dh_fg = FreqaiDataKitchen(self.config, self.data_drawer, - self.live, metadata["pair"]) - dh = self.start_live(dataframe, metadata, strategy, self.dh_fg, trainable=False) + self.dh = FreqaiDataKitchen(self.config, self.data_drawer, + self.live, metadata["pair"]) + dh = self.start_live(dataframe, metadata, strategy, self.dh) # For backtesting, each pair enters and then gets trained for each window along the # sliding window defined by "train_period" (training window) and "backtest_period" @@ -114,8 +108,36 @@ class IFreqaiModel(ABC): dh = self.start_backtesting(dataframe, metadata, self.dh) return self.return_values(dataframe, dh) - # return (dh.full_predictions, dh.full_do_predict, - # dh.full_target_mean, dh.full_target_std) + + @threaded + def start_scanning(self, strategy: IStrategy) -> None: + while 1: + for pair in self.config.get('exchange', {}).get('pair_whitelist'): + if self.data_drawer.pair_dict[pair]['priority'] != 1: + continue + dh = FreqaiDataKitchen(self.config, self.data_drawer, + self.live, pair) + + (model_filename, + trained_timestamp, + _, _) = self.data_drawer.get_pair_dict_info(pair) + + file_exists = False + + # dh.set_paths(pair, trained_timestamp) + file_exists = self.model_exists(pair, + dh, + trained_timestamp=trained_timestamp, + model_filename=model_filename) + + (self.retrain, + new_trained_timerange, + data_load_timerange) = dh.check_if_new_training_required(trained_timestamp) + dh.set_paths(pair, new_trained_timerange.stopts) + + if self.retrain or not file_exists: + self.train_model_in_series(new_trained_timerange, pair, + strategy, dh, data_load_timerange) def start_backtesting(self, dataframe: DataFrame, metadata: dict, dh: FreqaiDataKitchen) -> FreqaiDataKitchen: @@ -142,7 +164,7 @@ class IFreqaiModel(ABC): for tr_train, tr_backtest in zip( dh.training_timeranges, dh.backtesting_timeranges ): - (_, _, _, _) = self.data_drawer.get_pair_dict_info(metadata) + (_, _, _, _) = self.data_drawer.get_pair_dict_info(metadata['pair']) gc.collect() dh.data = {} # clean the pair specific data between training window sliding self.training_timerange = tr_train @@ -163,7 +185,7 @@ class IFreqaiModel(ABC): str(int(trained_timestamp.stopts)))) if not self.model_exists(metadata["pair"], dh, trained_timestamp=trained_timestamp.stopts): - self.model = self.train(dataframe_train, metadata, dh) + self.model = self.train(dataframe_train, metadata['pair'], dh) self.data_drawer.pair_dict[metadata['pair']][ 'trained_timestamp'] = trained_timestamp.stopts dh.set_new_model_names(metadata, trained_timestamp) @@ -184,8 +206,7 @@ class IFreqaiModel(ABC): return dh def start_live(self, dataframe: DataFrame, metadata: dict, - strategy: IStrategy, dh: FreqaiDataKitchen, - trainable: bool) -> FreqaiDataKitchen: + strategy: IStrategy, dh: FreqaiDataKitchen) -> FreqaiDataKitchen: """ The main broad execution for dry/live. This function will check if a retraining should be performed, and if so, retrain and reset the model. @@ -203,10 +224,10 @@ class IFreqaiModel(ABC): self.data_drawer.update_follower_metadata() # get the model metadata associated with the current pair - (model_filename, + (_, trained_timestamp, coin_first, - return_null_array) = self.data_drawer.get_pair_dict_info(metadata) + return_null_array) = self.data_drawer.get_pair_dict_info(metadata['pair']) # if the metadata doesnt exist, the follower returns null arrays to strategy if self.follow_mode and return_null_array: @@ -222,20 +243,18 @@ class IFreqaiModel(ABC): # if trainable, check if model needs training, if so compute new timerange, # then save model and metadata. # if not trainable, load existing data - if (trainable or coin_first) and not self.follow_mode: - file_exists = False - - if trained_timestamp != 0: # historical model available - dh.set_paths(metadata, trained_timestamp) - file_exists = self.model_exists(metadata['pair'], - dh, - trained_timestamp=trained_timestamp, - model_filename=model_filename) + if not self.follow_mode: + # if trained_timestamp != 0: # historical model available + # dh.set_paths(metadata['pair'], trained_timestamp) + # # file_exists = self.model_exists(metadata['pair'], + # # dh, + # # trained_timestamp=trained_timestamp, + # # model_filename=model_filename) (self.retrain, new_trained_timerange, data_load_timerange) = dh.check_if_new_training_required(trained_timestamp) - dh.set_paths(metadata, new_trained_timerange.stopts) + dh.set_paths(metadata['pair'], new_trained_timerange.stopts) # download candle history if it is not already in memory if not self.data_drawer.historic_data: @@ -246,20 +265,17 @@ class IFreqaiModel(ABC): dh.load_all_pair_histories(data_load_timerange) # train the model on the trained timerange - if self.retrain or not file_exists: - if coin_first: - self.train_model_in_series(new_trained_timerange, metadata, - strategy, dh, data_load_timerange) - else: - self.training_on_separate_thread = True # acts like a lock - self.retrain_model_on_separate_thread(new_trained_timerange, - metadata, strategy, - dh, data_load_timerange) + if coin_first and not self.scanning: + self.train_model_in_series(new_trained_timerange, metadata['pair'], + strategy, dh, data_load_timerange) + elif not coin_first and not self.scanning: + self.scanning = True + self.start_scanning(strategy) - elif not trainable and not self.follow_mode: - logger.info(f'{metadata["pair"]} holds spot ' - f'{self.data_drawer.pair_dict[metadata["pair"]]["priority"]} ' - 'in training queue') + # elif not trainable and not self.follow_mode: + # logger.info(f'{metadata["pair"]} holds spot ' + # f'{self.data_drawer.pair_dict[metadata["pair"]]["priority"]} ' + # 'in training queue') elif self.follow_mode: dh.set_paths(metadata, trained_timestamp) logger.info('FreqAI instance set to follow_mode, finding existing pair' @@ -382,7 +398,7 @@ class IFreqaiModel(ABC): str(self.freqai_info.get('identifier'))) @threaded - def retrain_model_on_separate_thread(self, new_trained_timerange: TimeRange, metadata: dict, + def retrain_model_on_separate_thread(self, new_trained_timerange: TimeRange, pair: str, strategy: IStrategy, dh: FreqaiDataKitchen, data_load_timerange: TimeRange): """ @@ -403,14 +419,14 @@ class IFreqaiModel(ABC): # metadata) corr_dataframes, base_dataframes = dh.get_base_and_corr_dataframes(data_load_timerange, - metadata) + pair) # protecting from common benign errors associated with grabbing new data from exchange: try: unfiltered_dataframe = dh.use_strategy_to_populate_indicators(strategy, corr_dataframes, base_dataframes, - metadata) + pair) unfiltered_dataframe = dh.slice_dataframe(new_trained_timerange, unfiltered_dataframe) except Exception as err: @@ -420,23 +436,23 @@ class IFreqaiModel(ABC): return try: - model = self.train(unfiltered_dataframe, metadata, dh) + model = self.train(unfiltered_dataframe, pair, dh) except ValueError: logger.warning('Value error encountered during training') self.training_on_separate_thread = False self.retrain = False return - self.data_drawer.pair_dict[metadata['pair']][ + self.data_drawer.pair_dict[pair][ 'trained_timestamp'] = new_trained_timerange.stopts - dh.set_new_model_names(metadata, new_trained_timerange) + dh.set_new_model_names(pair, new_trained_timerange) # logger.info('Training queue' # f'{sorted(self.data_drawer.pair_dict.items(), key=lambda item: item[1])}') - if self.data_drawer.pair_dict[metadata['pair']]['priority'] == 1: + if self.data_drawer.pair_dict[pair]['priority'] == 1: with self.lock: - self.data_drawer.pair_to_end_of_training_queue(metadata['pair']) - dh.save_data(model, coin=metadata['pair']) + self.data_drawer.pair_to_end_of_training_queue(pair) + dh.save_data(model, coin=pair) self.training_on_separate_thread = False self.retrain = False @@ -446,7 +462,7 @@ class IFreqaiModel(ABC): return - def train_model_in_series(self, new_trained_timerange: TimeRange, metadata: dict, + def train_model_in_series(self, new_trained_timerange: TimeRange, pair: str, strategy: IStrategy, dh: FreqaiDataKitchen, data_load_timerange: TimeRange): """ @@ -464,29 +480,32 @@ class IFreqaiModel(ABC): # corr_dataframes, base_dataframes = dh.load_pairs_histories(data_load_timerange, # metadata) corr_dataframes, base_dataframes = dh.get_base_and_corr_dataframes(data_load_timerange, - metadata) + pair) unfiltered_dataframe = dh.use_strategy_to_populate_indicators(strategy, corr_dataframes, base_dataframes, - metadata) + pair) unfiltered_dataframe = dh.slice_dataframe(new_trained_timerange, unfiltered_dataframe) - model = self.train(unfiltered_dataframe, metadata, dh) + model = self.train(unfiltered_dataframe, pair, dh) - self.data_drawer.pair_dict[metadata['pair']][ + self.data_drawer.pair_dict[pair][ 'trained_timestamp'] = new_trained_timerange.stopts - dh.set_new_model_names(metadata, new_trained_timerange) - self.data_drawer.pair_dict[metadata['pair']]['first'] = False - dh.save_data(model, coin=metadata['pair']) + dh.set_new_model_names(pair, new_trained_timerange) + self.data_drawer.pair_dict[pair]['first'] = False + if self.data_drawer.pair_dict[pair]['priority'] == 1 and self.scanning: + with self.lock: + self.data_drawer.pair_to_end_of_training_queue(pair) + dh.save_data(model, coin=pair) self.retrain = False # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModlel.py for an example. @abstractmethod - def train(self, unfiltered_dataframe: DataFrame, metadata: dict, dh: FreqaiDataKitchen) -> Any: + def train(self, unfiltered_dataframe: DataFrame, pair: str, dh: FreqaiDataKitchen) -> Any: """ Filter the training data and train a model to it. Train makes heavy use of the datahandler for storing, saving, loading, and analyzing the data. diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 6237e3397..bfecad495 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -532,7 +532,7 @@ class IStrategy(ABC, HyperStrategyMixin): """ return None - def populate_any_indicators(self, metadata: dict, pair: str, df: DataFrame, tf: str, + def populate_any_indicators(self, basepair: str, pair: str, df: DataFrame, tf: str, informative: DataFrame = None, coin: str = "") -> DataFrame: """ Function designed to automatically generate, name and merge features From c5de0c49e4741b73dfdb37b41b5d202cc38f4271 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 16 Jun 2022 00:21:15 +0200 Subject: [PATCH 2/4] first functional scanning commit --- freqtrade/freqai/data_drawer.py | 22 +++++----- freqtrade/freqai/data_kitchen.py | 7 +++- freqtrade/freqai/freqai_interface.py | 41 +++++++++++-------- .../CatboostPredictionModel.py | 6 +-- freqtrade/templates/FreqaiExampleStrategy.py | 3 +- 5 files changed, 45 insertions(+), 34 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 0fb399b58..5071b87ca 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -107,7 +107,7 @@ class FreqaiDataDrawer: if isinstance(object, np.generic): return object.item() - def get_pair_dict_info(self, metadata: dict) -> Tuple[str, int, bool, bool]: + def get_pair_dict_info(self, pair: str) -> Tuple[str, int, bool, bool]: """ Locate and load existing model metadata from persistent storage. If not located, create a new one and append the current pair to it and prepare it for its first @@ -120,22 +120,22 @@ class FreqaiDataDrawer: coin_first: bool = If the coin is fresh without metadata return_null_array: bool = Follower could not find pair metadata """ - pair_in_dict = self.pair_dict.get(metadata['pair']) - data_path_set = self.pair_dict.get(metadata['pair'], {}).get('data_path', None) + pair_in_dict = self.pair_dict.get(pair) + data_path_set = self.pair_dict.get(pair, {}).get('data_path', None) return_null_array = False if pair_in_dict: - model_filename = self.pair_dict[metadata['pair']]['model_filename'] - trained_timestamp = self.pair_dict[metadata['pair']]['trained_timestamp'] - coin_first = self.pair_dict[metadata['pair']]['first'] + model_filename = self.pair_dict[pair]['model_filename'] + trained_timestamp = self.pair_dict[pair]['trained_timestamp'] + coin_first = self.pair_dict[pair]['first'] elif not self.follow_mode: - self.pair_dict[metadata['pair']] = {} - model_filename = self.pair_dict[metadata['pair']]['model_filename'] = '' - coin_first = self.pair_dict[metadata['pair']]['first'] = True - trained_timestamp = self.pair_dict[metadata['pair']]['trained_timestamp'] = 0 + self.pair_dict[pair] = {} + model_filename = self.pair_dict[pair]['model_filename'] = '' + coin_first = self.pair_dict[pair]['first'] = True + trained_timestamp = self.pair_dict[pair]['trained_timestamp'] = 0 if not data_path_set and self.follow_mode: - logger.warning(f'Follower could not find current pair {metadata["pair"]} in ' + logger.warning(f'Follower could not find current pair {pair} in ' f'pair_dictionary at path {self.full_path}, sending null values ' 'back to strategy.') return_null_array = True diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index afc55a1a2..82c7fd921 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -151,6 +151,9 @@ class FreqaiDataKitchen: :model: User trained model which can be inferenced for new predictions """ + if not self.data_drawer.pair_dict[coin]['model_filename']: + return None + if self.live: self.model_filename = self.data_drawer.pair_dict[coin]['model_filename'] self.data_path = Path(self.data_drawer.pair_dict[coin]['data_path']) @@ -747,7 +750,7 @@ class FreqaiDataKitchen: logger.warning('FreqAI could not detect max timeframe and therefore may not ' 'download the proper amount of data for training') - logger.info(f'Extending data download by {additional_seconds/SECONDS_IN_DAY:.2f} days') + # logger.info(f'Extending data download by {additional_seconds/SECONDS_IN_DAY:.2f} days') if trained_timestamp != 0: elapsed_time = (time - trained_timestamp) / SECONDS_IN_DAY @@ -937,7 +940,7 @@ class FreqaiDataKitchen: for tf in self.freqai_config.get('timeframes'): base_dataframes[tf] = self.slice_dataframe( timerange, - historic_data[metadata['pair']][tf] + historic_data[pair][tf] ) if pairs: for p in pairs: diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index dc7b3a750..8bd8fe334 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -124,18 +124,19 @@ class IFreqaiModel(ABC): file_exists = False - # dh.set_paths(pair, trained_timestamp) + dh.set_paths(pair, trained_timestamp) file_exists = self.model_exists(pair, dh, trained_timestamp=trained_timestamp, - model_filename=model_filename) + model_filename=model_filename, + scanning=True) - (self.retrain, + (retrain, new_trained_timerange, data_load_timerange) = dh.check_if_new_training_required(trained_timestamp) dh.set_paths(pair, new_trained_timerange.stopts) - if self.retrain or not file_exists: + if retrain or not file_exists: self.train_model_in_series(new_trained_timerange, pair, strategy, dh, data_load_timerange) @@ -226,7 +227,7 @@ class IFreqaiModel(ABC): # get the model metadata associated with the current pair (_, trained_timestamp, - coin_first, + _, return_null_array) = self.data_drawer.get_pair_dict_info(metadata['pair']) # if the metadata doesnt exist, the follower returns null arrays to strategy @@ -264,14 +265,18 @@ class IFreqaiModel(ABC): dh.download_all_data_for_training(data_load_timerange) dh.load_all_pair_histories(data_load_timerange) - # train the model on the trained timerange - if coin_first and not self.scanning: - self.train_model_in_series(new_trained_timerange, metadata['pair'], - strategy, dh, data_load_timerange) - elif not coin_first and not self.scanning: + if not self.scanning: self.scanning = True self.start_scanning(strategy) + # train the model on the trained timerange + # if coin_first and not self.scanning: + # self.train_model_in_series(new_trained_timerange, metadata['pair'], + # strategy, dh, data_load_timerange) + # elif not coin_first and not self.scanning: + # self.scanning = True + # self.start_scanning(strategy) + # elif not trainable and not self.follow_mode: # logger.info(f'{metadata["pair"]} holds spot ' # f'{self.data_drawer.pair_dict[metadata["pair"]]["priority"]} ' @@ -283,6 +288,10 @@ class IFreqaiModel(ABC): # load the model and associated data into the data kitchen self.model = dh.load_data(coin=metadata['pair']) + if not self.model: + logger.warning('No model ready, returning null values to strategy.') + self.data_drawer.return_null_values_to_strategy(dataframe, dh) + return dh # ensure user is feeding the correct indicators to the model self.check_if_feature_list_matches_strategy(dataframe, dh) @@ -373,7 +382,7 @@ class IFreqaiModel(ABC): # dh.remove_outliers(predict=True) # creates dropped index def model_exists(self, pair: str, dh: FreqaiDataKitchen, trained_timestamp: int = None, - model_filename: str = '') -> bool: + model_filename: str = '', scanning: bool = False) -> bool: """ Given a pair and path, check if a model already exists :param pair: pair e.g. BTC/USD @@ -386,9 +395,9 @@ class IFreqaiModel(ABC): path_to_modelfile = Path(dh.data_path / str(model_filename + "_model.joblib")) file_exists = path_to_modelfile.is_file() - if file_exists: + if file_exists and not scanning: logger.info("Found model at %s", dh.data_path / dh.model_filename) - else: + elif not scanning: logger.info("Could not find model at %s", dh.data_path / dh.model_filename) return file_exists @@ -453,8 +462,8 @@ class IFreqaiModel(ABC): with self.lock: self.data_drawer.pair_to_end_of_training_queue(pair) dh.save_data(model, coin=pair) - self.training_on_separate_thread = False - self.retrain = False + # self.training_on_separate_thread = False + # self.retrain = False # each time we finish a training, we check the directory to purge old models. if self.freqai_info.get('purge_old_models', False): @@ -499,7 +508,7 @@ class IFreqaiModel(ABC): with self.lock: self.data_drawer.pair_to_end_of_training_queue(pair) dh.save_data(model, coin=pair) - self.retrain = False + # self.retrain = False # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModlel.py for an example. diff --git a/freqtrade/freqai/prediction_models/CatboostPredictionModel.py b/freqtrade/freqai/prediction_models/CatboostPredictionModel.py index ac37c7e28..84fb58157 100644 --- a/freqtrade/freqai/prediction_models/CatboostPredictionModel.py +++ b/freqtrade/freqai/prediction_models/CatboostPredictionModel.py @@ -48,7 +48,7 @@ class CatboostPredictionModel(IFreqaiModel): return dataframe["s"] def train(self, unfiltered_dataframe: DataFrame, - metadata: dict, dh: FreqaiDataKitchen) -> Tuple[DataFrame, DataFrame]: + pair: str, dh: FreqaiDataKitchen) -> Tuple[DataFrame, DataFrame]: """ Filter the training data and train a model to it. Train makes heavy use of the datahkitchen for storing, saving, loading, and analyzing the data. @@ -60,7 +60,7 @@ class CatboostPredictionModel(IFreqaiModel): """ logger.info('--------------------Starting training ' - f'{metadata["pair"]} --------------------') + f'{pair} --------------------') # create the full feature list based on user config info dh.training_features_list = dh.find_features(unfiltered_dataframe) @@ -88,7 +88,7 @@ class CatboostPredictionModel(IFreqaiModel): model = self.fit(data_dictionary) - logger.info(f'--------------------done training {metadata["pair"]}--------------------') + logger.info(f'--------------------done training {pair}--------------------') return model diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index e2b5622c9..50e729d75 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -116,7 +116,6 @@ class FreqaiExampleStrategy(IStrategy): informative[f"{coin}bb_upperband-period_{t}"] - informative[f"{coin}bb_lowerband-period_{t}"] ) / informative[f"{coin}bb_middleband-period_{t}"] - informative[f"%-{coin}close-bb_lower-period_{t}"] = ( informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] ) @@ -153,7 +152,7 @@ class FreqaiExampleStrategy(IStrategy): # Add generalized indicators here (because in live, it will call this # function to populate indicators during training). Notice how we ensure not to # add them multiple times - if pair == metadata["pair"] and tf == self.timeframe: + if pair == self.freqai_info['corr_pairlist'][0] and tf == self.timeframe: df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 From 0b0688a91ebf970b42d2ff78230cbff8e6006cc9 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 16 Jun 2022 16:12:38 +0200 Subject: [PATCH 3/4] ensure scanning purges models --- freqtrade/freqai/freqai_interface.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 8bd8fe334..7e0f69b98 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -508,6 +508,9 @@ class IFreqaiModel(ABC): with self.lock: self.data_drawer.pair_to_end_of_training_queue(pair) dh.save_data(model, coin=pair) + + if self.freqai_info.get('purge_old_models', False): + self.data_drawer.purge_old_models() # self.retrain = False # Following methods which are overridden by user made prediction models. From f631ae911b82e11cb4621528f2a2131b56a98c9c Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 17 Jun 2022 14:55:40 +0200 Subject: [PATCH 4/4] add model expiration feature, fix bug in DI return values --- docs/freqai.md | 18 ++++++++++ freqtrade/freqai/data_drawer.py | 20 +++++++---- freqtrade/freqai/data_kitchen.py | 12 +++++-- freqtrade/freqai/freqai_interface.py | 35 ++++++++++++++----- .../CatboostPredictionModel.py | 2 +- 5 files changed, 69 insertions(+), 18 deletions(-) diff --git a/docs/freqai.md b/docs/freqai.md index 8d54a7535..647ffcecd 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -452,6 +452,24 @@ config: which will automatically purge all models older than the two most recently trained ones. +## Defining model expirations + +During dry/live, FreqAI trains each pair sequentially (on separate threads/GPU from the main +Freqtrade bot). This means there is always an age discrepancy between models. If a user is training +on 50 pairs, and each pair requires 5 minutes to train, the oldest model will be over 4 hours old. +This may be undesirable if the characteristic time scale (read trade duration target) for a strategy +is much less than 4 hours. The user can decide to only make trade entries if the model is less than +a certain number of hours in age by setting the `expiration_hours` in the config file: + +```json + "freqai": { + "expiration_hours": 0.5, + } +``` + +In the present example, the user will only allow predictions on models that are less than 1/2 hours +old. +