From c5de0c49e4741b73dfdb37b41b5d202cc38f4271 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 16 Jun 2022 00:21:15 +0200 Subject: [PATCH] first functional scanning commit --- freqtrade/freqai/data_drawer.py | 22 +++++----- freqtrade/freqai/data_kitchen.py | 7 +++- freqtrade/freqai/freqai_interface.py | 41 +++++++++++-------- .../CatboostPredictionModel.py | 6 +-- freqtrade/templates/FreqaiExampleStrategy.py | 3 +- 5 files changed, 45 insertions(+), 34 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 0fb399b58..5071b87ca 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -107,7 +107,7 @@ class FreqaiDataDrawer: if isinstance(object, np.generic): return object.item() - def get_pair_dict_info(self, metadata: dict) -> Tuple[str, int, bool, bool]: + def get_pair_dict_info(self, pair: str) -> Tuple[str, int, bool, bool]: """ Locate and load existing model metadata from persistent storage. If not located, create a new one and append the current pair to it and prepare it for its first @@ -120,22 +120,22 @@ class FreqaiDataDrawer: coin_first: bool = If the coin is fresh without metadata return_null_array: bool = Follower could not find pair metadata """ - pair_in_dict = self.pair_dict.get(metadata['pair']) - data_path_set = self.pair_dict.get(metadata['pair'], {}).get('data_path', None) + pair_in_dict = self.pair_dict.get(pair) + data_path_set = self.pair_dict.get(pair, {}).get('data_path', None) return_null_array = False if pair_in_dict: - model_filename = self.pair_dict[metadata['pair']]['model_filename'] - trained_timestamp = self.pair_dict[metadata['pair']]['trained_timestamp'] - coin_first = self.pair_dict[metadata['pair']]['first'] + model_filename = self.pair_dict[pair]['model_filename'] + trained_timestamp = self.pair_dict[pair]['trained_timestamp'] + coin_first = self.pair_dict[pair]['first'] elif not self.follow_mode: - self.pair_dict[metadata['pair']] = {} - model_filename = self.pair_dict[metadata['pair']]['model_filename'] = '' - coin_first = self.pair_dict[metadata['pair']]['first'] = True - trained_timestamp = self.pair_dict[metadata['pair']]['trained_timestamp'] = 0 + self.pair_dict[pair] = {} + model_filename = self.pair_dict[pair]['model_filename'] = '' + coin_first = self.pair_dict[pair]['first'] = True + trained_timestamp = self.pair_dict[pair]['trained_timestamp'] = 0 if not data_path_set and self.follow_mode: - logger.warning(f'Follower could not find current pair {metadata["pair"]} in ' + logger.warning(f'Follower could not find current pair {pair} in ' f'pair_dictionary at path {self.full_path}, sending null values ' 'back to strategy.') return_null_array = True diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index afc55a1a2..82c7fd921 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -151,6 +151,9 @@ class FreqaiDataKitchen: :model: User trained model which can be inferenced for new predictions """ + if not self.data_drawer.pair_dict[coin]['model_filename']: + return None + if self.live: self.model_filename = self.data_drawer.pair_dict[coin]['model_filename'] self.data_path = Path(self.data_drawer.pair_dict[coin]['data_path']) @@ -747,7 +750,7 @@ class FreqaiDataKitchen: logger.warning('FreqAI could not detect max timeframe and therefore may not ' 'download the proper amount of data for training') - logger.info(f'Extending data download by {additional_seconds/SECONDS_IN_DAY:.2f} days') + # logger.info(f'Extending data download by {additional_seconds/SECONDS_IN_DAY:.2f} days') if trained_timestamp != 0: elapsed_time = (time - trained_timestamp) / SECONDS_IN_DAY @@ -937,7 +940,7 @@ class FreqaiDataKitchen: for tf in self.freqai_config.get('timeframes'): base_dataframes[tf] = self.slice_dataframe( timerange, - historic_data[metadata['pair']][tf] + historic_data[pair][tf] ) if pairs: for p in pairs: diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index dc7b3a750..8bd8fe334 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -124,18 +124,19 @@ class IFreqaiModel(ABC): file_exists = False - # dh.set_paths(pair, trained_timestamp) + dh.set_paths(pair, trained_timestamp) file_exists = self.model_exists(pair, dh, trained_timestamp=trained_timestamp, - model_filename=model_filename) + model_filename=model_filename, + scanning=True) - (self.retrain, + (retrain, new_trained_timerange, data_load_timerange) = dh.check_if_new_training_required(trained_timestamp) dh.set_paths(pair, new_trained_timerange.stopts) - if self.retrain or not file_exists: + if retrain or not file_exists: self.train_model_in_series(new_trained_timerange, pair, strategy, dh, data_load_timerange) @@ -226,7 +227,7 @@ class IFreqaiModel(ABC): # get the model metadata associated with the current pair (_, trained_timestamp, - coin_first, + _, return_null_array) = self.data_drawer.get_pair_dict_info(metadata['pair']) # if the metadata doesnt exist, the follower returns null arrays to strategy @@ -264,14 +265,18 @@ class IFreqaiModel(ABC): dh.download_all_data_for_training(data_load_timerange) dh.load_all_pair_histories(data_load_timerange) - # train the model on the trained timerange - if coin_first and not self.scanning: - self.train_model_in_series(new_trained_timerange, metadata['pair'], - strategy, dh, data_load_timerange) - elif not coin_first and not self.scanning: + if not self.scanning: self.scanning = True self.start_scanning(strategy) + # train the model on the trained timerange + # if coin_first and not self.scanning: + # self.train_model_in_series(new_trained_timerange, metadata['pair'], + # strategy, dh, data_load_timerange) + # elif not coin_first and not self.scanning: + # self.scanning = True + # self.start_scanning(strategy) + # elif not trainable and not self.follow_mode: # logger.info(f'{metadata["pair"]} holds spot ' # f'{self.data_drawer.pair_dict[metadata["pair"]]["priority"]} ' @@ -283,6 +288,10 @@ class IFreqaiModel(ABC): # load the model and associated data into the data kitchen self.model = dh.load_data(coin=metadata['pair']) + if not self.model: + logger.warning('No model ready, returning null values to strategy.') + self.data_drawer.return_null_values_to_strategy(dataframe, dh) + return dh # ensure user is feeding the correct indicators to the model self.check_if_feature_list_matches_strategy(dataframe, dh) @@ -373,7 +382,7 @@ class IFreqaiModel(ABC): # dh.remove_outliers(predict=True) # creates dropped index def model_exists(self, pair: str, dh: FreqaiDataKitchen, trained_timestamp: int = None, - model_filename: str = '') -> bool: + model_filename: str = '', scanning: bool = False) -> bool: """ Given a pair and path, check if a model already exists :param pair: pair e.g. BTC/USD @@ -386,9 +395,9 @@ class IFreqaiModel(ABC): path_to_modelfile = Path(dh.data_path / str(model_filename + "_model.joblib")) file_exists = path_to_modelfile.is_file() - if file_exists: + if file_exists and not scanning: logger.info("Found model at %s", dh.data_path / dh.model_filename) - else: + elif not scanning: logger.info("Could not find model at %s", dh.data_path / dh.model_filename) return file_exists @@ -453,8 +462,8 @@ class IFreqaiModel(ABC): with self.lock: self.data_drawer.pair_to_end_of_training_queue(pair) dh.save_data(model, coin=pair) - self.training_on_separate_thread = False - self.retrain = False + # self.training_on_separate_thread = False + # self.retrain = False # each time we finish a training, we check the directory to purge old models. if self.freqai_info.get('purge_old_models', False): @@ -499,7 +508,7 @@ class IFreqaiModel(ABC): with self.lock: self.data_drawer.pair_to_end_of_training_queue(pair) dh.save_data(model, coin=pair) - self.retrain = False + # self.retrain = False # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModlel.py for an example. diff --git a/freqtrade/freqai/prediction_models/CatboostPredictionModel.py b/freqtrade/freqai/prediction_models/CatboostPredictionModel.py index ac37c7e28..84fb58157 100644 --- a/freqtrade/freqai/prediction_models/CatboostPredictionModel.py +++ b/freqtrade/freqai/prediction_models/CatboostPredictionModel.py @@ -48,7 +48,7 @@ class CatboostPredictionModel(IFreqaiModel): return dataframe["s"] def train(self, unfiltered_dataframe: DataFrame, - metadata: dict, dh: FreqaiDataKitchen) -> Tuple[DataFrame, DataFrame]: + pair: str, dh: FreqaiDataKitchen) -> Tuple[DataFrame, DataFrame]: """ Filter the training data and train a model to it. Train makes heavy use of the datahkitchen for storing, saving, loading, and analyzing the data. @@ -60,7 +60,7 @@ class CatboostPredictionModel(IFreqaiModel): """ logger.info('--------------------Starting training ' - f'{metadata["pair"]} --------------------') + f'{pair} --------------------') # create the full feature list based on user config info dh.training_features_list = dh.find_features(unfiltered_dataframe) @@ -88,7 +88,7 @@ class CatboostPredictionModel(IFreqaiModel): model = self.fit(data_dictionary) - logger.info(f'--------------------done training {metadata["pair"]}--------------------') + logger.info(f'--------------------done training {pair}--------------------') return model diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index e2b5622c9..50e729d75 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -116,7 +116,6 @@ class FreqaiExampleStrategy(IStrategy): informative[f"{coin}bb_upperband-period_{t}"] - informative[f"{coin}bb_lowerband-period_{t}"] ) / informative[f"{coin}bb_middleband-period_{t}"] - informative[f"%-{coin}close-bb_lower-period_{t}"] = ( informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] ) @@ -153,7 +152,7 @@ class FreqaiExampleStrategy(IStrategy): # Add generalized indicators here (because in live, it will call this # function to populate indicators during training). Notice how we ensure not to # add them multiple times - if pair == metadata["pair"] and tf == self.timeframe: + if pair == self.freqai_info['corr_pairlist'][0] and tf == self.timeframe: df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25