first functional scanning commit

freqtrade/freqai/data_drawer.py

@@ -107,7 +107,7 @@ class FreqaiDataDrawer:
         if isinstance(object, np.generic):
             return object.item()
 
-    def get_pair_dict_info(self, metadata: dict) -> Tuple[str, int, bool, bool]:
+    def get_pair_dict_info(self, pair: str) -> Tuple[str, int, bool, bool]:
         """
         Locate and load existing model metadata from persistent storage. If not located,
         create a new one and append the current pair to it and prepare it for its first
@@ -120,22 +120,22 @@ class FreqaiDataDrawer:
         coin_first: bool = If the coin is fresh without metadata
         return_null_array: bool = Follower could not find pair metadata
         """
-        pair_in_dict = self.pair_dict.get(metadata['pair'])
+        pair_in_dict = self.pair_dict.get(pair)
-        data_path_set = self.pair_dict.get(metadata['pair'], {}).get('data_path', None)
+        data_path_set = self.pair_dict.get(pair, {}).get('data_path', None)
         return_null_array = False
 
         if pair_in_dict:
-            model_filename = self.pair_dict[metadata['pair']]['model_filename']
+            model_filename = self.pair_dict[pair]['model_filename']
-            trained_timestamp = self.pair_dict[metadata['pair']]['trained_timestamp']
+            trained_timestamp = self.pair_dict[pair]['trained_timestamp']
-            coin_first = self.pair_dict[metadata['pair']]['first']
+            coin_first = self.pair_dict[pair]['first']
         elif not self.follow_mode:
-            self.pair_dict[metadata['pair']] = {}
+            self.pair_dict[pair] = {}
-            model_filename = self.pair_dict[metadata['pair']]['model_filename'] = ''
+            model_filename = self.pair_dict[pair]['model_filename'] = ''
-            coin_first = self.pair_dict[metadata['pair']]['first'] = True
+            coin_first = self.pair_dict[pair]['first'] = True
-            trained_timestamp = self.pair_dict[metadata['pair']]['trained_timestamp'] = 0
+            trained_timestamp = self.pair_dict[pair]['trained_timestamp'] = 0
 
         if not data_path_set and self.follow_mode:
-            logger.warning(f'Follower could not find current pair {metadata["pair"]} in '
+            logger.warning(f'Follower could not find current pair {pair} in '
                            f'pair_dictionary at path {self.full_path}, sending null values '
                            'back to strategy.')
             return_null_array = True

freqtrade/freqai/data_kitchen.py

@@ -151,6 +151,9 @@ class FreqaiDataKitchen:
         :model: User trained model which can be inferenced for new predictions
         """
 
+        if not self.data_drawer.pair_dict[coin]['model_filename']:
+            return None
+
         if self.live:
             self.model_filename = self.data_drawer.pair_dict[coin]['model_filename']
             self.data_path = Path(self.data_drawer.pair_dict[coin]['data_path'])
@@ -747,7 +750,7 @@ class FreqaiDataKitchen:
                 logger.warning('FreqAI could not detect max timeframe and therefore may not '
                                'download the proper amount of data for training')
 
-        logger.info(f'Extending data download by {additional_seconds/SECONDS_IN_DAY:.2f} days')
+        # logger.info(f'Extending data download by {additional_seconds/SECONDS_IN_DAY:.2f} days')
 
         if trained_timestamp != 0:
             elapsed_time = (time - trained_timestamp) / SECONDS_IN_DAY
@@ -937,7 +940,7 @@ class FreqaiDataKitchen:
             for tf in self.freqai_config.get('timeframes'):
                 base_dataframes[tf] = self.slice_dataframe(
                                                         timerange,
-                                                        historic_data[metadata['pair']][tf]
+                                                        historic_data[pair][tf]
                                                         )
                 if pairs:
                     for p in pairs:

freqtrade/freqai/freqai_interface.py

@@ -124,18 +124,19 @@ class IFreqaiModel(ABC):
 
                 file_exists = False
 
-                # dh.set_paths(pair, trained_timestamp)
+                dh.set_paths(pair, trained_timestamp)
                 file_exists = self.model_exists(pair,
                                                 dh,
                                                 trained_timestamp=trained_timestamp,
-                                                model_filename=model_filename)
+                                                model_filename=model_filename,
+                                                scanning=True)
 
-                (self.retrain,
+                (retrain,
                  new_trained_timerange,
                  data_load_timerange) = dh.check_if_new_training_required(trained_timestamp)
                 dh.set_paths(pair, new_trained_timerange.stopts)
 
-                if self.retrain or not file_exists:
+                if retrain or not file_exists:
                     self.train_model_in_series(new_trained_timerange, pair,
                                                strategy, dh, data_load_timerange)
 
@@ -226,7 +227,7 @@ class IFreqaiModel(ABC):
         # get the model metadata associated with the current pair
         (_,
          trained_timestamp,
-         coin_first,
+         _,
          return_null_array) = self.data_drawer.get_pair_dict_info(metadata['pair'])
 
         # if the metadata doesnt exist, the follower returns null arrays to strategy
@@ -264,14 +265,18 @@ class IFreqaiModel(ABC):
                 dh.download_all_data_for_training(data_load_timerange)
                 dh.load_all_pair_histories(data_load_timerange)
 
-            # train the model on the trained timerange
+            if not self.scanning:
-            if coin_first and not self.scanning:
-                self.train_model_in_series(new_trained_timerange, metadata['pair'],
-                                           strategy, dh, data_load_timerange)
-            elif not coin_first and not self.scanning:
                 self.scanning = True
                 self.start_scanning(strategy)
 
+            # train the model on the trained timerange
+            # if coin_first and not self.scanning:
+            #     self.train_model_in_series(new_trained_timerange, metadata['pair'],
+            #                                strategy, dh, data_load_timerange)
+            # elif not coin_first and not self.scanning:
+            #     self.scanning = True
+            #     self.start_scanning(strategy)
+
         # elif not trainable and not self.follow_mode:
         #     logger.info(f'{metadata["pair"]} holds spot '
         #                 f'{self.data_drawer.pair_dict[metadata["pair"]]["priority"]} '
@@ -283,6 +288,10 @@ class IFreqaiModel(ABC):
 
         # load the model and associated data into the data kitchen
         self.model = dh.load_data(coin=metadata['pair'])
+        if not self.model:
+            logger.warning('No model ready, returning null values to strategy.')
+            self.data_drawer.return_null_values_to_strategy(dataframe, dh)
+            return dh
 
         # ensure user is feeding the correct indicators to the model
         self.check_if_feature_list_matches_strategy(dataframe, dh)
@@ -373,7 +382,7 @@ class IFreqaiModel(ABC):
         #     dh.remove_outliers(predict=True)  # creates dropped index
 
     def model_exists(self, pair: str, dh: FreqaiDataKitchen, trained_timestamp: int = None,
-                     model_filename: str = '') -> bool:
+                     model_filename: str = '', scanning: bool = False) -> bool:
         """
         Given a pair and path, check if a model already exists
         :param pair: pair e.g. BTC/USD
@@ -386,9 +395,9 @@ class IFreqaiModel(ABC):
 
         path_to_modelfile = Path(dh.data_path / str(model_filename + "_model.joblib"))
         file_exists = path_to_modelfile.is_file()
-        if file_exists:
+        if file_exists and not scanning:
             logger.info("Found model at %s", dh.data_path / dh.model_filename)
-        else:
+        elif not scanning:
             logger.info("Could not find model at %s", dh.data_path / dh.model_filename)
         return file_exists
 
@@ -453,8 +462,8 @@ class IFreqaiModel(ABC):
             with self.lock:
                 self.data_drawer.pair_to_end_of_training_queue(pair)
         dh.save_data(model, coin=pair)
-        self.training_on_separate_thread = False
+        # self.training_on_separate_thread = False
-        self.retrain = False
+        # self.retrain = False
 
         # each time we finish a training, we check the directory to purge old models.
         if self.freqai_info.get('purge_old_models', False):
@@ -499,7 +508,7 @@ class IFreqaiModel(ABC):
             with self.lock:
                 self.data_drawer.pair_to_end_of_training_queue(pair)
         dh.save_data(model, coin=pair)
-        self.retrain = False
+        # self.retrain = False
 
     # Following methods which are overridden by user made prediction models.
     # See freqai/prediction_models/CatboostPredictionModlel.py for an example.

freqtrade/freqai/prediction_models/CatboostPredictionModel.py

@@ -48,7 +48,7 @@ class CatboostPredictionModel(IFreqaiModel):
         return dataframe["s"]
 
     def train(self, unfiltered_dataframe: DataFrame,
-              metadata: dict, dh: FreqaiDataKitchen) -> Tuple[DataFrame, DataFrame]:
+              pair: str, dh: FreqaiDataKitchen) -> Tuple[DataFrame, DataFrame]:
         """
         Filter the training data and train a model to it. Train makes heavy use of the datahkitchen
         for storing, saving, loading, and analyzing the data.
@@ -60,7 +60,7 @@ class CatboostPredictionModel(IFreqaiModel):
         """
 
         logger.info('--------------------Starting training '
-                    f'{metadata["pair"]} --------------------')
+                    f'{pair} --------------------')
 
         # create the full feature list based on user config info
         dh.training_features_list = dh.find_features(unfiltered_dataframe)
@@ -88,7 +88,7 @@ class CatboostPredictionModel(IFreqaiModel):
 
         model = self.fit(data_dictionary)
 
-        logger.info(f'--------------------done training {metadata["pair"]}--------------------')
+        logger.info(f'--------------------done training {pair}--------------------')
 
         return model
 

freqtrade/templates/FreqaiExampleStrategy.py

@@ -116,7 +116,6 @@ class FreqaiExampleStrategy(IStrategy):
                     informative[f"{coin}bb_upperband-period_{t}"]
                     - informative[f"{coin}bb_lowerband-period_{t}"]
                 ) / informative[f"{coin}bb_middleband-period_{t}"]
-
                 informative[f"%-{coin}close-bb_lower-period_{t}"] = (
                     informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"]
                 )
@@ -153,7 +152,7 @@ class FreqaiExampleStrategy(IStrategy):
             # Add generalized indicators here (because in live, it will call this
             # function to populate indicators during training). Notice how we ensure not to
             # add them multiple times
-            if pair == metadata["pair"] and tf == self.timeframe:
+            if pair == self.freqai_info['corr_pairlist'][0] and tf == self.timeframe:
                 df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
                 df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25