diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 848fb20eb..cb5b0f0fb 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -59,7 +59,7 @@ class FreqaiDataDrawer: Juha Nykänen @suikula, Wagner Costa @wagnercosta, Johan Vlugt @Jooopieeert """ - def __init__(self, full_path: Path, config: Config, follow_mode: bool = False): + def __init__(self, full_path: Path, config: Config): self.config = config self.freqai_info = config.get("freqai", {}) @@ -84,9 +84,6 @@ class FreqaiDataDrawer: self.pair_dictionary_path = Path(self.full_path / "pair_dictionary.json") self.global_metadata_path = Path(self.full_path / "global_metadata.json") self.metric_tracker_path = Path(self.full_path / "metric_tracker.json") - self.follow_mode = follow_mode - if follow_mode: - self.create_follower_dict() self.load_drawer_from_disk() self.load_historic_predictions_from_disk() self.metric_tracker: Dict[str, Dict[str, Dict[str, list]]] = {} @@ -149,13 +146,8 @@ class FreqaiDataDrawer: if exists: with open(self.pair_dictionary_path, "r") as fp: self.pair_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE) - elif not self.follow_mode: - logger.info("Could not find existing datadrawer, starting from scratch") else: - logger.warning( - f"Follower could not find pair_dictionary at {self.full_path} " - "sending null values back to strategy" - ) + logger.info("Could not find existing datadrawer, starting from scratch") def load_metric_tracker_from_disk(self): """ @@ -193,13 +185,8 @@ class FreqaiDataDrawer: self.historic_predictions = cloudpickle.load(fp) logger.warning('FreqAI successfully loaded the backup historical predictions file.') - elif not self.follow_mode: - logger.info("Could not find existing historic_predictions, starting from scratch") else: - logger.warning( - f"Follower could not find historic predictions at {self.full_path} " - "sending null values back to strategy" - ) + logger.info("Could not find existing historic_predictions, starting from scratch") return exists @@ -248,23 +235,6 @@ class FreqaiDataDrawer: rapidjson.dump(metadata, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE) - def create_follower_dict(self): - """ - Create or dictionary for each follower to maintain unique persistent prediction targets - """ - - whitelist_pairs = self.config.get("exchange", {}).get("pair_whitelist") - - exists = self.follower_dict_path.is_file() - - if exists: - logger.info("Found an existing follower dictionary") - - for pair in whitelist_pairs: - self.follower_dict[pair] = {} - - self.save_follower_dict_to_disk() - def np_encoder(self, object): if isinstance(object, np.generic): return object.item() @@ -282,27 +252,17 @@ class FreqaiDataDrawer: """ pair_dict = self.pair_dict.get(pair) - data_path_set = self.pair_dict.get(pair, self.empty_pair_dict).get("data_path", "") + # data_path_set = self.pair_dict.get(pair, self.empty_pair_dict).get("data_path", "") return_null_array = False if pair_dict: model_filename = pair_dict["model_filename"] trained_timestamp = pair_dict["trained_timestamp"] - elif not self.follow_mode: + else: self.pair_dict[pair] = self.empty_pair_dict.copy() model_filename = "" trained_timestamp = 0 - if not data_path_set and self.follow_mode: - logger.warning( - f"Follower could not find current pair {pair} in " - f"pair_dictionary at path {self.full_path}, sending null values " - "back to strategy." - ) - trained_timestamp = 0 - model_filename = '' - return_null_array = True - return model_filename, trained_timestamp, return_null_array def set_pair_dict_info(self, metadata: dict) -> None: @@ -311,7 +271,6 @@ class FreqaiDataDrawer: return else: self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy() - return def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None: diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 830970ba0..34928f7c2 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -66,12 +66,11 @@ class IFreqaiModel(ABC): self.retrain = False self.first = True self.set_full_path() - self.follow_mode: bool = self.freqai_info.get("follow_mode", False) self.save_backtest_models: bool = self.freqai_info.get("save_backtest_models", True) if self.save_backtest_models: logger.info('Backtesting module configured to save all models.') - self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode) + self.dd = FreqaiDataDrawer(Path(self.full_path), self.config) # set current candle to arbitrary historical date self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc) self.dd.current_candle = self.current_candle @@ -153,7 +152,7 @@ class IFreqaiModel(ABC): # (backtest window, i.e. window immediately following the training window). # FreqAI slides the window and sequentially builds the backtesting results before returning # the concatenated results for the full backtesting period back to the strategy. - elif not self.follow_mode: + else: self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"]) if not self.config.get("freqai_backtest_live_models", False): logger.info(f"Training {len(self.dk.training_timeranges)} timeranges") @@ -379,46 +378,28 @@ class IFreqaiModel(ABC): :returns: dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only """ - # update follower - if self.follow_mode: - self.dd.update_follower_metadata() # get the model metadata associated with the current pair (_, trained_timestamp, return_null_array) = self.dd.get_pair_dict_info(metadata["pair"]) - # if the metadata doesn't exist, the follower returns null arrays to strategy - if self.follow_mode and return_null_array: - logger.info("Returning null array from follower to strategy") - self.dd.return_null_values_to_strategy(dataframe, dk) - return dk - # append the historic data once per round if self.dd.historic_data: self.dd.update_historic_data(strategy, dk) logger.debug(f'Updating historic data on pair {metadata["pair"]}') self.track_current_candle() - if not self.follow_mode: + (_, new_trained_timerange, data_load_timerange) = dk.check_if_new_training_required( + trained_timestamp + ) + dk.set_paths(metadata["pair"], new_trained_timerange.stopts) - (_, new_trained_timerange, data_load_timerange) = dk.check_if_new_training_required( - trained_timestamp - ) - dk.set_paths(metadata["pair"], new_trained_timerange.stopts) + # load candle history into memory if it is not yet. + if not self.dd.historic_data: + self.dd.load_all_pair_histories(data_load_timerange, dk) - # load candle history into memory if it is not yet. - if not self.dd.historic_data: - self.dd.load_all_pair_histories(data_load_timerange, dk) - - if not self.scanning: - self.scanning = True - self.start_scanning(strategy) - - elif self.follow_mode: - dk.set_paths(metadata["pair"], trained_timestamp) - logger.info( - "FreqAI instance set to follow_mode, finding existing pair " - f"using { self.identifier }" - ) + if not self.scanning: + self.scanning = True + self.start_scanning(strategy) # load the model and associated data into the data kitchen self.model = self.dd.load_data(metadata["pair"], dk) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 4ef99720a..79c04e6b3 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -376,57 +376,6 @@ def test_backtesting_fit_live_predictions(mocker, freqai_conf, caplog): shutil.rmtree(Path(freqai.dk.full_path)) -def test_follow_mode(mocker, freqai_conf): - freqai_conf.update({"timerange": "20180110-20180130"}) - - strategy = get_patched_freqai_strategy(mocker, freqai_conf) - exchange = get_patched_exchange(mocker, freqai_conf) - strategy.dp = DataProvider(freqai_conf, exchange) - strategy.freqai_info = freqai_conf.get("freqai", {}) - freqai = strategy.freqai - freqai.live = True - freqai.dk = FreqaiDataKitchen(freqai_conf) - timerange = TimeRange.parse_timerange("20180110-20180130") - freqai.dd.load_all_pair_histories(timerange, freqai.dk) - - metadata = {"pair": "ADA/BTC"} - freqai.dd.set_pair_dict_info(metadata) - - data_load_timerange = TimeRange.parse_timerange("20180110-20180130") - new_timerange = TimeRange.parse_timerange("20180120-20180130") - - freqai.extract_data_and_train_model( - new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) - - assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").is_file() - assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file() - assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file() - assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").is_file() - - # start the follower and ask it to predict on existing files - - freqai_conf.get("freqai", {}).update({"follow_mode": "true"}) - - strategy = get_patched_freqai_strategy(mocker, freqai_conf) - exchange = get_patched_exchange(mocker, freqai_conf) - strategy.dp = DataProvider(freqai_conf, exchange) - strategy.freqai_info = freqai_conf.get("freqai", {}) - freqai = strategy.freqai - freqai.live = True - freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.live) - timerange = TimeRange.parse_timerange("20180110-20180130") - freqai.dd.load_all_pair_histories(timerange, freqai.dk) - - df = strategy.dp.get_pair_dataframe('ADA/BTC', '5m') - - freqai.dk.pair = "ADA/BTC" - freqai.start_live(df, metadata, strategy, freqai.dk) - - assert len(freqai.dk.return_dataframe.index) == 5702 - - shutil.rmtree(Path(freqai.dk.full_path)) - - def test_principal_component_analysis(mocker, freqai_conf): freqai_conf.update({"timerange": "20180110-20180130"}) freqai_conf.get("freqai", {}).get("feature_parameters", {}).update(