From 0c34104e45f4fbeae66036452ade47fff13ff058 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 17 Aug 2022 15:18:44 +0200 Subject: [PATCH] extract download-data from freqai to prepare for future async changes --- freqtrade/freqai/data_kitchen.py | 102 ++++++++++++++++++++------- freqtrade/freqai/freqai_interface.py | 12 ++-- freqtrade/strategy/interface.py | 13 +++- 3 files changed, 94 insertions(+), 33 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 35f51baed..4554a5c1a 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -816,7 +816,7 @@ class FreqaiDataKitchen: return False def check_if_new_training_required( - self, trained_timestamp: int + self, trained_timestamp: int = 0 ) -> Tuple[bool, TimeRange, TimeRange]: time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() @@ -889,31 +889,6 @@ class FreqaiDataKitchen: self.model_filename = f"cb_{coin.lower()}_{int(trained_timerange.stopts)}" - def download_all_data_for_training(self, timerange: TimeRange, dp: DataProvider) -> None: - """ - Called only once upon start of bot to download the necessary data for - populating indicators and training the model. - :param timerange: TimeRange = The full data timerange for populating the indicators - and training the model. - :param dp: DataProvider instance attached to the strategy - """ - new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) - if not dp._exchange: - # Not realistic - this is only called in live mode. - raise OperationalException("Dataprovider did not have an exchange attached.") - refresh_backtest_ohlcv_data( - dp._exchange, - pairs=self.all_pairs, - timeframes=self.freqai_config["feature_parameters"].get("include_timeframes"), - datadir=self.config["datadir"], - timerange=timerange, - new_pairs_days=new_pairs_days, - erase=False, - data_format=self.config.get("dataformat_ohlcv", "json"), - trading_mode=self.config.get("trading_mode", "spot"), - prepend=self.config.get("prepend_data", False), - ) - def set_all_pairs(self) -> None: self.all_pairs = copy.deepcopy( @@ -1027,3 +1002,78 @@ class FreqaiDataKitchen: if self.unique_classes: for label in self.unique_classes: self.unique_class_list += list(self.unique_classes[label]) + +# Methods called by interface.py (load_freqai_model()) + + +def download_all_data_for_training(timerange: TimeRange, + dp: DataProvider, config: dict) -> None: + """ + Called only once upon start of bot to download the necessary data for + populating indicators and training the model. + :param timerange: TimeRange = The full data timerange for populating the indicators + and training the model. + :param dp: DataProvider instance attached to the strategy + """ + all_pairs = copy.deepcopy( + config["freqai"]["feature_parameters"].get("include_corr_pairlist", []) + ) + for pair in config.get("exchange", "").get("pair_whitelist"): + if pair not in all_pairs: + all_pairs.append(pair) + + new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) + if not dp._exchange: + # Not realistic - this is only called in live mode. + raise OperationalException("Dataprovider did not have an exchange attached.") + refresh_backtest_ohlcv_data( + dp._exchange, + pairs=all_pairs, + timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"), + datadir=config["datadir"], + timerange=timerange, + new_pairs_days=new_pairs_days, + erase=False, + data_format=config.get("dataformat_ohlcv", "json"), + trading_mode=config.get("trading_mode", "spot"), + prepend=config.get("prepend_data", False), + ) + + +def get_required_data_timerange( + config: dict +) -> TimeRange: + """ + Used by interface.py to pre-download necessary data for FreqAI + user. + """ + time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() + trained_timerange = TimeRange() + data_load_timerange = TimeRange() + + timeframes = config["freqai"]["feature_parameters"].get("include_timeframes") + + max_tf_seconds = 0 + for tf in timeframes: + secs = timeframe_to_seconds(tf) + if secs > max_tf_seconds: + max_tf_seconds = secs + + max_period = config["freqai"]["feature_parameters"].get( + "indicator_max_period_candles", 20 + ) * 2 + additional_seconds = max_period * max_tf_seconds + + trained_timerange.startts = int( + time - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY + ) + trained_timerange.stopts = int(time) + + data_load_timerange.startts = int( + time + - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY + - additional_seconds + ) + data_load_timerange.stopts = int(time) + + return data_load_timerange diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 49e4ce5c3..5d85cc225 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -278,12 +278,12 @@ class IFreqaiModel(ABC): # download candle history if it is not already in memory if not self.dd.historic_data: - logger.info( - "Downloading all training data for all pairs in whitelist and " - "corr_pairlist, this may take a while if you do not have the " - "data saved" - ) - dk.download_all_data_for_training(data_load_timerange, strategy.dp) + # logger.info( + # "Downloading all training data for all pairs in whitelist and " + # "corr_pairlist, this may take a while if you do not have the " + # "data saved" + # ) + # dk.download_all_data_for_training(data_load_timerange, strategy.dp) self.dd.load_all_pair_histories(data_load_timerange, dk) if not self.scanning: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 79dbd4c69..20a35ac3e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -149,9 +149,20 @@ class IStrategy(ABC, HyperStrategyMixin): if self.config.get('freqai', {}).get('enabled', False): # Import here to avoid importing this if freqAI is disabled from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver - + from freqtrade.freqai.data_kitchen import (get_required_data_timerange, + download_all_data_for_training) self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) self.freqai_info = self.config["freqai"] + + # download the desired data in dry/live + if self.config.get('runmode') in (RunMode.DRY_RUN, RunMode.LIVE): + logger.info( + "Downloading all training data for all pairs in whitelist and " + "corr_pairlist, this may take a while if you do not have the " + "data saved" + ) + data_load_timerange = get_required_data_timerange(self.config) + download_all_data_for_training(data_load_timerange, self.dp, self.config) else: # Gracious failures if freqAI is disabled but "start" is called. class DummyClass():