From 0c34104e45f4fbeae66036452ade47fff13ff058 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 17 Aug 2022 15:18:44 +0200 Subject: [PATCH 01/11] extract download-data from freqai to prepare for future async changes --- freqtrade/freqai/data_kitchen.py | 102 ++++++++++++++++++++------- freqtrade/freqai/freqai_interface.py | 12 ++-- freqtrade/strategy/interface.py | 13 +++- 3 files changed, 94 insertions(+), 33 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 35f51baed..4554a5c1a 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -816,7 +816,7 @@ class FreqaiDataKitchen: return False def check_if_new_training_required( - self, trained_timestamp: int + self, trained_timestamp: int = 0 ) -> Tuple[bool, TimeRange, TimeRange]: time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() @@ -889,31 +889,6 @@ class FreqaiDataKitchen: self.model_filename = f"cb_{coin.lower()}_{int(trained_timerange.stopts)}" - def download_all_data_for_training(self, timerange: TimeRange, dp: DataProvider) -> None: - """ - Called only once upon start of bot to download the necessary data for - populating indicators and training the model. - :param timerange: TimeRange = The full data timerange for populating the indicators - and training the model. - :param dp: DataProvider instance attached to the strategy - """ - new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) - if not dp._exchange: - # Not realistic - this is only called in live mode. - raise OperationalException("Dataprovider did not have an exchange attached.") - refresh_backtest_ohlcv_data( - dp._exchange, - pairs=self.all_pairs, - timeframes=self.freqai_config["feature_parameters"].get("include_timeframes"), - datadir=self.config["datadir"], - timerange=timerange, - new_pairs_days=new_pairs_days, - erase=False, - data_format=self.config.get("dataformat_ohlcv", "json"), - trading_mode=self.config.get("trading_mode", "spot"), - prepend=self.config.get("prepend_data", False), - ) - def set_all_pairs(self) -> None: self.all_pairs = copy.deepcopy( @@ -1027,3 +1002,78 @@ class FreqaiDataKitchen: if self.unique_classes: for label in self.unique_classes: self.unique_class_list += list(self.unique_classes[label]) + +# Methods called by interface.py (load_freqai_model()) + + +def download_all_data_for_training(timerange: TimeRange, + dp: DataProvider, config: dict) -> None: + """ + Called only once upon start of bot to download the necessary data for + populating indicators and training the model. + :param timerange: TimeRange = The full data timerange for populating the indicators + and training the model. + :param dp: DataProvider instance attached to the strategy + """ + all_pairs = copy.deepcopy( + config["freqai"]["feature_parameters"].get("include_corr_pairlist", []) + ) + for pair in config.get("exchange", "").get("pair_whitelist"): + if pair not in all_pairs: + all_pairs.append(pair) + + new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) + if not dp._exchange: + # Not realistic - this is only called in live mode. + raise OperationalException("Dataprovider did not have an exchange attached.") + refresh_backtest_ohlcv_data( + dp._exchange, + pairs=all_pairs, + timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"), + datadir=config["datadir"], + timerange=timerange, + new_pairs_days=new_pairs_days, + erase=False, + data_format=config.get("dataformat_ohlcv", "json"), + trading_mode=config.get("trading_mode", "spot"), + prepend=config.get("prepend_data", False), + ) + + +def get_required_data_timerange( + config: dict +) -> TimeRange: + """ + Used by interface.py to pre-download necessary data for FreqAI + user. + """ + time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() + trained_timerange = TimeRange() + data_load_timerange = TimeRange() + + timeframes = config["freqai"]["feature_parameters"].get("include_timeframes") + + max_tf_seconds = 0 + for tf in timeframes: + secs = timeframe_to_seconds(tf) + if secs > max_tf_seconds: + max_tf_seconds = secs + + max_period = config["freqai"]["feature_parameters"].get( + "indicator_max_period_candles", 20 + ) * 2 + additional_seconds = max_period * max_tf_seconds + + trained_timerange.startts = int( + time - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY + ) + trained_timerange.stopts = int(time) + + data_load_timerange.startts = int( + time + - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY + - additional_seconds + ) + data_load_timerange.stopts = int(time) + + return data_load_timerange diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 49e4ce5c3..5d85cc225 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -278,12 +278,12 @@ class IFreqaiModel(ABC): # download candle history if it is not already in memory if not self.dd.historic_data: - logger.info( - "Downloading all training data for all pairs in whitelist and " - "corr_pairlist, this may take a while if you do not have the " - "data saved" - ) - dk.download_all_data_for_training(data_load_timerange, strategy.dp) + # logger.info( + # "Downloading all training data for all pairs in whitelist and " + # "corr_pairlist, this may take a while if you do not have the " + # "data saved" + # ) + # dk.download_all_data_for_training(data_load_timerange, strategy.dp) self.dd.load_all_pair_histories(data_load_timerange, dk) if not self.scanning: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 79dbd4c69..20a35ac3e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -149,9 +149,20 @@ class IStrategy(ABC, HyperStrategyMixin): if self.config.get('freqai', {}).get('enabled', False): # Import here to avoid importing this if freqAI is disabled from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver - + from freqtrade.freqai.data_kitchen import (get_required_data_timerange, + download_all_data_for_training) self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) self.freqai_info = self.config["freqai"] + + # download the desired data in dry/live + if self.config.get('runmode') in (RunMode.DRY_RUN, RunMode.LIVE): + logger.info( + "Downloading all training data for all pairs in whitelist and " + "corr_pairlist, this may take a while if you do not have the " + "data saved" + ) + data_load_timerange = get_required_data_timerange(self.config) + download_all_data_for_training(data_load_timerange, self.dp, self.config) else: # Gracious failures if freqAI is disabled but "start" is called. class DummyClass(): From 5155afb4e7adb62b219fab65df86ac29524e405a Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 17 Aug 2022 15:22:48 +0200 Subject: [PATCH 02/11] clean up code remnants --- freqtrade/freqai/data_kitchen.py | 2 +- freqtrade/freqai/freqai_interface.py | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 4554a5c1a..6541261eb 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -816,7 +816,7 @@ class FreqaiDataKitchen: return False def check_if_new_training_required( - self, trained_timestamp: int = 0 + self, trained_timestamp: int ) -> Tuple[bool, TimeRange, TimeRange]: time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 5d85cc225..1a9e549f6 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -276,14 +276,8 @@ class IFreqaiModel(ABC): ) dk.set_paths(metadata["pair"], new_trained_timerange.stopts) - # download candle history if it is not already in memory + # load candle history into memory if it is not yet. if not self.dd.historic_data: - # logger.info( - # "Downloading all training data for all pairs in whitelist and " - # "corr_pairlist, this may take a while if you do not have the " - # "data saved" - # ) - # dk.download_all_data_for_training(data_load_timerange, strategy.dp) self.dd.load_all_pair_histories(data_load_timerange, dk) if not self.scanning: From 88dd9920ea9dd66c17a55fad4d6fb69cacefb8c2 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 17 Aug 2022 16:38:09 +0200 Subject: [PATCH 03/11] sort imports for isort --- freqtrade/strategy/interface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 20a35ac3e..1e51701f7 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -148,9 +148,9 @@ class IStrategy(ABC, HyperStrategyMixin): def load_freqAI_model(self) -> None: if self.config.get('freqai', {}).get('enabled', False): # Import here to avoid importing this if freqAI is disabled + from freqtrade.freqai.data_kitchen import (download_all_data_for_training, + get_required_data_timerange) from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver - from freqtrade.freqai.data_kitchen import (get_required_data_timerange, - download_all_data_for_training) self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) self.freqai_info = self.config["freqai"] From ac42c0153da716e0a70abd7a8024d8d3162e4ba6 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Mon, 22 Aug 2022 18:19:07 +0200 Subject: [PATCH 04/11] deprecate indicator_max_period_candles, automatically compute startup candles for FreqAI backtesting. --- config_examples/config_freqai.example.json | 2 +- docs/freqai.md | 2 +- docs/strategy-customization.md | 2 +- freqtrade/data/dataprovider.py | 13 +++++++- freqtrade/freqai/data_kitchen.py | 32 +++++++++---------- freqtrade/optimize/backtesting.py | 33 ++++++++++++++------ freqtrade/strategy/interface.py | 1 + freqtrade/templates/FreqaiExampleStrategy.py | 3 +- tests/freqai/conftest.py | 1 - tests/freqai/test_freqai_backtesting.py | 10 +++--- 10 files changed, 61 insertions(+), 38 deletions(-) diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json index aeb1cb13d..093e11b2a 100644 --- a/config_examples/config_freqai.example.json +++ b/config_examples/config_freqai.example.json @@ -9,6 +9,7 @@ "dry_run": true, "timeframe": "3m", "dry_run_wallet": 1000, + "startup_candle_count": 20, "cancel_open_orders_on_exit": true, "unfilledtimeout": { "entry": 10, @@ -53,7 +54,6 @@ ], "freqai": { "enabled": true, - "startup_candles": 10000, "purge_old_models": true, "train_period_days": 15, "backtest_period_days": 7, diff --git a/docs/freqai.md b/docs/freqai.md index b22e1cd31..f3c9021ed 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -113,7 +113,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `use_SVM_to_remove_outliers` | Ask FreqAI to train a support vector machine to detect and remove outliers from the training data set as well as from incoming data points.
**Datatype:** boolean. | `svm_params` | All parameters available in Sklearn's `SGDOneClassSVM()`. E.g. `nu` *Very* broadly, is the percentage of data points that should be considered outliers. `shuffle` is by default false to maintain reproducibility. But these and all others can be added/changed in this dictionary.
**Datatype:** dictionary. | `stratify_training_data` | This value is used to indicate the stratification of the data. e.g. 2 would set every 2nd data point into a separate dataset to be pulled from during training/testing.
**Datatype:** positive integer. -| `indicator_max_period_candles` | The maximum *period* used in `populate_any_indicators()` for indicator creation. FreqAI uses this information in combination with the maximum timeframe to calculate how many data points it should download so that the first data point does not have a NaN
**Datatype:** positive integer. +| `indicator_max_period_candles` | **Deprecated in favor of** strategy set `startup_candle_count`, however, both configuration parameters provide the same functionality; the maximum *period* used in `populate_any_indicators()` for indicator creation (timeframe independent). FreqAI uses this information in combination with the maximum timeframe to calculate how many data points it should download so that the first data point does not have a NaN
**Datatype:** positive integer. | `indicator_periods_candles` | A list of integers used to duplicate all indicators according to a set of periods and add them to the feature set.
**Datatype:** list of positive integers. | `use_DBSCAN_to_remove_outliers` | Inactive by default. If true, FreqAI clusters data using DBSCAN to identify and remove outliers from training and prediction data.
**Datatype:** float (fraction of 1). | | **Data split parameters** diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 260e253c4..a452b8f05 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -166,7 +166,7 @@ Additional technical libraries can be installed as necessary, or custom indicato Most indicators have an instable startup period, in which they are either not available (NaN), or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be. To account for this, the strategy can be assigned the `startup_candle_count` attribute. -This should be set to the maximum number of candles that the strategy requires to calculate stable indicators. +This should be set to the maximum number of candles that the strategy requires to calculate stable indicators. In the case where a user includes higher timeframes with informative pairs, the `startup_candle_count` does not necessarily change. The value is the maximum period (in candles) that any of the informatives timeframes need to compute stable indicators. In this example strategy, this should be set to 100 (`startup_candle_count = 100`), since the longest needed history is 100 candles. diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 21cead77f..529a12690 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -92,7 +92,7 @@ class DataProvider: 'timerange') is None else str(self._config.get('timerange'))) # Move informative start time respecting startup_candle_count timerange.subtract_start( - timeframe_to_seconds(str(timeframe)) * self._config.get('startup_candle_count', 0) + self.get_required_startup_seconds(str(timeframe)) ) self.__cached_pairs_backtesting[saved_pair] = load_pair_history( pair=pair, @@ -105,6 +105,17 @@ class DataProvider: ) return self.__cached_pairs_backtesting[saved_pair].copy() + def get_required_startup_seconds(self, timeframe: str) -> int: + tf_seconds = timeframe_to_seconds(timeframe) + base_seconds = tf_seconds * self._config.get('startup_candle_count', 0) + if not self._config['freqai']['enabled']: + return base_seconds + else: + train_seconds = self._config['freqai']['train_period_days'] * 86400 + # multiplied by safety factor of 2 because FreqAI users + # typically do not know the correct window. + return base_seconds * 2 + int(train_seconds) + def get_pair_dataframe( self, pair: str, diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 6541261eb..c768fc30e 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -20,6 +20,8 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds +from freqtrade.exchange.exchange import market_is_active +from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.strategy.interface import IStrategy @@ -834,9 +836,7 @@ class FreqaiDataKitchen: # We notice that users like to use exotic indicators where # they do not know the required timeperiod. Here we include a factor # of safety by multiplying the user considered "max" by 2. - max_period = self.freqai_config["feature_parameters"].get( - "indicator_max_period_candles", 20 - ) * 2 + max_period = self.config.get('startup_candle_count', 20) * 2 additional_seconds = max_period * max_tf_seconds if trained_timestamp != 0: @@ -1015,12 +1015,15 @@ def download_all_data_for_training(timerange: TimeRange, and training the model. :param dp: DataProvider instance attached to the strategy """ - all_pairs = copy.deepcopy( - config["freqai"]["feature_parameters"].get("include_corr_pairlist", []) - ) - for pair in config.get("exchange", "").get("pair_whitelist"): - if pair not in all_pairs: - all_pairs.append(pair) + + if dp._exchange is not None: + markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) + or config.get('include_inactive')] + else: + # This should not occur: + raise OperationalException('No exchange object found.') + + all_pairs = dynamic_expand_pairlist(config, markets) new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) if not dp._exchange: @@ -1048,7 +1051,6 @@ def get_required_data_timerange( user. """ time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() - trained_timerange = TimeRange() data_load_timerange = TimeRange() timeframes = config["freqai"]["feature_parameters"].get("include_timeframes") @@ -1059,15 +1061,9 @@ def get_required_data_timerange( if secs > max_tf_seconds: max_tf_seconds = secs - max_period = config["freqai"]["feature_parameters"].get( - "indicator_max_period_candles", 20 - ) * 2 - additional_seconds = max_period * max_tf_seconds + max_period = config.get('startup_candle_count', 20) * 2 - trained_timerange.startts = int( - time - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY - ) - trained_timerange.stopts = int(time) + additional_seconds = max_period * max_tf_seconds data_load_timerange.startts = int( time diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6528481d5..8f0302ada 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -211,21 +211,21 @@ class Backtesting: """ self.progress.init_step(BacktestState.DATALOAD, 1) - if self.config.get('freqai', {}).get('enabled', False): - startup_candles = int(self.config.get('freqai', {}).get('startup_candles', 0)) - if not startup_candles: - raise OperationalException('FreqAI backtesting module requires user set ' - 'startup_candles in config.') - self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 0)) - logger.info(f'Increasing startup_candle_count for freqai to {self.required_startup}') - self.config['startup_candle_count'] = self.required_startup + # if self.config.get('freqai', {}).get('enabled', False): + # startup_candles = int(self.config.get('freqai', {}).get('startup_candles', 0)) + # if not startup_candles: + # raise OperationalException('FreqAI backtesting module requires user set ' + # 'startup_candles in config.') + # self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 0)) + # logger.info(f'Increasing startup_candle_count for freqai to {self.required_startup}') + # self.config['startup_candle_count'] = self.required_startup data = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, timeframe=self.timeframe, timerange=self.timerange, - startup_candles=self.required_startup, + startup_candles=self.get_required_startup(self.timeframe), fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), candle_type=self.config.get('candle_type_def', CandleType.SPOT) @@ -244,6 +244,21 @@ class Backtesting: self.progress.set_new_value(1) return data, self.timerange + def get_required_startup(self, timeframe: str) -> int: + if not self.config['freqai']['enabled']: + return self.required_startup + else: + if not self.config['startup_candle_count']: + raise OperationalException('FreqAI backtesting module requires strategy ' + 'set startup_candle_count.') + tf_seconds = timeframe_to_seconds(timeframe) + train_candles = self.config['freqai']['train_period_days'] * 86400 / tf_seconds + # multiplied by safety factor of 2 because FreqAI users + # typically do not know the correct window. + total_candles = self.required_startup * 2 + train_candles + logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') + return total_candles + def load_bt_data_detail(self) -> None: """ Loads backtest detail data (smaller timeframe) if necessary. diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 1e51701f7..284727d2b 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -163,6 +163,7 @@ class IStrategy(ABC, HyperStrategyMixin): ) data_load_timerange = get_required_data_timerange(self.config) download_all_data_for_training(data_load_timerange, self.dp, self.config) + else: # Gracious failures if freqAI is disabled but "start" is called. class DummyClass(): diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 5810e7881..aa584bfbc 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -43,7 +43,8 @@ class FreqaiExampleStrategy(IStrategy): process_only_new_candles = True stoploss = -0.05 use_exit_signal = True - startup_candle_count: int = 300 + # this is the maximum period fed to talib (timeframe independent) + startup_candle_count: int = 20 can_short = False linear_roi_offset = DecimalParameter( diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index 6ace13677..113cb3a79 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -44,7 +44,6 @@ def freqai_conf(default_conf, tmpdir): "principal_component_analysis": False, "use_SVM_to_remove_outliers": True, "stratify_training_data": 0, - "indicator_max_period_candles": 10, "indicator_periods_candles": [10], }, "data_split_parameters": {"test_size": 0.33, "random_state": 1}, diff --git a/tests/freqai/test_freqai_backtesting.py b/tests/freqai/test_freqai_backtesting.py index 273791609..c8a51edb0 100644 --- a/tests/freqai/test_freqai_backtesting.py +++ b/tests/freqai/test_freqai_backtesting.py @@ -48,10 +48,10 @@ def test_freqai_backtest_load_data(freqai_conf, mocker, caplog): assert log_has_re('Increasing startup_candle_count for freqai to.*', caplog) - del freqai_conf['freqai']['startup_candles'] - backtesting = Backtesting(freqai_conf) - with pytest.raises(OperationalException, - match=r'FreqAI backtesting module.*startup_candles in config.'): - backtesting.load_bt_data() + # del freqai_conf['freqai']['startup_candles'] + # backtesting = Backtesting(freqai_conf) + # with pytest.raises(OperationalException, + # match=r'FreqAI backtesting module.*startup_candles in config.'): + # backtesting.load_bt_data() Backtesting.cleanup() From 4b7e640f31f2a35ba3b73a3bfbbfb2882ecb7a81 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 26 Aug 2022 13:56:44 +0200 Subject: [PATCH 05/11] reduce code duplication, optimize auto data download per tf --- freqtrade/data/dataprovider.py | 26 ++++++------ freqtrade/freqai/data_kitchen.py | 67 +++++++++++-------------------- freqtrade/optimize/backtesting.py | 26 +----------- freqtrade/strategy/interface.py | 7 ++-- 4 files changed, 41 insertions(+), 85 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 529a12690..a21114901 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -91,9 +91,9 @@ class DataProvider: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) # Move informative start time respecting startup_candle_count - timerange.subtract_start( - self.get_required_startup_seconds(str(timeframe)) - ) + startup_candles = self.get_required_startup(str(timeframe)) + tf_seconds = timeframe_to_seconds(str(timeframe)) + timerange.subtract_start(tf_seconds * startup_candles) self.__cached_pairs_backtesting[saved_pair] = load_pair_history( pair=pair, timeframe=timeframe or self._config['timeframe'], @@ -105,16 +105,18 @@ class DataProvider: ) return self.__cached_pairs_backtesting[saved_pair].copy() - def get_required_startup_seconds(self, timeframe: str) -> int: - tf_seconds = timeframe_to_seconds(timeframe) - base_seconds = tf_seconds * self._config.get('startup_candle_count', 0) - if not self._config['freqai']['enabled']: - return base_seconds + def get_required_startup(self, timeframe: str) -> int: + if not self._config.get('freqai', {}).get('enabled', False): + return self._config.get('startup_candle_count', 0) else: - train_seconds = self._config['freqai']['train_period_days'] * 86400 - # multiplied by safety factor of 2 because FreqAI users - # typically do not know the correct window. - return base_seconds * 2 + int(train_seconds) + if not self._config['startup_candle_count']: + raise OperationalException('FreqAI backtesting module requires strategy ' + 'set startup_candle_count.') + tf_seconds = timeframe_to_seconds(timeframe) + train_candles = self._config['freqai']['train_period_days'] * 86400 / tf_seconds + total_candles = int(self._config.get('startup_candle_count', 0) + train_candles) + logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') + return total_candles def get_pair_dataframe( self, diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index c768fc30e..1a8063add 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1006,8 +1006,7 @@ class FreqaiDataKitchen: # Methods called by interface.py (load_freqai_model()) -def download_all_data_for_training(timerange: TimeRange, - dp: DataProvider, config: dict) -> None: +def download_all_data_for_training(dp: DataProvider, config: dict) -> None: """ Called only once upon start of bot to download the necessary data for populating indicators and training the model. @@ -1025,51 +1024,31 @@ def download_all_data_for_training(timerange: TimeRange, all_pairs = dynamic_expand_pairlist(config, markets) - new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) if not dp._exchange: # Not realistic - this is only called in live mode. raise OperationalException("Dataprovider did not have an exchange attached.") - refresh_backtest_ohlcv_data( - dp._exchange, - pairs=all_pairs, - timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"), - datadir=config["datadir"], - timerange=timerange, - new_pairs_days=new_pairs_days, - erase=False, - data_format=config.get("dataformat_ohlcv", "json"), - trading_mode=config.get("trading_mode", "spot"), - prepend=config.get("prepend_data", False), - ) - -def get_required_data_timerange( - config: dict -) -> TimeRange: - """ - Used by interface.py to pre-download necessary data for FreqAI - user. - """ time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() - data_load_timerange = TimeRange() - timeframes = config["freqai"]["feature_parameters"].get("include_timeframes") - - max_tf_seconds = 0 - for tf in timeframes: - secs = timeframe_to_seconds(tf) - if secs > max_tf_seconds: - max_tf_seconds = secs - - max_period = config.get('startup_candle_count', 20) * 2 - - additional_seconds = max_period * max_tf_seconds - - data_load_timerange.startts = int( - time - - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY - - additional_seconds - ) - data_load_timerange.stopts = int(time) - - return data_load_timerange + for tf in config["freqai"]["feature_parameters"].get("include_timeframes"): + timerange = TimeRange() + timerange.startts = int(time) + timerange.stopts = int(time) + startup_candles = dp.get_required_startup(str(tf)) + tf_seconds = timeframe_to_seconds(str(tf)) + timerange.subtract_start(tf_seconds * startup_candles) + new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) + # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function + # redownloads the funding rate for each pair. + refresh_backtest_ohlcv_data( + dp._exchange, + pairs=all_pairs, + timeframes=[tf], + datadir=config["datadir"], + timerange=timerange, + new_pairs_days=new_pairs_days, + erase=False, + data_format=config.get("dataformat_ohlcv", "json"), + trading_mode=config.get("trading_mode", "spot"), + prepend=config.get("prepend_data", False), + ) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8f0302ada..3d715c82d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -211,21 +211,12 @@ class Backtesting: """ self.progress.init_step(BacktestState.DATALOAD, 1) - # if self.config.get('freqai', {}).get('enabled', False): - # startup_candles = int(self.config.get('freqai', {}).get('startup_candles', 0)) - # if not startup_candles: - # raise OperationalException('FreqAI backtesting module requires user set ' - # 'startup_candles in config.') - # self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 0)) - # logger.info(f'Increasing startup_candle_count for freqai to {self.required_startup}') - # self.config['startup_candle_count'] = self.required_startup - data = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, timeframe=self.timeframe, timerange=self.timerange, - startup_candles=self.get_required_startup(self.timeframe), + startup_candles=self.dataprovider.get_required_startup(self.timeframe), fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), candle_type=self.config.get('candle_type_def', CandleType.SPOT) @@ -244,21 +235,6 @@ class Backtesting: self.progress.set_new_value(1) return data, self.timerange - def get_required_startup(self, timeframe: str) -> int: - if not self.config['freqai']['enabled']: - return self.required_startup - else: - if not self.config['startup_candle_count']: - raise OperationalException('FreqAI backtesting module requires strategy ' - 'set startup_candle_count.') - tf_seconds = timeframe_to_seconds(timeframe) - train_candles = self.config['freqai']['train_period_days'] * 86400 / tf_seconds - # multiplied by safety factor of 2 because FreqAI users - # typically do not know the correct window. - total_candles = self.required_startup * 2 + train_candles - logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') - return total_candles - def load_bt_data_detail(self) -> None: """ Loads backtest detail data (smaller timeframe) if necessary. diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 284727d2b..9124a0427 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -148,8 +148,7 @@ class IStrategy(ABC, HyperStrategyMixin): def load_freqAI_model(self) -> None: if self.config.get('freqai', {}).get('enabled', False): # Import here to avoid importing this if freqAI is disabled - from freqtrade.freqai.data_kitchen import (download_all_data_for_training, - get_required_data_timerange) + from freqtrade.freqai.data_kitchen import (download_all_data_for_training) from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) self.freqai_info = self.config["freqai"] @@ -161,8 +160,8 @@ class IStrategy(ABC, HyperStrategyMixin): "corr_pairlist, this may take a while if you do not have the " "data saved" ) - data_load_timerange = get_required_data_timerange(self.config) - download_all_data_for_training(data_load_timerange, self.dp, self.config) + # data_load_timerange = get_required_data_timerange(self.config) + download_all_data_for_training(self.dp, self.config) else: # Gracious failures if freqAI is disabled but "start" is called. From 65b552e310fe751989e43848498e03157cc50232 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 26 Aug 2022 15:30:01 +0200 Subject: [PATCH 06/11] make docs reflect reality, move download_all_data to new utils.py file, automatic startup_candle detection --- docs/freqai.md | 7 +++-- freqtrade/data/dataprovider.py | 14 +++++---- freqtrade/freqai/data_kitchen.py | 54 -------------------------------- freqtrade/strategy/interface.py | 2 +- 4 files changed, 14 insertions(+), 63 deletions(-) diff --git a/docs/freqai.md b/docs/freqai.md index f3c9021ed..bd746c984 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -113,7 +113,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `use_SVM_to_remove_outliers` | Ask FreqAI to train a support vector machine to detect and remove outliers from the training data set as well as from incoming data points.
**Datatype:** boolean. | `svm_params` | All parameters available in Sklearn's `SGDOneClassSVM()`. E.g. `nu` *Very* broadly, is the percentage of data points that should be considered outliers. `shuffle` is by default false to maintain reproducibility. But these and all others can be added/changed in this dictionary.
**Datatype:** dictionary. | `stratify_training_data` | This value is used to indicate the stratification of the data. e.g. 2 would set every 2nd data point into a separate dataset to be pulled from during training/testing.
**Datatype:** positive integer. -| `indicator_max_period_candles` | **Deprecated in favor of** strategy set `startup_candle_count`, however, both configuration parameters provide the same functionality; the maximum *period* used in `populate_any_indicators()` for indicator creation (timeframe independent). FreqAI uses this information in combination with the maximum timeframe to calculate how many data points it should download so that the first data point does not have a NaN
**Datatype:** positive integer. +| `indicator_max_period_candles` | **No longer used**. User must use the strategy set `startup_candle_count` which defines the maximum *period* used in `populate_any_indicators()` for indicator creation (timeframe independent). FreqAI uses this information in combination with the maximum timeframe to calculate how many data points it should download so that the first data point does not have a NaN
**Datatype:** positive integer. | `indicator_periods_candles` | A list of integers used to duplicate all indicators according to a set of periods and add them to the feature set.
**Datatype:** list of positive integers. | `use_DBSCAN_to_remove_outliers` | Inactive by default. If true, FreqAI clusters data using DBSCAN to identify and remove outliers from training and prediction data.
**Datatype:** float (fraction of 1). | | **Data split parameters** @@ -162,7 +162,6 @@ The user interface is isolated to the typical config file. A typical FreqAI conf "label_period_candles": 24, "include_shifted_candles": 2, "weight_factor": 0, - "indicator_max_period_candles": 20, "indicator_periods_candles": [10, 20] }, "data_split_parameters" : { @@ -387,6 +386,10 @@ The FreqAI strategy requires the user to include the following lines of code in ```python + # user should define the maximum startup candle count (the largest number of candles + # passed to any single indicator) + startup_candle_count: int = 20 + def informative_pairs(self): whitelist_pairs = self.dp.current_whitelist() corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index a21114901..4151b7419 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -106,15 +106,17 @@ class DataProvider: return self.__cached_pairs_backtesting[saved_pair].copy() def get_required_startup(self, timeframe: str) -> int: - if not self._config.get('freqai', {}).get('enabled', False): + freqai_config = self._config.get('freqai', {}) + if not freqai_config.get('enabled', False): return self._config.get('startup_candle_count', 0) else: - if not self._config['startup_candle_count']: - raise OperationalException('FreqAI backtesting module requires strategy ' - 'set startup_candle_count.') + startup_candles = self._config.get('startup_candle_count', 0) + indicator_periods = freqai_config['feature_parameters']['indicator_periods_candles'] + # make sure the startupcandles is at least the set maximum indicator periods + self._config['startup_candle_count'] = max(startup_candles, max(indicator_periods)) tf_seconds = timeframe_to_seconds(timeframe) - train_candles = self._config['freqai']['train_period_days'] * 86400 / tf_seconds - total_candles = int(self._config.get('startup_candle_count', 0) + train_candles) + train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds + total_candles = int(self._config['startup_candle_count'] + train_candles) logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') return total_candles diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 1a8063add..1b88405c1 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -16,12 +16,8 @@ from sklearn.model_selection import train_test_split from sklearn.neighbors import NearestNeighbors from freqtrade.configuration import TimeRange -from freqtrade.data.dataprovider import DataProvider -from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds -from freqtrade.exchange.exchange import market_is_active -from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.strategy.interface import IStrategy @@ -1002,53 +998,3 @@ class FreqaiDataKitchen: if self.unique_classes: for label in self.unique_classes: self.unique_class_list += list(self.unique_classes[label]) - -# Methods called by interface.py (load_freqai_model()) - - -def download_all_data_for_training(dp: DataProvider, config: dict) -> None: - """ - Called only once upon start of bot to download the necessary data for - populating indicators and training the model. - :param timerange: TimeRange = The full data timerange for populating the indicators - and training the model. - :param dp: DataProvider instance attached to the strategy - """ - - if dp._exchange is not None: - markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) - or config.get('include_inactive')] - else: - # This should not occur: - raise OperationalException('No exchange object found.') - - all_pairs = dynamic_expand_pairlist(config, markets) - - if not dp._exchange: - # Not realistic - this is only called in live mode. - raise OperationalException("Dataprovider did not have an exchange attached.") - - time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() - - for tf in config["freqai"]["feature_parameters"].get("include_timeframes"): - timerange = TimeRange() - timerange.startts = int(time) - timerange.stopts = int(time) - startup_candles = dp.get_required_startup(str(tf)) - tf_seconds = timeframe_to_seconds(str(tf)) - timerange.subtract_start(tf_seconds * startup_candles) - new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) - # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function - # redownloads the funding rate for each pair. - refresh_backtest_ohlcv_data( - dp._exchange, - pairs=all_pairs, - timeframes=[tf], - datadir=config["datadir"], - timerange=timerange, - new_pairs_days=new_pairs_days, - erase=False, - data_format=config.get("dataformat_ohlcv", "json"), - trading_mode=config.get("trading_mode", "spot"), - prepend=config.get("prepend_data", False), - ) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 9124a0427..c9ec466de 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -148,7 +148,7 @@ class IStrategy(ABC, HyperStrategyMixin): def load_freqAI_model(self) -> None: if self.config.get('freqai', {}).get('enabled', False): # Import here to avoid importing this if freqAI is disabled - from freqtrade.freqai.data_kitchen import (download_all_data_for_training) + from freqtrade.freqai.utils import download_all_data_for_training from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) self.freqai_info = self.config["freqai"] From e7261cf51577dc30b530370f81df620c898f6a11 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 26 Aug 2022 15:30:28 +0200 Subject: [PATCH 07/11] add freqai utils.py file --- freqtrade/freqai/utils.py | 56 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 freqtrade/freqai/utils.py diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py new file mode 100644 index 000000000..056458115 --- /dev/null +++ b/freqtrade/freqai/utils.py @@ -0,0 +1,56 @@ +from freqtrade.data.dataprovider import DataProvider +from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist +from freqtrade.exchange.exchange import market_is_active +from freqtrade.exchange import timeframe_to_seconds +from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data +from datetime import datetime, timezone +from freqtrade.exceptions import OperationalException +from freqtrade.configuration import TimeRange + + +def download_all_data_for_training(dp: DataProvider, config: dict) -> None: + """ + Called only once upon start of bot to download the necessary data for + populating indicators and training a FreqAI model. + :param timerange: TimeRange = The full data timerange for populating the indicators + and training the model. + :param dp: DataProvider instance attached to the strategy + """ + + if dp._exchange is not None: + markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) + or config.get('include_inactive')] + else: + # This should not occur: + raise OperationalException('No exchange object found.') + + all_pairs = dynamic_expand_pairlist(config, markets) + + if not dp._exchange: + # Not realistic - this is only called in live mode. + raise OperationalException("Dataprovider did not have an exchange attached.") + + time = datetime.now(tz=timezone.utc).timestamp() + + for tf in config["freqai"]["feature_parameters"].get("include_timeframes"): + timerange = TimeRange() + timerange.startts = int(time) + timerange.stopts = int(time) + startup_candles = dp.get_required_startup(str(tf)) + tf_seconds = timeframe_to_seconds(str(tf)) + timerange.subtract_start(tf_seconds * startup_candles) + new_pairs_days = int((timerange.stopts - timerange.startts) / 86400) + # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function + # redownloads the funding rate for each pair. + refresh_backtest_ohlcv_data( + dp._exchange, + pairs=all_pairs, + timeframes=[tf], + datadir=config["datadir"], + timerange=timerange, + new_pairs_days=new_pairs_days, + erase=False, + data_format=config.get("dataformat_ohlcv", "json"), + trading_mode=config.get("trading_mode", "spot"), + prepend=config.get("prepend_data", False), + ) From bb3523f3838686f92420b27e58cc1b5a37df6b9e Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 26 Aug 2022 18:51:42 +0200 Subject: [PATCH 08/11] download data homogeneously across timeframes --- freqtrade/freqai/utils.py | 140 +++++++++++++++++++++++++------- freqtrade/strategy/interface.py | 6 +- 2 files changed, 113 insertions(+), 33 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 056458115..d56702049 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -1,17 +1,22 @@ -from freqtrade.data.dataprovider import DataProvider -from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist -from freqtrade.exchange.exchange import market_is_active -from freqtrade.exchange import timeframe_to_seconds -from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data +import logging from datetime import datetime, timezone -from freqtrade.exceptions import OperationalException + from freqtrade.configuration import TimeRange +from freqtrade.data.dataprovider import DataProvider +from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import timeframe_to_seconds +from freqtrade.exchange.exchange import market_is_active +from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist + + +logger = logging.getLogger(__name__) def download_all_data_for_training(dp: DataProvider, config: dict) -> None: """ Called only once upon start of bot to download the necessary data for - populating indicators and training a FreqAI model. + populating indicators and training the model. :param timerange: TimeRange = The full data timerange for populating the indicators and training the model. :param dp: DataProvider instance attached to the strategy @@ -26,31 +31,108 @@ def download_all_data_for_training(dp: DataProvider, config: dict) -> None: all_pairs = dynamic_expand_pairlist(config, markets) + timerange = get_required_data_timerange(config) + + new_pairs_days = int((timerange.stopts - timerange.startts) / 86400) if not dp._exchange: # Not realistic - this is only called in live mode. raise OperationalException("Dataprovider did not have an exchange attached.") + refresh_backtest_ohlcv_data( + dp._exchange, + pairs=all_pairs, + timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"), + datadir=config["datadir"], + timerange=timerange, + new_pairs_days=new_pairs_days, + erase=False, + data_format=config.get("dataformat_ohlcv", "json"), + trading_mode=config.get("trading_mode", "spot"), + prepend=config.get("prepend_data", False), + ) + +def get_required_data_timerange( + config: dict +) -> TimeRange: + """ + Used to compute the required data download time range + for auto data-download in FreqAI + """ time = datetime.now(tz=timezone.utc).timestamp() + data_load_timerange = TimeRange() - for tf in config["freqai"]["feature_parameters"].get("include_timeframes"): - timerange = TimeRange() - timerange.startts = int(time) - timerange.stopts = int(time) - startup_candles = dp.get_required_startup(str(tf)) - tf_seconds = timeframe_to_seconds(str(tf)) - timerange.subtract_start(tf_seconds * startup_candles) - new_pairs_days = int((timerange.stopts - timerange.startts) / 86400) - # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function - # redownloads the funding rate for each pair. - refresh_backtest_ohlcv_data( - dp._exchange, - pairs=all_pairs, - timeframes=[tf], - datadir=config["datadir"], - timerange=timerange, - new_pairs_days=new_pairs_days, - erase=False, - data_format=config.get("dataformat_ohlcv", "json"), - trading_mode=config.get("trading_mode", "spot"), - prepend=config.get("prepend_data", False), - ) + timeframes = config["freqai"]["feature_parameters"].get("include_timeframes") + + max_tf_seconds = 0 + for tf in timeframes: + secs = timeframe_to_seconds(tf) + if secs > max_tf_seconds: + max_tf_seconds = secs + + startup_candles = config.get('startup_candle_count', 0) + indicator_periods = config["freqai"]["feature_parameters"]["indicator_periods_candles"] + + # factor the max_period as a factor of safety. + max_period = int(max(startup_candles, max(indicator_periods)) * 1.5) + config['startup_candle_count'] = max_period + logger.info(f'FreqAI auto-downloader using {max_period} startup candles.') + + additional_seconds = max_period * max_tf_seconds + + data_load_timerange.startts = int( + time + - config["freqai"].get("train_period_days", 0) * 86400 + - additional_seconds + ) + data_load_timerange.stopts = int(time) + + return data_load_timerange + + +# Keep below for when we wish to download heterogeneously lengthed data for FreqAI. +# def download_all_data_for_training(dp: DataProvider, config: dict) -> None: +# """ +# Called only once upon start of bot to download the necessary data for +# populating indicators and training a FreqAI model. +# :param timerange: TimeRange = The full data timerange for populating the indicators +# and training the model. +# :param dp: DataProvider instance attached to the strategy +# """ + +# if dp._exchange is not None: +# markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) +# or config.get('include_inactive')] +# else: +# # This should not occur: +# raise OperationalException('No exchange object found.') + +# all_pairs = dynamic_expand_pairlist(config, markets) + +# if not dp._exchange: +# # Not realistic - this is only called in live mode. +# raise OperationalException("Dataprovider did not have an exchange attached.") + +# time = datetime.now(tz=timezone.utc).timestamp() + +# for tf in config["freqai"]["feature_parameters"].get("include_timeframes"): +# timerange = TimeRange() +# timerange.startts = int(time) +# timerange.stopts = int(time) +# startup_candles = dp.get_required_startup(str(tf)) +# tf_seconds = timeframe_to_seconds(str(tf)) +# timerange.subtract_start(tf_seconds * startup_candles) +# new_pairs_days = int((timerange.stopts - timerange.startts) / 86400) +# # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function +# # redownloads the funding rate for each pair. +# refresh_backtest_ohlcv_data( +# dp._exchange, +# pairs=all_pairs, +# timeframes=[tf], +# datadir=config["datadir"], +# timerange=timerange, +# new_pairs_days=new_pairs_days, +# erase=False, +# data_format=config.get("dataformat_ohlcv", "json"), +# trading_mode=config.get("trading_mode", "spot"), +# prepend=config.get("prepend_data", False), +# ) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index c9ec466de..3ea1a3fae 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -157,12 +157,10 @@ class IStrategy(ABC, HyperStrategyMixin): if self.config.get('runmode') in (RunMode.DRY_RUN, RunMode.LIVE): logger.info( "Downloading all training data for all pairs in whitelist and " - "corr_pairlist, this may take a while if you do not have the " - "data saved" + "corr_pairlist, this may take a while if the data is not " + "already on disk." ) - # data_load_timerange = get_required_data_timerange(self.config) download_all_data_for_training(self.dp, self.config) - else: # Gracious failures if freqAI is disabled but "start" is called. class DummyClass(): From 7ba4fda5d7c4f472bb61bf03fe91e7f2b1564762 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 Aug 2022 10:26:47 +0000 Subject: [PATCH 09/11] Implement PR feedback --- freqtrade/freqai/utils.py | 12 ++++-------- tests/freqai/test_freqai_backtesting.py | 6 ------ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index d56702049..6081b6ce5 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -22,21 +22,17 @@ def download_all_data_for_training(dp: DataProvider, config: dict) -> None: :param dp: DataProvider instance attached to the strategy """ - if dp._exchange is not None: - markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) - or config.get('include_inactive')] - else: - # This should not occur: + if dp._exchange is None: raise OperationalException('No exchange object found.') + markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) + or config.get('include_inactive')] all_pairs = dynamic_expand_pairlist(config, markets) timerange = get_required_data_timerange(config) new_pairs_days = int((timerange.stopts - timerange.startts) / 86400) - if not dp._exchange: - # Not realistic - this is only called in live mode. - raise OperationalException("Dataprovider did not have an exchange attached.") + refresh_backtest_ohlcv_data( dp._exchange, pairs=all_pairs, diff --git a/tests/freqai/test_freqai_backtesting.py b/tests/freqai/test_freqai_backtesting.py index c8a51edb0..ea127fa99 100644 --- a/tests/freqai/test_freqai_backtesting.py +++ b/tests/freqai/test_freqai_backtesting.py @@ -48,10 +48,4 @@ def test_freqai_backtest_load_data(freqai_conf, mocker, caplog): assert log_has_re('Increasing startup_candle_count for freqai to.*', caplog) - # del freqai_conf['freqai']['startup_candles'] - # backtesting = Backtesting(freqai_conf) - # with pytest.raises(OperationalException, - # match=r'FreqAI backtesting module.*startup_candles in config.'): - # backtesting.load_bt_data() - Backtesting.cleanup() From 13ccd940d5e9d9bacedb896cbe4859217f487dde Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 Aug 2022 10:26:58 +0000 Subject: [PATCH 10/11] Remove startup_candle_count from freqai sample config to avoid confusion --- config_examples/config_freqai.example.json | 8 +++++--- freqtrade/templates/FreqaiHybridExampleStrategy.py | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json index 7112fc225..13c7a94ea 100644 --- a/config_examples/config_freqai.example.json +++ b/config_examples/config_freqai.example.json @@ -9,7 +9,6 @@ "dry_run": true, "timeframe": "3m", "dry_run_wallet": 1000, - "startup_candle_count": 20, "cancel_open_orders_on_exit": true, "unfilledtimeout": { "entry": 10, @@ -76,7 +75,10 @@ "principal_component_analysis": false, "use_SVM_to_remove_outliers": true, "indicator_max_period_candles": 20, - "indicator_periods_candles": [10, 20] + "indicator_periods_candles": [ + 10, + 20 + ] }, "data_split_parameters": { "test_size": 0.33, @@ -92,4 +94,4 @@ "internals": { "process_throttle_secs": 5 } -} +} \ No newline at end of file diff --git a/freqtrade/templates/FreqaiHybridExampleStrategy.py b/freqtrade/templates/FreqaiHybridExampleStrategy.py index 0a91455f5..5d1e149dd 100644 --- a/freqtrade/templates/FreqaiHybridExampleStrategy.py +++ b/freqtrade/templates/FreqaiHybridExampleStrategy.py @@ -45,7 +45,6 @@ class FreqaiExampleHybridStrategy(IStrategy): "weight_factor": 0.9, "principal_component_analysis": false, "use_SVM_to_remove_outliers": true, - "indicator_max_period_candles": 20, "indicator_periods_candles": [10, 20] }, "data_split_parameters": { From 57ff6f8ac592678c5399e83e75656c66076bd863 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 Aug 2022 10:28:31 +0000 Subject: [PATCH 11/11] Init timerange object properly --- freqtrade/freqai/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 6081b6ce5..6a70f050f 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -55,7 +55,6 @@ def get_required_data_timerange( for auto data-download in FreqAI """ time = datetime.now(tz=timezone.utc).timestamp() - data_load_timerange = TimeRange() timeframes = config["freqai"]["feature_parameters"].get("include_timeframes") @@ -75,12 +74,13 @@ def get_required_data_timerange( additional_seconds = max_period * max_tf_seconds - data_load_timerange.startts = int( + startts = int( time - config["freqai"].get("train_period_days", 0) * 86400 - additional_seconds ) - data_load_timerange.stopts = int(time) + stopts = int(time) + data_load_timerange = TimeRange('date', 'date', startts, stopts) return data_load_timerange