From 0c34104e45f4fbeae66036452ade47fff13ff058 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 17 Aug 2022 15:18:44 +0200 Subject: [PATCH 01/26] extract download-data from freqai to prepare for future async changes --- freqtrade/freqai/data_kitchen.py | 102 ++++++++++++++++++++------- freqtrade/freqai/freqai_interface.py | 12 ++-- freqtrade/strategy/interface.py | 13 +++- 3 files changed, 94 insertions(+), 33 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 35f51baed..4554a5c1a 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -816,7 +816,7 @@ class FreqaiDataKitchen: return False def check_if_new_training_required( - self, trained_timestamp: int + self, trained_timestamp: int = 0 ) -> Tuple[bool, TimeRange, TimeRange]: time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() @@ -889,31 +889,6 @@ class FreqaiDataKitchen: self.model_filename = f"cb_{coin.lower()}_{int(trained_timerange.stopts)}" - def download_all_data_for_training(self, timerange: TimeRange, dp: DataProvider) -> None: - """ - Called only once upon start of bot to download the necessary data for - populating indicators and training the model. - :param timerange: TimeRange = The full data timerange for populating the indicators - and training the model. - :param dp: DataProvider instance attached to the strategy - """ - new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) - if not dp._exchange: - # Not realistic - this is only called in live mode. - raise OperationalException("Dataprovider did not have an exchange attached.") - refresh_backtest_ohlcv_data( - dp._exchange, - pairs=self.all_pairs, - timeframes=self.freqai_config["feature_parameters"].get("include_timeframes"), - datadir=self.config["datadir"], - timerange=timerange, - new_pairs_days=new_pairs_days, - erase=False, - data_format=self.config.get("dataformat_ohlcv", "json"), - trading_mode=self.config.get("trading_mode", "spot"), - prepend=self.config.get("prepend_data", False), - ) - def set_all_pairs(self) -> None: self.all_pairs = copy.deepcopy( @@ -1027,3 +1002,78 @@ class FreqaiDataKitchen: if self.unique_classes: for label in self.unique_classes: self.unique_class_list += list(self.unique_classes[label]) + +# Methods called by interface.py (load_freqai_model()) + + +def download_all_data_for_training(timerange: TimeRange, + dp: DataProvider, config: dict) -> None: + """ + Called only once upon start of bot to download the necessary data for + populating indicators and training the model. + :param timerange: TimeRange = The full data timerange for populating the indicators + and training the model. + :param dp: DataProvider instance attached to the strategy + """ + all_pairs = copy.deepcopy( + config["freqai"]["feature_parameters"].get("include_corr_pairlist", []) + ) + for pair in config.get("exchange", "").get("pair_whitelist"): + if pair not in all_pairs: + all_pairs.append(pair) + + new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) + if not dp._exchange: + # Not realistic - this is only called in live mode. + raise OperationalException("Dataprovider did not have an exchange attached.") + refresh_backtest_ohlcv_data( + dp._exchange, + pairs=all_pairs, + timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"), + datadir=config["datadir"], + timerange=timerange, + new_pairs_days=new_pairs_days, + erase=False, + data_format=config.get("dataformat_ohlcv", "json"), + trading_mode=config.get("trading_mode", "spot"), + prepend=config.get("prepend_data", False), + ) + + +def get_required_data_timerange( + config: dict +) -> TimeRange: + """ + Used by interface.py to pre-download necessary data for FreqAI + user. + """ + time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() + trained_timerange = TimeRange() + data_load_timerange = TimeRange() + + timeframes = config["freqai"]["feature_parameters"].get("include_timeframes") + + max_tf_seconds = 0 + for tf in timeframes: + secs = timeframe_to_seconds(tf) + if secs > max_tf_seconds: + max_tf_seconds = secs + + max_period = config["freqai"]["feature_parameters"].get( + "indicator_max_period_candles", 20 + ) * 2 + additional_seconds = max_period * max_tf_seconds + + trained_timerange.startts = int( + time - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY + ) + trained_timerange.stopts = int(time) + + data_load_timerange.startts = int( + time + - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY + - additional_seconds + ) + data_load_timerange.stopts = int(time) + + return data_load_timerange diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 49e4ce5c3..5d85cc225 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -278,12 +278,12 @@ class IFreqaiModel(ABC): # download candle history if it is not already in memory if not self.dd.historic_data: - logger.info( - "Downloading all training data for all pairs in whitelist and " - "corr_pairlist, this may take a while if you do not have the " - "data saved" - ) - dk.download_all_data_for_training(data_load_timerange, strategy.dp) + # logger.info( + # "Downloading all training data for all pairs in whitelist and " + # "corr_pairlist, this may take a while if you do not have the " + # "data saved" + # ) + # dk.download_all_data_for_training(data_load_timerange, strategy.dp) self.dd.load_all_pair_histories(data_load_timerange, dk) if not self.scanning: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 79dbd4c69..20a35ac3e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -149,9 +149,20 @@ class IStrategy(ABC, HyperStrategyMixin): if self.config.get('freqai', {}).get('enabled', False): # Import here to avoid importing this if freqAI is disabled from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver - + from freqtrade.freqai.data_kitchen import (get_required_data_timerange, + download_all_data_for_training) self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) self.freqai_info = self.config["freqai"] + + # download the desired data in dry/live + if self.config.get('runmode') in (RunMode.DRY_RUN, RunMode.LIVE): + logger.info( + "Downloading all training data for all pairs in whitelist and " + "corr_pairlist, this may take a while if you do not have the " + "data saved" + ) + data_load_timerange = get_required_data_timerange(self.config) + download_all_data_for_training(data_load_timerange, self.dp, self.config) else: # Gracious failures if freqAI is disabled but "start" is called. class DummyClass(): From 5155afb4e7adb62b219fab65df86ac29524e405a Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 17 Aug 2022 15:22:48 +0200 Subject: [PATCH 02/26] clean up code remnants --- freqtrade/freqai/data_kitchen.py | 2 +- freqtrade/freqai/freqai_interface.py | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 4554a5c1a..6541261eb 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -816,7 +816,7 @@ class FreqaiDataKitchen: return False def check_if_new_training_required( - self, trained_timestamp: int = 0 + self, trained_timestamp: int ) -> Tuple[bool, TimeRange, TimeRange]: time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 5d85cc225..1a9e549f6 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -276,14 +276,8 @@ class IFreqaiModel(ABC): ) dk.set_paths(metadata["pair"], new_trained_timerange.stopts) - # download candle history if it is not already in memory + # load candle history into memory if it is not yet. if not self.dd.historic_data: - # logger.info( - # "Downloading all training data for all pairs in whitelist and " - # "corr_pairlist, this may take a while if you do not have the " - # "data saved" - # ) - # dk.download_all_data_for_training(data_load_timerange, strategy.dp) self.dd.load_all_pair_histories(data_load_timerange, dk) if not self.scanning: From 88dd9920ea9dd66c17a55fad4d6fb69cacefb8c2 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 17 Aug 2022 16:38:09 +0200 Subject: [PATCH 03/26] sort imports for isort --- freqtrade/strategy/interface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 20a35ac3e..1e51701f7 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -148,9 +148,9 @@ class IStrategy(ABC, HyperStrategyMixin): def load_freqAI_model(self) -> None: if self.config.get('freqai', {}).get('enabled', False): # Import here to avoid importing this if freqAI is disabled + from freqtrade.freqai.data_kitchen import (download_all_data_for_training, + get_required_data_timerange) from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver - from freqtrade.freqai.data_kitchen import (get_required_data_timerange, - download_all_data_for_training) self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) self.freqai_info = self.config["freqai"] From ac42c0153da716e0a70abd7a8024d8d3162e4ba6 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Mon, 22 Aug 2022 18:19:07 +0200 Subject: [PATCH 04/26] deprecate indicator_max_period_candles, automatically compute startup candles for FreqAI backtesting. --- config_examples/config_freqai.example.json | 2 +- docs/freqai.md | 2 +- docs/strategy-customization.md | 2 +- freqtrade/data/dataprovider.py | 13 +++++++- freqtrade/freqai/data_kitchen.py | 32 +++++++++---------- freqtrade/optimize/backtesting.py | 33 ++++++++++++++------ freqtrade/strategy/interface.py | 1 + freqtrade/templates/FreqaiExampleStrategy.py | 3 +- tests/freqai/conftest.py | 1 - tests/freqai/test_freqai_backtesting.py | 10 +++--- 10 files changed, 61 insertions(+), 38 deletions(-) diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json index aeb1cb13d..093e11b2a 100644 --- a/config_examples/config_freqai.example.json +++ b/config_examples/config_freqai.example.json @@ -9,6 +9,7 @@ "dry_run": true, "timeframe": "3m", "dry_run_wallet": 1000, + "startup_candle_count": 20, "cancel_open_orders_on_exit": true, "unfilledtimeout": { "entry": 10, @@ -53,7 +54,6 @@ ], "freqai": { "enabled": true, - "startup_candles": 10000, "purge_old_models": true, "train_period_days": 15, "backtest_period_days": 7, diff --git a/docs/freqai.md b/docs/freqai.md index b22e1cd31..f3c9021ed 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -113,7 +113,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `use_SVM_to_remove_outliers` | Ask FreqAI to train a support vector machine to detect and remove outliers from the training data set as well as from incoming data points.
**Datatype:** boolean. | `svm_params` | All parameters available in Sklearn's `SGDOneClassSVM()`. E.g. `nu` *Very* broadly, is the percentage of data points that should be considered outliers. `shuffle` is by default false to maintain reproducibility. But these and all others can be added/changed in this dictionary.
**Datatype:** dictionary. | `stratify_training_data` | This value is used to indicate the stratification of the data. e.g. 2 would set every 2nd data point into a separate dataset to be pulled from during training/testing.
**Datatype:** positive integer. -| `indicator_max_period_candles` | The maximum *period* used in `populate_any_indicators()` for indicator creation. FreqAI uses this information in combination with the maximum timeframe to calculate how many data points it should download so that the first data point does not have a NaN
**Datatype:** positive integer. +| `indicator_max_period_candles` | **Deprecated in favor of** strategy set `startup_candle_count`, however, both configuration parameters provide the same functionality; the maximum *period* used in `populate_any_indicators()` for indicator creation (timeframe independent). FreqAI uses this information in combination with the maximum timeframe to calculate how many data points it should download so that the first data point does not have a NaN
**Datatype:** positive integer. | `indicator_periods_candles` | A list of integers used to duplicate all indicators according to a set of periods and add them to the feature set.
**Datatype:** list of positive integers. | `use_DBSCAN_to_remove_outliers` | Inactive by default. If true, FreqAI clusters data using DBSCAN to identify and remove outliers from training and prediction data.
**Datatype:** float (fraction of 1). | | **Data split parameters** diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 260e253c4..a452b8f05 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -166,7 +166,7 @@ Additional technical libraries can be installed as necessary, or custom indicato Most indicators have an instable startup period, in which they are either not available (NaN), or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be. To account for this, the strategy can be assigned the `startup_candle_count` attribute. -This should be set to the maximum number of candles that the strategy requires to calculate stable indicators. +This should be set to the maximum number of candles that the strategy requires to calculate stable indicators. In the case where a user includes higher timeframes with informative pairs, the `startup_candle_count` does not necessarily change. The value is the maximum period (in candles) that any of the informatives timeframes need to compute stable indicators. In this example strategy, this should be set to 100 (`startup_candle_count = 100`), since the longest needed history is 100 candles. diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 21cead77f..529a12690 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -92,7 +92,7 @@ class DataProvider: 'timerange') is None else str(self._config.get('timerange'))) # Move informative start time respecting startup_candle_count timerange.subtract_start( - timeframe_to_seconds(str(timeframe)) * self._config.get('startup_candle_count', 0) + self.get_required_startup_seconds(str(timeframe)) ) self.__cached_pairs_backtesting[saved_pair] = load_pair_history( pair=pair, @@ -105,6 +105,17 @@ class DataProvider: ) return self.__cached_pairs_backtesting[saved_pair].copy() + def get_required_startup_seconds(self, timeframe: str) -> int: + tf_seconds = timeframe_to_seconds(timeframe) + base_seconds = tf_seconds * self._config.get('startup_candle_count', 0) + if not self._config['freqai']['enabled']: + return base_seconds + else: + train_seconds = self._config['freqai']['train_period_days'] * 86400 + # multiplied by safety factor of 2 because FreqAI users + # typically do not know the correct window. + return base_seconds * 2 + int(train_seconds) + def get_pair_dataframe( self, pair: str, diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 6541261eb..c768fc30e 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -20,6 +20,8 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds +from freqtrade.exchange.exchange import market_is_active +from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.strategy.interface import IStrategy @@ -834,9 +836,7 @@ class FreqaiDataKitchen: # We notice that users like to use exotic indicators where # they do not know the required timeperiod. Here we include a factor # of safety by multiplying the user considered "max" by 2. - max_period = self.freqai_config["feature_parameters"].get( - "indicator_max_period_candles", 20 - ) * 2 + max_period = self.config.get('startup_candle_count', 20) * 2 additional_seconds = max_period * max_tf_seconds if trained_timestamp != 0: @@ -1015,12 +1015,15 @@ def download_all_data_for_training(timerange: TimeRange, and training the model. :param dp: DataProvider instance attached to the strategy """ - all_pairs = copy.deepcopy( - config["freqai"]["feature_parameters"].get("include_corr_pairlist", []) - ) - for pair in config.get("exchange", "").get("pair_whitelist"): - if pair not in all_pairs: - all_pairs.append(pair) + + if dp._exchange is not None: + markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) + or config.get('include_inactive')] + else: + # This should not occur: + raise OperationalException('No exchange object found.') + + all_pairs = dynamic_expand_pairlist(config, markets) new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) if not dp._exchange: @@ -1048,7 +1051,6 @@ def get_required_data_timerange( user. """ time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() - trained_timerange = TimeRange() data_load_timerange = TimeRange() timeframes = config["freqai"]["feature_parameters"].get("include_timeframes") @@ -1059,15 +1061,9 @@ def get_required_data_timerange( if secs > max_tf_seconds: max_tf_seconds = secs - max_period = config["freqai"]["feature_parameters"].get( - "indicator_max_period_candles", 20 - ) * 2 - additional_seconds = max_period * max_tf_seconds + max_period = config.get('startup_candle_count', 20) * 2 - trained_timerange.startts = int( - time - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY - ) - trained_timerange.stopts = int(time) + additional_seconds = max_period * max_tf_seconds data_load_timerange.startts = int( time diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6528481d5..8f0302ada 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -211,21 +211,21 @@ class Backtesting: """ self.progress.init_step(BacktestState.DATALOAD, 1) - if self.config.get('freqai', {}).get('enabled', False): - startup_candles = int(self.config.get('freqai', {}).get('startup_candles', 0)) - if not startup_candles: - raise OperationalException('FreqAI backtesting module requires user set ' - 'startup_candles in config.') - self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 0)) - logger.info(f'Increasing startup_candle_count for freqai to {self.required_startup}') - self.config['startup_candle_count'] = self.required_startup + # if self.config.get('freqai', {}).get('enabled', False): + # startup_candles = int(self.config.get('freqai', {}).get('startup_candles', 0)) + # if not startup_candles: + # raise OperationalException('FreqAI backtesting module requires user set ' + # 'startup_candles in config.') + # self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 0)) + # logger.info(f'Increasing startup_candle_count for freqai to {self.required_startup}') + # self.config['startup_candle_count'] = self.required_startup data = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, timeframe=self.timeframe, timerange=self.timerange, - startup_candles=self.required_startup, + startup_candles=self.get_required_startup(self.timeframe), fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), candle_type=self.config.get('candle_type_def', CandleType.SPOT) @@ -244,6 +244,21 @@ class Backtesting: self.progress.set_new_value(1) return data, self.timerange + def get_required_startup(self, timeframe: str) -> int: + if not self.config['freqai']['enabled']: + return self.required_startup + else: + if not self.config['startup_candle_count']: + raise OperationalException('FreqAI backtesting module requires strategy ' + 'set startup_candle_count.') + tf_seconds = timeframe_to_seconds(timeframe) + train_candles = self.config['freqai']['train_period_days'] * 86400 / tf_seconds + # multiplied by safety factor of 2 because FreqAI users + # typically do not know the correct window. + total_candles = self.required_startup * 2 + train_candles + logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') + return total_candles + def load_bt_data_detail(self) -> None: """ Loads backtest detail data (smaller timeframe) if necessary. diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 1e51701f7..284727d2b 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -163,6 +163,7 @@ class IStrategy(ABC, HyperStrategyMixin): ) data_load_timerange = get_required_data_timerange(self.config) download_all_data_for_training(data_load_timerange, self.dp, self.config) + else: # Gracious failures if freqAI is disabled but "start" is called. class DummyClass(): diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 5810e7881..aa584bfbc 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -43,7 +43,8 @@ class FreqaiExampleStrategy(IStrategy): process_only_new_candles = True stoploss = -0.05 use_exit_signal = True - startup_candle_count: int = 300 + # this is the maximum period fed to talib (timeframe independent) + startup_candle_count: int = 20 can_short = False linear_roi_offset = DecimalParameter( diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index 6ace13677..113cb3a79 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -44,7 +44,6 @@ def freqai_conf(default_conf, tmpdir): "principal_component_analysis": False, "use_SVM_to_remove_outliers": True, "stratify_training_data": 0, - "indicator_max_period_candles": 10, "indicator_periods_candles": [10], }, "data_split_parameters": {"test_size": 0.33, "random_state": 1}, diff --git a/tests/freqai/test_freqai_backtesting.py b/tests/freqai/test_freqai_backtesting.py index 273791609..c8a51edb0 100644 --- a/tests/freqai/test_freqai_backtesting.py +++ b/tests/freqai/test_freqai_backtesting.py @@ -48,10 +48,10 @@ def test_freqai_backtest_load_data(freqai_conf, mocker, caplog): assert log_has_re('Increasing startup_candle_count for freqai to.*', caplog) - del freqai_conf['freqai']['startup_candles'] - backtesting = Backtesting(freqai_conf) - with pytest.raises(OperationalException, - match=r'FreqAI backtesting module.*startup_candles in config.'): - backtesting.load_bt_data() + # del freqai_conf['freqai']['startup_candles'] + # backtesting = Backtesting(freqai_conf) + # with pytest.raises(OperationalException, + # match=r'FreqAI backtesting module.*startup_candles in config.'): + # backtesting.load_bt_data() Backtesting.cleanup() From 4b7e640f31f2a35ba3b73a3bfbbfb2882ecb7a81 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 26 Aug 2022 13:56:44 +0200 Subject: [PATCH 05/26] reduce code duplication, optimize auto data download per tf --- freqtrade/data/dataprovider.py | 26 ++++++------ freqtrade/freqai/data_kitchen.py | 67 +++++++++++-------------------- freqtrade/optimize/backtesting.py | 26 +----------- freqtrade/strategy/interface.py | 7 ++-- 4 files changed, 41 insertions(+), 85 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 529a12690..a21114901 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -91,9 +91,9 @@ class DataProvider: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) # Move informative start time respecting startup_candle_count - timerange.subtract_start( - self.get_required_startup_seconds(str(timeframe)) - ) + startup_candles = self.get_required_startup(str(timeframe)) + tf_seconds = timeframe_to_seconds(str(timeframe)) + timerange.subtract_start(tf_seconds * startup_candles) self.__cached_pairs_backtesting[saved_pair] = load_pair_history( pair=pair, timeframe=timeframe or self._config['timeframe'], @@ -105,16 +105,18 @@ class DataProvider: ) return self.__cached_pairs_backtesting[saved_pair].copy() - def get_required_startup_seconds(self, timeframe: str) -> int: - tf_seconds = timeframe_to_seconds(timeframe) - base_seconds = tf_seconds * self._config.get('startup_candle_count', 0) - if not self._config['freqai']['enabled']: - return base_seconds + def get_required_startup(self, timeframe: str) -> int: + if not self._config.get('freqai', {}).get('enabled', False): + return self._config.get('startup_candle_count', 0) else: - train_seconds = self._config['freqai']['train_period_days'] * 86400 - # multiplied by safety factor of 2 because FreqAI users - # typically do not know the correct window. - return base_seconds * 2 + int(train_seconds) + if not self._config['startup_candle_count']: + raise OperationalException('FreqAI backtesting module requires strategy ' + 'set startup_candle_count.') + tf_seconds = timeframe_to_seconds(timeframe) + train_candles = self._config['freqai']['train_period_days'] * 86400 / tf_seconds + total_candles = int(self._config.get('startup_candle_count', 0) + train_candles) + logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') + return total_candles def get_pair_dataframe( self, diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index c768fc30e..1a8063add 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1006,8 +1006,7 @@ class FreqaiDataKitchen: # Methods called by interface.py (load_freqai_model()) -def download_all_data_for_training(timerange: TimeRange, - dp: DataProvider, config: dict) -> None: +def download_all_data_for_training(dp: DataProvider, config: dict) -> None: """ Called only once upon start of bot to download the necessary data for populating indicators and training the model. @@ -1025,51 +1024,31 @@ def download_all_data_for_training(timerange: TimeRange, all_pairs = dynamic_expand_pairlist(config, markets) - new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) if not dp._exchange: # Not realistic - this is only called in live mode. raise OperationalException("Dataprovider did not have an exchange attached.") - refresh_backtest_ohlcv_data( - dp._exchange, - pairs=all_pairs, - timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"), - datadir=config["datadir"], - timerange=timerange, - new_pairs_days=new_pairs_days, - erase=False, - data_format=config.get("dataformat_ohlcv", "json"), - trading_mode=config.get("trading_mode", "spot"), - prepend=config.get("prepend_data", False), - ) - -def get_required_data_timerange( - config: dict -) -> TimeRange: - """ - Used by interface.py to pre-download necessary data for FreqAI - user. - """ time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() - data_load_timerange = TimeRange() - timeframes = config["freqai"]["feature_parameters"].get("include_timeframes") - - max_tf_seconds = 0 - for tf in timeframes: - secs = timeframe_to_seconds(tf) - if secs > max_tf_seconds: - max_tf_seconds = secs - - max_period = config.get('startup_candle_count', 20) * 2 - - additional_seconds = max_period * max_tf_seconds - - data_load_timerange.startts = int( - time - - config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY - - additional_seconds - ) - data_load_timerange.stopts = int(time) - - return data_load_timerange + for tf in config["freqai"]["feature_parameters"].get("include_timeframes"): + timerange = TimeRange() + timerange.startts = int(time) + timerange.stopts = int(time) + startup_candles = dp.get_required_startup(str(tf)) + tf_seconds = timeframe_to_seconds(str(tf)) + timerange.subtract_start(tf_seconds * startup_candles) + new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) + # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function + # redownloads the funding rate for each pair. + refresh_backtest_ohlcv_data( + dp._exchange, + pairs=all_pairs, + timeframes=[tf], + datadir=config["datadir"], + timerange=timerange, + new_pairs_days=new_pairs_days, + erase=False, + data_format=config.get("dataformat_ohlcv", "json"), + trading_mode=config.get("trading_mode", "spot"), + prepend=config.get("prepend_data", False), + ) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8f0302ada..3d715c82d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -211,21 +211,12 @@ class Backtesting: """ self.progress.init_step(BacktestState.DATALOAD, 1) - # if self.config.get('freqai', {}).get('enabled', False): - # startup_candles = int(self.config.get('freqai', {}).get('startup_candles', 0)) - # if not startup_candles: - # raise OperationalException('FreqAI backtesting module requires user set ' - # 'startup_candles in config.') - # self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 0)) - # logger.info(f'Increasing startup_candle_count for freqai to {self.required_startup}') - # self.config['startup_candle_count'] = self.required_startup - data = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, timeframe=self.timeframe, timerange=self.timerange, - startup_candles=self.get_required_startup(self.timeframe), + startup_candles=self.dataprovider.get_required_startup(self.timeframe), fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), candle_type=self.config.get('candle_type_def', CandleType.SPOT) @@ -244,21 +235,6 @@ class Backtesting: self.progress.set_new_value(1) return data, self.timerange - def get_required_startup(self, timeframe: str) -> int: - if not self.config['freqai']['enabled']: - return self.required_startup - else: - if not self.config['startup_candle_count']: - raise OperationalException('FreqAI backtesting module requires strategy ' - 'set startup_candle_count.') - tf_seconds = timeframe_to_seconds(timeframe) - train_candles = self.config['freqai']['train_period_days'] * 86400 / tf_seconds - # multiplied by safety factor of 2 because FreqAI users - # typically do not know the correct window. - total_candles = self.required_startup * 2 + train_candles - logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') - return total_candles - def load_bt_data_detail(self) -> None: """ Loads backtest detail data (smaller timeframe) if necessary. diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 284727d2b..9124a0427 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -148,8 +148,7 @@ class IStrategy(ABC, HyperStrategyMixin): def load_freqAI_model(self) -> None: if self.config.get('freqai', {}).get('enabled', False): # Import here to avoid importing this if freqAI is disabled - from freqtrade.freqai.data_kitchen import (download_all_data_for_training, - get_required_data_timerange) + from freqtrade.freqai.data_kitchen import (download_all_data_for_training) from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) self.freqai_info = self.config["freqai"] @@ -161,8 +160,8 @@ class IStrategy(ABC, HyperStrategyMixin): "corr_pairlist, this may take a while if you do not have the " "data saved" ) - data_load_timerange = get_required_data_timerange(self.config) - download_all_data_for_training(data_load_timerange, self.dp, self.config) + # data_load_timerange = get_required_data_timerange(self.config) + download_all_data_for_training(self.dp, self.config) else: # Gracious failures if freqAI is disabled but "start" is called. From 65b552e310fe751989e43848498e03157cc50232 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 26 Aug 2022 15:30:01 +0200 Subject: [PATCH 06/26] make docs reflect reality, move download_all_data to new utils.py file, automatic startup_candle detection --- docs/freqai.md | 7 +++-- freqtrade/data/dataprovider.py | 14 +++++---- freqtrade/freqai/data_kitchen.py | 54 -------------------------------- freqtrade/strategy/interface.py | 2 +- 4 files changed, 14 insertions(+), 63 deletions(-) diff --git a/docs/freqai.md b/docs/freqai.md index f3c9021ed..bd746c984 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -113,7 +113,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `use_SVM_to_remove_outliers` | Ask FreqAI to train a support vector machine to detect and remove outliers from the training data set as well as from incoming data points.
**Datatype:** boolean. | `svm_params` | All parameters available in Sklearn's `SGDOneClassSVM()`. E.g. `nu` *Very* broadly, is the percentage of data points that should be considered outliers. `shuffle` is by default false to maintain reproducibility. But these and all others can be added/changed in this dictionary.
**Datatype:** dictionary. | `stratify_training_data` | This value is used to indicate the stratification of the data. e.g. 2 would set every 2nd data point into a separate dataset to be pulled from during training/testing.
**Datatype:** positive integer. -| `indicator_max_period_candles` | **Deprecated in favor of** strategy set `startup_candle_count`, however, both configuration parameters provide the same functionality; the maximum *period* used in `populate_any_indicators()` for indicator creation (timeframe independent). FreqAI uses this information in combination with the maximum timeframe to calculate how many data points it should download so that the first data point does not have a NaN
**Datatype:** positive integer. +| `indicator_max_period_candles` | **No longer used**. User must use the strategy set `startup_candle_count` which defines the maximum *period* used in `populate_any_indicators()` for indicator creation (timeframe independent). FreqAI uses this information in combination with the maximum timeframe to calculate how many data points it should download so that the first data point does not have a NaN
**Datatype:** positive integer. | `indicator_periods_candles` | A list of integers used to duplicate all indicators according to a set of periods and add them to the feature set.
**Datatype:** list of positive integers. | `use_DBSCAN_to_remove_outliers` | Inactive by default. If true, FreqAI clusters data using DBSCAN to identify and remove outliers from training and prediction data.
**Datatype:** float (fraction of 1). | | **Data split parameters** @@ -162,7 +162,6 @@ The user interface is isolated to the typical config file. A typical FreqAI conf "label_period_candles": 24, "include_shifted_candles": 2, "weight_factor": 0, - "indicator_max_period_candles": 20, "indicator_periods_candles": [10, 20] }, "data_split_parameters" : { @@ -387,6 +386,10 @@ The FreqAI strategy requires the user to include the following lines of code in ```python + # user should define the maximum startup candle count (the largest number of candles + # passed to any single indicator) + startup_candle_count: int = 20 + def informative_pairs(self): whitelist_pairs = self.dp.current_whitelist() corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index a21114901..4151b7419 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -106,15 +106,17 @@ class DataProvider: return self.__cached_pairs_backtesting[saved_pair].copy() def get_required_startup(self, timeframe: str) -> int: - if not self._config.get('freqai', {}).get('enabled', False): + freqai_config = self._config.get('freqai', {}) + if not freqai_config.get('enabled', False): return self._config.get('startup_candle_count', 0) else: - if not self._config['startup_candle_count']: - raise OperationalException('FreqAI backtesting module requires strategy ' - 'set startup_candle_count.') + startup_candles = self._config.get('startup_candle_count', 0) + indicator_periods = freqai_config['feature_parameters']['indicator_periods_candles'] + # make sure the startupcandles is at least the set maximum indicator periods + self._config['startup_candle_count'] = max(startup_candles, max(indicator_periods)) tf_seconds = timeframe_to_seconds(timeframe) - train_candles = self._config['freqai']['train_period_days'] * 86400 / tf_seconds - total_candles = int(self._config.get('startup_candle_count', 0) + train_candles) + train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds + total_candles = int(self._config['startup_candle_count'] + train_candles) logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') return total_candles diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 1a8063add..1b88405c1 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -16,12 +16,8 @@ from sklearn.model_selection import train_test_split from sklearn.neighbors import NearestNeighbors from freqtrade.configuration import TimeRange -from freqtrade.data.dataprovider import DataProvider -from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds -from freqtrade.exchange.exchange import market_is_active -from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.strategy.interface import IStrategy @@ -1002,53 +998,3 @@ class FreqaiDataKitchen: if self.unique_classes: for label in self.unique_classes: self.unique_class_list += list(self.unique_classes[label]) - -# Methods called by interface.py (load_freqai_model()) - - -def download_all_data_for_training(dp: DataProvider, config: dict) -> None: - """ - Called only once upon start of bot to download the necessary data for - populating indicators and training the model. - :param timerange: TimeRange = The full data timerange for populating the indicators - and training the model. - :param dp: DataProvider instance attached to the strategy - """ - - if dp._exchange is not None: - markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) - or config.get('include_inactive')] - else: - # This should not occur: - raise OperationalException('No exchange object found.') - - all_pairs = dynamic_expand_pairlist(config, markets) - - if not dp._exchange: - # Not realistic - this is only called in live mode. - raise OperationalException("Dataprovider did not have an exchange attached.") - - time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp() - - for tf in config["freqai"]["feature_parameters"].get("include_timeframes"): - timerange = TimeRange() - timerange.startts = int(time) - timerange.stopts = int(time) - startup_candles = dp.get_required_startup(str(tf)) - tf_seconds = timeframe_to_seconds(str(tf)) - timerange.subtract_start(tf_seconds * startup_candles) - new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY) - # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function - # redownloads the funding rate for each pair. - refresh_backtest_ohlcv_data( - dp._exchange, - pairs=all_pairs, - timeframes=[tf], - datadir=config["datadir"], - timerange=timerange, - new_pairs_days=new_pairs_days, - erase=False, - data_format=config.get("dataformat_ohlcv", "json"), - trading_mode=config.get("trading_mode", "spot"), - prepend=config.get("prepend_data", False), - ) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 9124a0427..c9ec466de 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -148,7 +148,7 @@ class IStrategy(ABC, HyperStrategyMixin): def load_freqAI_model(self) -> None: if self.config.get('freqai', {}).get('enabled', False): # Import here to avoid importing this if freqAI is disabled - from freqtrade.freqai.data_kitchen import (download_all_data_for_training) + from freqtrade.freqai.utils import download_all_data_for_training from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) self.freqai_info = self.config["freqai"] From e7261cf51577dc30b530370f81df620c898f6a11 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 26 Aug 2022 15:30:28 +0200 Subject: [PATCH 07/26] add freqai utils.py file --- freqtrade/freqai/utils.py | 56 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 freqtrade/freqai/utils.py diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py new file mode 100644 index 000000000..056458115 --- /dev/null +++ b/freqtrade/freqai/utils.py @@ -0,0 +1,56 @@ +from freqtrade.data.dataprovider import DataProvider +from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist +from freqtrade.exchange.exchange import market_is_active +from freqtrade.exchange import timeframe_to_seconds +from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data +from datetime import datetime, timezone +from freqtrade.exceptions import OperationalException +from freqtrade.configuration import TimeRange + + +def download_all_data_for_training(dp: DataProvider, config: dict) -> None: + """ + Called only once upon start of bot to download the necessary data for + populating indicators and training a FreqAI model. + :param timerange: TimeRange = The full data timerange for populating the indicators + and training the model. + :param dp: DataProvider instance attached to the strategy + """ + + if dp._exchange is not None: + markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) + or config.get('include_inactive')] + else: + # This should not occur: + raise OperationalException('No exchange object found.') + + all_pairs = dynamic_expand_pairlist(config, markets) + + if not dp._exchange: + # Not realistic - this is only called in live mode. + raise OperationalException("Dataprovider did not have an exchange attached.") + + time = datetime.now(tz=timezone.utc).timestamp() + + for tf in config["freqai"]["feature_parameters"].get("include_timeframes"): + timerange = TimeRange() + timerange.startts = int(time) + timerange.stopts = int(time) + startup_candles = dp.get_required_startup(str(tf)) + tf_seconds = timeframe_to_seconds(str(tf)) + timerange.subtract_start(tf_seconds * startup_candles) + new_pairs_days = int((timerange.stopts - timerange.startts) / 86400) + # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function + # redownloads the funding rate for each pair. + refresh_backtest_ohlcv_data( + dp._exchange, + pairs=all_pairs, + timeframes=[tf], + datadir=config["datadir"], + timerange=timerange, + new_pairs_days=new_pairs_days, + erase=False, + data_format=config.get("dataformat_ohlcv", "json"), + trading_mode=config.get("trading_mode", "spot"), + prepend=config.get("prepend_data", False), + ) From bb3523f3838686f92420b27e58cc1b5a37df6b9e Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 26 Aug 2022 18:51:42 +0200 Subject: [PATCH 08/26] download data homogeneously across timeframes --- freqtrade/freqai/utils.py | 140 +++++++++++++++++++++++++------- freqtrade/strategy/interface.py | 6 +- 2 files changed, 113 insertions(+), 33 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 056458115..d56702049 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -1,17 +1,22 @@ -from freqtrade.data.dataprovider import DataProvider -from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist -from freqtrade.exchange.exchange import market_is_active -from freqtrade.exchange import timeframe_to_seconds -from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data +import logging from datetime import datetime, timezone -from freqtrade.exceptions import OperationalException + from freqtrade.configuration import TimeRange +from freqtrade.data.dataprovider import DataProvider +from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import timeframe_to_seconds +from freqtrade.exchange.exchange import market_is_active +from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist + + +logger = logging.getLogger(__name__) def download_all_data_for_training(dp: DataProvider, config: dict) -> None: """ Called only once upon start of bot to download the necessary data for - populating indicators and training a FreqAI model. + populating indicators and training the model. :param timerange: TimeRange = The full data timerange for populating the indicators and training the model. :param dp: DataProvider instance attached to the strategy @@ -26,31 +31,108 @@ def download_all_data_for_training(dp: DataProvider, config: dict) -> None: all_pairs = dynamic_expand_pairlist(config, markets) + timerange = get_required_data_timerange(config) + + new_pairs_days = int((timerange.stopts - timerange.startts) / 86400) if not dp._exchange: # Not realistic - this is only called in live mode. raise OperationalException("Dataprovider did not have an exchange attached.") + refresh_backtest_ohlcv_data( + dp._exchange, + pairs=all_pairs, + timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"), + datadir=config["datadir"], + timerange=timerange, + new_pairs_days=new_pairs_days, + erase=False, + data_format=config.get("dataformat_ohlcv", "json"), + trading_mode=config.get("trading_mode", "spot"), + prepend=config.get("prepend_data", False), + ) + +def get_required_data_timerange( + config: dict +) -> TimeRange: + """ + Used to compute the required data download time range + for auto data-download in FreqAI + """ time = datetime.now(tz=timezone.utc).timestamp() + data_load_timerange = TimeRange() - for tf in config["freqai"]["feature_parameters"].get("include_timeframes"): - timerange = TimeRange() - timerange.startts = int(time) - timerange.stopts = int(time) - startup_candles = dp.get_required_startup(str(tf)) - tf_seconds = timeframe_to_seconds(str(tf)) - timerange.subtract_start(tf_seconds * startup_candles) - new_pairs_days = int((timerange.stopts - timerange.startts) / 86400) - # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function - # redownloads the funding rate for each pair. - refresh_backtest_ohlcv_data( - dp._exchange, - pairs=all_pairs, - timeframes=[tf], - datadir=config["datadir"], - timerange=timerange, - new_pairs_days=new_pairs_days, - erase=False, - data_format=config.get("dataformat_ohlcv", "json"), - trading_mode=config.get("trading_mode", "spot"), - prepend=config.get("prepend_data", False), - ) + timeframes = config["freqai"]["feature_parameters"].get("include_timeframes") + + max_tf_seconds = 0 + for tf in timeframes: + secs = timeframe_to_seconds(tf) + if secs > max_tf_seconds: + max_tf_seconds = secs + + startup_candles = config.get('startup_candle_count', 0) + indicator_periods = config["freqai"]["feature_parameters"]["indicator_periods_candles"] + + # factor the max_period as a factor of safety. + max_period = int(max(startup_candles, max(indicator_periods)) * 1.5) + config['startup_candle_count'] = max_period + logger.info(f'FreqAI auto-downloader using {max_period} startup candles.') + + additional_seconds = max_period * max_tf_seconds + + data_load_timerange.startts = int( + time + - config["freqai"].get("train_period_days", 0) * 86400 + - additional_seconds + ) + data_load_timerange.stopts = int(time) + + return data_load_timerange + + +# Keep below for when we wish to download heterogeneously lengthed data for FreqAI. +# def download_all_data_for_training(dp: DataProvider, config: dict) -> None: +# """ +# Called only once upon start of bot to download the necessary data for +# populating indicators and training a FreqAI model. +# :param timerange: TimeRange = The full data timerange for populating the indicators +# and training the model. +# :param dp: DataProvider instance attached to the strategy +# """ + +# if dp._exchange is not None: +# markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) +# or config.get('include_inactive')] +# else: +# # This should not occur: +# raise OperationalException('No exchange object found.') + +# all_pairs = dynamic_expand_pairlist(config, markets) + +# if not dp._exchange: +# # Not realistic - this is only called in live mode. +# raise OperationalException("Dataprovider did not have an exchange attached.") + +# time = datetime.now(tz=timezone.utc).timestamp() + +# for tf in config["freqai"]["feature_parameters"].get("include_timeframes"): +# timerange = TimeRange() +# timerange.startts = int(time) +# timerange.stopts = int(time) +# startup_candles = dp.get_required_startup(str(tf)) +# tf_seconds = timeframe_to_seconds(str(tf)) +# timerange.subtract_start(tf_seconds * startup_candles) +# new_pairs_days = int((timerange.stopts - timerange.startts) / 86400) +# # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function +# # redownloads the funding rate for each pair. +# refresh_backtest_ohlcv_data( +# dp._exchange, +# pairs=all_pairs, +# timeframes=[tf], +# datadir=config["datadir"], +# timerange=timerange, +# new_pairs_days=new_pairs_days, +# erase=False, +# data_format=config.get("dataformat_ohlcv", "json"), +# trading_mode=config.get("trading_mode", "spot"), +# prepend=config.get("prepend_data", False), +# ) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index c9ec466de..3ea1a3fae 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -157,12 +157,10 @@ class IStrategy(ABC, HyperStrategyMixin): if self.config.get('runmode') in (RunMode.DRY_RUN, RunMode.LIVE): logger.info( "Downloading all training data for all pairs in whitelist and " - "corr_pairlist, this may take a while if you do not have the " - "data saved" + "corr_pairlist, this may take a while if the data is not " + "already on disk." ) - # data_load_timerange = get_required_data_timerange(self.config) download_all_data_for_training(self.dp, self.config) - else: # Gracious failures if freqAI is disabled but "start" is called. class DummyClass(): From 7ba4fda5d7c4f472bb61bf03fe91e7f2b1564762 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 Aug 2022 10:26:47 +0000 Subject: [PATCH 09/26] Implement PR feedback --- freqtrade/freqai/utils.py | 12 ++++-------- tests/freqai/test_freqai_backtesting.py | 6 ------ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index d56702049..6081b6ce5 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -22,21 +22,17 @@ def download_all_data_for_training(dp: DataProvider, config: dict) -> None: :param dp: DataProvider instance attached to the strategy """ - if dp._exchange is not None: - markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) - or config.get('include_inactive')] - else: - # This should not occur: + if dp._exchange is None: raise OperationalException('No exchange object found.') + markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) + or config.get('include_inactive')] all_pairs = dynamic_expand_pairlist(config, markets) timerange = get_required_data_timerange(config) new_pairs_days = int((timerange.stopts - timerange.startts) / 86400) - if not dp._exchange: - # Not realistic - this is only called in live mode. - raise OperationalException("Dataprovider did not have an exchange attached.") + refresh_backtest_ohlcv_data( dp._exchange, pairs=all_pairs, diff --git a/tests/freqai/test_freqai_backtesting.py b/tests/freqai/test_freqai_backtesting.py index c8a51edb0..ea127fa99 100644 --- a/tests/freqai/test_freqai_backtesting.py +++ b/tests/freqai/test_freqai_backtesting.py @@ -48,10 +48,4 @@ def test_freqai_backtest_load_data(freqai_conf, mocker, caplog): assert log_has_re('Increasing startup_candle_count for freqai to.*', caplog) - # del freqai_conf['freqai']['startup_candles'] - # backtesting = Backtesting(freqai_conf) - # with pytest.raises(OperationalException, - # match=r'FreqAI backtesting module.*startup_candles in config.'): - # backtesting.load_bt_data() - Backtesting.cleanup() From 13ccd940d5e9d9bacedb896cbe4859217f487dde Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 Aug 2022 10:26:58 +0000 Subject: [PATCH 10/26] Remove startup_candle_count from freqai sample config to avoid confusion --- config_examples/config_freqai.example.json | 8 +++++--- freqtrade/templates/FreqaiHybridExampleStrategy.py | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json index 7112fc225..13c7a94ea 100644 --- a/config_examples/config_freqai.example.json +++ b/config_examples/config_freqai.example.json @@ -9,7 +9,6 @@ "dry_run": true, "timeframe": "3m", "dry_run_wallet": 1000, - "startup_candle_count": 20, "cancel_open_orders_on_exit": true, "unfilledtimeout": { "entry": 10, @@ -76,7 +75,10 @@ "principal_component_analysis": false, "use_SVM_to_remove_outliers": true, "indicator_max_period_candles": 20, - "indicator_periods_candles": [10, 20] + "indicator_periods_candles": [ + 10, + 20 + ] }, "data_split_parameters": { "test_size": 0.33, @@ -92,4 +94,4 @@ "internals": { "process_throttle_secs": 5 } -} +} \ No newline at end of file diff --git a/freqtrade/templates/FreqaiHybridExampleStrategy.py b/freqtrade/templates/FreqaiHybridExampleStrategy.py index 0a91455f5..5d1e149dd 100644 --- a/freqtrade/templates/FreqaiHybridExampleStrategy.py +++ b/freqtrade/templates/FreqaiHybridExampleStrategy.py @@ -45,7 +45,6 @@ class FreqaiExampleHybridStrategy(IStrategy): "weight_factor": 0.9, "principal_component_analysis": false, "use_SVM_to_remove_outliers": true, - "indicator_max_period_candles": 20, "indicator_periods_candles": [10, 20] }, "data_split_parameters": { From 57ff6f8ac592678c5399e83e75656c66076bd863 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 Aug 2022 10:28:31 +0000 Subject: [PATCH 11/26] Init timerange object properly --- freqtrade/freqai/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 6081b6ce5..6a70f050f 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -55,7 +55,6 @@ def get_required_data_timerange( for auto data-download in FreqAI """ time = datetime.now(tz=timezone.utc).timestamp() - data_load_timerange = TimeRange() timeframes = config["freqai"]["feature_parameters"].get("include_timeframes") @@ -75,12 +74,13 @@ def get_required_data_timerange( additional_seconds = max_period * max_tf_seconds - data_load_timerange.startts = int( + startts = int( time - config["freqai"].get("train_period_days", 0) * 86400 - additional_seconds ) - data_load_timerange.stopts = int(time) + stopts = int(time) + data_load_timerange = TimeRange('date', 'date', startts, stopts) return data_load_timerange From df51da22ee699e9a362d980747dba28e578d6c47 Mon Sep 17 00:00:00 2001 From: Wagner Costa Santos Date: Wed, 31 Aug 2022 11:23:48 -0300 Subject: [PATCH 12/26] refactoring freqai backtesting --- freqtrade/freqai/data_kitchen.py | 37 ++++++++++- freqtrade/freqai/freqai_interface.py | 98 ++++++++++++++++++++++++---- 2 files changed, 119 insertions(+), 16 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 763a07375..80b795b8e 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1,6 +1,7 @@ import copy import datetime import logging +import os import shutil from pathlib import Path from typing import Any, Dict, List, Tuple @@ -780,9 +781,10 @@ class FreqaiDataKitchen: weights = np.exp(-np.arange(num_weights) / (wfactor * num_weights))[::-1] return weights - def append_predictions(self, predictions: DataFrame, do_predict: npt.ArrayLike) -> None: + def get_predictions_to_append(self, predictions: DataFrame, + do_predict: npt.ArrayLike) -> DataFrame: """ - Append backtest prediction from current backtest period to all previous periods + Get backtest prediction from current backtest period """ append_df = DataFrame() @@ -797,12 +799,19 @@ class FreqaiDataKitchen: if self.freqai_config["feature_parameters"].get("DI_threshold", 0) > 0: append_df["DI_values"] = self.DI_values + return append_df + + def append_predictions(self, append_df: DataFrame) -> None: + """ + Append backtest prediction from current backtest period to all previous periods + """ + if self.full_df.empty: self.full_df = append_df else: self.full_df = pd.concat([self.full_df, append_df], axis=0) - return + return append_df def fill_predictions(self, dataframe): """ @@ -1089,3 +1098,25 @@ class FreqaiDataKitchen: if self.unique_classes: for label in self.unique_classes: self.unique_class_list += list(self.unique_classes[label]) + + def save_backtesting_prediction( + self, file_name: str, root_folder: str, append_df: DataFrame + ) -> None: + + """ + Save prediction dataframe from backtesting to h5 file format + :param file_name: h5 file name + :param root_folder: folder to save h5 file + """ + os.makedirs(root_folder, exist_ok=True) + append_df.to_hdf(file_name, key='append_df', mode='w') + + def get_backtesting_prediction(self, prediction_file_name: str) -> DataFrame: + """ + Retrive from disk the prediction dataframe + :param prediction_file_name: prediction file full path + :return: + :Dataframe: Backtesting prediction from current backtesting period + """ + append_df = pd.read_hdf(prediction_file_name) + return append_df diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 4106f24e0..d396113e8 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -224,28 +224,50 @@ class IFreqaiModel(ABC): "trains" ) + trained_timestamp_int = int(trained_timestamp.stopts) dk.data_path = Path( dk.full_path / - f"sub-train-{metadata['pair'].split('/')[0]}_{int(trained_timestamp.stopts)}" + f"sub-train-{metadata['pair'].split('/')[0]}_{trained_timestamp_int}" ) - if not self.model_exists( - metadata["pair"], dk, trained_timestamp=int(trained_timestamp.stopts) + + if self.backtest_prediction_exists( + metadata["pair"], dk, trained_timestamp=trained_timestamp_int ): - dk.find_features(dataframe_train) - self.model = self.train(dataframe_train, metadata["pair"], dk) - self.dd.pair_dict[metadata["pair"]]["trained_timestamp"] = int( - trained_timestamp.stopts) - dk.set_new_model_names(metadata["pair"], trained_timestamp) - self.dd.save_data(self.model, metadata["pair"], dk) + prediction_filename, _ = self.get_backtesting_prediction_file_name( + metadata["pair"], + dk, + trained_timestamp=int(trained_timestamp.stopts)) + + append_df = dk.get_backtesting_prediction(prediction_filename) + dk.append_predictions(append_df) else: - self.model = self.dd.load_data(metadata["pair"], dk) + if not self.model_exists( + metadata["pair"], dk, trained_timestamp=trained_timestamp_int + ): + dk.find_features(dataframe_train) + self.model = self.train(dataframe_train, metadata["pair"], dk) + self.dd.pair_dict[metadata["pair"]]["trained_timestamp"] = int( + trained_timestamp.stopts) + dk.set_new_model_names(metadata["pair"], trained_timestamp) + self.dd.save_data(self.model, metadata["pair"], dk) + else: + self.model = self.dd.load_data(metadata["pair"], dk) - self.check_if_feature_list_matches_strategy(dataframe_train, dk) + self.check_if_feature_list_matches_strategy(dataframe_train, dk) - pred_df, do_preds = self.predict(dataframe_backtest, dk) + pred_df, do_preds = self.predict(dataframe_backtest, dk) + append_df = dk.get_predictions_to_append(pred_df, do_preds) + dk.append_predictions(append_df) - dk.append_predictions(pred_df, do_preds) + prediction_file_name, root_prediction = self.get_backtesting_prediction_file_name( + metadata["pair"], + dk, + trained_timestamp_int) + + dk.save_backtesting_prediction(prediction_file_name, + root_prediction, + append_df) dk.fill_predictions(dataframe) @@ -643,6 +665,56 @@ class IFreqaiModel(ABC): self.train_time = 0 return + def backtest_prediction_exists( + self, + pair: str, + dk: FreqaiDataKitchen, + trained_timestamp: int, + scanning: bool = False, + ) -> bool: + """ + Given a pair and path, check if a backtesting prediction already exists + :param pair: pair e.g. BTC/USD + :param path: path to prediction + :return: + :boolean: whether the prediction file exists or not. + """ + if not self.live: + prediction_file_name, _ = self.get_backtesting_prediction_file_name( + pair, dk, trained_timestamp + ) + path_to_predictionfile = Path(prediction_file_name) + + file_exists = path_to_predictionfile.is_file() + if file_exists and not scanning: + logger.info("Found backtesting prediction file at %s", prediction_file_name) + elif not scanning: + logger.info( + "Could not find backtesting prediction file at %s", prediction_file_name + ) + return file_exists + else: + return False + + def get_backtesting_prediction_file_name( + self, pair: str, dk: FreqaiDataKitchen, trained_timestamp: int + ): + """ + Given a pair, path and a trained timestamp, + returns the path and name of the predictions file + :param pair: pair e.g. BTC/USD + :param dk: FreqaiDataKitchen + :trained_timestamp: current backtesting timestamp period + :return: + :str: prediction file name + :str: prediction root path + """ + coin, _ = pair.split("/") + prediction_base_filename = f"{coin.lower()}_{trained_timestamp}" + root_prediction = f'{dk.full_path}/backtesting_predictions' + prediction_file_name = f"{root_prediction}/{prediction_base_filename}_predictions.h5" + return prediction_file_name, root_prediction + # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModel.py for an example. From 7bed0450d2c6ae90dd00d98a51b18701be6c4874 Mon Sep 17 00:00:00 2001 From: Wagner Costa Santos Date: Wed, 31 Aug 2022 15:36:29 -0300 Subject: [PATCH 13/26] pr review - refactoring backtesting freqai --- freqtrade/freqai/data_kitchen.py | 19 ++++++++++++++----- freqtrade/freqai/freqai_interface.py | 12 ++++++------ tests/freqai/test_freqai_interface.py | 8 ++++---- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 80b795b8e..8dc94e9ec 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1,7 +1,6 @@ import copy import datetime import logging -import os import shutil from pathlib import Path from typing import Any, Dict, List, Tuple @@ -1108,15 +1107,25 @@ class FreqaiDataKitchen: :param file_name: h5 file name :param root_folder: folder to save h5 file """ - os.makedirs(root_folder, exist_ok=True) - append_df.to_hdf(file_name, key='append_df', mode='w') + backtesting_root = Path( + self.full_path + / root_folder + ) + if not backtesting_root.is_dir(): + backtesting_root.mkdir(parents=True, exist_ok=True) - def get_backtesting_prediction(self, prediction_file_name: str) -> DataFrame: + full_file_path = Path(self.full_path / root_folder / file_name) + append_df.to_hdf(full_file_path, key='append_df', mode='w') + + def get_backtesting_prediction( + self, root_prediction: str, prediction_file_name: str + ) -> DataFrame: """ Retrive from disk the prediction dataframe :param prediction_file_name: prediction file full path :return: :Dataframe: Backtesting prediction from current backtesting period """ - append_df = pd.read_hdf(prediction_file_name) + prediction_path = Path(self.full_path / root_prediction / prediction_file_name) + append_df = pd.read_hdf(prediction_path) return append_df diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index d396113e8..ad64588a7 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -234,12 +234,12 @@ class IFreqaiModel(ABC): if self.backtest_prediction_exists( metadata["pair"], dk, trained_timestamp=trained_timestamp_int ): - prediction_filename, _ = self.get_backtesting_prediction_file_name( + prediction_filename, root_prediction = self.get_backtesting_prediction_file_name( metadata["pair"], dk, trained_timestamp=int(trained_timestamp.stopts)) - append_df = dk.get_backtesting_prediction(prediction_filename) + append_df = dk.get_backtesting_prediction(root_prediction, prediction_filename) dk.append_predictions(append_df) else: if not self.model_exists( @@ -680,10 +680,10 @@ class IFreqaiModel(ABC): :boolean: whether the prediction file exists or not. """ if not self.live: - prediction_file_name, _ = self.get_backtesting_prediction_file_name( + prediction_file_name, root_prediction = self.get_backtesting_prediction_file_name( pair, dk, trained_timestamp ) - path_to_predictionfile = Path(prediction_file_name) + path_to_predictionfile = Path(dk.full_path / root_prediction / prediction_file_name) file_exists = path_to_predictionfile.is_file() if file_exists and not scanning: @@ -711,8 +711,8 @@ class IFreqaiModel(ABC): """ coin, _ = pair.split("/") prediction_base_filename = f"{coin.lower()}_{trained_timestamp}" - root_prediction = f'{dk.full_path}/backtesting_predictions' - prediction_file_name = f"{root_prediction}/{prediction_base_filename}_predictions.h5" + root_prediction = 'backtesting_predictions' + prediction_file_name = f"{prediction_base_filename}_predictions.h5" return prediction_file_name, root_prediction # Following methods which are overridden by user made prediction models. diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 792ffc467..09f5d27ff 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -192,7 +192,7 @@ def test_start_backtesting(mocker, freqai_conf): freqai.start_backtesting(df, metadata, freqai.dk) model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] - assert len(model_folders) == 5 + assert len(model_folders) == 6 shutil.rmtree(Path(freqai.dk.full_path)) @@ -217,7 +217,7 @@ def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf): metadata = {"pair": "LTC/BTC"} freqai.start_backtesting(df, metadata, freqai.dk) model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] - assert len(model_folders) == 8 + assert len(model_folders) == 9 shutil.rmtree(Path(freqai.dk.full_path)) @@ -242,7 +242,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): freqai.start_backtesting(df, metadata, freqai.dk) model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] - assert len(model_folders) == 5 + assert len(model_folders) == 6 # without deleting the exiting folder structure, re-run @@ -263,7 +263,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): freqai.start_backtesting(df, metadata, freqai.dk) assert log_has_re( - "Found model at ", + "Found backtesting prediction ", caplog, ) From ba2eb7cf0f9f517f526a44b3efc792674911a5da Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 1 Sep 2022 06:35:24 +0200 Subject: [PATCH 14/26] Fix BNB fee bug when selling thanks @epigramx, for reporting and for the detailed data. --- freqtrade/freqtradebot.py | 16 +++++++++++----- tests/test_freqtradebot.py | 8 +++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5393e3d39..37bc6dfed 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1778,7 +1778,7 @@ class FreqtradeBot(LoggingMixin): self.rpc.send_msg(msg) def apply_fee_conditional(self, trade: Trade, trade_base_currency: str, - amount: float, fee_abs: float) -> Optional[float]: + amount: float, fee_abs: float, order_obj: Order) -> Optional[float]: """ Applies the fee to amount (either from Order or from Trades). Can eat into dust if more than the required asset is available. @@ -1786,7 +1786,12 @@ class FreqtradeBot(LoggingMixin): never in base currency. """ self.wallets.update() - if fee_abs != 0 and self.wallets.get_free(trade_base_currency) >= amount: + amount_ = amount + if order_obj.ft_order_side == trade.exit_side or order_obj.ft_order_side == 'stoploss': + # check against remaining amount! + amount_ = trade.amount - amount + + if fee_abs != 0 and self.wallets.get_free(trade_base_currency) >= amount_: # Eat into dust if we own more than base currency logger.info(f"Fee amount for {trade} was in base currency - " f"Eating Fee {fee_abs} into dust.") @@ -1833,7 +1838,8 @@ class FreqtradeBot(LoggingMixin): if trade_base_currency == fee_currency: # Apply fee to amount return self.apply_fee_conditional(trade, trade_base_currency, - amount=order_amount, fee_abs=fee_cost) + amount=order_amount, fee_abs=fee_cost, + order_obj=order_obj) return None return self.fee_detection_from_trades( trade, order, order_obj, order_amount, order.get('trades', [])) @@ -1892,8 +1898,8 @@ class FreqtradeBot(LoggingMixin): raise DependencyException("Half bought? Amounts don't match") if fee_abs != 0: - return self.apply_fee_conditional(trade, trade_base_currency, - amount=amount, fee_abs=fee_abs) + return self.apply_fee_conditional( + trade, trade_base_currency, amount=amount, fee_abs=fee_abs, order_obj=order_obj) return None def get_valid_price(self, custom_price: float, proposed_price: float) -> float: diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e6c6e7978..aff0504b3 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -4650,11 +4650,17 @@ def test_apply_fee_conditional(default_conf_usdt, fee, mocker, fee_close=fee.return_value, open_order_id="123456" ) + order = Order( + ft_order_side='buy', + order_id='100', + ft_pair=trade.pair, + ft_is_open=True, + ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) walletmock.reset_mock() # Amount is kept as is - assert freqtrade.apply_fee_conditional(trade, 'LTC', amount, fee_abs) == amount_exp + assert freqtrade.apply_fee_conditional(trade, 'LTC', amount, fee_abs, order) == amount_exp assert walletmock.call_count == 1 From f3c73189d55be454845a4e9cc7d8b084a2b0853c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 1 Sep 2022 06:49:51 +0200 Subject: [PATCH 15/26] Remove pointless default on wallet_balance argument --- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_exchange.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 9d08d3d19..c3dca43a8 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2600,7 +2600,7 @@ class Exchange: is_short: bool, amount: float, # Absolute value of position size stake_amount: float, - wallet_balance: float = 0.0, + wallet_balance: float, mm_ex_1: float = 0.0, # (Binance) Cross only upnl_ex_1: float = 0.0, # (Binance) Cross only ) -> Optional[float]: diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 5456b3098..3352019a9 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -4985,6 +4985,7 @@ def test_get_liquidation_price1(mocker, default_conf): is_short=False, amount=0.8, stake_amount=18.884 * 0.8, + wallet_balance=18.884 * 0.8, ) assert liq_price == 17.47 @@ -4996,6 +4997,7 @@ def test_get_liquidation_price1(mocker, default_conf): is_short=False, amount=0.8, stake_amount=18.884 * 0.8, + wallet_balance=18.884 * 0.8, ) assert liq_price == 17.540699999999998 @@ -5007,6 +5009,7 @@ def test_get_liquidation_price1(mocker, default_conf): is_short=False, amount=0.8, stake_amount=18.884 * 0.8, + wallet_balance=18.884 * 0.8, ) assert liq_price is None default_conf['trading_mode'] = 'margin' @@ -5019,6 +5022,7 @@ def test_get_liquidation_price1(mocker, default_conf): is_short=False, amount=0.8, stake_amount=18.884 * 0.8, + wallet_balance=18.884 * 0.8, ) From d6e115178a117e6f22f648b1d0def25b90eec471 Mon Sep 17 00:00:00 2001 From: Wagner Costa Santos Date: Thu, 1 Sep 2022 07:09:23 -0300 Subject: [PATCH 16/26] refactoring freqai backtesting - remove duplicate code --- freqtrade/freqai/data_kitchen.py | 32 ++++++-------- freqtrade/freqai/freqai_interface.py | 62 ++++++---------------------- 2 files changed, 24 insertions(+), 70 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 361d9872d..f88e20223 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -69,6 +69,8 @@ class FreqaiDataKitchen: self.label_list: List = [] self.training_features_list: List = [] self.model_filename: str = "" + self.backtesting_results_path = Path() + self.backtesting_prediction_folder: str = "backtesting_predictions" self.live = live self.pair = pair @@ -808,8 +810,6 @@ class FreqaiDataKitchen: else: self.full_df = pd.concat([self.full_df, append_df], axis=0) - return append_df - def fill_predictions(self, dataframe): """ Back fill values to before the backtesting range so that the dataframe matches size @@ -1070,33 +1070,25 @@ class FreqaiDataKitchen: self.unique_class_list += list(self.unique_classes[label]) def save_backtesting_prediction( - self, file_name: str, root_folder: str, append_df: DataFrame + self, append_df: DataFrame ) -> None: """ Save prediction dataframe from backtesting to h5 file format - :param file_name: h5 file name - :param root_folder: folder to save h5 file + :param append_df: dataframe for backtesting period """ - backtesting_root = Path( - self.full_path - / root_folder - ) - if not backtesting_root.is_dir(): - backtesting_root.mkdir(parents=True, exist_ok=True) + full_predictions_folder = Path(self.full_path / self.backtesting_prediction_folder) + if not full_predictions_folder.is_dir(): + full_predictions_folder.mkdir(parents=True, exist_ok=True) - full_file_path = Path(self.full_path / root_folder / file_name) - append_df.to_hdf(full_file_path, key='append_df', mode='w') + append_df.to_hdf(self.backtesting_results_path, key='append_df', mode='w') def get_backtesting_prediction( - self, root_prediction: str, prediction_file_name: str + self ) -> DataFrame: + """ - Retrive from disk the prediction dataframe - :param prediction_file_name: prediction file full path - :return: - :Dataframe: Backtesting prediction from current backtesting period + Get prediction dataframe from h5 file format """ - prediction_path = Path(self.full_path / root_prediction / prediction_file_name) - append_df = pd.read_hdf(prediction_path) + append_df = pd.read_hdf(self.backtesting_results_path) return append_df diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 2297811b4..0a63e36ea 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -231,15 +231,11 @@ class IFreqaiModel(ABC): f"sub-train-{metadata['pair'].split('/')[0]}_{trained_timestamp_int}" ) - if self.backtest_prediction_exists( - metadata["pair"], dk, trained_timestamp=trained_timestamp_int - ): - prediction_filename, root_prediction = self.get_backtesting_prediction_file_name( - metadata["pair"], - dk, - trained_timestamp=int(trained_timestamp.stopts)) + coin, _ = metadata["pair"].split("/") + dk.model_filename = f"cb_{coin.lower()}_{trained_timestamp_int}" - append_df = dk.get_backtesting_prediction(root_prediction, prediction_filename) + if self.backtest_prediction_exists(dk): + append_df = dk.get_backtesting_prediction() dk.append_predictions(append_df) else: if not self.model_exists( @@ -259,15 +255,7 @@ class IFreqaiModel(ABC): pred_df, do_preds = self.predict(dataframe_backtest, dk) append_df = dk.get_predictions_to_append(pred_df, do_preds) dk.append_predictions(append_df) - - prediction_file_name, root_prediction = self.get_backtesting_prediction_file_name( - metadata["pair"], - dk, - trained_timestamp_int) - - dk.save_backtesting_prediction(prediction_file_name, - root_prediction, - append_df) + dk.save_backtesting_prediction(append_df) dk.fill_predictions(dataframe) @@ -478,11 +466,6 @@ class IFreqaiModel(ABC): :return: :boolean: whether the model file exists or not. """ - coin, _ = pair.split("/") - - if not self.live: - dk.model_filename = model_filename = f"cb_{coin.lower()}_{trained_timestamp}" - path_to_modelfile = Path(dk.data_path / f"{model_filename}_model.joblib") file_exists = path_to_modelfile.is_file() if file_exists and not scanning: @@ -661,23 +644,21 @@ class IFreqaiModel(ABC): def backtest_prediction_exists( self, - pair: str, dk: FreqaiDataKitchen, - trained_timestamp: int, scanning: bool = False, ) -> bool: """ - Given a pair and path, check if a backtesting prediction already exists - :param pair: pair e.g. BTC/USD - :param path: path to prediction + Check if a backtesting prediction already exists + :param dk: FreqaiDataKitchen :return: :boolean: whether the prediction file exists or not. """ if not self.live: - prediction_file_name, root_prediction = self.get_backtesting_prediction_file_name( - pair, dk, trained_timestamp - ) - path_to_predictionfile = Path(dk.full_path / root_prediction / prediction_file_name) + prediction_file_name = dk.model_filename + path_to_predictionfile = Path(dk.full_path / + dk.backtesting_prediction_folder / + f"{prediction_file_name}_prediction.h5") + dk.backtesting_results_path = path_to_predictionfile file_exists = path_to_predictionfile.is_file() if file_exists and not scanning: @@ -690,25 +671,6 @@ class IFreqaiModel(ABC): else: return False - def get_backtesting_prediction_file_name( - self, pair: str, dk: FreqaiDataKitchen, trained_timestamp: int - ): - """ - Given a pair, path and a trained timestamp, - returns the path and name of the predictions file - :param pair: pair e.g. BTC/USD - :param dk: FreqaiDataKitchen - :trained_timestamp: current backtesting timestamp period - :return: - :str: prediction file name - :str: prediction root path - """ - coin, _ = pair.split("/") - prediction_base_filename = f"{coin.lower()}_{trained_timestamp}" - root_prediction = 'backtesting_predictions' - prediction_file_name = f"{prediction_base_filename}_predictions.h5" - return prediction_file_name, root_prediction - # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModel.py for an example. From 61d5fc0e0836aa880ce00bf60981027a0bce7f40 Mon Sep 17 00:00:00 2001 From: epigramx Date: Thu, 1 Sep 2022 17:22:34 +0300 Subject: [PATCH 17/26] Make the recommendation for Binance/Kucoin blacklisting more accurate. Now that a recent bug regarding selling BNB is fixed, it should be safe to trade it, but with a warning that the user may have to manually maintain extra BNB. Also the old text implied those features are always unabled so this texts makes it clear those fee-related features can be also disabled. I'm not sure if it's still true that an "eaten by fees" position becomes unsellable but I left that as it is. --- docs/exchanges.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index 407a67d70..b55eaac74 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -61,8 +61,8 @@ Binance supports [time_in_force](configuration.md#understand-order_time_in_force ### Binance Blacklist -For Binance, please add `"BNB/"` to your blacklist to avoid issues. -Accounts having BNB accounts use this to pay for fees - if your first trade happens to be on `BNB`, further trades will consume this position and make the initial BNB trade unsellable as the expected amount is not there anymore. +For Binance, it is suggested to add `"BNB/"` to your blacklist to avoid issues, unless you are willing to maintain enough extra `BNB` on the account or, unless you're willing to disable using `BNB` for fees. +Binance accounts may use `BNB` for fees, and if a trade happens to be on `BNB`, further trades may consume this position and make the initial BNB trade unsellable as the expected amount is not there anymore. ### Binance Futures @@ -205,8 +205,8 @@ Kucoin supports [time_in_force](configuration.md#understand-order_time_in_force) ### Kucoin Blacklists -For Kucoin, please add `"KCS/"` to your blacklist to avoid issues. -Accounts having KCS accounts use this to pay for fees - if your first trade happens to be on `KCS`, further trades will consume this position and make the initial KCS trade unsellable as the expected amount is not there anymore. +For Kucoin, it is suggested to add `"KCS/"` to your blacklist to avoid issues, unless you are willing to maintain enough extra `KCS` on the account or, unless you're willing to disable using `KCS` for fees. +Kucoin accounts may use `KCS` for fees, and if a trade happens to be on `KCS`, further trades may consume this position and make the initial `KCS` trade unsellable as the expected amount is not there anymore. ## Huobi From 11fbfd3402996db93870b5d4beadc2b0840bc37c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 1 Sep 2022 19:39:20 +0200 Subject: [PATCH 18/26] Remove unnecessary assignment --- freqtrade/persistence/trade_model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 23997f835..1f14f110e 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -648,7 +648,6 @@ class LocalTrade(): """ self.close_rate = rate self.close_date = self.close_date or datetime.utcnow() - self.close_profit_abs = self.calc_profit(rate) + self.realized_profit self.is_open = False self.exit_order_status = 'closed' self.open_order_id = None From b53791fef20990d2a46b9dbf6795fb8cfc490c2f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 2 Sep 2022 07:11:20 +0200 Subject: [PATCH 19/26] Futures volumepairlist to account for contract size --- freqtrade/plugins/pairlist/VolumePairList.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 8138a5fb6..d7cc6e5ec 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -186,6 +186,7 @@ class VolumePairList(IPairList): needed_pairs, since_ms=since_ms, cache=False ) for i, p in enumerate(filtered_tickers): + contract_size = self._exchange.markets[p['symbol']].get('contractSize', 1.0) or 1.0 pair_candles = candles[ (p['symbol'], self._lookback_timeframe, self._def_candletype) ] if ( @@ -199,6 +200,7 @@ class VolumePairList(IPairList): pair_candles['quoteVolume'] = ( pair_candles['volume'] * pair_candles['typical_price'] + * contract_size ) else: # Exchange ohlcv data is in quote volume already. From a948e51389108d7b827d4ad4551fd861b88bdb71 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 2 Sep 2022 19:56:12 +0200 Subject: [PATCH 20/26] Update futures docs to define pair namings #7334, #7136, ... --- docs/leverage.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/leverage.md b/docs/leverage.md index 491e6eda0..429aff86c 100644 --- a/docs/leverage.md +++ b/docs/leverage.md @@ -13,7 +13,7 @@ Please only use advanced trading modes when you know how freqtrade (and your strategy) works. Also, never risk more than what you can afford to lose. -Please read the [strategy migration guide](strategy_migration.md#strategy-migration-between-v2-and-v3) to migrate your strategy from a freqtrade v2 strategy, to v3 strategy that can short and trade futures. +If you already have an existing strategy, please read the [strategy migration guide](strategy_migration.md#strategy-migration-between-v2-and-v3) to migrate your strategy from a freqtrade v2 strategy, to strategy of version 3 which can short and trade futures. ## Shorting @@ -62,6 +62,13 @@ You will also have to pick a "margin mode" (explanation below) - with freqtrade "margin_mode": "isolated" ``` +##### Pair namings + +Freqtrade follows the [ccxt naming conventions for futures](https://docs.ccxt.com/en/latest/manual.html?#perpetual-swap-perpetual-future). +A futures pair will therefore have the naming of `base/quote:settle` (e.g. `ETH/USDT:USDT`). + +Binance is currently still an exception to this naming scheme, where pairs are named `ETH/USDT` also for futures markets, but will be aligned as soon as CCXT is ready. + ### Margin mode On top of `trading_mode` - you will also have to configure your `margin_mode`. From b26126cb57300cefb51222ddf185a4f929bdbba8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 2 Sep 2022 20:08:38 +0200 Subject: [PATCH 21/26] Don't use ticker['symbol'] but use "pair" instead closes #7262 --- docs/exchanges.md | 4 ++-- freqtrade/plugins/pairlist/PrecisionFilter.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index b55eaac74..b5470f65a 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -61,7 +61,7 @@ Binance supports [time_in_force](configuration.md#understand-order_time_in_force ### Binance Blacklist -For Binance, it is suggested to add `"BNB/"` to your blacklist to avoid issues, unless you are willing to maintain enough extra `BNB` on the account or, unless you're willing to disable using `BNB` for fees. +For Binance, it is suggested to add `"BNB/"` to your blacklist to avoid issues, unless you are willing to maintain enough extra `BNB` on the account or unless you're willing to disable using `BNB` for fees. Binance accounts may use `BNB` for fees, and if a trade happens to be on `BNB`, further trades may consume this position and make the initial BNB trade unsellable as the expected amount is not there anymore. ### Binance Futures @@ -205,7 +205,7 @@ Kucoin supports [time_in_force](configuration.md#understand-order_time_in_force) ### Kucoin Blacklists -For Kucoin, it is suggested to add `"KCS/"` to your blacklist to avoid issues, unless you are willing to maintain enough extra `KCS` on the account or, unless you're willing to disable using `KCS` for fees. +For Kucoin, it is suggested to add `"KCS/"` to your blacklist to avoid issues, unless you are willing to maintain enough extra `KCS` on the account or unless you're willing to disable using `KCS` for fees. Kucoin accounts may use `KCS` for fees, and if a trade happens to be on `KCS`, further trades may consume this position and make the initial `KCS` trade unsellable as the expected amount is not there anymore. ## Huobi diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index dcd153d8e..61150f03d 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -52,7 +52,7 @@ class PrecisionFilter(IPairList): :return: True if the pair can stay, false if it should be removed """ if ticker.get('last', None) is None: - self.log_once(f"Removed {ticker['symbol']} from whitelist, because " + self.log_once(f"Removed {pair} from whitelist, because " "ticker['last'] is empty (Usually no trade in the last 24h).", logger.info) return False @@ -62,10 +62,10 @@ class PrecisionFilter(IPairList): sp = self._exchange.price_to_precision(pair, stop_price) stop_gap_price = self._exchange.price_to_precision(pair, stop_price * 0.99) - logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") + logger.debug(f"{pair} - {sp} : {stop_gap_price}") if sp <= stop_gap_price: - self.log_once(f"Removed {ticker['symbol']} from whitelist, because " + self.log_once(f"Removed {pair} from whitelist, because " f"stop price {sp} would be <= stop limit {stop_gap_price}", logger.info) return False From af5460cebf2b17ea440f1d7f037a7b8c88681d6a Mon Sep 17 00:00:00 2001 From: Wagner Costa Santos Date: Fri, 2 Sep 2022 22:01:53 -0300 Subject: [PATCH 22/26] Add option to keep models only in memory for backtest --- config_examples/config_freqai.example.json | 3 ++- docs/freqai.md | 1 + freqtrade/freqai/freqai_interface.py | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json index 13c7a94ea..846d37a82 100644 --- a/config_examples/config_freqai.example.json +++ b/config_examples/config_freqai.example.json @@ -56,6 +56,7 @@ "purge_old_models": true, "train_period_days": 15, "backtest_period_days": 7, + "backtest_save_model": true, "live_retrain_hours": 0, "identifier": "uniqe-id", "feature_parameters": { @@ -94,4 +95,4 @@ "internals": { "process_throttle_secs": 5 } -} \ No newline at end of file +} diff --git a/docs/freqai.md b/docs/freqai.md index 482a56d2b..6ee124b9b 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -93,6 +93,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `purge_old_models` | Delete obsolete models (otherwise, all historic models will remain on disk).
**Datatype:** Boolean. Default: `False`. | `train_period_days` | **Required.**
Number of days to use for the training data (width of the sliding window).
**Datatype:** Positive integer. | `backtest_period_days` | **Required.**
Number of days to inference from the trained model before sliding the window defined above, and retraining the model. This can be fractional days, but beware that the user-provided `timerange` will be divided by this number to yield the number of trainings necessary to complete the backtest.
**Datatype:** Float. +| `backtest_save_model` | Saves models to disk when running backtesting.
**Datatype:** Boolean. Default: `True`. | `identifier` | **Required.**
A unique name for the current model. This can be reused to reload pre-trained models/data.
**Datatype:** String. | `live_retrain_hours` | Frequency of retraining during dry/live runs.
Default set to 0, which means the model will retrain as often as possible.
**Datatype:** Float > 0. | `expiration_hours` | Avoid making predictions if a model is more than `expiration_hours` old.
Defaults set to 0, which means models never expire.
**Datatype:** Positive integer. diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 0a63e36ea..9c7ef05a7 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -71,6 +71,7 @@ class IFreqaiModel(ABC): self.first = True self.set_full_path() self.follow_mode: bool = self.freqai_info.get("follow_mode", False) + self.backtest_save_model: bool = self.freqai_info.get("backtest_save_model", True) self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode) self.identifier: str = self.freqai_info.get("identifier", "no_id_provided") self.scanning = False @@ -246,7 +247,8 @@ class IFreqaiModel(ABC): self.dd.pair_dict[metadata["pair"]]["trained_timestamp"] = int( trained_timestamp.stopts) dk.set_new_model_names(metadata["pair"], trained_timestamp) - self.dd.save_data(self.model, metadata["pair"], dk) + if self.backtest_save_model: + self.dd.save_data(self.model, metadata["pair"], dk) else: self.model = self.dd.load_data(metadata["pair"], dk) From 966de1961180e39061277ffef00fdf78f1d86bdf Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Sep 2022 08:16:33 +0200 Subject: [PATCH 23/26] Improve test resiliance by properly setting Order object --- tests/plugins/test_protections.py | 2 ++ tests/test_persistence.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index acfe124a8..820eced20 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -37,6 +37,7 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool, trade.orders.append(Order( ft_order_side=trade.entry_side, order_id=f'{pair}-{trade.entry_side}-{trade.open_date}', + ft_is_open=False, ft_pair=pair, amount=trade.amount, filled=trade.amount, @@ -51,6 +52,7 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool, trade.orders.append(Order( ft_order_side=trade.exit_side, order_id=f'{pair}-{trade.exit_side}-{trade.close_date}', + ft_is_open=False, ft_pair=pair, amount=trade.amount, filled=trade.amount, diff --git a/tests/test_persistence.py b/tests/test_persistence.py index f16c8b054..3ce8a0a2c 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -654,6 +654,7 @@ def test_trade_close(fee): trade.orders.append(Order( ft_order_side=trade.entry_side, order_id=f'{trade.pair}-{trade.entry_side}-{trade.open_date}', + ft_is_open=False, ft_pair=trade.pair, amount=trade.amount, filled=trade.amount, @@ -667,6 +668,7 @@ def test_trade_close(fee): trade.orders.append(Order( ft_order_side=trade.exit_side, order_id=f'{trade.pair}-{trade.exit_side}-{trade.open_date}', + ft_is_open=False, ft_pair=trade.pair, amount=trade.amount, filled=trade.amount, From be192fae910612b745c9b062788d54903bf1c95c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Sep 2022 10:53:51 +0200 Subject: [PATCH 24/26] Test should use proper Order objects --- tests/test_persistence.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 3ce8a0a2c..23ccc67f3 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -581,25 +581,25 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, @pytest.mark.parametrize( 'exchange,is_short,lev,open_value,close_value,profit,profit_ratio,trading_mode,funding_fees', [ ("binance", False, 1, 60.15, 65.835, 5.685, 0.09451371, spot, 0.0), - ("binance", True, 1, 59.850, 66.1663784375, -6.3163784375, -0.1055368, margin, 0.0), + ("binance", True, 1, 65.835, 60.151253125, 5.68374687, 0.08633321, margin, 0.0), ("binance", False, 3, 60.15, 65.83416667, 5.68416667, 0.28349958, margin, 0.0), - ("binance", True, 3, 59.85, 66.1663784375, -6.3163784375, -0.31661044, margin, 0.0), + ("binance", True, 3, 65.835, 60.151253125, 5.68374687, 0.25899963, margin, 0.0), ("kraken", False, 1, 60.15, 65.835, 5.685, 0.09451371, spot, 0.0), - ("kraken", True, 1, 59.850, 66.231165, -6.381165, -0.1066192, margin, 0.0), + ("kraken", True, 1, 65.835, 60.21015, 5.62485, 0.0854386, margin, 0.0), ("kraken", False, 3, 60.15, 65.795, 5.645, 0.28154613, margin, 0.0), - ("kraken", True, 3, 59.850, 66.231165, -6.381165, -0.3198578, margin, 0.0), + ("kraken", True, 3, 65.835, 60.21015, 5.62485, 0.25631579, margin, 0.0), ("binance", False, 1, 60.15, 65.835, 5.685, 0.09451371, futures, 0.0), ("binance", False, 1, 60.15, 66.835, 6.685, 0.11113881, futures, 1.0), - ("binance", True, 1, 59.85, 66.165, -6.315, -0.10551378, futures, 0.0), - ("binance", True, 1, 59.85, 67.165, -7.315, -0.12222222, futures, -1.0), + ("binance", True, 1, 65.835, 60.15, 5.685, 0.08635224, futures, 0.0), + ("binance", True, 1, 65.835, 61.15, 4.685, 0.07116276, futures, -1.0), + ("binance", True, 3, 65.835, 59.15, 6.685, 0.3046252, futures, 1.0), ("binance", False, 3, 60.15, 64.835, 4.685, 0.23366583, futures, -1.0), - ("binance", True, 3, 59.85, 65.165, -5.315, -0.26641604, futures, 1.0), ]) @pytest.mark.usefixtures("init_persistence") def test_calc_open_close_trade_price( - limit_buy_order_usdt, limit_sell_order_usdt, fee, exchange, is_short, lev, + limit_order, fee, exchange, is_short, lev, open_value, close_value, profit, profit_ratio, trading_mode, funding_fees ): trade: Trade = Trade( @@ -617,22 +617,24 @@ def test_calc_open_close_trade_price( trading_mode=trading_mode, funding_fees=funding_fees ) - + entry_order = limit_order[trade.entry_side] + exit_order = limit_order[trade.exit_side] trade.open_order_id = f'something-{is_short}-{lev}-{exchange}' - oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') + oobj = Order.parse_from_ccxt_object(entry_order, 'ADA/USDT', trade.entry_side) + trade.orders.append(oobj) trade.update_trade(oobj) - oobj = Order.parse_from_ccxt_object(limit_sell_order_usdt, 'ADA/USDT', 'sell') + oobj = Order.parse_from_ccxt_object(exit_order, 'ADA/USDT', trade.exit_side) + trade.orders.append(oobj) trade.update_trade(oobj) - trade.open_rate = 2.0 - trade.close_rate = 2.2 - trade.recalc_open_trade_value() + assert trade.is_open is False + assert pytest.approx(trade._calc_open_trade_value(trade.amount, trade.open_rate)) == open_value assert pytest.approx(trade.calc_close_trade_value(trade.close_rate)) == close_value - assert pytest.approx(trade.calc_profit(trade.close_rate)) == round(profit, 8) - assert pytest.approx(trade.calc_profit_ratio(trade.close_rate)) == profit_ratio + assert pytest.approx(trade.close_profit_abs) == profit + assert pytest.approx(trade.close_profit) == profit_ratio @pytest.mark.usefixtures("init_persistence") From 599c1c79fb8170a7ea4bf9d250a4a3db0a3234ba Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sat, 3 Sep 2022 14:00:01 +0200 Subject: [PATCH 25/26] reorganized backtest utilities, test new functionality, improve/update doc --- config_examples/config_freqai.example.json | 2 - docs/freqai.md | 32 +++++++------ freqtrade/freqai/data_kitchen.py | 27 ++++++++++- freqtrade/freqai/freqai_interface.py | 50 +++++--------------- freqtrade/templates/FreqaiExampleStrategy.py | 2 +- tests/freqai/test_freqai_interface.py | 9 +++- 6 files changed, 63 insertions(+), 59 deletions(-) diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json index 846d37a82..12eb30128 100644 --- a/config_examples/config_freqai.example.json +++ b/config_examples/config_freqai.example.json @@ -56,7 +56,6 @@ "purge_old_models": true, "train_period_days": 15, "backtest_period_days": 7, - "backtest_save_model": true, "live_retrain_hours": 0, "identifier": "uniqe-id", "feature_parameters": { @@ -75,7 +74,6 @@ "weight_factor": 0.9, "principal_component_analysis": false, "use_SVM_to_remove_outliers": true, - "indicator_max_period_candles": 20, "indicator_periods_candles": [ 10, 20 diff --git a/docs/freqai.md b/docs/freqai.md index 6ee124b9b..3646362c3 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -89,11 +89,10 @@ Mandatory parameters are marked as **Required**, which means that they are requi |------------|-------------| | | **General configuration parameters** | `freqai` | **Required.**
The parent dictionary containing all the parameters for controlling FreqAI.
**Datatype:** Dictionary. -| `startup_candles` | Number of candles needed for *backtesting only* to ensure all indicators are non NaNs at the start of the first train period.
**Datatype:** Positive integer. | `purge_old_models` | Delete obsolete models (otherwise, all historic models will remain on disk).
**Datatype:** Boolean. Default: `False`. | `train_period_days` | **Required.**
Number of days to use for the training data (width of the sliding window).
**Datatype:** Positive integer. | `backtest_period_days` | **Required.**
Number of days to inference from the trained model before sliding the window defined above, and retraining the model. This can be fractional days, but beware that the user-provided `timerange` will be divided by this number to yield the number of trainings necessary to complete the backtest.
**Datatype:** Float. -| `backtest_save_model` | Saves models to disk when running backtesting.
**Datatype:** Boolean. Default: `True`. +| `save_backtest_models` | Backtesting operates most efficiently by saving the prediction data and reusing them directly for subsequent runs (when users wish to tune entry/exit parameters). If a user wishes to save models to disk when running backtesting, they should activate `save_backtest_models`. A user may wish to do this if they plan to use the same model files for starting a dry/live instance with the same `identifier`.
**Datatype:** Boolean. Default: `False`. | `identifier` | **Required.**
A unique name for the current model. This can be reused to reload pre-trained models/data.
**Datatype:** String. | `live_retrain_hours` | Frequency of retraining during dry/live runs.
Default set to 0, which means the model will retrain as often as possible.
**Datatype:** Float > 0. | `expiration_hours` | Avoid making predictions if a model is more than `expiration_hours` old.
Defaults set to 0, which means models never expire.
**Datatype:** Positive integer. @@ -280,6 +279,17 @@ The FreqAI strategy requires the user to include the following lines of code in Notice how the `populate_any_indicators()` is where the user adds their own features ([more information](#feature-engineering)) and labels ([more information](#setting-classifier-targets)). See a full example at `templates/FreqaiExampleStrategy.py`. +### Setting the `startup_candle_count` +Users need to take care to set the `startup_candle_count` in their strategy the same way they would for any normal Freqtrade strategy (see details [here](strategy-customization.md/#strategy-startup-period)). This value is used by Freqtrade to ensure that a sufficient amount of data is provided when calling on the `dataprovider` to avoid any NaNs at the beginning of the first training. Users can easily set this value by identifying the longest period (in candle units) that they pass to their indicator creation functions (e.g. talib functions). In the present example, the user would pass 20 to as this value (since it is the maximum value in their `indicators_periods_candles`). + +!!! Note + Typically it is best for users to be safe and multiply their expected `startup_candle_count` by 2. There are instances where the talib functions actually require more data than just the passed `period`. Anecdotally, multiplying the `startup_candle_count` by 2 always leads to a fully NaN free training dataset. Look out for this log message to confirm that your data is clean: + + ``` + 2022-08-31 15:14:04 - freqtrade.freqai.data_kitchen - INFO - dropped 0 training points due to NaNs in populated dataset 4319. + ``` + + ## Creating a dynamic target The `&*_std/mean` return values describe the statistical fit of the user defined label *during the most recent training*. This value allows the user to know the rarity of a given prediction. For example, `templates/FreqaiExampleStrategy.py`, creates a `target_roi` which is based on filtering out predictions that are below a given z-score of 1.25. @@ -505,7 +515,7 @@ and if a full `live_retrain_hours` has elapsed since the end of the loaded model The FreqAI backtesting module can be executed with the following command: ```bash -freqtrade backtesting --strategy FreqaiExampleStrategy --config config_freqai.example.json --freqaimodel LightGBMRegressor --timerange 20210501-20210701 +freqtrade backtesting --strategy FreqaiExampleStrategy --config config_examples/config_freqai.example.json --freqaimodel LightGBMRegressor --timerange 20210501-20210701 ``` Backtesting mode requires the user to have the data pre-downloaded (unlike in dry/live mode where FreqAI automatically downloads the necessary data). The user should be careful to consider that the time range of the downloaded data is more than the backtesting time range. This is because FreqAI needs data prior to the desired backtesting time range in order to train a model to be ready to make predictions on the first candle of the user-set backtesting time range. More details on how to calculate the data to download can be found [here](#deciding-the-sliding-training-window-and-backtesting-duration). @@ -532,20 +542,14 @@ the user is asking FreqAI to use a training period of 30 days and backtest on th This means that if the user sets `--timerange 20210501-20210701`, FreqAI will train have trained 8 separate models at the end of `--timerange` (because the full range comprises 8 weeks). After the training of the model, FreqAI will backtest the subsequent 7 days. The "sliding window" then moves one week forward (emulating FreqAI retraining once per week in live mode) and the new model uses the previous 30 days (including the 7 days used for backtesting by the previous model) to train. This is repeated until the end of `--timerange`. -In live mode, the required training data is automatically computed and downloaded. However, in backtesting mode, -the user must manually enter the required number of `startup_candles` in the config. This value -is used to increase the data to FreqAI, which should be sufficient to enable all indicators -to be NaN free at the beginning of the first training. This is done by identifying the -longest timeframe (`4h` in presented example config) and the longest indicator period (`20` days in presented example config) -and adding this to the `train_period_days`. The units need to be in the base candle time frame: -`startup_candles` = ( 4 hours * 20 max period * 60 minutes/hour + 30 day train_period_days * 1440 minutes per day ) / 5 min (base time frame) = 9360. - -!!! Note - In dry/live mode, this is all precomputed and handled automatically. Thus, `startup_candle` has no influence on dry/live mode. - !!! Note Although fractional `backtest_period_days` is allowed, the user should be aware that the `--timerange` is divided by this value to determine the number of models that FreqAI will need to train in order to backtest the full range. For example, if the user wants to set a `--timerange` of 10 days, and asks for a `backtest_period_days` of 0.1, FreqAI will need to train 100 models per pair to complete the full backtest. Because of this, a true backtest of FreqAI adaptive training would take a *very* long time. The best way to fully test a model is to run it dry and let it constantly train. In this case, backtesting would take the exact same amount of time as a dry run. +### Downloading data for backtesting +Live/dry instances will download the data automatically for the user, but users who wish to use backtesting functionality still need to download the necessary data using `download-data` (details [here](data-download/#data-downloading)). FreqAI users need to pay careful attention to understanding how much *additional* data needs to be downloaded to ensure that they have a sufficient amount of training data *before* the start of their backtesting timerange. The amount of additional data can be roughly estimated by taking subtracting `train_period_days` and the `startup_candle_count` ([details](#setting-the-startupcandlecount)) from the beginning of the desired backtesting timerange. + +As an example, if we wish to backtest the `--timerange` above of `20210501-20210701`, and we use the example config which sets `train_period_days` to 15. The startup candle count is 40 on a maximum `include_timeframes` of 1h. We would need 20210501 - 15 days - 40 * 1h / 24 hours = 20210414 (16.7 days earlier than the start of the desired training timerange). + ### Defining model expirations During dry/live mode, FreqAI trains each coin pair sequentially (on separate threads/GPU from the main Freqtrade bot). This means that there is always an age discrepancy between models. If a user is training on 50 pairs, and each pair requires 5 minutes to train, the oldest model will be over 4 hours old. This may be undesirable if the characteristic time scale (the trade duration target) for a strategy is less than 4 hours. The user can decide to only make trade entries if the model is less than diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index f88e20223..13af1e0d2 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -70,7 +70,7 @@ class FreqaiDataKitchen: self.training_features_list: List = [] self.model_filename: str = "" self.backtesting_results_path = Path() - self.backtesting_prediction_folder: str = "backtesting_predictions" + self.backtest_predictions_folder: str = "backtesting_predictions" self.live = live self.pair = pair @@ -1077,7 +1077,7 @@ class FreqaiDataKitchen: Save prediction dataframe from backtesting to h5 file format :param append_df: dataframe for backtesting period """ - full_predictions_folder = Path(self.full_path / self.backtesting_prediction_folder) + full_predictions_folder = Path(self.full_path / self.backtest_predictions_folder) if not full_predictions_folder.is_dir(): full_predictions_folder.mkdir(parents=True, exist_ok=True) @@ -1092,3 +1092,26 @@ class FreqaiDataKitchen: """ append_df = pd.read_hdf(self.backtesting_results_path) return append_df + + def check_if_backtest_prediction_exists( + self + ) -> bool: + """ + Check if a backtesting prediction already exists + :param dk: FreqaiDataKitchen + :return: + :boolean: whether the prediction file exists or not. + """ + path_to_predictionfile = Path(self.full_path / + self.backtest_predictions_folder / + f"{self.model_filename}_prediction.h5") + self.backtesting_results_path = path_to_predictionfile + + file_exists = path_to_predictionfile.is_file() + if file_exists: + logger.info(f"Found backtesting prediction file at {path_to_predictionfile}") + else: + logger.info( + f"Could not find backtesting prediction file at {path_to_predictionfile}" + ) + return file_exists diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 9c7ef05a7..399568c7d 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -71,7 +71,9 @@ class IFreqaiModel(ABC): self.first = True self.set_full_path() self.follow_mode: bool = self.freqai_info.get("follow_mode", False) - self.backtest_save_model: bool = self.freqai_info.get("backtest_save_model", True) + self.save_backtest_models: bool = self.freqai_info.get("save_backtest_models", False) + if self.save_backtest_models: + logger.info('Backtesting module configured to save all models.') self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode) self.identifier: str = self.freqai_info.get("identifier", "no_id_provided") self.scanning = False @@ -125,10 +127,9 @@ class IFreqaiModel(ABC): elif not self.follow_mode: self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"]) logger.info(f"Training {len(self.dk.training_timeranges)} timeranges") - with self.analysis_lock: - dataframe = self.dk.use_strategy_to_populate_indicators( - strategy, prediction_dataframe=dataframe, pair=metadata["pair"] - ) + dataframe = self.dk.use_strategy_to_populate_indicators( + strategy, prediction_dataframe=dataframe, pair=metadata["pair"] + ) dk = self.start_backtesting(dataframe, metadata, self.dk) dataframe = dk.remove_features_from_df(dk.return_dataframe) @@ -232,10 +233,9 @@ class IFreqaiModel(ABC): f"sub-train-{metadata['pair'].split('/')[0]}_{trained_timestamp_int}" ) - coin, _ = metadata["pair"].split("/") - dk.model_filename = f"cb_{coin.lower()}_{trained_timestamp_int}" + dk.set_new_model_names(metadata["pair"], trained_timestamp) - if self.backtest_prediction_exists(dk): + if dk.check_if_backtest_prediction_exists(): append_df = dk.get_backtesting_prediction() dk.append_predictions(append_df) else: @@ -246,8 +246,9 @@ class IFreqaiModel(ABC): self.model = self.train(dataframe_train, metadata["pair"], dk) self.dd.pair_dict[metadata["pair"]]["trained_timestamp"] = int( trained_timestamp.stopts) - dk.set_new_model_names(metadata["pair"], trained_timestamp) - if self.backtest_save_model: + + if self.save_backtest_models: + logger.info('Saving backtest model to disk.') self.dd.save_data(self.model, metadata["pair"], dk) else: self.model = self.dd.load_data(metadata["pair"], dk) @@ -644,35 +645,6 @@ class IFreqaiModel(ABC): self.train_time = 0 return - def backtest_prediction_exists( - self, - dk: FreqaiDataKitchen, - scanning: bool = False, - ) -> bool: - """ - Check if a backtesting prediction already exists - :param dk: FreqaiDataKitchen - :return: - :boolean: whether the prediction file exists or not. - """ - if not self.live: - prediction_file_name = dk.model_filename - path_to_predictionfile = Path(dk.full_path / - dk.backtesting_prediction_folder / - f"{prediction_file_name}_prediction.h5") - dk.backtesting_results_path = path_to_predictionfile - - file_exists = path_to_predictionfile.is_file() - if file_exists and not scanning: - logger.info("Found backtesting prediction file at %s", prediction_file_name) - elif not scanning: - logger.info( - "Could not find backtesting prediction file at %s", prediction_file_name - ) - return file_exists - else: - return False - # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModel.py for an example. diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index aa584bfbc..0e822a028 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -44,7 +44,7 @@ class FreqaiExampleStrategy(IStrategy): stoploss = -0.05 use_exit_signal = True # this is the maximum period fed to talib (timeframe independent) - startup_candle_count: int = 20 + startup_candle_count: int = 40 can_short = False linear_roi_offset = DecimalParameter( diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 09f5d27ff..5441b3c24 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -174,6 +174,7 @@ def test_train_model_in_series_LightGBMClassifier(mocker, freqai_conf): def test_start_backtesting(mocker, freqai_conf): freqai_conf.update({"timerange": "20180120-20180130"}) + freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) strategy.dp = DataProvider(freqai_conf, exchange) @@ -200,6 +201,7 @@ def test_start_backtesting(mocker, freqai_conf): def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf): freqai_conf.update({"timerange": "20180120-20180124"}) freqai_conf.get("freqai", {}).update({"backtest_period_days": 0.5}) + freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) strategy.dp = DataProvider(freqai_conf, exchange) @@ -224,6 +226,7 @@ def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf): def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): freqai_conf.update({"timerange": "20180120-20180130"}) + freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) strategy.dp = DataProvider(freqai_conf, exchange) @@ -263,10 +266,14 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): freqai.start_backtesting(df, metadata, freqai.dk) assert log_has_re( - "Found backtesting prediction ", + "Found backtesting prediction file ", caplog, ) + path = (freqai.dd.full_path / freqai.dk.backtest_predictions_folder) + prediction_files = [x for x in path.iterdir() if x.is_file()] + assert len(prediction_files) == 5 + shutil.rmtree(Path(freqai.dk.full_path)) From 80b5f035aba23f382363dc81499477977cea1e27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Sep 2022 15:01:28 +0200 Subject: [PATCH 26/26] Remove typo in log message --- freqtrade/freqai/freqai_interface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 6b4ac183a..0bd88c64f 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -613,8 +613,8 @@ class IFreqaiModel(ABC): logger.info( f'Total time spent inferencing pairlist {self.inference_time:.2f} seconds') if self.inference_time > 0.25 * self.base_tf_seconds: - logger.warning('Inference took over 25/% of the candle time. Reduce pairlist to' - ' avoid blinding open trades and degrading performance.') + logger.warning("Inference took over 25% of the candle time. Reduce pairlist to" + " avoid blinding open trades and degrading performance.") self.pair_it = 0 self.inference_time = 0 return