From 800b2eeaf0e5629c5f5fe8877ae414859db25e50 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 3 Aug 2021 06:38:15 +0200 Subject: [PATCH 01/40] Load protections as part of backtest() this enables different values in hyperopt per epoch --- freqtrade/optimize/backtesting.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 45e60e013..3079e326d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -146,6 +146,8 @@ class Backtesting: # since a "perfect" stoploss-sell is assumed anyway # And the regular "stoploss" function would not apply to that case self.strategy.order_types['stoploss_on_exchange'] = False + + def _load_protections(self, strategy: IStrategy): if self.config.get('enable_protections', False): conf = self.config if hasattr(strategy, 'protections'): @@ -194,6 +196,7 @@ class Backtesting: Trade.reset_trades() self.rejected_trades = 0 self.dataprovider.clear_cache() + self._load_protections(self.strategy) def check_abort(self): """ From 544e0da6c2c84cc918978dce0146089b011744f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 3 Aug 2021 07:10:04 +0200 Subject: [PATCH 02/40] Add protection parameter space --- freqtrade/commands/cli_options.py | 2 +- freqtrade/optimize/hyperopt.py | 17 +++++++++++++++-- freqtrade/optimize/hyperopt_auto.py | 3 +++ freqtrade/optimize/hyperopt_interface.py | 7 +++++++ freqtrade/strategy/hyper.py | 16 +++++++++++----- 5 files changed, 37 insertions(+), 8 deletions(-) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index f56a2bf18..215ed3f6e 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -218,7 +218,7 @@ AVAILABLE_CLI_OPTIONS = { "spaces": Arg( '--spaces', help='Specify which parameters to hyperopt. Space-separated list.', - choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'default'], + choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'protection', 'default'], nargs='+', default='default', ), diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a69e5a5a2..18ed66b4a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -66,6 +66,7 @@ class Hyperopt: def __init__(self, config: Dict[str, Any]) -> None: self.buy_space: List[Dimension] = [] self.sell_space: List[Dimension] = [] + self.protection_space: List[Dimension] = [] self.roi_space: List[Dimension] = [] self.stoploss_space: List[Dimension] = [] self.trailing_space: List[Dimension] = [] @@ -191,6 +192,8 @@ class Hyperopt: result['buy'] = {p.name: params.get(p.name) for p in self.buy_space} if HyperoptTools.has_space(self.config, 'sell'): result['sell'] = {p.name: params.get(p.name) for p in self.sell_space} + if HyperoptTools.has_space(self.config, 'protection'): + result['protection'] = {p.name: params.get(p.name) for p in self.protection_space} if HyperoptTools.has_space(self.config, 'roi'): result['roi'] = {str(k): v for k, v in self.custom_hyperopt.generate_roi_table(params).items()} @@ -241,6 +244,10 @@ class Hyperopt: """ Assign the dimensions in the hyperoptimization space. """ + if self.auto_hyperopt and HyperoptTools.has_space(self.config, 'protection'): + # Protections can only be optimized when using the Parameter interface + logger.debug("Hyperopt has 'protection' space") + self.protection_space = self.custom_hyperopt.protection_space() if HyperoptTools.has_space(self.config, 'buy'): logger.debug("Hyperopt has 'buy' space") @@ -261,8 +268,8 @@ class Hyperopt: if HyperoptTools.has_space(self.config, 'trailing'): logger.debug("Hyperopt has 'trailing' space") self.trailing_space = self.custom_hyperopt.trailing_space() - self.dimensions = (self.buy_space + self.sell_space + self.roi_space + - self.stoploss_space + self.trailing_space) + self.dimensions = (self.buy_space + self.sell_space + self.protection_space + + self.roi_space + self.stoploss_space + self.trailing_space) def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict: """ @@ -282,6 +289,12 @@ class Hyperopt: self.backtesting.strategy.advise_sell = ( # type: ignore self.custom_hyperopt.sell_strategy_generator(params_dict)) + if HyperoptTools.has_space(self.config, 'protection'): + for attr_name, attr in self.backtesting.strategy.enumerate_parameters('protection'): + if attr.optimize: + # noinspection PyProtectedMember + attr.value = params_dict[attr_name] + if HyperoptTools.has_space(self.config, 'roi'): self.backtesting.strategy.minimal_roi = ( # type: ignore self.custom_hyperopt.generate_roi_table(params_dict)) diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py index f86204406..03f7dd21e 100644 --- a/freqtrade/optimize/hyperopt_auto.py +++ b/freqtrade/optimize/hyperopt_auto.py @@ -73,6 +73,9 @@ class HyperOptAuto(IHyperOpt): def sell_indicator_space(self) -> List['Dimension']: return self._get_indicator_space('sell', 'sell_indicator_space') + def protection_space(self) -> List['Dimension']: + return self._get_indicator_space('protection', 'indicator_space') + def generate_roi_table(self, params: Dict) -> Dict[int, float]: return self._get_func('generate_roi_table')(params) diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 889854cad..500798627 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -57,6 +57,13 @@ class IHyperOpt(ABC): """ raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell')) + def protection_space(self) -> List[Dimension]: + """ + Create a protection space. + Only supported by the Parameter interface. + """ + raise OperationalException(_format_exception_message('indicator_space', 'protection')) + def indicator_space(self) -> List[Dimension]: """ Create an indicator space. diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index b067e19d5..e143dda7e 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -283,6 +283,7 @@ class HyperStrategyMixin(object): self.config = config self.ft_buy_params: List[BaseParameter] = [] self.ft_sell_params: List[BaseParameter] = [] + self.ft_protection_params: List[BaseParameter] = [] self._load_hyper_params(config.get('runmode') == RunMode.HYPEROPT) @@ -292,11 +293,11 @@ class HyperStrategyMixin(object): :param category: :return: """ - if category not in ('buy', 'sell', None): - raise OperationalException('Category must be one of: "buy", "sell", None.') + if category not in ('buy', 'sell', 'protection', None): + raise OperationalException('Category must be one of: "buy", "sell", "protection", None.') if category is None: - params = self.ft_buy_params + self.ft_sell_params + params = self.ft_buy_params + self.ft_sell_params + self.ft_protection_params else: params = getattr(self, f"ft_{category}_params") @@ -324,9 +325,10 @@ class HyperStrategyMixin(object): params: Dict = { 'buy': list(cls.detect_parameters('buy')), 'sell': list(cls.detect_parameters('sell')), + 'protection': list(cls.detect_parameters('protection')), } params.update({ - 'count': len(params['buy'] + params['sell']) + 'count': len(params['buy'] + params['sell'] + params['protection']) }) return params @@ -340,9 +342,12 @@ class HyperStrategyMixin(object): self._ft_params_from_file = params buy_params = deep_merge_dicts(params.get('buy', {}), getattr(self, 'buy_params', {})) sell_params = deep_merge_dicts(params.get('sell', {}), getattr(self, 'sell_params', {})) + protection_params = deep_merge_dicts(params.get('protection', {}), + getattr(self, 'protection_params', {})) self._load_params(buy_params, 'buy', hyperopt) self._load_params(sell_params, 'sell', hyperopt) + self._load_params(protection_params, 'protection', hyperopt) def load_params_from_file(self) -> Dict: filename_str = getattr(self, '__file__', '') @@ -397,7 +402,8 @@ class HyperStrategyMixin(object): """ params = { 'buy': {}, - 'sell': {} + 'sell': {}, + 'protection': {}, } for name, p in self.enumerate_parameters(): if not p.optimize or not p.in_space: From 091bf7c4d2940e72ed00639b362b8aff1d009f84 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 3 Aug 2021 07:14:31 +0200 Subject: [PATCH 03/40] Output protection space --- freqtrade/optimize/hyperopt_tools.py | 8 +++++--- freqtrade/plugins/protections/iprotection.py | 6 +++--- tests/optimize/test_hyperopt.py | 3 ++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 439016c14..94b724d9a 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -82,8 +82,8 @@ class HyperoptTools(): """ Tell if the space value is contained in the configuration """ - # The 'trailing' space is not included in the 'default' set of spaces - if space == 'trailing': + # 'trailing' and 'protection spaces are not included in the 'default' set of spaces + if space in ('trailing', 'protection'): return any(s in config['spaces'] for s in [space, 'all']) else: return any(s in config['spaces'] for s in [space, 'all', 'default']) @@ -149,7 +149,7 @@ class HyperoptTools(): if print_json: result_dict: Dict = {} - for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']: + for s in ['buy', 'sell', 'protection', 'roi', 'stoploss', 'trailing']: HyperoptTools._params_update_for_json(result_dict, params, non_optimized, s) print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE)) @@ -158,6 +158,8 @@ class HyperoptTools(): non_optimized) HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:", non_optimized) + HyperoptTools._params_pretty_print(params, 'protection', + "Protection hyperspace params:", non_optimized) HyperoptTools._params_pretty_print(params, 'roi', "ROI table:", non_optimized) HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:", non_optimized) HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:", non_optimized) diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index d034beefc..e6bb49064 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -27,17 +27,17 @@ class IProtection(LoggingMixin, ABC): self._protection_config = protection_config tf_in_min = timeframe_to_minutes(config['timeframe']) if 'stop_duration_candles' in protection_config: - self._stop_duration_candles = protection_config.get('stop_duration_candles', 1) + self._stop_duration_candles = int(protection_config.get('stop_duration_candles', 1)) self._stop_duration = (tf_in_min * self._stop_duration_candles) else: self._stop_duration_candles = None self._stop_duration = protection_config.get('stop_duration', 60) if 'lookback_period_candles' in protection_config: - self._lookback_period_candles = protection_config.get('lookback_period_candles', 1) + self._lookback_period_candles = int(protection_config.get('lookback_period_candles', 1)) self._lookback_period = tf_in_min * self._lookback_period_candles else: self._lookback_period_candles = None - self._lookback_period = protection_config.get('lookback_period', 60) + self._lookback_period = int(protection_config.get('lookback_period', 60)) LoggingMixin.__init__(self, logger) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 14fea573f..ef4b85beb 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -577,6 +577,7 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: "20.0": 0.02, "50.0": 0.01, "110.0": 0}, + 'protection': {}, 'sell': {'sell-adx-enabled': False, 'sell-adx-value': 0, 'sell-fastd-enabled': True, @@ -592,7 +593,7 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: 'trailing_stop_positive': 0.02, 'trailing_stop_positive_offset': 0.07}}, 'params_dict': optimizer_param, - 'params_not_optimized': {'buy': {}, 'sell': {}}, + 'params_not_optimized': {'buy': {}, 'protection': {}, 'sell': {}}, 'results_metrics': ANY, 'total_profit': 3.1e-08 } From a6454cfc39453dbceb9fbba2bf896b45d64334ee Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 4 Aug 2021 07:14:47 +0200 Subject: [PATCH 04/40] Autoenable protections when protection-space is selected --- freqtrade/optimize/hyperopt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 18ed66b4a..3eab709da 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -247,6 +247,8 @@ class Hyperopt: if self.auto_hyperopt and HyperoptTools.has_space(self.config, 'protection'): # Protections can only be optimized when using the Parameter interface logger.debug("Hyperopt has 'protection' space") + # Enable Protections if protection space is selected. + self.config['enable_protections'] = True self.protection_space = self.custom_hyperopt.protection_space() if HyperoptTools.has_space(self.config, 'buy'): From ceed3c663b3eb5498d2eeff232f4122945e58126 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 4 Aug 2021 07:15:14 +0200 Subject: [PATCH 05/40] Document using protections --- docs/hyperopt.md | 90 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 4fba925d0..95a1cfcc5 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -326,7 +326,7 @@ There are four parameter types each suited for different purposes. !!! Warning Hyperoptable parameters cannot be used in `populate_indicators` - as hyperopt does not recalculate indicators for each epoch, so the starting value would be used in this case. -### Optimizing an indicator parameter +## Optimizing an indicator parameter Assuming you have a simple strategy in mind - a EMA cross strategy (2 Moving averages crossing) - and you'd like to find the ideal parameters for this strategy. @@ -413,6 +413,94 @@ While this strategy is most likely too simple to provide consistent profit, it s While this may slow down the hyperopt startup speed, the overall performance will increase as the Hyperopt execution itself may pick the same value for multiple epochs (changing other values). You should however try to use space ranges as small as possible. Every new column will require more memory, and every possibility hyperopt can try will increase the search space. +## Optimizing protections + +Freqtrade can also optimize protections. How you optimize protections is up to you, and the following should be considered as example only. + +The strategy will simply need to define the "protections" entry as property returning a list of protection configurations. + +``` python +from pandas import DataFrame +from functools import reduce + +import talib.abstract as ta + +from freqtrade.strategy import IStrategy +from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter +import freqtrade.vendor.qtpylib.indicators as qtpylib + +class MyAwesomeStrategy(IStrategy): + stoploss = -0.05 + timeframe = '15m' + # Define the parameter spaces + coolback_lookback = IntParameter(2, 48, default=5, space="protection", optimize=True) + stop_duration = IntParameter(12, 200, default=5, space="protection", optimize=True) + use_stop_protection = CategoricalParameter([True, False], default=True, space="protection", optimize=True) + + + @property + def protections(self): + prot = [] + + prot.append({ + "method": "CooldownPeriod", + "stop_duration_candles": self.coolback_lookback.value + }) + if self.use_stop_protection.value: + prot.append({ + "method": "StoplossGuard", + "lookback_period_candles": 24 * 3, + "trade_limit": 4, + "stop_duration_candles": self.stop_duration.value, + "only_per_pair": False + }) + + return protection + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # ... + +``` + +You can then run hyperopt as follows: +`freqtrade hyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy MyAwesomeStrategy --spaces protection` + +!!! Note + The protection space is not part of the default space, and is only available with the Parameters Hyperopt interface, not with the legacy hyperopt interface (which required separate hyperopt files). + Freqtrade will also automatically change the "--enable-protections" flag if the protection space is selected. + +### Migrating from previous property setups + +A migration from a previous setup is pretty simple, and can be accomplished by converting the protections entry to a property. +In simple terms, the following configuration will be converted to the below. + +``` python +class MyAwesomeStrategy(IStrategy): + protections = [ + { + "method": "CooldownPeriod", + "stop_duration_candles": 4 + } + ] +``` + +Result + +``` python +class MyAwesomeStrategy(IStrategy): + + @property + def protections(self): + return [ + { + "method": "CooldownPeriod", + "stop_duration_candles": 4 + } + ] +``` + +You will then obviously also change potential interesting entries to parameters to allow hyper-optimization. + ## Loss-functions Each hyperparameter tuning requires a target. This is usually defined as a loss function (sometimes also called objective function), which should decrease for more desirable results, and increase for bad results. From a661e0db6e373ade53690a6214f2e7b2b84ea333 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 4 Aug 2021 19:43:16 +0200 Subject: [PATCH 06/40] Deprecate protections from config --- docs/hyperopt.md | 4 + docs/includes/protections.md | 149 ++++++++++-------- .../configuration/deprecated_settings.py | 3 + tests/test_configuration.py | 13 +- 4 files changed, 101 insertions(+), 68 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 95a1cfcc5..e11c93748 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -469,6 +469,10 @@ You can then run hyperopt as follows: The protection space is not part of the default space, and is only available with the Parameters Hyperopt interface, not with the legacy hyperopt interface (which required separate hyperopt files). Freqtrade will also automatically change the "--enable-protections" flag if the protection space is selected. +!!! Warning + If protections are defined as property, entries from the configuration will be ignored. + It is therefore recommended to not define protections in the configuration. + ### Migrating from previous property setups A migration from a previous setup is pretty simple, and can be accomplished by converting the protections entry to a property. diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 5dcc83738..0757d2f6d 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -15,6 +15,10 @@ All protection end times are rounded up to the next candle to avoid sudden, unex !!! Note "Backtesting" Protections are supported by backtesting and hyperopt, but must be explicitly enabled by using the `--enable-protections` flag. +!!! Warning "Setting protections from the configuration" + Setting protections from the configuration via `"protections": [],` key should be considered deprecated and will be removed in a future version. + It is also no longer guaranteed that your protections apply to the strategy in cases where the strategy defines [protections as property](hyperopt.md#optimizing-protections). + ### Available Protections * [`StoplossGuard`](#stoploss-guard) Stop trading if a certain amount of stoploss occurred within a certain time window. @@ -47,15 +51,17 @@ This applies across all pairs, unless `only_per_pair` is set to true, which will The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles. ``` python -protections = [ - { - "method": "StoplossGuard", - "lookback_period_candles": 24, - "trade_limit": 4, - "stop_duration_candles": 4, - "only_per_pair": False - } -] +@property +def protections(self): + return [ + { + "method": "StoplossGuard", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 4, + "only_per_pair": False + } + ] ``` !!! Note @@ -69,15 +75,17 @@ protections = [ The below sample stops trading for 12 candles if max-drawdown is > 20% considering all pairs - with a minimum of `trade_limit` trades - within the last 48 candles. If desired, `lookback_period` and/or `stop_duration` can be used. ``` python -protections = [ - { - "method": "MaxDrawdown", - "lookback_period_candles": 48, - "trade_limit": 20, - "stop_duration_candles": 12, - "max_allowed_drawdown": 0.2 - }, -] +@property +def protections(self): + return [ + { + "method": "MaxDrawdown", + "lookback_period_candles": 48, + "trade_limit": 20, + "stop_duration_candles": 12, + "max_allowed_drawdown": 0.2 + }, + ] ``` #### Low Profit Pairs @@ -88,15 +96,17 @@ If that ratio is below `required_profit`, that pair will be locked for `stop_dur The below example will stop trading a pair for 60 minutes if the pair does not have a required profit of 2% (and a minimum of 2 trades) within the last 6 candles. ``` python -protections = [ - { - "method": "LowProfitPairs", - "lookback_period_candles": 6, - "trade_limit": 2, - "stop_duration": 60, - "required_profit": 0.02 - } -] +@property +def protections(self): + return [ + { + "method": "LowProfitPairs", + "lookback_period_candles": 6, + "trade_limit": 2, + "stop_duration": 60, + "required_profit": 0.02 + } + ] ``` #### Cooldown Period @@ -106,12 +116,14 @@ protections = [ The below example will stop trading a pair for 2 candles after closing a trade, allowing this pair to "cool down". ``` python -protections = [ - { - "method": "CooldownPeriod", - "stop_duration_candles": 2 - } -] +@property +def protections(self): + return [ + { + "method": "CooldownPeriod", + "stop_duration_candles": 2 + } + ] ``` !!! Note @@ -136,39 +148,42 @@ from freqtrade.strategy import IStrategy class AwesomeStrategy(IStrategy) timeframe = '1h' - protections = [ - { - "method": "CooldownPeriod", - "stop_duration_candles": 5 - }, - { - "method": "MaxDrawdown", - "lookback_period_candles": 48, - "trade_limit": 20, - "stop_duration_candles": 4, - "max_allowed_drawdown": 0.2 - }, - { - "method": "StoplossGuard", - "lookback_period_candles": 24, - "trade_limit": 4, - "stop_duration_candles": 2, - "only_per_pair": False - }, - { - "method": "LowProfitPairs", - "lookback_period_candles": 6, - "trade_limit": 2, - "stop_duration_candles": 60, - "required_profit": 0.02 - }, - { - "method": "LowProfitPairs", - "lookback_period_candles": 24, - "trade_limit": 4, - "stop_duration_candles": 2, - "required_profit": 0.01 - } - ] + + @property + def protections(self): + return [ + { + "method": "CooldownPeriod", + "stop_duration_candles": 5 + }, + { + "method": "MaxDrawdown", + "lookback_period_candles": 48, + "trade_limit": 20, + "stop_duration_candles": 4, + "max_allowed_drawdown": 0.2 + }, + { + "method": "StoplossGuard", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 2, + "only_per_pair": False + }, + { + "method": "LowProfitPairs", + "lookback_period_candles": 6, + "trade_limit": 2, + "stop_duration_candles": 60, + "required_profit": 0.02 + }, + { + "method": "LowProfitPairs", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 2, + "required_profit": 0.01 + } + ] # ... ``` diff --git a/freqtrade/configuration/deprecated_settings.py b/freqtrade/configuration/deprecated_settings.py index 1b162f7c9..529d4cd6a 100644 --- a/freqtrade/configuration/deprecated_settings.py +++ b/freqtrade/configuration/deprecated_settings.py @@ -110,3 +110,6 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None: "Please remove 'ticker_interval' from your configuration to continue operating." ) config['timeframe'] = config['ticker_interval'] + + if 'protections' in config: + logger.warning("DEPRECATED: Setting 'protections' in the configuration is deprecated.") diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 7012333e9..9dce304c7 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1330,7 +1330,7 @@ def test_process_removed_setting(mocker, default_conf, caplog): 'sectionB', 'somesetting') -def test_process_deprecated_ticker_interval(mocker, default_conf, caplog): +def test_process_deprecated_ticker_interval(default_conf, caplog): message = "DEPRECATED: Please use 'timeframe' instead of 'ticker_interval." config = deepcopy(default_conf) process_temporary_deprecated_settings(config) @@ -1352,6 +1352,17 @@ def test_process_deprecated_ticker_interval(mocker, default_conf, caplog): process_temporary_deprecated_settings(config) +def test_process_deprecated_protections(default_conf, caplog): + message = "DEPRECATED: Setting 'protections' in the configuration is deprecated." + config = deepcopy(default_conf) + process_temporary_deprecated_settings(config) + assert not log_has(message, caplog) + + config['protections'] = [] + process_temporary_deprecated_settings(config) + assert log_has(message, caplog) + + def test_flat_vars_to_nested_dict(caplog): test_args = { From 3c412233338afbd1648786e0b5dbaf484d0988e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 4 Aug 2021 20:01:28 +0200 Subject: [PATCH 07/40] Add test for protections-hyperopt --- docs/hyperopt.md | 4 ++-- freqtrade/strategy/hyper.py | 3 ++- tests/optimize/test_hyperopt.py | 8 ++++++++ tests/strategy/strats/hyperoptable_strategy.py | 11 +++++++++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index e11c93748..f869b9ac1 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -433,7 +433,7 @@ class MyAwesomeStrategy(IStrategy): stoploss = -0.05 timeframe = '15m' # Define the parameter spaces - coolback_lookback = IntParameter(2, 48, default=5, space="protection", optimize=True) + cooldown_lookback = IntParameter(2, 48, default=5, space="protection", optimize=True) stop_duration = IntParameter(12, 200, default=5, space="protection", optimize=True) use_stop_protection = CategoricalParameter([True, False], default=True, space="protection", optimize=True) @@ -444,7 +444,7 @@ class MyAwesomeStrategy(IStrategy): prot.append({ "method": "CooldownPeriod", - "stop_duration_candles": self.coolback_lookback.value + "stop_duration_candles": self.cooldown_lookback.value }) if self.use_stop_protection.value: prot.append({ diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index e143dda7e..b9f586ac5 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -294,7 +294,8 @@ class HyperStrategyMixin(object): :return: """ if category not in ('buy', 'sell', 'protection', None): - raise OperationalException('Category must be one of: "buy", "sell", "protection", None.') + raise OperationalException( + 'Category must be one of: "buy", "sell", "protection", None.') if category is None: params = self.ft_buy_params + self.ft_sell_params + self.ft_protection_params diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index ef4b85beb..077fb516d 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1003,6 +1003,8 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: hyperopt_conf.update({ 'strategy': 'HyperoptableStrategy', 'user_data_dir': Path(tmpdir), + 'hyperopt_random_state': 42, + 'spaces': ['buy', 'sell', 'protection'] }) hyperopt = Hyperopt(hyperopt_conf) assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto) @@ -1010,12 +1012,18 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: assert hyperopt.backtesting.strategy.buy_rsi.in_space is True assert hyperopt.backtesting.strategy.buy_rsi.value == 35 + assert hyperopt.backtesting.strategy.sell_rsi.value == 74 + assert hyperopt.backtesting.strategy.protection_cooldown_lookback.value == 30 buy_rsi_range = hyperopt.backtesting.strategy.buy_rsi.range assert isinstance(buy_rsi_range, range) # Range from 0 - 50 (inclusive) assert len(list(buy_rsi_range)) == 51 hyperopt.start() + # All values should've changed. + assert hyperopt.backtesting.strategy.protection_cooldown_lookback.value != 30 + assert hyperopt.backtesting.strategy.buy_rsi.value != 35 + assert hyperopt.backtesting.strategy.sell_rsi.value != 74 def test_SKDecimal(): diff --git a/tests/strategy/strats/hyperoptable_strategy.py b/tests/strategy/strats/hyperoptable_strategy.py index cc4734e13..9d332e243 100644 --- a/tests/strategy/strats/hyperoptable_strategy.py +++ b/tests/strategy/strats/hyperoptable_strategy.py @@ -64,6 +64,17 @@ class HyperoptableStrategy(IStrategy): sell_rsi = IntParameter(low=50, high=100, default=70, space='sell') sell_minusdi = DecimalParameter(low=0, high=1, default=0.5001, decimals=3, space='sell', load=False) + protection_cooldown_lookback = IntParameter([0, 50], default=30) + + @property + def protections(self): + prot = [] + + prot.append({ + "method": "CooldownPeriod", + "stop_duration_candles": self.protection_cooldown_lookback.value + }) + return prot def informative_pairs(self): """ From b73768acd1244d5863bcf27fe868f6dfaa11f4f0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 4 Aug 2021 20:10:32 +0200 Subject: [PATCH 08/40] Fix bug in property overwriting prevention --- freqtrade/resolvers/strategy_resolver.py | 2 +- tests/optimize/test_hyperopt.py | 2 +- tests/strategy/test_interface.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 1239b78b3..0a18809b4 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -119,7 +119,7 @@ class StrategyResolver(IResolver): - default (if not None) """ if (attribute in config - and not isinstance(getattr(type(strategy), 'my_property', None), property)): + and not isinstance(getattr(type(strategy), attribute, None), property)): # Ensure Properties are not overwritten setattr(strategy, attribute, config[attribute]) logger.info("Override strategy '%s' with value in config file: %s.", diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 077fb516d..4fba80bc3 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1004,7 +1004,7 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: 'strategy': 'HyperoptableStrategy', 'user_data_dir': Path(tmpdir), 'hyperopt_random_state': 42, - 'spaces': ['buy', 'sell', 'protection'] + 'spaces': ['all'] }) hyperopt = Hyperopt(hyperopt_conf) assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index d8c87506c..9593cd02e 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -734,7 +734,7 @@ def test_auto_hyperopt_interface(default_conf): assert isinstance(all_params, dict) assert len(all_params['buy']) == 2 assert len(all_params['sell']) == 2 - assert all_params['count'] == 4 + assert all_params['count'] == 5 strategy.__class__.sell_rsi = IntParameter([0, 10], default=5, space='buy') From ad0e4a8567592edb9078a27cbc2c0a8bd5fd178a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 4 Aug 2021 20:52:56 +0200 Subject: [PATCH 09/40] Add BooleanParameter --- docs/hyperopt.md | 13 ++++++----- freqtrade/plugins/protections/iprotection.py | 3 +++ freqtrade/strategy/__init__.py | 4 ++-- freqtrade/strategy/hyper.py | 22 +++++++++++++++++++ freqtrade/templates/base_strategy.py.j2 | 4 ++-- freqtrade/templates/sample_strategy.py | 4 ++-- .../strategy/strats/hyperoptable_strategy.py | 14 +++++++----- tests/strategy/test_interface.py | 18 ++++++++++++--- 8 files changed, 61 insertions(+), 21 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index f869b9ac1..96f9ff177 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -253,7 +253,7 @@ We continue to define hyperoptable parameters: class MyAwesomeStrategy(IStrategy): buy_adx = DecimalParameter(20, 40, decimals=1, default=30.1, space="buy") buy_rsi = IntParameter(20, 40, default=30, space="buy") - buy_adx_enabled = CategoricalParameter([True, False], default=True, space="buy") + buy_adx_enabled = BooleanParameter(default=True, space="buy") buy_rsi_enabled = CategoricalParameter([True, False], default=False, space="buy") buy_trigger = CategoricalParameter(["bb_lower", "macd_cross_signal"], default="bb_lower", space="buy") ``` @@ -316,6 +316,7 @@ There are four parameter types each suited for different purposes. * `DecimalParameter` - defines a floating point parameter with a limited number of decimals (default 3). Should be preferred instead of `RealParameter` in most cases. * `RealParameter` - defines a floating point parameter with upper and lower boundaries and no precision limit. Rarely used as it creates a space with a near infinite number of possibilities. * `CategoricalParameter` - defines a parameter with a predetermined number of choices. +* `BooleanParameter` - Shorthand for `CategoricalParameter([True, False])` - great for "enable" parameters. !!! Tip "Disabling parameter optimization" Each parameter takes two boolean parameters: @@ -336,8 +337,8 @@ from functools import reduce import talib.abstract as ta -from freqtrade.strategy import IStrategy -from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter +from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter, + IStrategy, IntParameter) import freqtrade.vendor.qtpylib.indicators as qtpylib class MyAwesomeStrategy(IStrategy): @@ -425,8 +426,8 @@ from functools import reduce import talib.abstract as ta -from freqtrade.strategy import IStrategy -from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter +from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter, + IStrategy, IntParameter) import freqtrade.vendor.qtpylib.indicators as qtpylib class MyAwesomeStrategy(IStrategy): @@ -435,7 +436,7 @@ class MyAwesomeStrategy(IStrategy): # Define the parameter spaces cooldown_lookback = IntParameter(2, 48, default=5, space="protection", optimize=True) stop_duration = IntParameter(12, 200, default=5, space="protection", optimize=True) - use_stop_protection = CategoricalParameter([True, False], default=True, space="protection", optimize=True) + use_stop_protection = BooleanParameter(default=True, space="protection", optimize=True) @property diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index e6bb49064..e0a89e334 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -25,6 +25,9 @@ class IProtection(LoggingMixin, ABC): def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: self._config = config self._protection_config = protection_config + self._stop_duration_candles: Optional[int] = None + self._lookback_period_candles: Optional[int] = None + tf_in_min = timeframe_to_minutes(config['timeframe']) if 'stop_duration_candles' in protection_config: self._stop_duration_candles = int(protection_config.get('stop_duration_candles', 1)) diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index bd49165df..be655fc33 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -1,7 +1,7 @@ # flake8: noqa: F401 from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date, timeframe_to_seconds) -from freqtrade.strategy.hyper import (CategoricalParameter, DecimalParameter, IntParameter, - RealParameter) +from freqtrade.strategy.hyper import (BooleanParameter, CategoricalParameter, DecimalParameter, + IntParameter, RealParameter) from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.strategy_helper import merge_informative_pair, stoploss_from_open diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index b9f586ac5..dad282d7e 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -270,6 +270,28 @@ class CategoricalParameter(BaseParameter): return [self.value] +class BooleanParameter(CategoricalParameter): + + def __init__(self, *, default: Optional[Any] = None, + space: Optional[str] = None, optimize: bool = True, load: bool = True, **kwargs): + """ + Initialize hyperopt-optimizable Boolean Parameter. + It's a shortcut to `CategoricalParameter([True, False])`. + :param default: A default value. If not specified, first item from specified space will be + used. + :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if + parameter field + name is prefixed with 'buy_' or 'sell_'. + :param optimize: Include parameter in hyperopt optimizations. + :param load: Load parameter value from {space}_params. + :param kwargs: Extra parameters to skopt.space.Categorical. + """ + + categories = [True, False] + super().__init__(categories=categories, default=default, space=space, optimize=optimize, + load=load, **kwargs) + + class HyperStrategyMixin(object): """ A helper base class which allows HyperOptAuto class to reuse implementations of buy/sell diff --git a/freqtrade/templates/base_strategy.py.j2 b/freqtrade/templates/base_strategy.py.j2 index 13fc0853a..06d7cbc5c 100644 --- a/freqtrade/templates/base_strategy.py.j2 +++ b/freqtrade/templates/base_strategy.py.j2 @@ -6,8 +6,8 @@ import numpy as np # noqa import pandas as pd # noqa from pandas import DataFrame -from freqtrade.strategy import IStrategy -from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter +from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter, + IStrategy, IntParameter) # -------------------------------- # Add your lib to import here diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index 282b2f8e2..574819949 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -6,8 +6,8 @@ import numpy as np # noqa import pandas as pd # noqa from pandas import DataFrame -from freqtrade.strategy import IStrategy -from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter +from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter, + IStrategy, IntParameter) # -------------------------------- # Add your lib to import here diff --git a/tests/strategy/strats/hyperoptable_strategy.py b/tests/strategy/strats/hyperoptable_strategy.py index 9d332e243..88bdd078e 100644 --- a/tests/strategy/strats/hyperoptable_strategy.py +++ b/tests/strategy/strats/hyperoptable_strategy.py @@ -4,7 +4,8 @@ import talib.abstract as ta from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy, RealParameter +from freqtrade.strategy import (BooleanParameter, DecimalParameter, IntParameter, IStrategy, + RealParameter) class HyperoptableStrategy(IStrategy): @@ -64,16 +65,17 @@ class HyperoptableStrategy(IStrategy): sell_rsi = IntParameter(low=50, high=100, default=70, space='sell') sell_minusdi = DecimalParameter(low=0, high=1, default=0.5001, decimals=3, space='sell', load=False) + protection_enabled = BooleanParameter(default=True) protection_cooldown_lookback = IntParameter([0, 50], default=30) @property def protections(self): prot = [] - - prot.append({ - "method": "CooldownPeriod", - "stop_duration_candles": self.protection_cooldown_lookback.value - }) + if self.protection_enabled.value: + prot.append({ + "method": "CooldownPeriod", + "stop_duration_candles": self.protection_cooldown_lookback.value + }) return prot def informative_pairs(self): diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 9593cd02e..0ad6d6f32 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -16,8 +16,8 @@ from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.optimize.space import SKDecimal from freqtrade.persistence import PairLocks, Trade from freqtrade.resolvers import StrategyResolver -from freqtrade.strategy.hyper import (BaseParameter, CategoricalParameter, DecimalParameter, - IntParameter, RealParameter) +from freqtrade.strategy.hyper import (BaseParameter, BooleanParameter, CategoricalParameter, + DecimalParameter, IntParameter, RealParameter) from freqtrade.strategy.interface import SellCheckTuple from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from tests.conftest import log_has, log_has_re @@ -717,6 +717,17 @@ def test_hyperopt_parameters(): assert len(list(catpar.range)) == 3 assert list(catpar.range) == ['buy_rsi', 'buy_macd', 'buy_none'] + boolpar = BooleanParameter(default=True, space='buy') + assert boolpar.value is True + assert isinstance(boolpar.get_space(''), Categorical) + assert isinstance(boolpar.range, list) + assert len(list(boolpar.range)) == 1 + + boolpar.in_space = True + assert len(list(boolpar.range)) == 2 + + assert list(boolpar.range) == [True, False] + def test_auto_hyperopt_interface(default_conf): default_conf.update({'strategy': 'HyperoptableStrategy'}) @@ -734,7 +745,8 @@ def test_auto_hyperopt_interface(default_conf): assert isinstance(all_params, dict) assert len(all_params['buy']) == 2 assert len(all_params['sell']) == 2 - assert all_params['count'] == 5 + # Number of Hyperoptable parameters + assert all_params['count'] == 6 strategy.__class__.sell_rsi = IntParameter([0, 10], default=5, space='buy') From b9356a55644979c494f65e3235ab3b38a8cee1b1 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 6 Aug 2021 16:19:36 -0600 Subject: [PATCH 10/40] Autopep8 formatting --- freqtrade/commands/build_config_commands.py | 2 +- freqtrade/commands/deploy_commands.py | 14 ++++----- freqtrade/commands/hyperopt_commands.py | 8 ++--- freqtrade/configuration/check_exchange.py | 8 ++--- freqtrade/configuration/config_validation.py | 4 +-- .../configuration/deprecated_settings.py | 2 +- freqtrade/constants.py | 2 +- freqtrade/edge/edge_positioning.py | 6 ++-- freqtrade/main.py | 2 +- freqtrade/optimize/hyperopt.py | 6 ++-- freqtrade/optimize/hyperopt_tools.py | 2 +- freqtrade/optimize/optimize_reports.py | 6 ++-- freqtrade/plot/plotting.py | 4 +-- freqtrade/plugins/pairlist/IPairList.py | 2 +- freqtrade/plugins/pairlist/VolumePairList.py | 8 ++--- freqtrade/plugins/pairlistmanager.py | 14 ++++----- .../plugins/protections/stoploss_guard.py | 6 ++-- freqtrade/resolvers/__init__.py | 3 -- freqtrade/resolvers/strategy_resolver.py | 2 +- freqtrade/rpc/api_server/api_v1.py | 4 +-- freqtrade/rpc/fiat_convert.py | 2 +- freqtrade/rpc/rpc_manager.py | 1 + freqtrade/rpc/telegram.py | 22 +++++++------- freqtrade/strategy/strategy_helper.py | 2 +- tests/config_test_comments.json | 26 ++++++++-------- tests/optimize/test_hyperopt.py | 2 +- tests/plugins/test_protections.py | 4 +-- tests/rpc/test_fiat_convert.py | 6 ++-- tests/rpc/test_rpc_apiserver.py | 6 ++-- tests/test_arguments.py | 2 +- tests/test_configuration.py | 30 +++++++++---------- 31 files changed, 104 insertions(+), 104 deletions(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index b3f912433..852cab92e 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -193,7 +193,7 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: selections['exchange'] = render_template( templatefile=f"subtemplates/exchange_{exchange_template}.j2", arguments=selections - ) + ) except TemplateNotFound: selections['exchange'] = render_template( templatefile="subtemplates/exchange_generic.j2", diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index cc0d653b9..eb65579e2 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -38,15 +38,15 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st indicators = render_template_with_fallback( templatefile=f"subtemplates/indicators_{subtemplate}.j2", templatefallbackfile=f"subtemplates/indicators_{fallback}.j2", - ) + ) buy_trend = render_template_with_fallback( templatefile=f"subtemplates/buy_trend_{subtemplate}.j2", templatefallbackfile=f"subtemplates/buy_trend_{fallback}.j2", - ) + ) sell_trend = render_template_with_fallback( templatefile=f"subtemplates/sell_trend_{subtemplate}.j2", templatefallbackfile=f"subtemplates/sell_trend_{fallback}.j2", - ) + ) plot_config = render_template_with_fallback( templatefile=f"subtemplates/plot_config_{subtemplate}.j2", templatefallbackfile=f"subtemplates/plot_config_{fallback}.j2", @@ -97,19 +97,19 @@ def deploy_new_hyperopt(hyperopt_name: str, hyperopt_path: Path, subtemplate: st buy_guards = render_template_with_fallback( templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2", templatefallbackfile=f"subtemplates/hyperopt_buy_guards_{fallback}.j2", - ) + ) sell_guards = render_template_with_fallback( templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2", templatefallbackfile=f"subtemplates/hyperopt_sell_guards_{fallback}.j2", - ) + ) buy_space = render_template_with_fallback( templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2", templatefallbackfile=f"subtemplates/hyperopt_buy_space_{fallback}.j2", - ) + ) sell_space = render_template_with_fallback( templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2", templatefallbackfile=f"subtemplates/hyperopt_sell_space_{fallback}.j2", - ) + ) strategy_text = render_template(templatefile='base_hyperopt.py.j2', arguments={"hyperopt": hyperopt_name, diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 5a2727795..4694d1111 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -187,7 +187,7 @@ def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> Li x for x in epochs if x['results_metrics'].get( 'trade_count', x['results_metrics'].get('total_trades') - ) < filteroptions['filter_max_trades'] + ) < filteroptions['filter_max_trades'] ] return epochs @@ -239,7 +239,7 @@ def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List: x for x in epochs if x['results_metrics'].get( 'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100 - ) < filteroptions['filter_max_avg_profit'] + ) < filteroptions['filter_max_avg_profit'] ] if filteroptions['filter_min_total_profit'] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) @@ -247,7 +247,7 @@ def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List: x for x in epochs if x['results_metrics'].get( 'profit', x['results_metrics'].get('profit_total_abs', 0) - ) > filteroptions['filter_min_total_profit'] + ) > filteroptions['filter_min_total_profit'] ] if filteroptions['filter_max_total_profit'] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) @@ -255,7 +255,7 @@ def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List: x for x in epochs if x['results_metrics'].get( 'profit', x['results_metrics'].get('profit_total_abs', 0) - ) < filteroptions['filter_max_total_profit'] + ) < filteroptions['filter_max_total_profit'] ] return epochs diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index f282447d4..c4f038103 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -51,10 +51,10 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool: if not is_exchange_known_ccxt(exchange): raise OperationalException( - f'Exchange "{exchange}" is not known to the ccxt library ' - f'and therefore not available for the bot.\n' - f'The following exchanges are available for Freqtrade: ' - f'{", ".join(available_exchanges())}' + f'Exchange "{exchange}" is not known to the ccxt library ' + f'and therefore not available for the bot.\n' + f'The following exchanges are available for Freqtrade: ' + f'{", ".join(available_exchanges())}' ) valid, reason = validate_exchange(exchange) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index aad03e983..85ff4408f 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -115,7 +115,7 @@ def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None: if conf.get('stoploss') == 0.0: raise OperationalException( 'The config stoploss needs to be different from 0 to avoid problems with sell orders.' - ) + ) # Skip if trailing stoploss is not activated if not conf.get('trailing_stop', False): return @@ -180,7 +180,7 @@ def _validate_protections(conf: Dict[str, Any]) -> None: raise OperationalException( "Protections must specify either `stop_duration` or `stop_duration_candles`.\n" f"Please fix the protection {prot.get('method')}" - ) + ) if ('lookback_period' in prot and 'lookback_period_candles' in prot): raise OperationalException( diff --git a/freqtrade/configuration/deprecated_settings.py b/freqtrade/configuration/deprecated_settings.py index 1b162f7c9..e59e51f87 100644 --- a/freqtrade/configuration/deprecated_settings.py +++ b/freqtrade/configuration/deprecated_settings.py @@ -108,5 +108,5 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None: raise OperationalException( "Both 'timeframe' and 'ticker_interval' detected." "Please remove 'ticker_interval' from your configuration to continue operating." - ) + ) config['timeframe'] = config['ticker_interval'] diff --git a/freqtrade/constants.py b/freqtrade/constants.py index b48644c58..de4bc99b4 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -280,7 +280,7 @@ CONF_SCHEMA = { 'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS, 'default': 'off' - }, + }, } }, 'reload': {'type': 'boolean'}, diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 977b7e4ec..243043d31 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -231,12 +231,12 @@ class Edge: 'Minimum expectancy and minimum winrate are met only for %s,' ' so other pairs are filtered out.', self._final_pairs - ) + ) else: logger.info( 'Edge removed all pairs as no pair with minimum expectancy ' 'and minimum winrate was found !' - ) + ) return self._final_pairs @@ -247,7 +247,7 @@ class Edge: final = [] for pair, info in self._cached_pairs.items(): if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ - info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)): + info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)): final.append({ 'Pair': pair, 'Winrate': info.winrate, diff --git a/freqtrade/main.py b/freqtrade/main.py index 84d4b24f8..2fd3d32bb 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -44,7 +44,7 @@ def main(sysargv: List[str] = None) -> None: "as `freqtrade trade [options...]`.\n" "To see the full list of options available, please use " "`freqtrade --help` or `freqtrade --help`." - ) + ) except SystemExit as e: return_code = e diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a69e5a5a2..d40bbb73b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -444,9 +444,9 @@ class Hyperopt: ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', ] with progressbar.ProgressBar( - max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False, - widgets=widgets - ) as pbar: + max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False, + widgets=widgets + ) as pbar: EVALS = ceil(self.total_epochs / jobs) for i in range(EVALS): # Correct the number of epochs to be processed for the last diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 439016c14..51f1f977a 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -203,7 +203,7 @@ class HyperoptTools(): elif space == "roi": result = result[:-1] + f'{appendix}\n' minimal_roi_result = rapidjson.dumps({ - str(k): v for k, v in (space_params or no_params).items() + str(k): v for k, v in (space_params or no_params).items() }, default=str, indent=4, number_mode=rapidjson.NM_NATIVE) result += f"minimal_roi = {minimal_roi_result}" elif space == "trailing": diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index eefacbbab..7bb60228a 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -31,7 +31,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N filename = Path.joinpath( recordfilename.parent, f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}' - ).with_suffix(recordfilename.suffix) + ).with_suffix(recordfilename.suffix) file_dump_json(filename, stats) latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN) @@ -173,7 +173,7 @@ def generate_strategy_comparison(all_results: Dict) -> List[Dict]: for strategy, results in all_results.items(): tabular_data.append(_generate_result_line( results['results'], results['config']['dry_run_wallet'], strategy) - ) + ) try: max_drawdown_per, _, _, _, _ = calculate_max_drawdown(results['results'], value_col='profit_ratio') @@ -604,7 +604,7 @@ def text_table_add_metrics(strat_results: Dict) -> str: strat_results['stake_currency']) stake_amount = round_coin_value( strat_results['stake_amount'], strat_results['stake_currency'] - ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited' + ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited' message = ("No trades made. " f"Your starting balance was {start_balance}, " diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 061460975..2fbf343ce 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -334,8 +334,8 @@ def add_areas(fig, row: int, data: pd.DataFrame, indicators) -> make_subplots: ) elif indicator_b not in data: logger.info( - 'fill_to: "%s" ignored. Reason: This indicator is not ' - 'in your strategy.', indicator_b + 'fill_to: "%s" ignored. Reason: This indicator is not ' + 'in your strategy.', indicator_b ) return fig diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 74348b1a7..bfde2ace0 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -144,7 +144,7 @@ class IPairList(LoggingMixin, ABC): markets = self._exchange.markets if not markets: raise OperationalException( - 'Markets not loaded. Make sure that exchange is initialized correctly.') + 'Markets not loaded. Make sure that exchange is initialized correctly.') sanitized_whitelist: List[str] = [] for pair in pairlist: diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index d6b8aaaa3..901fde2d0 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -120,9 +120,9 @@ class VolumePairList(IPairList): # Use fresh pairlist # Check if pair quote currency equals to the stake currency. filtered_tickers = [ - v for k, v in tickers.items() - if (self._exchange.get_pair_quote_currency(k) == self._stake_currency - and v[self._sort_key] is not None)] + v for k, v in tickers.items() + if (self._exchange.get_pair_quote_currency(k) == self._stake_currency + and v[self._sort_key] is not None)] pairlist = [s['symbol'] for s in filtered_tickers] pairlist = self.filter_pairlist(pairlist, tickers) @@ -197,7 +197,7 @@ class VolumePairList(IPairList): if self._min_value > 0: filtered_tickers = [ - v for v in filtered_tickers if v[self._sort_key] > self._min_value] + v for v in filtered_tickers if v[self._sort_key] > self._min_value] sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key]) diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 03f4760b8..face79729 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -28,13 +28,13 @@ class PairListManager(): self._tickers_needed = False for pairlist_handler_config in self._config.get('pairlists', None): pairlist_handler = PairListResolver.load_pairlist( - pairlist_handler_config['method'], - exchange=exchange, - pairlistmanager=self, - config=config, - pairlistconfig=pairlist_handler_config, - pairlist_pos=len(self._pairlist_handlers) - ) + pairlist_handler_config['method'], + exchange=exchange, + pairlistmanager=self, + config=config, + pairlistconfig=pairlist_handler_config, + pairlist_pos=len(self._pairlist_handlers) + ) self._tickers_needed |= pairlist_handler.needstickers self._pairlist_handlers.append(pairlist_handler) diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 45d393411..40edf1204 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -54,9 +54,9 @@ class StoplossGuard(IProtection): trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) trades = [trade for trade in trades1 if (str(trade.sell_reason) in ( - SellType.TRAILING_STOP_LOSS.value, SellType.STOP_LOSS.value, - SellType.STOPLOSS_ON_EXCHANGE.value) - and trade.close_profit and trade.close_profit < 0)] + SellType.TRAILING_STOP_LOSS.value, SellType.STOP_LOSS.value, + SellType.STOPLOSS_ON_EXCHANGE.value) + and trade.close_profit and trade.close_profit < 0)] if len(trades) < self._trade_limit: return False, None, None diff --git a/freqtrade/resolvers/__init__.py b/freqtrade/resolvers/__init__.py index ef24bf481..2f70a788a 100644 --- a/freqtrade/resolvers/__init__.py +++ b/freqtrade/resolvers/__init__.py @@ -8,6 +8,3 @@ from freqtrade.resolvers.exchange_resolver import ExchangeResolver from freqtrade.resolvers.pairlist_resolver import PairListResolver from freqtrade.resolvers.protection_resolver import ProtectionResolver from freqtrade.resolvers.strategy_resolver import StrategyResolver - - - diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 1239b78b3..82942bd68 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -50,7 +50,7 @@ class StrategyResolver(IResolver): if 'timeframe' not in config: logger.warning( "DEPRECATED: Please migrate to using 'timeframe' instead of 'ticker_interval'." - ) + ) strategy.timeframe = strategy.ticker_interval if strategy._ft_params_from_file: diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 61d69707e..f2361fda8 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -199,8 +199,8 @@ def pair_history(pair: str, timeframe: str, timerange: str, strategy: str, config=Depends(get_config)): config = deepcopy(config) config.update({ - 'strategy': strategy, - }) + 'strategy': strategy, + }) return RPC._rpc_analysed_history_full(config, pair, timeframe, timerange) diff --git a/freqtrade/rpc/fiat_convert.py b/freqtrade/rpc/fiat_convert.py index 199e6a7db..cdc09b437 100644 --- a/freqtrade/rpc/fiat_convert.py +++ b/freqtrade/rpc/fiat_convert.py @@ -62,7 +62,7 @@ class CryptoToFiatConverter: # If the request is not a 429 error we want to raise the normal error logger.error( "Could not load FIAT Cryptocurrency map for the following problem: {}".format( - request_exception + request_exception ) ) except (Exception) as exception: diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 67842e849..8085ece94 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -15,6 +15,7 @@ class RPCManager: """ Class to manage RPC objects (Telegram, API, ...) """ + def __init__(self, freqtrade) -> None: """ Initializes all enabled rpc modules """ self.registered_modules: List[RPCHandler] = [] diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a1f6a7e33..a988d2b60 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -77,7 +77,6 @@ class Telegram(RPCHandler): """ This class handles all telegram communication """ def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: - """ Init the Telegram call, and init the super class RPCHandler :param rpc: instance of RPC Helper class @@ -270,7 +269,7 @@ class Telegram(RPCHandler): noti = '' if msg_type == RPCMessageType.SELL: sell_noti = self._config['telegram'] \ - .get('notification_settings', {}).get(str(msg_type), {}) + .get('notification_settings', {}).get(str(msg_type), {}) # For backward compatibility sell still can be string if isinstance(sell_noti, str): noti = sell_noti @@ -278,7 +277,7 @@ class Telegram(RPCHandler): noti = sell_noti.get(str(msg['sell_reason']), default_noti) else: noti = self._config['telegram'] \ - .get('notification_settings', {}).get(str(msg_type), default_noti) + .get('notification_settings', {}).get(str(msg_type), default_noti) if noti == 'off': logger.info(f"Notification '{msg_type}' not sent.") @@ -541,7 +540,7 @@ class Telegram(RPCHandler): f"`{first_trade_date}`\n" f"*Latest Trade opened:* `{latest_trade_date}\n`" f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`" - ) + ) if stats['closed_trade_count'] > 0: markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n" f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`") @@ -576,13 +575,14 @@ class Telegram(RPCHandler): sell_reasons_msg = tabulate( sell_reasons_tabulate, headers=['Sell Reason', 'Sells', 'Wins', 'Losses'] - ) + ) durations = stats['durations'] - duration_msg = tabulate([ - ['Wins', str(timedelta(seconds=durations['wins'])) - if durations['wins'] != 'N/A' else 'N/A'], - ['Losses', str(timedelta(seconds=durations['losses'])) - if durations['losses'] != 'N/A' else 'N/A'] + duration_msg = tabulate( + [ + ['Wins', str(timedelta(seconds=durations['wins'])) + if durations['wins'] != 'N/A' else 'N/A'], + ['Losses', str(timedelta(seconds=durations['losses'])) + if durations['losses'] != 'N/A' else 'N/A'] ], headers=['', 'Avg. Duration'] ) @@ -1100,7 +1100,7 @@ class Telegram(RPCHandler): if reload_able: reply_markup = InlineKeyboardMarkup([ [InlineKeyboardButton("Refresh", callback_data=callback_path)], - ]) + ]) else: reply_markup = InlineKeyboardMarkup([[]]) msg += "\nUpdated: {}".format(datetime.now().ctime()) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index 22b6f0be5..e089ebf31 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -38,7 +38,7 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, # Detailed explanation in https://github.com/freqtrade/freqtrade/issues/4073 informative['date_merge'] = ( informative["date"] + pd.to_timedelta(minutes_inf, 'm') - pd.to_timedelta(minutes, 'm') - ) + ) else: raise ValueError("Tried to merge a faster timeframe to a slower timeframe." "This would create new rows, and can throw off your regular indicators.") diff --git a/tests/config_test_comments.json b/tests/config_test_comments.json index 48a087dec..19d82c454 100644 --- a/tests/config_test_comments.json +++ b/tests/config_test_comments.json @@ -6,8 +6,8 @@ */ "stake_currency": "BTC", "stake_amount": 0.05, - "fiat_display_currency": "USD", // C++-style comment - "amount_reserve_percent" : 0.05, // And more, tabs before this comment + "fiat_display_currency": "USD", // C++-style comment + "amount_reserve_percent": 0.05, // And more, tabs before this comment "dry_run": false, "timeframe": "5m", "trailing_stop": false, @@ -15,15 +15,15 @@ "trailing_stop_positive_offset": 0.0051, "trailing_only_offset_is_reached": false, "minimal_roi": { - "40": 0.0, - "30": 0.01, - "20": 0.02, - "0": 0.04 + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 }, "stoploss": -0.10, "unfilledtimeout": { "buy": 10, - "sell": 30, // Trailing comma should also be accepted now + "sell": 30, // Trailing comma should also be accepted now }, "bid_strategy": { "use_order_book": false, @@ -34,7 +34,7 @@ "bids_to_ask_delta": 1 } }, - "ask_strategy":{ + "ask_strategy": { "use_order_book": false, "order_book_min": 1, "order_book_max": 9 @@ -64,7 +64,9 @@ "key": "your_exchange_key", "secret": "your_exchange_secret", "password": "", - "ccxt_config": {"enableRateLimit": true}, + "ccxt_config": { + "enableRateLimit": true + }, "ccxt_async_config": { "enableRateLimit": false, "rateLimit": 500, @@ -103,8 +105,8 @@ "remove_pumps": false }, "telegram": { -// We can now comment out some settings -// "enabled": true, + // We can now comment out some settings + // "enabled": true, "enabled": false, "token": "your_telegram_token", "chat_id": "your_telegram_chat_id" @@ -124,4 +126,4 @@ }, "strategy": "DefaultStrategy", "strategy_path": "user_data/strategies/" -} +} \ No newline at end of file diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 14fea573f..b5197e73f 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -399,7 +399,7 @@ def test_hyperopt_format_results(hyperopt): 'rejected_signals': 2, 'backtest_start_time': 1619718665, 'backtest_end_time': 1619718665, - } + } results_metrics = generate_strategy_stats({'XRP/BTC': None}, '', bt_result, Arrow(2017, 11, 14, 19, 32, 00), Arrow(2017, 12, 14, 19, 32, 00), market_change=0) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 9ec47dade..c0a9ae72a 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -93,7 +93,7 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): Trade.query.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=200, min_ago_close=30, - )) + )) assert not freqtrade.protections.global_stop() assert not log_has_re(message, caplog) @@ -150,7 +150,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair Trade.query.session.add(generate_mock_trade( pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=200, min_ago_close=30, profit_rate=0.9, - )) + )) assert not freqtrade.protections.stop_per_pair(pair) assert not freqtrade.protections.global_stop() diff --git a/tests/rpc/test_fiat_convert.py b/tests/rpc/test_fiat_convert.py index 5174f9416..9fb1122f5 100644 --- a/tests/rpc/test_fiat_convert.py +++ b/tests/rpc/test_fiat_convert.py @@ -139,9 +139,9 @@ def test_fiat_too_many_requests_response(mocker, caplog): assert length_cryptomap == 0 assert fiat_convert._backoff > datetime.datetime.now().timestamp() assert log_has( - 'Too many requests for Coingecko API, backing off and trying again later.', - caplog - ) + 'Too many requests for Coingecko API, backing off and trying again later.', + caplog + ) def test_fiat_invalid_response(mocker, caplog): diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 68f23e0fd..1517b6fcc 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -942,7 +942,7 @@ def test_api_whitelist(botclient): "whitelist": ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'], "length": 4, "method": ["StaticPairList"] - } + } def test_api_forcebuy(botclient, mocker, fee): @@ -1033,7 +1033,7 @@ def test_api_forcebuy(botclient, mocker, fee): 'buy_tag': None, 'timeframe': 5, 'exchange': 'binance', - } + } def test_api_forcesell(botclient, mocker, ticker, fee, markets): @@ -1215,7 +1215,7 @@ def test_api_strategies(botclient): 'DefaultStrategy', 'HyperoptableStrategy', 'TestStrategyLegacy' - ]} + ]} def test_api_strategy(botclient): diff --git a/tests/test_arguments.py b/tests/test_arguments.py index fd6f162fd..5374881fa 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -125,7 +125,7 @@ def test_parse_args_backtesting_custom() -> None: '--strategy-list', 'DefaultStrategy', 'SampleStrategy' - ] + ] call_args = Arguments(args).get_parsed_arg() assert call_args['config'] == ['test_conf.json'] assert call_args['verbosity'] == 0 diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 7012333e9..f97ccd488 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1130,17 +1130,17 @@ def test_pairlist_resolving_fallback(mocker): @pytest.mark.parametrize("setting", [ - ("ask_strategy", "use_sell_signal", True, - None, "use_sell_signal", False), - ("ask_strategy", "sell_profit_only", True, - None, "sell_profit_only", False), - ("ask_strategy", "sell_profit_offset", 0.1, - None, "sell_profit_offset", 0.01), - ("ask_strategy", "ignore_roi_if_buy_signal", True, - None, "ignore_roi_if_buy_signal", False), - ("ask_strategy", "ignore_buying_expired_candle_after", 5, - None, "ignore_buying_expired_candle_after", 6), - ]) + ("ask_strategy", "use_sell_signal", True, + None, "use_sell_signal", False), + ("ask_strategy", "sell_profit_only", True, + None, "sell_profit_only", False), + ("ask_strategy", "sell_profit_offset", 0.1, + None, "sell_profit_offset", 0.01), + ("ask_strategy", "ignore_roi_if_buy_signal", True, + None, "ignore_roi_if_buy_signal", False), + ("ask_strategy", "ignore_buying_expired_candle_after", 5, + None, "ignore_buying_expired_candle_after", 6), +]) def test_process_temporary_deprecated_settings(mocker, default_conf, setting, caplog): patched_configuration_load_config_file(mocker, default_conf) @@ -1180,10 +1180,10 @@ def test_process_temporary_deprecated_settings(mocker, default_conf, setting, ca @pytest.mark.parametrize("setting", [ - ("experimental", "use_sell_signal", False), - ("experimental", "sell_profit_only", True), - ("experimental", "ignore_roi_if_buy_signal", True), - ]) + ("experimental", "use_sell_signal", False), + ("experimental", "sell_profit_only", True), + ("experimental", "ignore_roi_if_buy_signal", True), +]) def test_process_removed_settings(mocker, default_conf, setting): patched_configuration_load_config_file(mocker, default_conf) From 5393c55b513028a3159da23a19e241050766651b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Aug 2021 16:01:20 +0200 Subject: [PATCH 11/40] Document min_value for VolumePairList closes #5260 --- docs/includes/pairlists.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 995e49a2d..b92b05af9 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -58,7 +58,7 @@ This option must be configured along with `exchange.skip_pair_validation` in the When used in the chain of Pairlist Handlers in a non-leading position (after StaticPairList and other Pairlist Filters), `VolumePairList` considers outputs of previous Pairlist Handlers, adding its sorting/selection of the pairs by the trading volume. -When used on the leading position of the chain of Pairlist Handlers, it does not consider `pair_whitelist` configuration setting, but selects the top assets from all available markets (with matching stake-currency) on the exchange. +When used in the leading position of the chain of Pairlist Handlers, the `pair_whitelist` configuration setting is ignored. Instead, `VolumePairList` selects the top assets from all available markets with matching stake-currency on the exchange. The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists. @@ -74,11 +74,14 @@ Filtering instances (not the first position in the list) will not apply any cach "method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume", + "min_value": 0, "refresh_period": 1800 } ], ``` +You can define a minimum volume with `min_value` - which will filter out pairs with a volume lower than the specified value in the specified timerange. + `VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles. For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days: @@ -89,6 +92,7 @@ For convenience `lookback_days` can be specified, which will imply that 1d candl "method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume", + "min_value": 0, "refresh_period": 86400, "lookback_days": 7 } From cf70b34ff0504dcbffc1106bfbefd521bc1c3357 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Aug 2021 16:15:42 +0200 Subject: [PATCH 12/40] Add min_value to all volumepairlist samples --- docs/includes/pairlists.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index b92b05af9..6e23c9003 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -113,6 +113,7 @@ More sophisticated approach can be used, by using `lookback_timeframe` for candl "method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume", + "min_value": 0, "refresh_period": 3600, "lookback_timeframe": "1h", "lookback_period": 72 From 7eaadb2630a99c8f467751bdff071b52cdb098b2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 Aug 2021 09:50:50 +0200 Subject: [PATCH 13/40] Add custom-* methods to bot-basics closes #5370 --- docs/bot-basics.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/bot-basics.md b/docs/bot-basics.md index 943af0362..44181abfa 100644 --- a/docs/bot-basics.md +++ b/docs/bot-basics.md @@ -35,7 +35,7 @@ By default, loop runs every few seconds (`internals.process_throttle_secs`) and * Calls `check_buy_timeout()` strategy callback for open buy orders. * Calls `check_sell_timeout()` strategy callback for open sell orders. * Verifies existing positions and eventually places sell orders. - * Considers stoploss, ROI and sell-signal. + * Considers stoploss, ROI and sell-signal, `custom_sell()` and `custom_stoploss()`. * Determine sell-price based on `ask_strategy` configuration setting. * Before a sell order is placed, `confirm_trade_exit()` strategy callback is called. * Check if trade-slots are still available (if `max_open_trades` is reached). @@ -52,9 +52,10 @@ This loop will be repeated again and again until the bot is stopped. * Load historic data for configured pairlist. * Calls `bot_loop_start()` once. * Calculate indicators (calls `populate_indicators()` once per pair). -* Calculate buy / sell signals (calls `populate_buy_trend()` and `populate_sell_trend()` once per pair) -* Confirm trade buy / sell (calls `confirm_trade_entry()` and `confirm_trade_exit()` if implemented in the strategy) +* Calculate buy / sell signals (calls `populate_buy_trend()` and `populate_sell_trend()` once per pair). * Loops per candle simulating entry and exit points. + * Confirm trade buy / sell (calls `confirm_trade_entry()` and `confirm_trade_exit()` if implemented in the strategy). + * Call `custom_stoploss()` and `custom_sell()` to find custom exit points. * Generate backtest report output !!! Note From 0ae4eccea5ad7cd773e0d0e309617bfcf6a1e4ae Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 Aug 2021 10:05:28 +0200 Subject: [PATCH 14/40] Refactor Hyperopt-list and hyperopt-show to reduce some duplicate code --- freqtrade/commands/hyperopt_commands.py | 180 +------------------ freqtrade/optimize/hyperopt_epoch_filters.py | 142 +++++++++++++++ freqtrade/optimize/hyperopt_tools.py | 28 ++- 3 files changed, 174 insertions(+), 176 deletions(-) create mode 100644 freqtrade/optimize/hyperopt_epoch_filters.py diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 4694d1111..089529d15 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -1,6 +1,6 @@ import logging from operator import itemgetter -from typing import Any, Dict, List +from typing import Any, Dict from colorama import init as colorama_init @@ -28,30 +28,12 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: no_details = config.get('hyperopt_list_no_details', False) no_header = False - filteroptions = { - 'only_best': config.get('hyperopt_list_best', False), - 'only_profitable': config.get('hyperopt_list_profitable', False), - 'filter_min_trades': config.get('hyperopt_list_min_trades', 0), - 'filter_max_trades': config.get('hyperopt_list_max_trades', 0), - 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None), - 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None), - 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), - 'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None), - 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None), - 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None), - 'filter_min_objective': config.get('hyperopt_list_min_objective', None), - 'filter_max_objective': config.get('hyperopt_list_max_objective', None), - } - results_file = get_latest_hyperopt_file( config['user_data_dir'] / 'hyperopt_results', config.get('hyperoptexportfilename')) # Previous evaluations - epochs = HyperoptTools.load_previous_results(results_file) - total_epochs = len(epochs) - - epochs = hyperopt_filter_epochs(epochs, filteroptions) + epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config) if print_colorized: colorama_init(autoreset=True) @@ -59,7 +41,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: if not export_csv: try: print(HyperoptTools.get_result_table(config, epochs, total_epochs, - not filteroptions['only_best'], + not config.get('hyperopt_list_best', False), print_colorized, 0)) except KeyboardInterrupt: print('User interrupted..') @@ -71,7 +53,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: if epochs and export_csv: HyperoptTools.export_csv_file( - config, epochs, total_epochs, not filteroptions['only_best'], export_csv + config, epochs, total_epochs, not config.get('hyperopt_list_best', False), export_csv ) @@ -91,26 +73,9 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: n = config.get('hyperopt_show_index', -1) - filteroptions = { - 'only_best': config.get('hyperopt_list_best', False), - 'only_profitable': config.get('hyperopt_list_profitable', False), - 'filter_min_trades': config.get('hyperopt_list_min_trades', 0), - 'filter_max_trades': config.get('hyperopt_list_max_trades', 0), - 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None), - 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None), - 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), - 'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None), - 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None), - 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None), - 'filter_min_objective': config.get('hyperopt_list_min_objective', None), - 'filter_max_objective': config.get('hyperopt_list_max_objective', None) - } - # Previous evaluations - epochs = HyperoptTools.load_previous_results(results_file) - total_epochs = len(epochs) + epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config) - epochs = hyperopt_filter_epochs(epochs, filteroptions) filtered_epochs = len(epochs) if n > filtered_epochs: @@ -137,138 +102,3 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header, header_str="Epoch details") - - -def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: - """ - Filter our items from the list of hyperopt results - TODO: after 2021.5 remove all "legacy" mode queries. - """ - if filteroptions['only_best']: - epochs = [x for x in epochs if x['is_best']] - if filteroptions['only_profitable']: - epochs = [x for x in epochs if x['results_metrics'].get( - 'profit', x['results_metrics'].get('profit_total', 0)) > 0] - - epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions) - - epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions) - - epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions) - - epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions) - - logger.info(f"{len(epochs)} " + - ("best " if filteroptions['only_best'] else "") + - ("profitable " if filteroptions['only_profitable'] else "") + - "epochs found.") - return epochs - - -def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int): - """ - Filter epochs with trade-counts > trades - """ - return [ - x for x in epochs - if x['results_metrics'].get( - 'trade_count', x['results_metrics'].get('total_trades', 0) - ) > trade_count - ] - - -def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List: - - if filteroptions['filter_min_trades'] > 0: - epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades']) - - if filteroptions['filter_max_trades'] > 0: - epochs = [ - x for x in epochs - if x['results_metrics'].get( - 'trade_count', x['results_metrics'].get('total_trades') - ) < filteroptions['filter_max_trades'] - ] - return epochs - - -def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List: - - def get_duration_value(x): - # Duration in minutes ... - if 'duration' in x['results_metrics']: - return x['results_metrics']['duration'] - else: - # New mode - if 'holding_avg_s' in x['results_metrics']: - avg = x['results_metrics']['holding_avg_s'] - return avg // 60 - raise OperationalException( - "Holding-average not available. Please omit the filter on average time, " - "or rerun hyperopt with this version") - - if filteroptions['filter_min_avg_time'] is not None: - epochs = _hyperopt_filter_epochs_trade(epochs, 0) - epochs = [ - x for x in epochs - if get_duration_value(x) > filteroptions['filter_min_avg_time'] - ] - if filteroptions['filter_max_avg_time'] is not None: - epochs = _hyperopt_filter_epochs_trade(epochs, 0) - epochs = [ - x for x in epochs - if get_duration_value(x) < filteroptions['filter_max_avg_time'] - ] - - return epochs - - -def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List: - - if filteroptions['filter_min_avg_profit'] is not None: - epochs = _hyperopt_filter_epochs_trade(epochs, 0) - epochs = [ - x for x in epochs - if x['results_metrics'].get( - 'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100 - ) > filteroptions['filter_min_avg_profit'] - ] - if filteroptions['filter_max_avg_profit'] is not None: - epochs = _hyperopt_filter_epochs_trade(epochs, 0) - epochs = [ - x for x in epochs - if x['results_metrics'].get( - 'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100 - ) < filteroptions['filter_max_avg_profit'] - ] - if filteroptions['filter_min_total_profit'] is not None: - epochs = _hyperopt_filter_epochs_trade(epochs, 0) - epochs = [ - x for x in epochs - if x['results_metrics'].get( - 'profit', x['results_metrics'].get('profit_total_abs', 0) - ) > filteroptions['filter_min_total_profit'] - ] - if filteroptions['filter_max_total_profit'] is not None: - epochs = _hyperopt_filter_epochs_trade(epochs, 0) - epochs = [ - x for x in epochs - if x['results_metrics'].get( - 'profit', x['results_metrics'].get('profit_total_abs', 0) - ) < filteroptions['filter_max_total_profit'] - ] - return epochs - - -def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List: - - if filteroptions['filter_min_objective'] is not None: - epochs = _hyperopt_filter_epochs_trade(epochs, 0) - - epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']] - if filteroptions['filter_max_objective'] is not None: - epochs = _hyperopt_filter_epochs_trade(epochs, 0) - - epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']] - - return epochs diff --git a/freqtrade/optimize/hyperopt_epoch_filters.py b/freqtrade/optimize/hyperopt_epoch_filters.py new file mode 100644 index 000000000..2084c5ba8 --- /dev/null +++ b/freqtrade/optimize/hyperopt_epoch_filters.py @@ -0,0 +1,142 @@ +import logging +from typing import List + +from freqtrade.exceptions import OperationalException + + +logger = logging.getLogger(__name__) + + +def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: + """ + Filter our items from the list of hyperopt results + TODO: after 2021.5 remove all "legacy" mode queries. + """ + if filteroptions['only_best']: + epochs = [x for x in epochs if x['is_best']] + if filteroptions['only_profitable']: + epochs = [x for x in epochs if x['results_metrics'].get( + 'profit', x['results_metrics'].get('profit_total', 0)) > 0] + + epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions) + + epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions) + + epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions) + + epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions) + + logger.info(f"{len(epochs)} " + + ("best " if filteroptions['only_best'] else "") + + ("profitable " if filteroptions['only_profitable'] else "") + + "epochs found.") + return epochs + + +def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int): + """ + Filter epochs with trade-counts > trades + """ + return [ + x for x in epochs + if x['results_metrics'].get( + 'trade_count', x['results_metrics'].get('total_trades', 0) + ) > trade_count + ] + + +def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List: + + if filteroptions['filter_min_trades'] > 0: + epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades']) + + if filteroptions['filter_max_trades'] > 0: + epochs = [ + x for x in epochs + if x['results_metrics'].get( + 'trade_count', x['results_metrics'].get('total_trades') + ) < filteroptions['filter_max_trades'] + ] + return epochs + + +def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List: + + def get_duration_value(x): + # Duration in minutes ... + if 'duration' in x['results_metrics']: + return x['results_metrics']['duration'] + else: + # New mode + if 'holding_avg_s' in x['results_metrics']: + avg = x['results_metrics']['holding_avg_s'] + return avg // 60 + raise OperationalException( + "Holding-average not available. Please omit the filter on average time, " + "or rerun hyperopt with this version") + + if filteroptions['filter_min_avg_time'] is not None: + epochs = _hyperopt_filter_epochs_trade(epochs, 0) + epochs = [ + x for x in epochs + if get_duration_value(x) > filteroptions['filter_min_avg_time'] + ] + if filteroptions['filter_max_avg_time'] is not None: + epochs = _hyperopt_filter_epochs_trade(epochs, 0) + epochs = [ + x for x in epochs + if get_duration_value(x) < filteroptions['filter_max_avg_time'] + ] + + return epochs + + +def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List: + + if filteroptions['filter_min_avg_profit'] is not None: + epochs = _hyperopt_filter_epochs_trade(epochs, 0) + epochs = [ + x for x in epochs + if x['results_metrics'].get( + 'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100 + ) > filteroptions['filter_min_avg_profit'] + ] + if filteroptions['filter_max_avg_profit'] is not None: + epochs = _hyperopt_filter_epochs_trade(epochs, 0) + epochs = [ + x for x in epochs + if x['results_metrics'].get( + 'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100 + ) < filteroptions['filter_max_avg_profit'] + ] + if filteroptions['filter_min_total_profit'] is not None: + epochs = _hyperopt_filter_epochs_trade(epochs, 0) + epochs = [ + x for x in epochs + if x['results_metrics'].get( + 'profit', x['results_metrics'].get('profit_total_abs', 0) + ) > filteroptions['filter_min_total_profit'] + ] + if filteroptions['filter_max_total_profit'] is not None: + epochs = _hyperopt_filter_epochs_trade(epochs, 0) + epochs = [ + x for x in epochs + if x['results_metrics'].get( + 'profit', x['results_metrics'].get('profit_total_abs', 0) + ) < filteroptions['filter_max_total_profit'] + ] + return epochs + + +def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List: + + if filteroptions['filter_min_objective'] is not None: + epochs = _hyperopt_filter_epochs_trade(epochs, 0) + + epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']] + if filteroptions['filter_max_objective'] is not None: + epochs = _hyperopt_filter_epochs_trade(epochs, 0) + + epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']] + + return epochs diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 52aa85c84..ed9583bdc 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -4,7 +4,7 @@ import logging from copy import deepcopy from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple import numpy as np import rapidjson @@ -15,6 +15,7 @@ from pandas import isna, json_normalize from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2 +from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs logger = logging.getLogger(__name__) @@ -130,6 +131,31 @@ class HyperoptTools(): logger.info(f"Loaded {len(epochs)} previous evaluations from disk.") return epochs + @staticmethod + def load_filtered_results(results_file: Path, config: Dict[str, Any]) -> Tuple[List, int]: + filteroptions = { + 'only_best': config.get('hyperopt_list_best', False), + 'only_profitable': config.get('hyperopt_list_profitable', False), + 'filter_min_trades': config.get('hyperopt_list_min_trades', 0), + 'filter_max_trades': config.get('hyperopt_list_max_trades', 0), + 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None), + 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None), + 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), + 'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None), + 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None), + 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None), + 'filter_min_objective': config.get('hyperopt_list_min_objective', None), + 'filter_max_objective': config.get('hyperopt_list_max_objective', None), + } + + # Previous evaluations + epochs = HyperoptTools.load_previous_results(results_file) + total_epochs = len(epochs) + + epochs = hyperopt_filter_epochs(epochs, filteroptions) + + return epochs, total_epochs + @staticmethod def show_epoch_details(results, total_epochs: int, print_json: bool, no_header: bool = False, header_str: str = None) -> None: From faf16a64e507dc684d527a804abef266e39c26c6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 Aug 2021 10:22:45 +0200 Subject: [PATCH 15/40] Remove legacy hyperopt file support --- freqtrade/optimize/hyperopt_epoch_filters.py | 50 +- tests/commands/test_commands.py | 464 +++++++++---------- tests/conftest.py | 132 ------ 3 files changed, 249 insertions(+), 397 deletions(-) diff --git a/freqtrade/optimize/hyperopt_epoch_filters.py b/freqtrade/optimize/hyperopt_epoch_filters.py index 2084c5ba8..b70db94af 100644 --- a/freqtrade/optimize/hyperopt_epoch_filters.py +++ b/freqtrade/optimize/hyperopt_epoch_filters.py @@ -10,13 +10,12 @@ logger = logging.getLogger(__name__) def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: """ Filter our items from the list of hyperopt results - TODO: after 2021.5 remove all "legacy" mode queries. """ if filteroptions['only_best']: epochs = [x for x in epochs if x['is_best']] if filteroptions['only_profitable']: - epochs = [x for x in epochs if x['results_metrics'].get( - 'profit', x['results_metrics'].get('profit_total', 0)) > 0] + epochs = [x for x in epochs + if x['results_metrics'].get('profit_total', 0) > 0] epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions) @@ -38,10 +37,7 @@ def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int): Filter epochs with trade-counts > trades """ return [ - x for x in epochs - if x['results_metrics'].get( - 'trade_count', x['results_metrics'].get('total_trades', 0) - ) > trade_count + x for x in epochs if x['results_metrics'].get('total_trades', 0) > trade_count ] @@ -53,9 +49,7 @@ def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> Li if filteroptions['filter_max_trades'] > 0: epochs = [ x for x in epochs - if x['results_metrics'].get( - 'trade_count', x['results_metrics'].get('total_trades') - ) < filteroptions['filter_max_trades'] + if x['results_metrics'].get('total_trades') < filteroptions['filter_max_trades'] ] return epochs @@ -64,16 +58,12 @@ def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List: def get_duration_value(x): # Duration in minutes ... - if 'duration' in x['results_metrics']: - return x['results_metrics']['duration'] - else: - # New mode - if 'holding_avg_s' in x['results_metrics']: - avg = x['results_metrics']['holding_avg_s'] - return avg // 60 - raise OperationalException( - "Holding-average not available. Please omit the filter on average time, " - "or rerun hyperopt with this version") + if 'holding_avg_s' in x['results_metrics']: + avg = x['results_metrics']['holding_avg_s'] + return avg // 60 + raise OperationalException( + "Holding-average not available. Please omit the filter on average time, " + "or rerun hyperopt with this version") if filteroptions['filter_min_avg_time'] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) @@ -97,33 +87,29 @@ def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List: epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ x for x in epochs - if x['results_metrics'].get( - 'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100 - ) > filteroptions['filter_min_avg_profit'] + if x['results_metrics'].get('profit_mean', 0) * 100 + > filteroptions['filter_min_avg_profit'] ] if filteroptions['filter_max_avg_profit'] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ x for x in epochs - if x['results_metrics'].get( - 'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100 - ) < filteroptions['filter_max_avg_profit'] + if x['results_metrics'].get('profit_mean', 0) * 100 + < filteroptions['filter_max_avg_profit'] ] if filteroptions['filter_min_total_profit'] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ x for x in epochs - if x['results_metrics'].get( - 'profit', x['results_metrics'].get('profit_total_abs', 0) - ) > filteroptions['filter_min_total_profit'] + if x['results_metrics'].get('profit_total_abs', 0) + > filteroptions['filter_min_total_profit'] ] if filteroptions['filter_max_total_profit'] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ x for x in epochs - if x['results_metrics'].get( - 'profit', x['results_metrics'].get('profit_total_abs', 0) - ) < filteroptions['filter_max_total_profit'] + if x['results_metrics'].get('profit_total_abs', 0) + < filteroptions['filter_max_total_profit'] ] return epochs diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index c0268038a..80dd04b27 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -938,241 +938,239 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): pytest.fail(f'Expected well formed JSON, but failed to parse: {captured.out}') -def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results, - saved_hyperopt_results_legacy, tmpdir): +def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results, tmpdir): csv_file = Path(tmpdir) / "test.csv" - for res in (saved_hyperopt_results, saved_hyperopt_results_legacy): - mocker.patch( - 'freqtrade.optimize.hyperopt_tools.HyperoptTools.load_previous_results', - MagicMock(return_value=res) - ) + mocker.patch( + 'freqtrade.optimize.hyperopt_tools.HyperoptTools.load_previous_results', + MagicMock(return_value=saved_hyperopt_results) + ) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", - " 6/12", " 7/12", " 8/12", " 9/12", " 10/12", - " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--best", - "--no-details", - "--no-color", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 1/12", " 5/12", " 10/12"]) - assert all(x not in captured.out - for x in [" 2/12", " 3/12", " 4/12", " 6/12", " 7/12", " 8/12", " 9/12", - " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--profitable", - "--no-details", - "--no-color", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 2/12", " 10/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", - " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--profitable", - "--no-color", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 2/12", " 10/12", "Best result:", "Buy hyperspace params", - "Sell hyperspace params", "ROI table", "Stoploss"]) - assert all(x not in captured.out - for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", - " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--min-trades", "20", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 3/12", " 6/12", " 7/12", " 9/12", " 11/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 2/12", " 4/12", " 5/12", " 8/12", " 10/12", " 12/12"]) - args = [ - "hyperopt-list", - "--profitable", - "--no-details", - "--no-color", - "--max-trades", "20", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 2/12", " 10/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", - " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--profitable", - "--no-details", - "--no-color", - "--min-avg-profit", "0.11", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 2/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", - " 10/12", " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--max-avg-profit", "0.10", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 1/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", - " 11/12"]) - assert all(x not in captured.out - for x in [" 2/12", " 4/12", " 10/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--min-total-profit", "0.4", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 10/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", - " 9/12", " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--max-total-profit", "0.4", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", - " 9/12", " 11/12"]) - assert all(x not in captured.out - for x in [" 4/12", " 10/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--min-objective", "0.1", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 10/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", - " 9/12", " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--max-objective", "0.1", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", - " 9/12", " 11/12"]) - assert all(x not in captured.out - for x in [" 4/12", " 10/12", " 12/12"]) - args = [ - "hyperopt-list", - "--profitable", - "--no-details", - "--no-color", - "--min-avg-time", "2000", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 10/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", - " 8/12", " 9/12", " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--max-avg-time", "1500", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 2/12", " 6/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 7/12", " 8/12" - " 9/12", " 10/12", " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--export-csv", - str(csv_file), - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - log_has("CSV file created: test_file.csv", caplog) - assert csv_file.is_file() - line = csv_file.read_text() - assert ('Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,"3,930.0 m",0.43662' in line - or "Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,2 days 17:30:00,0.43662" in line) - csv_file.unlink() + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", + " 6/12", " 7/12", " 8/12", " 9/12", " 10/12", + " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--best", + "--no-details", + "--no-color", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 5/12", " 10/12"]) + assert all(x not in captured.out + for x in [" 2/12", " 3/12", " 4/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--no-color", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-color", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 10/12", "Best result:", "Buy hyperspace params", + "Sell hyperspace params", "ROI table", "Stoploss"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--min-trades", "20", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 3/12", " 6/12", " 7/12", " 9/12", " 11/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 4/12", " 5/12", " 8/12", " 10/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--no-color", + "--max-trades", "20", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--no-color", + "--min-avg-profit", "0.11", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 10/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--max-avg-profit", "0.10", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12"]) + assert all(x not in captured.out + for x in [" 2/12", " 4/12", " 10/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--min-total-profit", "0.4", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--max-total-profit", "0.4", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12"]) + assert all(x not in captured.out + for x in [" 4/12", " 10/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--min-objective", "0.1", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--max-objective", "0.1", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12"]) + assert all(x not in captured.out + for x in [" 4/12", " 10/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--no-color", + "--min-avg-time", "2000", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", + " 8/12", " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--max-avg-time", "1500", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 6/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 7/12", " 8/12" + " 9/12", " 10/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--export-csv", + str(csv_file), + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + log_has("CSV file created: test_file.csv", caplog) + assert csv_file.is_file() + line = csv_file.read_text() + assert ('Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,"3,930.0 m",0.43662' in line + or "Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,2 days 17:30:00,0.43662" in line) + csv_file.unlink() def test_hyperopt_show(mocker, capsys, saved_hyperopt_results): diff --git a/tests/conftest.py b/tests/conftest.py index 1924e1f95..0c9a96e2b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1814,138 +1814,6 @@ def open_trade(): ) -@pytest.fixture -def saved_hyperopt_results_legacy(): - return [ - { - 'loss': 0.4366182531161519, - 'params_dict': { - 'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1190, 'roi_t2': 541, 'roi_t3': 408, 'roi_p1': 0.026035863879169705, 'roi_p2': 0.12508730043628782, 'roi_p3': 0.27766427921605896, 'stoploss': -0.2562930402099556}, # noqa: E501 - 'params_details': {'buy': {'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.4287874435315165, 408: 0.15112316431545753, 949: 0.026035863879169705, 2139: 0}, 'stoploss': {'stoploss': -0.2562930402099556}}, # noqa: E501 - 'results_metrics': {'trade_count': 2, 'avg_profit': -1.254995, 'median_profit': -1.2222, 'total_profit': -0.00125625, 'profit': -2.50999, 'duration': 3930.0}, # noqa: E501 - 'results_explanation': ' 2 trades. Avg profit -1.25%. Total profit -0.00125625 BTC ( -2.51Σ%). Avg duration 3930.0 min.', # noqa: E501 - 'total_profit': -0.00125625, - 'current_epoch': 1, - 'is_initial_point': True, - 'is_best': True - }, { - 'loss': 20.0, - 'params_dict': { - 'mfi-value': 17, 'fastd-value': 38, 'adx-value': 48, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 96, 'sell-fastd-value': 68, 'sell-adx-value': 63, 'sell-rsi-value': 81, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal', 'roi_t1': 334, 'roi_t2': 683, 'roi_t3': 140, 'roi_p1': 0.06403981740598495, 'roi_p2': 0.055519840060645045, 'roi_p3': 0.3253712811342459, 'stoploss': -0.338070047333259}, # noqa: E501 - 'params_details': { - 'buy': {'mfi-value': 17, 'fastd-value': 38, 'adx-value': 48, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, # noqa: E501 - 'sell': {'sell-mfi-value': 96, 'sell-fastd-value': 68, 'sell-adx-value': 63, 'sell-rsi-value': 81, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'}, # noqa: E501 - 'roi': {0: 0.4449309386008759, 140: 0.11955965746663, 823: 0.06403981740598495, 1157: 0}, # noqa: E501 - 'stoploss': {'stoploss': -0.338070047333259}}, - 'results_metrics': {'trade_count': 1, 'avg_profit': 0.12357, 'median_profit': -1.2222, 'total_profit': 6.185e-05, 'profit': 0.12357, 'duration': 1200.0}, # noqa: E501 - 'results_explanation': ' 1 trades. Avg profit 0.12%. Total profit 0.00006185 BTC ( 0.12Σ%). Avg duration 1200.0 min.', # noqa: E501 - 'total_profit': 6.185e-05, - 'current_epoch': 2, - 'is_initial_point': True, - 'is_best': False - }, { - 'loss': 14.241196856510731, - 'params_dict': {'mfi-value': 25, 'fastd-value': 16, 'adx-value': 29, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 98, 'sell-fastd-value': 72, 'sell-adx-value': 51, 'sell-rsi-value': 82, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 889, 'roi_t2': 533, 'roi_t3': 263, 'roi_p1': 0.04759065393663096, 'roi_p2': 0.1488819964638463, 'roi_p3': 0.4102801822104605, 'stoploss': -0.05394588767607611}, # noqa: E501 - 'params_details': {'buy': {'mfi-value': 25, 'fastd-value': 16, 'adx-value': 29, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 98, 'sell-fastd-value': 72, 'sell-adx-value': 51, 'sell-rsi-value': 82, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.6067528326109377, 263: 0.19647265040047726, 796: 0.04759065393663096, 1685: 0}, 'stoploss': {'stoploss': -0.05394588767607611}}, # noqa: E501 - 'results_metrics': {'trade_count': 621, 'avg_profit': -0.43883302093397747, 'median_profit': -1.2222, 'total_profit': -0.13639474, 'profit': -272.515306, 'duration': 1691.207729468599}, # noqa: E501 - 'results_explanation': ' 621 trades. Avg profit -0.44%. Total profit -0.13639474 BTC (-272.52Σ%). Avg duration 1691.2 min.', # noqa: E501 - 'total_profit': -0.13639474, - 'current_epoch': 3, - 'is_initial_point': True, - 'is_best': False - }, { - 'loss': 100000, - 'params_dict': {'mfi-value': 13, 'fastd-value': 35, 'adx-value': 39, 'rsi-value': 29, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 87, 'sell-fastd-value': 54, 'sell-adx-value': 63, 'sell-rsi-value': 93, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1402, 'roi_t2': 676, 'roi_t3': 215, 'roi_p1': 0.06264755784937427, 'roi_p2': 0.14258587851894644, 'roi_p3': 0.20671291201040828, 'stoploss': -0.11818343570194478}, # noqa: E501 - 'params_details': {'buy': {'mfi-value': 13, 'fastd-value': 35, 'adx-value': 39, 'rsi-value': 29, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 54, 'sell-adx-value': 63, 'sell-rsi-value': 93, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.411946348378729, 215: 0.2052334363683207, 891: 0.06264755784937427, 2293: 0}, 'stoploss': {'stoploss': -0.11818343570194478}}, # noqa: E501 - 'results_metrics': {'trade_count': 0, 'avg_profit': None, 'median_profit': None, 'total_profit': 0, 'profit': 0.0, 'duration': None}, # noqa: E501 - 'results_explanation': ' 0 trades. Avg profit nan%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration nan min.', # noqa: E501 - 'total_profit': 0, 'current_epoch': 4, 'is_initial_point': True, 'is_best': False - }, { - 'loss': 0.22195522184191518, - 'params_dict': {'mfi-value': 17, 'fastd-value': 21, 'adx-value': 38, 'rsi-value': 33, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 87, 'sell-fastd-value': 82, 'sell-adx-value': 78, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 1269, 'roi_t2': 601, 'roi_t3': 444, 'roi_p1': 0.07280999507931168, 'roi_p2': 0.08946698095898986, 'roi_p3': 0.1454876733325284, 'stoploss': -0.18181041180901014}, # noqa: E501 - 'params_details': {'buy': {'mfi-value': 17, 'fastd-value': 21, 'adx-value': 38, 'rsi-value': 33, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 82, 'sell-adx-value': 78, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.3077646493708299, 444: 0.16227697603830155, 1045: 0.07280999507931168, 2314: 0}, 'stoploss': {'stoploss': -0.18181041180901014}}, # noqa: E501 - 'results_metrics': {'trade_count': 14, 'avg_profit': -0.3539515, 'median_profit': -1.2222, 'total_profit': -0.002480140000000001, 'profit': -4.955321, 'duration': 3402.8571428571427}, # noqa: E501 - 'results_explanation': ' 14 trades. Avg profit -0.35%. Total profit -0.00248014 BTC ( -4.96Σ%). Avg duration 3402.9 min.', # noqa: E501 - 'total_profit': -0.002480140000000001, - 'current_epoch': 5, - 'is_initial_point': True, - 'is_best': True - }, { - 'loss': 0.545315889154162, - 'params_dict': {'mfi-value': 22, 'fastd-value': 43, 'adx-value': 46, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'bb_lower', 'sell-mfi-value': 87, 'sell-fastd-value': 65, 'sell-adx-value': 94, 'sell-rsi-value': 63, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 319, 'roi_t2': 556, 'roi_t3': 216, 'roi_p1': 0.06251955472249589, 'roi_p2': 0.11659519602202795, 'roi_p3': 0.0953744132197762, 'stoploss': -0.024551752215582423}, # noqa: E501 - 'params_details': {'buy': {'mfi-value': 22, 'fastd-value': 43, 'adx-value': 46, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 65, 'sell-adx-value': 94, 'sell-rsi-value': 63, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.2744891639643, 216: 0.17911475074452382, 772: 0.06251955472249589, 1091: 0}, 'stoploss': {'stoploss': -0.024551752215582423}}, # noqa: E501 - 'results_metrics': {'trade_count': 39, 'avg_profit': -0.21400679487179478, 'median_profit': -1.2222, 'total_profit': -0.0041773, 'profit': -8.346264999999997, 'duration': 636.9230769230769}, # noqa: E501 - 'results_explanation': ' 39 trades. Avg profit -0.21%. Total profit -0.00417730 BTC ( -8.35Σ%). Avg duration 636.9 min.', # noqa: E501 - 'total_profit': -0.0041773, - 'current_epoch': 6, - 'is_initial_point': True, - 'is_best': False - }, { - 'loss': 4.713497421432944, - 'params_dict': {'mfi-value': 13, 'fastd-value': 41, 'adx-value': 21, 'rsi-value': 29, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower', 'sell-mfi-value': 99, 'sell-fastd-value': 60, 'sell-adx-value': 81, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 771, 'roi_t2': 620, 'roi_t3': 145, 'roi_p1': 0.0586919200378493, 'roi_p2': 0.04984118697312542, 'roi_p3': 0.37521058680247044, 'stoploss': -0.14613268022709905}, # noqa: E501 - 'params_details': { - 'buy': {'mfi-value': 13, 'fastd-value': 41, 'adx-value': 21, 'rsi-value': 29, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 99, 'sell-fastd-value': 60, 'sell-adx-value': 81, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.4837436938134452, 145: 0.10853310701097472, 765: 0.0586919200378493, 1536: 0}, # noqa: E501 - 'stoploss': {'stoploss': -0.14613268022709905}}, # noqa: E501 - 'results_metrics': {'trade_count': 318, 'avg_profit': -0.39833954716981146, 'median_profit': -1.2222, 'total_profit': -0.06339929, 'profit': -126.67197600000004, 'duration': 3140.377358490566}, # noqa: E501 - 'results_explanation': ' 318 trades. Avg profit -0.40%. Total profit -0.06339929 BTC (-126.67Σ%). Avg duration 3140.4 min.', # noqa: E501 - 'total_profit': -0.06339929, - 'current_epoch': 7, - 'is_initial_point': True, - 'is_best': False - }, { - 'loss': 20.0, # noqa: E501 - 'params_dict': {'mfi-value': 24, 'fastd-value': 43, 'adx-value': 33, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'sar_reversal', 'sell-mfi-value': 89, 'sell-fastd-value': 74, 'sell-adx-value': 70, 'sell-rsi-value': 70, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': False, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal', 'roi_t1': 1149, 'roi_t2': 375, 'roi_t3': 289, 'roi_p1': 0.05571820757172588, 'roi_p2': 0.0606240398618907, 'roi_p3': 0.1729012220156157, 'stoploss': -0.1588514289110401}, # noqa: E501 - 'params_details': {'buy': {'mfi-value': 24, 'fastd-value': 43, 'adx-value': 33, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'sar_reversal'}, 'sell': {'sell-mfi-value': 89, 'sell-fastd-value': 74, 'sell-adx-value': 70, 'sell-rsi-value': 70, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': False, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'}, 'roi': {0: 0.2892434694492323, 289: 0.11634224743361658, 664: 0.05571820757172588, 1813: 0}, 'stoploss': {'stoploss': -0.1588514289110401}}, # noqa: E501 - 'results_metrics': {'trade_count': 1, 'avg_profit': 0.0, 'median_profit': 0.0, 'total_profit': 0.0, 'profit': 0.0, 'duration': 5340.0}, # noqa: E501 - 'results_explanation': ' 1 trades. Avg profit 0.00%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration 5340.0 min.', # noqa: E501 - 'total_profit': 0.0, - 'current_epoch': 8, - 'is_initial_point': True, - 'is_best': False - }, { - 'loss': 2.4731817780991223, - 'params_dict': {'mfi-value': 22, 'fastd-value': 20, 'adx-value': 29, 'rsi-value': 40, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'sar_reversal', 'sell-mfi-value': 97, 'sell-fastd-value': 65, 'sell-adx-value': 81, 'sell-rsi-value': 64, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1012, 'roi_t2': 584, 'roi_t3': 422, 'roi_p1': 0.036764323603472565, 'roi_p2': 0.10335480573205287, 'roi_p3': 0.10322347377503042, 'stoploss': -0.2780610808108503}, # noqa: E501 - 'params_details': {'buy': {'mfi-value': 22, 'fastd-value': 20, 'adx-value': 29, 'rsi-value': 40, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'sar_reversal'}, 'sell': {'sell-mfi-value': 97, 'sell-fastd-value': 65, 'sell-adx-value': 81, 'sell-rsi-value': 64, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.2433426031105559, 422: 0.14011912933552545, 1006: 0.036764323603472565, 2018: 0}, 'stoploss': {'stoploss': -0.2780610808108503}}, # noqa: E501 - 'results_metrics': {'trade_count': 229, 'avg_profit': -0.38433433624454144, 'median_profit': -1.2222, 'total_profit': -0.044050070000000004, 'profit': -88.01256299999999, 'duration': 6505.676855895196}, # noqa: E501 - 'results_explanation': ' 229 trades. Avg profit -0.38%. Total profit -0.04405007 BTC ( -88.01Σ%). Avg duration 6505.7 min.', # noqa: E501 - 'total_profit': -0.044050070000000004, # noqa: E501 - 'current_epoch': 9, - 'is_initial_point': True, - 'is_best': False - }, { - 'loss': -0.2604606005845212, # noqa: E501 - 'params_dict': {'mfi-value': 23, 'fastd-value': 24, 'adx-value': 22, 'rsi-value': 24, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 97, 'sell-fastd-value': 70, 'sell-adx-value': 64, 'sell-rsi-value': 80, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal', 'roi_t1': 792, 'roi_t2': 464, 'roi_t3': 215, 'roi_p1': 0.04594053535385903, 'roi_p2': 0.09623192684243963, 'roi_p3': 0.04428219070850663, 'stoploss': -0.16992287161634415}, # noqa: E501 - 'params_details': {'buy': {'mfi-value': 23, 'fastd-value': 24, 'adx-value': 22, 'rsi-value': 24, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 97, 'sell-fastd-value': 70, 'sell-adx-value': 64, 'sell-rsi-value': 80, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'}, 'roi': {0: 0.18645465290480528, 215: 0.14217246219629864, 679: 0.04594053535385903, 1471: 0}, 'stoploss': {'stoploss': -0.16992287161634415}}, # noqa: E501 - 'results_metrics': {'trade_count': 4, 'avg_profit': 0.1080385, 'median_profit': -1.2222, 'total_profit': 0.00021629, 'profit': 0.432154, 'duration': 2850.0}, # noqa: E501 - 'results_explanation': ' 4 trades. Avg profit 0.11%. Total profit 0.00021629 BTC ( 0.43Σ%). Avg duration 2850.0 min.', # noqa: E501 - 'total_profit': 0.00021629, - 'current_epoch': 10, - 'is_initial_point': True, - 'is_best': True - }, { - 'loss': 4.876465945994304, # noqa: E501 - 'params_dict': {'mfi-value': 20, 'fastd-value': 32, 'adx-value': 49, 'rsi-value': 23, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower', 'sell-mfi-value': 75, 'sell-fastd-value': 56, 'sell-adx-value': 61, 'sell-rsi-value': 62, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 579, 'roi_t2': 614, 'roi_t3': 273, 'roi_p1': 0.05307643172744114, 'roi_p2': 0.1352282078262871, 'roi_p3': 0.1913307406325751, 'stoploss': -0.25728526022513887}, # noqa: E501 - 'params_details': {'buy': {'mfi-value': 20, 'fastd-value': 32, 'adx-value': 49, 'rsi-value': 23, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 75, 'sell-fastd-value': 56, 'sell-adx-value': 61, 'sell-rsi-value': 62, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.3796353801863034, 273: 0.18830463955372825, 887: 0.05307643172744114, 1466: 0}, 'stoploss': {'stoploss': -0.25728526022513887}}, # noqa: E501 - 'results_metrics': {'trade_count': 117, 'avg_profit': -1.2698609145299145, 'median_profit': -1.2222, 'total_profit': -0.07436117, 'profit': -148.573727, 'duration': 4282.5641025641025}, # noqa: E501 - 'results_explanation': ' 117 trades. Avg profit -1.27%. Total profit -0.07436117 BTC (-148.57Σ%). Avg duration 4282.6 min.', # noqa: E501 - 'total_profit': -0.07436117, - 'current_epoch': 11, - 'is_initial_point': True, - 'is_best': False - }, { - 'loss': 100000, - 'params_dict': {'mfi-value': 10, 'fastd-value': 36, 'adx-value': 31, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'sar_reversal', 'sell-mfi-value': 80, 'sell-fastd-value': 71, 'sell-adx-value': 60, 'sell-rsi-value': 85, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1156, 'roi_t2': 581, 'roi_t3': 408, 'roi_p1': 0.06860454019988212, 'roi_p2': 0.12473718444931989, 'roi_p3': 0.2896360635226823, 'stoploss': -0.30889015124682806}, # noqa: E501 - 'params_details': {'buy': {'mfi-value': 10, 'fastd-value': 36, 'adx-value': 31, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'sar_reversal'}, 'sell': {'sell-mfi-value': 80, 'sell-fastd-value': 71, 'sell-adx-value': 60, 'sell-rsi-value': 85, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.4829777881718843, 408: 0.19334172464920202, 989: 0.06860454019988212, 2145: 0}, 'stoploss': {'stoploss': -0.30889015124682806}}, # noqa: E501 - 'results_metrics': {'trade_count': 0, 'avg_profit': None, 'median_profit': None, 'total_profit': 0, 'profit': 0.0, 'duration': None}, # noqa: E501 - 'results_explanation': ' 0 trades. Avg profit nan%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration nan min.', # noqa: E501 - 'total_profit': 0, - 'current_epoch': 12, - 'is_initial_point': True, - 'is_best': False - } - ] - - @pytest.fixture def saved_hyperopt_results(): hyperopt_res = [ From be240566ba1df4eb282b79f17e69adbd02a2e401 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 Aug 2021 10:57:20 +0200 Subject: [PATCH 16/40] Fix random test failure --- tests/strategy/test_interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 0ad6d6f32..dc51f0811 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -552,6 +552,7 @@ def test__analyze_ticker_internal_skip_analyze(ohlcv_history, mocker, caplog) -> def test_is_pair_locked(default_conf): default_conf.update({'strategy': 'DefaultStrategy'}) PairLocks.timeframe = default_conf['timeframe'] + PairLocks.use_db = True strategy = StrategyResolver.load_strategy(default_conf) # No lock should be present assert len(PairLocks.get_pair_locks(None)) == 0 From 3bd0c3d0098bbcab9a9b8444e785ff43fbfc9fe7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 Aug 2021 11:02:54 +0200 Subject: [PATCH 17/40] Remove legacy code from export to csv --- freqtrade/optimize/hyperopt_tools.py | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index ed9583bdc..b51db4db2 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -459,21 +459,14 @@ class HyperoptTools(): trials['Best'] = '' trials['Stake currency'] = config['stake_currency'] - if 'results_metrics.total_trades' in trials: - base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades', - 'results_metrics.profit_mean', 'results_metrics.profit_median', - 'results_metrics.profit_total', - 'Stake currency', - 'results_metrics.profit_total_abs', 'results_metrics.holding_avg', - 'loss', 'is_initial_point', 'is_best'] - perc_multi = 100 - else: - perc_multi = 1 - base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count', - 'results_metrics.avg_profit', 'results_metrics.median_profit', - 'results_metrics.total_profit', - 'Stake currency', 'results_metrics.profit', 'results_metrics.duration', - 'loss', 'is_initial_point', 'is_best'] + base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades', + 'results_metrics.profit_mean', 'results_metrics.profit_median', + 'results_metrics.profit_total', + 'Stake currency', + 'results_metrics.profit_total_abs', 'results_metrics.holding_avg', + 'loss', 'is_initial_point', 'is_best'] + perc_multi = 100 + param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()] trials = trials[base_metrics + param_metrics] @@ -501,11 +494,6 @@ class HyperoptTools(): trials['Avg profit'] = trials['Avg profit'].apply( lambda x: f'{x * perc_multi:,.2f}%' if not isna(x) else "" ) - if perc_multi == 1: - trials['Avg duration'] = trials['Avg duration'].apply( - lambda x: f'{x:,.1f} m' if isinstance( - x, float) else f"{x.total_seconds() // 60:,.1f} m" if not isna(x) else "" - ) trials['Objective'] = trials['Objective'].apply( lambda x: f'{x:,.5f}' if x != 100000 else "" ) From 32e8e3b24258e1a7d12bde1d107d222ef1ee8fd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 03:01:19 +0000 Subject: [PATCH 18/40] Bump types-cachetools from 0.1.9 to 0.1.10 Bumps [types-cachetools](https://github.com/python/typeshed) from 0.1.9 to 0.1.10. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cachetools dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c1f7d6486..7b7727bd5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -19,7 +19,7 @@ isort==5.9.3 nbconvert==6.1.0 # mypy types -types-cachetools==0.1.9 +types-cachetools==0.1.10 types-filelock==0.1.4 types-requests==2.25.1 types-tabulate==0.1.1 From bad25b753c679218557fce23488b13bfe20bc609 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 03:01:25 +0000 Subject: [PATCH 19/40] Bump ccxt from 1.54.24 to 1.54.62 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.54.24 to 1.54.62. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.54.24...1.54.62) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6c2ef56c3..60175c12f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.21.1 pandas==1.3.1 -ccxt==1.54.24 +ccxt==1.54.62 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 From 59626b4ffcaa7f348d8e6986b274d32a65b54009 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 03:01:27 +0000 Subject: [PATCH 20/40] Bump types-tabulate from 0.1.1 to 0.8.2 Bumps [types-tabulate](https://github.com/python/typeshed) from 0.1.1 to 0.8.2. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-tabulate dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c1f7d6486..cbb59be4e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,4 +22,4 @@ nbconvert==6.1.0 types-cachetools==0.1.9 types-filelock==0.1.4 types-requests==2.25.1 -types-tabulate==0.1.1 +types-tabulate==0.8.2 From b89a993890224ead95a8652bc0a83dde89c87838 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 04:47:41 +0000 Subject: [PATCH 21/40] Bump types-filelock from 0.1.4 to 0.1.5 Bumps [types-filelock](https://github.com/python/typeshed) from 0.1.4 to 0.1.5. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-filelock dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 8f0cc0b34..7128721c5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,6 +20,6 @@ nbconvert==6.1.0 # mypy types types-cachetools==0.1.10 -types-filelock==0.1.4 +types-filelock==0.1.5 types-requests==2.25.1 types-tabulate==0.8.2 From 47f641d12f9d7813f80369f2691a1633168331a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 Aug 2021 07:03:13 +0200 Subject: [PATCH 22/40] Remove hyperopt-pickle result support --- freqtrade/optimize/hyperopt_tools.py | 17 ++++------------- tests/optimize/test_hyperopt_tools.py | 27 +++------------------------ 2 files changed, 7 insertions(+), 37 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index b51db4db2..0bb6aba15 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -89,18 +89,6 @@ class HyperoptTools(): else: return any(s in config['spaces'] for s in [space, 'all', 'default']) - @staticmethod - def _read_results_pickle(results_file: Path) -> List: - """ - Read hyperopt results from pickle file - LEGACY method - new files are written as json and cannot be read with this method. - """ - from joblib import load - - logger.info(f"Reading pickled epochs from '{results_file}'") - data = load(results_file) - return data - @staticmethod def _read_results(results_file: Path) -> List: """ @@ -120,7 +108,10 @@ class HyperoptTools(): epochs: List = [] if results_file.is_file() and results_file.stat().st_size > 0: if results_file.suffix == '.pickle': - epochs = HyperoptTools._read_results_pickle(results_file) + raise OperationalException( + "Legacy hyperopt results are no longer supported." + "Please rerun hyperopt or use an older version to load this file." + ) else: epochs = HyperoptTools._read_results(results_file) # Detection of some old format, without 'is_best' field saved diff --git a/tests/optimize/test_hyperopt_tools.py b/tests/optimize/test_hyperopt_tools.py index 44b4a7a03..d59a44da7 100644 --- a/tests/optimize/test_hyperopt_tools.py +++ b/tests/optimize/test_hyperopt_tools.py @@ -10,7 +10,7 @@ import rapidjson from freqtrade.constants import FTHYPT_FILEVERSION from freqtrade.exceptions import OperationalException from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer -from tests.conftest import log_has, log_has_re +from tests.conftest import log_has # Functions for recurrent object patching @@ -37,31 +37,10 @@ def test_save_results_saves_epochs(hyperopt, tmpdir, caplog) -> None: assert len(hyperopt_epochs) == 2 -def test_load_previous_results(testdatadir, caplog) -> None: - - results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle' - - hyperopt_epochs = HyperoptTools.load_previous_results(results_file) - - assert len(hyperopt_epochs) == 5 - assert log_has_re(r"Reading pickled epochs from .*", caplog) - - caplog.clear() - - # Modern version - results_file = testdatadir / 'strategy_SampleStrategy.fthypt' - - hyperopt_epochs = HyperoptTools.load_previous_results(results_file) - - assert len(hyperopt_epochs) == 5 - assert log_has_re(r"Reading epochs from .*", caplog) - - def test_load_previous_results2(mocker, testdatadir, caplog) -> None: - mocker.patch('freqtrade.optimize.hyperopt_tools.HyperoptTools._read_results_pickle', - return_value=[{'asdf': '222'}]) results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle' - with pytest.raises(OperationalException, match=r"The file .* incompatible.*"): + with pytest.raises(OperationalException, + match=r"Legacy hyperopt results are no longer supported.*"): HyperoptTools.load_previous_results(results_file) From 5919992ad2ab67ad3aaffd92bdaa6ccec30fd537 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 05:24:31 +0000 Subject: [PATCH 23/40] Bump types-requests from 2.25.1 to 2.25.6 Bumps [types-requests](https://github.com/python/typeshed) from 2.25.1 to 2.25.6. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7128721c5..9629bbea1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -21,5 +21,5 @@ nbconvert==6.1.0 # mypy types types-cachetools==0.1.10 types-filelock==0.1.5 -types-requests==2.25.1 +types-requests==2.25.6 types-tabulate==0.8.2 From f17942b68fa23a5dc727cba463e92f0a71e8cfb3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 Aug 2021 11:18:18 +0200 Subject: [PATCH 24/40] Fix random test failure --- freqtrade/optimize/backtesting.py | 3 +++ tests/optimize/test_backtesting.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3079e326d..85137a8ef 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -130,6 +130,9 @@ class Backtesting: self.abort = False def __del__(self): + self.cleanup() + + def cleanup(self): LoggingMixin.show_output = True PairLocks.use_db = True Trade.use_db = True diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index deaaf9f2f..b859e9017 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -535,6 +535,8 @@ def test_backtest__enter_trade(default_conf, fee, mocker) -> None: trade = backtesting._enter_trade(pair, row=row) assert trade is None + backtesting.cleanup() + def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: default_conf['use_sell_signal'] = False From 927ac24f82f66c3932936ea451f39efde193adfd Mon Sep 17 00:00:00 2001 From: pcassimans Date: Mon, 9 Aug 2021 14:21:59 +0200 Subject: [PATCH 25/40] Update README.md Fix Typo of Kukoin to Kucoin --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 78ea3cecd..309fab94b 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even Exchanges confirmed working by the community: - [X] [Bitvavo](https://bitvavo.com/) -- [X] [Kukoin](https://www.kucoin.com/) +- [X] [Kucoin](https://www.kucoin.com/) ## Documentation From 519c256b88e31f8347bd5bf7395a08d0c774fdae Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 Aug 2021 14:43:35 +0200 Subject: [PATCH 26/40] Fix kucoin typo in index.md as well --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 05eaa7552..fd3b8f224 100644 --- a/docs/index.md +++ b/docs/index.md @@ -47,7 +47,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual, Exchanges confirmed working by the community: - [X] [Bitvavo](https://bitvavo.com/) -- [X] [Kukoin](https://www.kucoin.com/) +- [X] [Kucoin](https://www.kucoin.com/) ## Requirements From a5f796bc9764040be80478deb246fe08b814be9f Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 Aug 2021 14:53:18 +0200 Subject: [PATCH 27/40] refactor ohlcvdata_to_dataframe to advise_all_indicators --- freqtrade/edge/edge_positioning.py | 2 +- freqtrade/optimize/backtesting.py | 5 +++-- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/strategy/interface.py | 2 +- tests/data/test_history.py | 6 +++--- tests/optimize/test_backtesting.py | 18 +++++++++--------- tests/optimize/test_hyperopt.py | 24 ++++++++++++------------ tests/strategy/test_interface.py | 8 ++++---- 8 files changed, 34 insertions(+), 33 deletions(-) diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 243043d31..f12b1b37d 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -151,7 +151,7 @@ class Edge: # Fake run-mode to Edge prior_rm = self.config['runmode'] self.config['runmode'] = RunMode.EDGE - preprocessed = self.strategy.ohlcvdata_to_dataframe(data) + preprocessed = self.strategy.advise_all_indicators(data) self.config['runmode'] = prior_rm # Print timeframe diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 85137a8ef..825d1dd25 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -533,7 +533,8 @@ class Backtesting: 'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']), } - def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange): + def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, DataFrame], + timerange: TimeRange): self.progress.init_step(BacktestState.ANALYZE, 0) logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) @@ -552,7 +553,7 @@ class Backtesting: max_open_trades = 0 # need to reprocess data every time to populate signals - preprocessed = self.strategy.ohlcvdata_to_dataframe(data) + preprocessed = self.strategy.advise_all_indicators(data) # Trim startup period from analyzed dataframe preprocessed_tmp = trim_dataframes(preprocessed, timerange, self.required_startup) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 0db78aa39..901900121 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -394,7 +394,7 @@ class Hyperopt: data, timerange = self.backtesting.load_bt_data() logger.info("Dataload complete. Calculating indicators") - preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data) + preprocessed = self.backtesting.strategy.advise_all_indicators(data) # Trim startup period from analyzed dataframe to get correct dates for output. processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index bf5cc10af..d4f10301a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -732,7 +732,7 @@ class IStrategy(ABC, HyperStrategyMixin): else: return current_profit > roi - def ohlcvdata_to_dataframe(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]: + def advise_all_indicators(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]: """ Populates indicators for given candle (OHLCV) data (for multiple pairs) Does not run advise_buy or advise_sell! diff --git a/tests/data/test_history.py b/tests/data/test_history.py index d203d0792..9cfe861ea 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -381,7 +381,7 @@ def test_get_timerange(default_conf, mocker, testdatadir) -> None: default_conf.update({'strategy': 'DefaultStrategy'}) strategy = StrategyResolver.load_strategy(default_conf) - data = strategy.ohlcvdata_to_dataframe( + data = strategy.advise_all_indicators( load_data( datadir=testdatadir, timeframe='1m', @@ -399,7 +399,7 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir) default_conf.update({'strategy': 'DefaultStrategy'}) strategy = StrategyResolver.load_strategy(default_conf) - data = strategy.ohlcvdata_to_dataframe( + data = strategy.advise_all_indicators( load_data( datadir=testdatadir, timeframe='1m', @@ -424,7 +424,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No strategy = StrategyResolver.load_strategy(default_conf) timerange = TimeRange('index', 'index', 200, 250) - data = strategy.ohlcvdata_to_dataframe( + data = strategy.advise_all_indicators( load_data( datadir=testdatadir, timeframe='5m', diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index b859e9017..8e3d4063a 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -85,7 +85,7 @@ def simple_backtest(config, contour, mocker, testdatadir) -> None: backtesting._set_strategy(backtesting.strategylist[0]) data = load_data_test(contour, testdatadir) - processed = backtesting.strategy.ohlcvdata_to_dataframe(data) + processed = backtesting.strategy.advise_all_indicators(data) min_date, max_date = get_timerange(processed) assert isinstance(processed, dict) results = backtesting.backtest( @@ -107,7 +107,7 @@ def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC'): patch_exchange(mocker) backtesting = Backtesting(conf) backtesting._set_strategy(backtesting.strategylist[0]) - processed = backtesting.strategy.ohlcvdata_to_dataframe(data) + processed = backtesting.strategy.advise_all_indicators(data) min_date, max_date = get_timerange(processed) return { 'processed': processed, @@ -289,7 +289,7 @@ def test_backtesting_init(mocker, default_conf, order_types) -> None: backtesting._set_strategy(backtesting.strategylist[0]) assert backtesting.config == default_conf assert backtesting.timeframe == '5m' - assert callable(backtesting.strategy.ohlcvdata_to_dataframe) + assert callable(backtesting.strategy.advise_all_indicators) assert callable(backtesting.strategy.advise_buy) assert callable(backtesting.strategy.advise_sell) assert isinstance(backtesting.strategy.dp, DataProvider) @@ -335,14 +335,14 @@ def test_data_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: fill_up_missing=True) backtesting = Backtesting(default_conf) backtesting._set_strategy(backtesting.strategylist[0]) - processed = backtesting.strategy.ohlcvdata_to_dataframe(data) + processed = backtesting.strategy.advise_all_indicators(data) assert len(processed['UNITTEST/BTC']) == 102 # Load strategy to compare the result between Backtesting function and strategy are the same default_conf.update({'strategy': 'DefaultStrategy'}) strategy = StrategyResolver.load_strategy(default_conf) - processed2 = strategy.ohlcvdata_to_dataframe(data) + processed2 = strategy.advise_all_indicators(data) assert processed['UNITTEST/BTC'].equals(processed2['UNITTEST/BTC']) @@ -549,7 +549,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: timerange = TimeRange('date', None, 1517227800, 0) data = history.load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], timerange=timerange) - processed = backtesting.strategy.ohlcvdata_to_dataframe(data) + processed = backtesting.strategy.advise_all_indicators(data) min_date, max_date = get_timerange(processed) result = backtesting.backtest( processed=processed, @@ -614,7 +614,7 @@ def test_backtest_1min_timeframe(default_conf, fee, mocker, testdatadir) -> None timerange = TimeRange.parse_timerange('1510688220-1510700340') data = history.load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC'], timerange=timerange) - processed = backtesting.strategy.ohlcvdata_to_dataframe(data) + processed = backtesting.strategy.advise_all_indicators(data) min_date, max_date = get_timerange(processed) results = backtesting.backtest( processed=processed, @@ -633,7 +633,7 @@ def test_processed(default_conf, mocker, testdatadir) -> None: backtesting._set_strategy(backtesting.strategylist[0]) dict_of_tickerrows = load_data_test('raise', testdatadir) - dataframes = backtesting.strategy.ohlcvdata_to_dataframe(dict_of_tickerrows) + dataframes = backtesting.strategy.advise_all_indicators(dict_of_tickerrows) dataframe = dataframes['UNITTEST/BTC'] cols = dataframe.columns # assert the dataframe got some of the indicator columns @@ -782,7 +782,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) backtesting.strategy.advise_buy = _trend_alternate_hold # Override backtesting.strategy.advise_sell = _trend_alternate_hold # Override - processed = backtesting.strategy.ohlcvdata_to_dataframe(data) + processed = backtesting.strategy.advise_all_indicators(data) min_date, max_date = get_timerange(processed) backtest_conf = { 'processed': processed, diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index d146e84f1..0ca79d268 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -351,7 +351,7 @@ def test_start_calls_optimizer(mocker, hyperopt_conf, capsys) -> None: del hyperopt_conf['timeframe'] hyperopt = Hyperopt(hyperopt_conf) - hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) hyperopt.start() @@ -426,7 +426,7 @@ def test_hyperopt_format_results(hyperopt): def test_populate_indicators(hyperopt, testdatadir) -> None: data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) - dataframes = hyperopt.backtesting.strategy.ohlcvdata_to_dataframe(data) + dataframes = hyperopt.backtesting.strategy.advise_all_indicators(data) dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) @@ -438,7 +438,7 @@ def test_populate_indicators(hyperopt, testdatadir) -> None: def test_buy_strategy_generator(hyperopt, testdatadir) -> None: data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) - dataframes = hyperopt.backtesting.strategy.ohlcvdata_to_dataframe(data) + dataframes = hyperopt.backtesting.strategy.advise_all_indicators(data) dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) @@ -463,7 +463,7 @@ def test_buy_strategy_generator(hyperopt, testdatadir) -> None: def test_sell_strategy_generator(hyperopt, testdatadir) -> None: data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) - dataframes = hyperopt.backtesting.strategy.ohlcvdata_to_dataframe(data) + dataframes = hyperopt.backtesting.strategy.advise_all_indicators(data) dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) @@ -660,7 +660,7 @@ def test_print_json_spaces_all(mocker, hyperopt_conf, capsys) -> None: }) hyperopt = Hyperopt(hyperopt_conf) - hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) hyperopt.start() @@ -713,7 +713,7 @@ def test_print_json_spaces_default(mocker, hyperopt_conf, capsys) -> None: hyperopt_conf.update({'print_json': True}) hyperopt = Hyperopt(hyperopt_conf) - hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) hyperopt.start() @@ -761,7 +761,7 @@ def test_print_json_spaces_roi_stoploss(mocker, hyperopt_conf, capsys) -> None: }) hyperopt = Hyperopt(hyperopt_conf) - hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) hyperopt.start() @@ -805,7 +805,7 @@ def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> Non hyperopt_conf.update({'spaces': 'roi stoploss'}) hyperopt = Hyperopt(hyperopt_conf) - hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) del hyperopt.custom_hyperopt.__class__.buy_strategy_generator @@ -844,7 +844,7 @@ def test_simplified_interface_all_failed(mocker, hyperopt_conf) -> None: hyperopt_conf.update({'spaces': 'all', }) hyperopt = Hyperopt(hyperopt_conf) - hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) del hyperopt.custom_hyperopt.__class__.buy_strategy_generator @@ -886,7 +886,7 @@ def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None: hyperopt_conf.update({'spaces': 'buy'}) hyperopt = Hyperopt(hyperopt_conf) - hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) # TODO: sell_strategy_generator() is actually not called because @@ -940,7 +940,7 @@ def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: hyperopt_conf.update({'spaces': 'sell', }) hyperopt = Hyperopt(hyperopt_conf) - hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) # TODO: buy_strategy_generator() is actually not called because @@ -985,7 +985,7 @@ def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> No hyperopt_conf.update({'spaces': space}) hyperopt = Hyperopt(hyperopt_conf) - hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) delattr(hyperopt.custom_hyperopt.__class__, method) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index dc51f0811..686a36c96 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -228,25 +228,25 @@ def test_assert_df(ohlcv_history, caplog): _STRATEGY.disable_dataframe_checks = False -def test_ohlcvdata_to_dataframe(default_conf, testdatadir) -> None: +def test_advise_all_indicators(default_conf, testdatadir) -> None: default_conf.update({'strategy': 'DefaultStrategy'}) strategy = StrategyResolver.load_strategy(default_conf) timerange = TimeRange.parse_timerange('1510694220-1510700340') data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, fill_up_missing=True) - processed = strategy.ohlcvdata_to_dataframe(data) + processed = strategy.advise_all_indicators(data) assert len(processed['UNITTEST/BTC']) == 102 # partial candle was removed -def test_ohlcvdata_to_dataframe_copy(mocker, default_conf, testdatadir) -> None: +def test_advise_all_indicators_copy(mocker, default_conf, testdatadir) -> None: default_conf.update({'strategy': 'DefaultStrategy'}) strategy = StrategyResolver.load_strategy(default_conf) aimock = mocker.patch('freqtrade.strategy.interface.IStrategy.advise_indicators') timerange = TimeRange.parse_timerange('1510694220-1510700340') data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, fill_up_missing=True) - strategy.ohlcvdata_to_dataframe(data) + strategy.advise_all_indicators(data) assert aimock.call_count == 1 # Ensure that a copy of the dataframe is passed to advice_indicators assert aimock.call_args_list[0][0][0] is not data From 895b912c719da2f613c5ae5da47cfb375bf771b8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 Aug 2021 14:40:02 +0200 Subject: [PATCH 28/40] Fix recently introduced lookahead bias in backtesting closes #5388 --- freqtrade/optimize/backtesting.py | 4 +++- tests/optimize/test_backtesting.py | 18 ++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 825d1dd25..3a864173f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -466,6 +466,8 @@ class Backtesting: for i, pair in enumerate(data): row_index = indexes[pair] try: + # Row is treated as "current incomplete candle". + # Buy / sell signals are shifted by 1 to compensate for this. row = data[pair][row_index] except IndexError: # missing Data for one pair at the end. @@ -476,8 +478,8 @@ class Backtesting: if row[DATE_IDX] > tmp: continue - row_index += 1 self.dataprovider._set_dataframe_max_index(row_index) + row_index += 1 indexes[pair] = row_index # without positionstacking, we can only have one open trade per pair. diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 8e3d4063a..ff9b81c30 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument import random +from datetime import timedelta from pathlib import Path from unittest.mock import MagicMock, PropertyMock @@ -741,8 +742,13 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): # 100 buys signals results = result['results'] assert len(results) == 100 - # Cached data should be 200 (no change since required_startup is 0) - assert len(backtesting.dataprovider.get_analyzed_dataframe('UNITTEST/BTC', '1m')[0]) == 200 + # Cached data should be 199 (missing 1 candle at the start) + analyzed_df = backtesting.dataprovider.get_analyzed_dataframe('UNITTEST/BTC', '1m')[0] + assert len(analyzed_df) == 199 + # Expect last candle to be 1 below end date (as the last candle is assumed as "incomplete" + # during backtesting) + expected_last_candle_date = backtest_conf['end_date'] - timedelta(minutes=1) + assert analyzed_df.iloc[-1]['date'].to_pydatetime() == expected_last_candle_date # One trade was force-closed at the end assert len(results.loc[results['is_open']]) == 0 @@ -774,7 +780,8 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) data = trim_dictlist(data, -500) # Remove data for one pair from the beginning of the data - data[pair] = data[pair][tres:].reset_index() + if tres > 0: + data[pair] = data[pair][tres:].reset_index() default_conf['timeframe'] = '5m' backtesting = Backtesting(default_conf) @@ -800,8 +807,11 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) assert len(evaluate_result_multi(results['results'], '5m', 3)) == 0 # Cached data correctly removed amounts - removed_candles = len(data[pair]) - 1 - backtesting.strategy.startup_candle_count + offset = 2 if tres == 0 else 1 + removed_candles = len(data[pair]) - offset - backtesting.strategy.startup_candle_count assert len(backtesting.dataprovider.get_analyzed_dataframe(pair, '5m')[0]) == removed_candles + assert len(backtesting.dataprovider.get_analyzed_dataframe( + 'NXT/BTC', '5m')[0]) == len(data['NXT/BTC']) - 2 - backtesting.strategy.startup_candle_count backtest_conf = { 'processed': processed, From 5bfb9edf02922c38aef0f501634eb116c74f7515 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 Aug 2021 15:42:17 +0200 Subject: [PATCH 29/40] Only query date once from list --- freqtrade/optimize/backtesting.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3a864173f..5efb5101f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -323,14 +323,14 @@ class Backtesting: return sell_row[OPEN_IDX] def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]: - + sell_candle_time = sell_row[DATE_IDX].to_pydatetime() sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], # type: ignore - sell_row[DATE_IDX].to_pydatetime(), sell_row[BUY_IDX], + sell_candle_time, sell_row[BUY_IDX], sell_row[SELL_IDX], low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX]) if sell.sell_flag: - trade.close_date = sell_row[DATE_IDX].to_pydatetime() + trade.close_date = sell_candle_time trade.sell_reason = sell.sell_reason trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60) closerate = self._get_close_rate(sell_row, trade, sell, trade_dur) @@ -342,7 +342,7 @@ class Backtesting: rate=closerate, time_in_force=time_in_force, sell_reason=sell.sell_reason, - current_time=sell_row[DATE_IDX].to_pydatetime()): + current_time=sell_candle_time): return None trade.close(closerate, show_msg=False) From cf27968b97cc1491b61923a0a95f2183aacf0592 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 Aug 2021 19:38:56 +0200 Subject: [PATCH 30/40] Properly preserve trade's low during backtesting --- freqtrade/persistence/models.py | 4 ++-- freqtrade/strategy/interface.py | 2 +- tests/optimize/test_backtesting.py | 2 +- tests/strategy/test_interface.py | 2 +- tests/test_persistence.py | 13 +++++++++---- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 43fbec8c0..a45274266 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -354,12 +354,12 @@ class LocalTrade(): LocalTrade.trades_open = [] LocalTrade.total_profit = 0 - def adjust_min_max_rates(self, current_price: float) -> None: + def adjust_min_max_rates(self, current_price: float, current_price_low: float) -> None: """ Adjust the max_rate and min_rate. """ self.max_rate = max(current_price, self.max_rate or self.open_rate) - self.min_rate = min(current_price, self.min_rate or self.open_rate) + self.min_rate = min(current_price_low, self.min_rate or self.open_rate) def _set_new_stoploss(self, new_loss: float, stoploss: float): """Assign new stop value""" diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index d4f10301a..bb8980a53 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -568,7 +568,7 @@ class IStrategy(ABC, HyperStrategyMixin): current_rate = rate current_profit = trade.calc_profit_ratio(current_rate) - trade.adjust_min_max_rates(high or current_rate) + trade.adjust_min_max_rates(high or current_rate, low or current_rate) stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, current_time=date, current_profit=current_profit, diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index ff9b81c30..57b2b8733 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -584,7 +584,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: 'initial_stop_loss_ratio': [-0.1, -0.1], 'stop_loss_abs': [0.0940005, 0.09272236], 'stop_loss_ratio': [-0.1, -0.1], - 'min_rate': [0.1038, 0.10302485], + 'min_rate': [0.10370188, 0.10300000000000001], 'max_rate': [0.10501, 0.1038888], 'is_open': [False, False], 'buy_tag': [None, None], diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 686a36c96..cb4b8bd63 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -398,7 +398,7 @@ def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, traili exchange='binance', open_rate=1, ) - trade.adjust_min_max_rates(trade.open_rate) + trade.adjust_min_max_rates(trade.open_rate, trade.open_rate) strategy.trailing_stop = trailing strategy.trailing_stop_positive = -0.05 strategy.use_custom_stoploss = custom diff --git a/tests/test_persistence.py b/tests/test_persistence.py index f7bcad806..105cee23a 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -799,25 +799,30 @@ def test_adjust_min_max_rates(fee): open_rate=1, ) - trade.adjust_min_max_rates(trade.open_rate) + trade.adjust_min_max_rates(trade.open_rate, trade.open_rate) assert trade.max_rate == 1 assert trade.min_rate == 1 # check min adjusted, max remained - trade.adjust_min_max_rates(0.96) + trade.adjust_min_max_rates(0.96, 0.96) assert trade.max_rate == 1 assert trade.min_rate == 0.96 # check max adjusted, min remains - trade.adjust_min_max_rates(1.05) + trade.adjust_min_max_rates(1.05, 1.05) assert trade.max_rate == 1.05 assert trade.min_rate == 0.96 # current rate "in the middle" - no adjustment - trade.adjust_min_max_rates(1.03) + trade.adjust_min_max_rates(1.03, 1.03) assert trade.max_rate == 1.05 assert trade.min_rate == 0.96 + # current rate "in the middle" - no adjustment + trade.adjust_min_max_rates(1.10, 0.91) + assert trade.max_rate == 1.10 + assert trade.min_rate == 0.91 + @pytest.mark.usefixtures("init_persistence") @pytest.mark.parametrize('use_db', [True, False]) From 3f160c71443281d4118a79ee5c11af3891c384d5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 10 Aug 2021 07:09:38 +0200 Subject: [PATCH 31/40] Cache dataframe before cutting the first candle This allows providing the "current closed" candle in all cases. --- freqtrade/optimize/backtesting.py | 6 +++--- tests/optimize/test_backtesting.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5efb5101f..fce27d39b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -246,11 +246,11 @@ class Backtesting: if has_buy_tag: df_analyzed.loc[:, 'buy_tag'] = df_analyzed.loc[:, 'buy_tag'].shift(1) - df_analyzed.drop(df_analyzed.head(1).index, inplace=True) - # Update dataprovider cache self.dataprovider._set_cached_df(pair, self.timeframe, df_analyzed) + df_analyzed = df_analyzed.drop(df_analyzed.head(1).index) + # Convert from Pandas to list for performance reasons # (Looping Pandas is slow.) data[pair] = df_analyzed[headers].values.tolist() @@ -478,9 +478,9 @@ class Backtesting: if row[DATE_IDX] > tmp: continue - self.dataprovider._set_dataframe_max_index(row_index) row_index += 1 indexes[pair] = row_index + self.dataprovider._set_dataframe_max_index(row_index) # without positionstacking, we can only have one open trade per pair. # max_open_trades must be respected diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 57b2b8733..998b2d837 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -742,9 +742,9 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): # 100 buys signals results = result['results'] assert len(results) == 100 - # Cached data should be 199 (missing 1 candle at the start) + # Cached data should be 200 analyzed_df = backtesting.dataprovider.get_analyzed_dataframe('UNITTEST/BTC', '1m')[0] - assert len(analyzed_df) == 199 + assert len(analyzed_df) == 200 # Expect last candle to be 1 below end date (as the last candle is assumed as "incomplete" # during backtesting) expected_last_candle_date = backtest_conf['end_date'] - timedelta(minutes=1) @@ -807,11 +807,11 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) assert len(evaluate_result_multi(results['results'], '5m', 3)) == 0 # Cached data correctly removed amounts - offset = 2 if tres == 0 else 1 + offset = 1 if tres == 0 else 0 removed_candles = len(data[pair]) - offset - backtesting.strategy.startup_candle_count assert len(backtesting.dataprovider.get_analyzed_dataframe(pair, '5m')[0]) == removed_candles assert len(backtesting.dataprovider.get_analyzed_dataframe( - 'NXT/BTC', '5m')[0]) == len(data['NXT/BTC']) - 2 - backtesting.strategy.startup_candle_count + 'NXT/BTC', '5m')[0]) == len(data['NXT/BTC']) - 1 - backtesting.strategy.startup_candle_count backtest_conf = { 'processed': processed, From 039d6384edf79d87556f5fa66b5917d1a77f9a6b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 10 Aug 2021 09:48:26 +0200 Subject: [PATCH 32/40] Stream hyperopt-result in small batches Avoiding memory-exhaustion on huge hyperopt results closes #5305 closes #5149 --- freqtrade/optimize/hyperopt_epoch_filters.py | 12 ++--- freqtrade/optimize/hyperopt_tools.py | 54 +++++++++++--------- tests/commands/test_commands.py | 24 +++++++-- tests/optimize/test_hyperopt_tools.py | 24 +++++++-- 4 files changed, 78 insertions(+), 36 deletions(-) diff --git a/freqtrade/optimize/hyperopt_epoch_filters.py b/freqtrade/optimize/hyperopt_epoch_filters.py index b70db94af..80cc89d4b 100644 --- a/freqtrade/optimize/hyperopt_epoch_filters.py +++ b/freqtrade/optimize/hyperopt_epoch_filters.py @@ -7,7 +7,7 @@ from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) -def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: +def hyperopt_filter_epochs(epochs: List, filteroptions: dict, log: bool = True) -> List: """ Filter our items from the list of hyperopt results """ @@ -24,11 +24,11 @@ def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions) epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions) - - logger.info(f"{len(epochs)} " + - ("best " if filteroptions['only_best'] else "") + - ("profitable " if filteroptions['only_profitable'] else "") + - "epochs found.") + if log: + logger.info(f"{len(epochs)} " + + ("best " if filteroptions['only_best'] else "") + + ("profitable " if filteroptions['only_profitable'] else "") + + "epochs found.") return epochs diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 0bb6aba15..b2e024f65 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -4,7 +4,7 @@ import logging from copy import deepcopy from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, Iterator, List, Optional, Tuple import numpy as np import rapidjson @@ -90,37 +90,33 @@ class HyperoptTools(): return any(s in config['spaces'] for s in [space, 'all', 'default']) @staticmethod - def _read_results(results_file: Path) -> List: + def _read_results(results_file: Path, batch_size: int = 10) -> Iterator[List[Any]]: """ - Read hyperopt results from file + Stream hyperopt results from file """ import rapidjson logger.info(f"Reading epochs from '{results_file}'") with results_file.open('r') as f: - data = [rapidjson.loads(line) for line in f] - return data + data = [] + for line in f: + data += [rapidjson.loads(line)] + if len(data) >= batch_size: + yield data + data = [] + yield data @staticmethod - def load_previous_results(results_file: Path) -> List: - """ - Load data for epochs from the file if we have one - """ - epochs: List = [] + def _test_hyperopt_results_exist(results_file) -> bool: if results_file.is_file() and results_file.stat().st_size > 0: if results_file.suffix == '.pickle': raise OperationalException( "Legacy hyperopt results are no longer supported." "Please rerun hyperopt or use an older version to load this file." ) - else: - epochs = HyperoptTools._read_results(results_file) - # Detection of some old format, without 'is_best' field saved - if epochs[0].get('is_best') is None: - raise OperationalException( - "The file with HyperoptTools results is incompatible with this version " - "of Freqtrade and cannot be loaded.") - logger.info(f"Loaded {len(epochs)} previous evaluations from disk.") - return epochs + return True + else: + # No file found. + return False @staticmethod def load_filtered_results(results_file: Path, config: Dict[str, Any]) -> Tuple[List, int]: @@ -138,12 +134,24 @@ class HyperoptTools(): 'filter_min_objective': config.get('hyperopt_list_min_objective', None), 'filter_max_objective': config.get('hyperopt_list_max_objective', None), } + if not HyperoptTools._test_hyperopt_results_exist(results_file): + # No file found. + return [], 0 - # Previous evaluations - epochs = HyperoptTools.load_previous_results(results_file) - total_epochs = len(epochs) + epochs = [] + total_epochs = 0 + for epochs_tmp in HyperoptTools._read_results(results_file): + if total_epochs == 0 and epochs_tmp[0].get('is_best') is None: + raise OperationalException( + "The file with HyperoptTools results is incompatible with this version " + "of Freqtrade and cannot be loaded.") + total_epochs += len(epochs_tmp) + epochs += hyperopt_filter_epochs(epochs_tmp, filteroptions, log=False) - epochs = hyperopt_filter_epochs(epochs, filteroptions) + logger.info(f"Loaded {total_epochs} previous evaluations from disk.") + + # Final filter run ... + epochs = hyperopt_filter_epochs(epochs, filteroptions, log=True) return epochs, total_epochs diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 80dd04b27..fc5101979 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -941,8 +941,16 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results, tmpdir): csv_file = Path(tmpdir) / "test.csv" mocker.patch( - 'freqtrade.optimize.hyperopt_tools.HyperoptTools.load_previous_results', - MagicMock(return_value=saved_hyperopt_results) + 'freqtrade.optimize.hyperopt_tools.HyperoptTools._test_hyperopt_results_exist', + return_value=True + ) + + def fake_iterator(*args, **kwargs): + yield from [saved_hyperopt_results] + + mocker.patch( + 'freqtrade.optimize.hyperopt_tools.HyperoptTools._read_results', + side_effect=fake_iterator ) args = [ @@ -1175,8 +1183,16 @@ def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results, tmpdir): def test_hyperopt_show(mocker, capsys, saved_hyperopt_results): mocker.patch( - 'freqtrade.optimize.hyperopt_tools.HyperoptTools.load_previous_results', - MagicMock(return_value=saved_hyperopt_results) + 'freqtrade.optimize.hyperopt_tools.HyperoptTools._test_hyperopt_results_exist', + return_value=True + ) + + def fake_iterator(*args, **kwargs): + yield from [saved_hyperopt_results] + + mocker.patch( + 'freqtrade.optimize.hyperopt_tools.HyperoptTools._read_results', + side_effect=fake_iterator ) mocker.patch('freqtrade.commands.hyperopt_commands.show_backtest_result') diff --git a/tests/optimize/test_hyperopt_tools.py b/tests/optimize/test_hyperopt_tools.py index d59a44da7..cbcb13384 100644 --- a/tests/optimize/test_hyperopt_tools.py +++ b/tests/optimize/test_hyperopt_tools.py @@ -20,9 +20,14 @@ def create_results() -> List[Dict]: def test_save_results_saves_epochs(hyperopt, tmpdir, caplog) -> None: + + hyperopt.results_file = Path(tmpdir / 'ut_results.fthypt') + + hyperopt_epochs = HyperoptTools.load_filtered_results(hyperopt.results_file, {}) + assert hyperopt_epochs == ([], 0) + # Test writing to temp dir and reading again epochs = create_results() - hyperopt.results_file = Path(tmpdir / 'ut_results.fthypt') caplog.set_level(logging.DEBUG) @@ -33,15 +38,28 @@ def test_save_results_saves_epochs(hyperopt, tmpdir, caplog) -> None: hyperopt._save_result(epochs[0]) assert log_has(f"2 epochs saved to '{hyperopt.results_file}'.", caplog) - hyperopt_epochs = HyperoptTools.load_previous_results(hyperopt.results_file) + hyperopt_epochs = HyperoptTools.load_filtered_results(hyperopt.results_file, {}) assert len(hyperopt_epochs) == 2 + assert hyperopt_epochs[1] == 2 + assert len(hyperopt_epochs[0]) == 2 + + result_gen = HyperoptTools._read_results(hyperopt.results_file, 1) + epoch = next(result_gen) + assert len(epoch) == 1 + assert epoch[0] == epochs[0] + epoch = next(result_gen) + assert len(epoch) == 1 + epoch = next(result_gen) + assert len(epoch) == 0 + with pytest.raises(StopIteration): + next(result_gen) def test_load_previous_results2(mocker, testdatadir, caplog) -> None: results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle' with pytest.raises(OperationalException, match=r"Legacy hyperopt results are no longer supported.*"): - HyperoptTools.load_previous_results(results_file) + HyperoptTools.load_filtered_results(results_file, {}) @pytest.mark.parametrize("spaces, expected_results", [ From 65d025923d9453b2173de63c898232847e31ad7a Mon Sep 17 00:00:00 2001 From: ipqhjjybj <250657661@qq.com> Date: Wed, 11 Aug 2021 14:35:16 +0800 Subject: [PATCH 33/40] add code --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fce27d39b..a8fd9c04a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -504,7 +504,7 @@ class Backtesting: open_trades[pair].append(trade) LocalTrade.add_bt_trade(trade) - for trade in open_trades[pair]: + for trade in list(open_trades[pair]): # also check the buying candle for sell conditions. trade_entry = self._get_sell_trade_entry(trade, row) # Sell occurred From 61c076563fb7c778b93d9e056f5403423aa9406f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Aug 2021 12:11:29 +0200 Subject: [PATCH 34/40] Add max-slippage limiting for dry-run orders to avoid insane market order fills --- freqtrade/exchange/exchange.py | 13 ++++++++++++- tests/exchange/test_exchange.py | 23 ++++++++++++++--------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index c6f60e08a..cde643cff 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -618,6 +618,8 @@ class Exchange: if self.exchange_has('fetchL2OrderBook'): ob = self.fetch_l2_order_book(pair, 20) ob_type = 'asks' if side == 'buy' else 'bids' + slippage = 0.05 + max_slippage_val = rate * ((1 + slippage) if side == 'buy' else (1 - slippage)) remaining_amount = amount filled_amount = 0 @@ -626,7 +628,9 @@ class Exchange: book_entry_coin_volume = book_entry[1] if remaining_amount > 0: if remaining_amount < book_entry_coin_volume: + # Orderbook at this slot bigger than remaining amount filled_amount += remaining_amount * book_entry_price + break else: filled_amount += book_entry_coin_volume * book_entry_price remaining_amount -= book_entry_coin_volume @@ -635,7 +639,14 @@ class Exchange: else: # If remaining_amount wasn't consumed completely (break was not called) filled_amount += remaining_amount * book_entry_price - forecast_avg_filled_price = filled_amount / amount + forecast_avg_filled_price = max(filled_amount, 0) / amount + # Limit max. slippage to specified value + if side == 'buy': + forecast_avg_filled_price = min(forecast_avg_filled_price, max_slippage_val) + + else: + forecast_avg_filled_price = max(forecast_avg_filled_price, max_slippage_val) + return self.price_to_precision(pair, forecast_avg_filled_price) return rate diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a3ebbe8bd..9ac9f84e5 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -984,16 +984,21 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, assert order['fee'] -@pytest.mark.parametrize("side,amount,endprice", [ - ("buy", 1, 25.566), - ("buy", 100, 25.5672), # Requires interpolation - ("buy", 1000, 25.575), # More than orderbook return - ("sell", 1, 25.563), - ("sell", 100, 25.5625), # Requires interpolation - ("sell", 1000, 25.5555), # More than orderbook return +@pytest.mark.parametrize("side,rate,amount,endprice", [ + # spread is 25.263-25.266 + ("buy", 25.564, 1, 25.566), + ("buy", 25.564, 100, 25.5672), # Requires interpolation + ("buy", 25.590, 100, 25.5672), # Price above spread ... average is lower + ("buy", 25.564, 1000, 25.575), # More than orderbook return + ("buy", 24.000, 100000, 25.200), # Run into max_slippage of 5% + ("sell", 25.564, 1, 25.563), + ("sell", 25.564, 100, 25.5625), # Requires interpolation + ("sell", 25.510, 100, 25.5625), # price below spread - average is higher + ("sell", 25.564, 1000, 25.5555), # More than orderbook return + ("sell", 27, 10000, 25.65), # max-slippage 5% ]) @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_create_dry_run_order_market_fill(default_conf, mocker, side, amount, endprice, +def test_create_dry_run_order_market_fill(default_conf, mocker, side, rate, amount, endprice, exchange_name, order_book_l2_usd): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) @@ -1003,7 +1008,7 @@ def test_create_dry_run_order_market_fill(default_conf, mocker, side, amount, en ) order = exchange.create_dry_run_order( - pair='LTC/USDT', ordertype='market', side=side, amount=amount, rate=25.5) + pair='LTC/USDT', ordertype='market', side=side, amount=amount, rate=rate) assert 'id' in order assert f'dry_run_{side}_' in order["id"] assert order["side"] == side From f6267c75143b35c0270428f70d7936f7880fc917 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Wed, 11 Aug 2021 10:18:25 +0300 Subject: [PATCH 35/40] Fix buy_tag not being saved to trade object. Column is mistakenly excluded because advise_buy() creating this column runs after code detecting presence of buy_tag column. --- freqtrade/optimize/backtesting.py | 10 +++------- tests/optimize/__init__.py | 2 ++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fce27d39b..06464d40b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -218,7 +218,7 @@ class Backtesting: """ # Every change to this headers list must evaluate further usages of the resulting tuple # and eventually change the constants for indexes at the top - headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high'] + headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high', 'buy_tag'] data: Dict = {} self.progress.init_step(BacktestState.CONVERT, len(processed)) @@ -226,13 +226,10 @@ class Backtesting: for pair, pair_data in processed.items(): self.check_abort() self.progress.increment() - has_buy_tag = 'buy_tag' in pair_data - headers = headers + ['buy_tag'] if has_buy_tag else headers if not pair_data.empty: pair_data.loc[:, 'buy'] = 0 # cleanup if buy_signal is exist pair_data.loc[:, 'sell'] = 0 # cleanup if sell_signal is exist - if has_buy_tag: - pair_data.loc[:, 'buy_tag'] = None # cleanup if buy_tag is exist + pair_data.loc[:, 'buy_tag'] = None # cleanup if buy_tag is exist df_analyzed = self.strategy.advise_sell( self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair}).copy() @@ -243,8 +240,7 @@ class Backtesting: # from the previous candle df_analyzed.loc[:, 'buy'] = df_analyzed.loc[:, 'buy'].shift(1) df_analyzed.loc[:, 'sell'] = df_analyzed.loc[:, 'sell'].shift(1) - if has_buy_tag: - df_analyzed.loc[:, 'buy_tag'] = df_analyzed.loc[:, 'buy_tag'].shift(1) + df_analyzed.loc[:, 'buy_tag'] = df_analyzed.loc[:, 'buy_tag'].shift(1) # Update dataprovider cache self.dataprovider._set_cached_df(pair, self.timeframe, df_analyzed) diff --git a/tests/optimize/__init__.py b/tests/optimize/__init__.py index f29d8d585..6ad2d300b 100644 --- a/tests/optimize/__init__.py +++ b/tests/optimize/__init__.py @@ -52,4 +52,6 @@ def _build_backtest_dataframe(data): # Ensure floats are in place for column in ['open', 'high', 'low', 'close', 'volume']: frame[column] = frame[column].astype('float64') + if 'buy_tag' not in columns: + frame['buy_tag'] = None return frame From fad253ad5141978b5d35d878414feeead837fab8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Aug 2021 06:53:55 +0200 Subject: [PATCH 36/40] Version bump ccxt to 1.54.74 closes #5401 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 60175c12f..0e107d8e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.21.1 pandas==1.3.1 -ccxt==1.54.62 +ccxt==1.54.74 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 From e03784d98dc016790cb42eb194adf2c2be752091 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Aug 2021 05:02:36 +0000 Subject: [PATCH 37/40] Fix filled exception closes #5404 --- freqtrade/persistence/models.py | 2 +- tests/test_persistence.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index a45274266..5eaca7966 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -161,7 +161,7 @@ class Order(_DECL_BASE): self.ft_is_open = True if self.status in ('closed', 'canceled', 'cancelled'): self.ft_is_open = False - if order.get('filled', 0) > 0: + if (order.get('filled', 0.0) or 0.0) > 0: self.order_filled_date = datetime.now(timezone.utc) self.order_update_date = datetime.now(timezone.utc) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 105cee23a..d036b045e 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1224,6 +1224,11 @@ def test_update_order_from_ccxt(caplog): assert o.ft_is_open assert o.order_filled_date is None + # Order is unfilled, "filled" not set + # https://github.com/freqtrade/freqtrade/issues/5404 + ccxt_order.update({'filled': None, 'remaining': 20.0, 'status': 'canceled'}) + o.update_from_ccxt_object(ccxt_order) + # Order has been closed ccxt_order.update({'filled': 20.0, 'remaining': 0.0, 'status': 'closed'}) o.update_from_ccxt_object(ccxt_order) From c7147311f8d75635cff229ceccc88656903da045 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 13 Aug 2021 17:14:38 +0200 Subject: [PATCH 38/40] Fix json syntax error in config template --- freqtrade/templates/base_config.json.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index 03a6c4855..a5782f7cd 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -25,7 +25,7 @@ "ask_strategy": { "price_side": "ask", "use_order_book": true, - "order_book_top": 1, + "order_book_top": 1 }, {{ exchange | indent(4) }}, "pairlists": [ From db5a9443967eae8c0ed97229d5a510a3ffe14715 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 Aug 2021 08:46:19 +0200 Subject: [PATCH 39/40] Cleanup GHA node after building images --- build_helpers/publish_docker_arm64.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/build_helpers/publish_docker_arm64.sh b/build_helpers/publish_docker_arm64.sh index 08793d339..e7b69b2dc 100755 --- a/build_helpers/publish_docker_arm64.sh +++ b/build_helpers/publish_docker_arm64.sh @@ -74,7 +74,5 @@ fi docker images -if [ $? -ne 0 ]; then - echo "failed building image" - return 1 -fi +# Cleanup old images from arm64 node. +docker image prune -a --force --filter "until=24h" From bb472ff98b06a8ae4059b742e56b22f1d2aca244 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 Aug 2021 08:47:14 +0200 Subject: [PATCH 40/40] Improve new-exchange documentation --- docs/developer.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/developer.md b/docs/developer.md index dd56a367c..bd138212b 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -240,11 +240,18 @@ The `IProtection` parent class provides a helper method for this in `calculate_l !!! Note This section is a Work in Progress and is not a complete guide on how to test a new exchange with Freqtrade. +!!! Note + Make sure to use an up-to-date version of CCXT before running any of the below tests. + You can get the latest version of ccxt by running `pip install -U ccxt` with activated virtual environment. + Native docker is not supported for these tests, however the available dev-container will support all required actions and eventually necessary changes. + Most exchanges supported by CCXT should work out of the box. To quickly test the public endpoints of an exchange, add a configuration for your exchange to `test_ccxt_compat.py` and run these tests with `pytest --longrun tests/exchange/test_ccxt_compat.py`. Completing these tests successfully a good basis point (it's a requirement, actually), however these won't guarantee correct exchange functioning, as this only tests public endpoints, but no private endpoint (like generate order or similar). +Also try to use `freqtrade download-data` for an extended timerange and verify that the data downloaded correctly (no holes, the specified timerange was actually downloaded). + ### Stoploss On Exchange Check if the new exchange supports Stoploss on Exchange orders through their API.