From 5fd85368a9388ad12fdf334ebc042fb01674da04 Mon Sep 17 00:00:00 2001 From: Antonio Della Fortuna Date: Wed, 4 Jan 2023 10:34:44 +0100 Subject: [PATCH 01/10] Added support for max_open_trades hyperopting --- docs/advanced-hyperopt.md | 6 +++ docs/configuration.md | 3 +- docs/hyperopt.md | 10 +++-- freqtrade/commands/cli_options.py | 3 +- freqtrade/constants.py | 2 +- freqtrade/data/btanalysis.py | 2 +- freqtrade/freqtradebot.py | 3 +- freqtrade/optimize/backtesting.py | 7 +-- freqtrade/optimize/hyperopt.py | 15 ++++++- freqtrade/optimize/hyperopt_auto.py | 3 ++ freqtrade/optimize/hyperopt_interface.py | 10 +++++ freqtrade/optimize/hyperopt_tools.py | 11 +++-- freqtrade/optimize/optimize_reports.py | 2 +- freqtrade/resolvers/strategy_resolver.py | 3 ++ freqtrade/strategy/hyper.py | 2 + freqtrade/strategy/interface.py | 3 ++ tests/optimize/test_hyperopt.py | 14 ++++-- tests/optimize/test_hyperopt_tools.py | 52 ++++++++++++++++------- tests/strategy/strats/strategy_test_v3.py | 3 ++ tests/strategy/test_strategy_loading.py | 37 ++++++++++++++++ 20 files changed, 155 insertions(+), 36 deletions(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 0dace9985..b958ca22e 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -123,6 +123,12 @@ class MyAwesomeStrategy(IStrategy): Categorical([True, False], name='trailing_only_offset_is_reached'), ] + + # Define a custom max_open_trades space + def trades_space(self) -> List[Dimension]: + return [ + Integer(1, 10, name='max_open_trades'), + ] ``` !!! Note diff --git a/docs/configuration.md b/docs/configuration.md index 83b23425c..c8c87fcff 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -134,7 +134,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | Parameter | Description | |------------|-------------| -| `max_open_trades` | **Required.** Number of open trades your bot is allowed to have. Only one open trade per pair is possible, so the length of your pairlist is another limitation that can apply. If -1 then it is ignored (i.e. potentially unlimited open trades, limited by the pairlist). [More information below](#configuring-amount-per-trade).
**Datatype:** Positive integer or -1. +| `max_open_trades` | **Required.** Number of open trades your bot is allowed to have. Only one open trade per pair is possible, so the length of your pairlist is another limitation that can apply. If -1 then it is ignored (i.e. potentially unlimited open trades, limited by the pairlist). [More information below](#configuring-amount-per-trade). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Positive integer or -1. | `stake_currency` | **Required.** Crypto-currency used for trading.
**Datatype:** String | `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#configuring-amount-per-trade).
**Datatype:** Positive float or `"unlimited"`. | `tradable_balance_ratio` | Ratio of the total account balance the bot is allowed to trade. [More information below](#configuring-amount-per-trade).
*Defaults to `0.99` 99%).*
**Datatype:** Positive float between `0.1` and `1.0`. @@ -263,6 +263,7 @@ Values set in the configuration file always overwrite values set in the strategy * `minimal_roi` * `timeframe` * `stoploss` +* `max_open_trades` * `trailing_stop` * `trailing_stop_positive` * `trailing_stop_positive_offset` diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 6b6c2a772..5e676c34b 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -50,7 +50,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--eps] [--dmmp] [--enable-protections] [--dry-run-wallet DRY_RUN_WALLET] [--timeframe-detail TIMEFRAME_DETAIL] [-e INT] - [--spaces {all,buy,sell,roi,stoploss,trailing,protection,default} [{all,buy,sell,roi,stoploss,trailing,protection,default} ...]] + [--spaces {all,buy,sell,roi,stoploss,trailing,protection,trades,default} [{all,buy,sell,roi,stoploss,trailing,protection,trades,default} ...]] [--print-all] [--no-color] [--print-json] [-j JOBS] [--random-state INT] [--min-trades INT] [--hyperopt-loss NAME] [--disable-param-export] @@ -96,7 +96,7 @@ optional arguments: Specify detail timeframe for backtesting (`1m`, `5m`, `30m`, `1h`, `1d`). -e INT, --epochs INT Specify number of epochs (default: 100). - --spaces {all,buy,sell,roi,stoploss,trailing,protection,default} [{all,buy,sell,roi,stoploss,trailing,protection,default} ...] + --spaces {all,buy,sell,roi,stoploss,trailing,protection,trades,default} [{all,buy,sell,roi,stoploss,trailing,protection,trades,default} ...] Specify which parameters to hyperopt. Space-separated list. --print-all Print all results, not only the best ones. @@ -180,6 +180,7 @@ Rarely you may also need to create a [nested class](advanced-hyperopt.md#overrid * `generate_roi_table` - for custom ROI optimization (if you need the ranges for the values in the ROI table that differ from default or the number of entries (steps) in the ROI table which differs from the default 4 steps) * `stoploss_space` - for custom stoploss optimization (if you need the range for the stoploss parameter in the optimization hyperspace that differs from default) * `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default) +* `trades_space` - for custom max_open_trades optimization (if you need the ranges for the max_open_trades parameter in the optimization hyperspace that differ from default) !!! Tip "Quickly optimize ROI, stoploss and trailing stoploss" You can quickly optimize the spaces `roi`, `stoploss` and `trailing` without changing anything in your strategy. @@ -643,6 +644,7 @@ Legal values are: * `roi`: just optimize the minimal profit table for your strategy * `stoploss`: search for the best stoploss value * `trailing`: search for the best trailing stop values +* `trades`: search for the best max open trades values * `protection`: search for the best protection parameters (read the [protections section](#optimizing-protections) on how to properly define these) * `default`: `all` except `trailing` and `protection` * space-separated list of any of the above values for example `--spaces roi stoploss` @@ -916,5 +918,5 @@ Once the optimized strategy has been implemented into your strategy, you should To achieve same the results (number of trades, their durations, profit, etc.) as during Hyperopt, please use the same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting. Should results not match, please double-check to make sure you transferred all conditions correctly. -Pay special care to the stoploss (and trailing stoploss) parameters, as these are often set in configuration files, which override changes to the strategy. -You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss` or `trailing_stop`). +Pay special care to the stoploss, max_open_trades and trailing stoploss parameters, as these are often set in configuration files, which override changes to the strategy. +You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss`, `max_open_trades` or `trailing_stop`). diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 91ac16365..c70073582 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -251,7 +251,8 @@ AVAILABLE_CLI_OPTIONS = { "spaces": Arg( '--spaces', help='Specify which parameters to hyperopt. Space-separated list.', - choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'protection', 'default'], + choices=['all', 'buy', 'sell', 'roi', 'stoploss', + 'trailing', 'protection', 'default', 'trades'], nargs='+', default='default', ), diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 397367216..95efa63b8 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -636,7 +636,6 @@ SCHEMA_TRADE_REQUIRED = [ SCHEMA_BACKTEST_REQUIRED = [ 'exchange', - 'max_open_trades', 'stake_currency', 'stake_amount', 'dry_run_wallet', @@ -646,6 +645,7 @@ SCHEMA_BACKTEST_REQUIRED = [ SCHEMA_BACKTEST_REQUIRED_FINAL = SCHEMA_BACKTEST_REQUIRED + [ 'stoploss', 'minimal_roi', + 'max_open_trades' ] SCHEMA_MINIMAL_REQUIRED = [ diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 3102683b2..bc28d9de0 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -332,7 +332,7 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF def evaluate_result_multi(results: pd.DataFrame, timeframe: str, - max_open_trades: int) -> pd.DataFrame: + max_open_trades: int | float) -> pd.DataFrame: """ Find overlapping trades by expanding each trade once per period it was open and then counting overlaps diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 258a45008..4bf208176 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -520,7 +520,8 @@ class FreqtradeBot(LoggingMixin): else: self.log_once(f"Pair {pair} is currently locked.", logger.info) return False - stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge) + stake_amount = self.wallets.get_trade_stake_amount( + pair, self.edge, self.config['max_open_trades']) bid_check_dom = self.config.get('entry_pricing', {}).get('check_depth_of_market', {}) if ((bid_check_dom.get('enabled', False)) and diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2b8b96cba..178b2c18d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -920,7 +920,7 @@ class Backtesting: trade.close(exit_row[OPEN_IDX], show_msg=False) LocalTrade.close_bt_trade(trade) - def trade_slot_available(self, max_open_trades: int, open_trade_count: int) -> bool: + def trade_slot_available(self, max_open_trades: int | float, open_trade_count: int) -> bool: # Always allow trades when max_open_trades is enabled. if max_open_trades <= 0 or open_trade_count < max_open_trades: return True @@ -1051,7 +1051,8 @@ class Backtesting: def backtest_loop( self, row: Tuple, pair: str, current_time: datetime, end_date: datetime, - max_open_trades: int, open_trade_count_start: int, is_first: bool = True) -> int: + max_open_trades: int | float, + open_trade_count_start: int, is_first: bool = True) -> int: """ NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized. @@ -1122,7 +1123,7 @@ class Backtesting: def backtest(self, processed: Dict, start_date: datetime, end_date: datetime, - max_open_trades: int = 0) -> Dict[str, Any]: + max_open_trades: int | float = 0) -> Dict[str, Any]: """ Implement backtesting functionality diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b459d59f2..aae7802a5 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -74,6 +74,7 @@ class Hyperopt: self.roi_space: List[Dimension] = [] self.stoploss_space: List[Dimension] = [] self.trailing_space: List[Dimension] = [] + self.trades_space: List[Dimension] = [] self.dimensions: List[Dimension] = [] self.config = config @@ -209,6 +210,8 @@ class Hyperopt: result['stoploss'] = {p.name: params.get(p.name) for p in self.stoploss_space} if HyperoptTools.has_space(self.config, 'trailing'): result['trailing'] = self.custom_hyperopt.generate_trailing_params(params) + if HyperoptTools.has_space(self.config, 'trades'): + result['max_open_trades'] = {p.name: params.get(p.name) for p in self.trades_space} return result @@ -229,6 +232,8 @@ class Hyperopt: 'trailing_stop_positive_offset': strategy.trailing_stop_positive_offset, 'trailing_only_offset_is_reached': strategy.trailing_only_offset_is_reached, } + if not HyperoptTools.has_space(self.config, 'trades'): + result['max_open_trades'] = {'max_open_trades': strategy.max_open_trades} return result def print_results(self, results) -> None: @@ -280,8 +285,13 @@ class Hyperopt: logger.debug("Hyperopt has 'trailing' space") self.trailing_space = self.custom_hyperopt.trailing_space() + if HyperoptTools.has_space(self.config, 'trades'): + logger.debug("Hyperopt has 'trades' space") + self.trades_space = self.custom_hyperopt.trades_space() + self.dimensions = (self.buy_space + self.sell_space + self.protection_space - + self.roi_space + self.stoploss_space + self.trailing_space) + + self.roi_space + self.stoploss_space + self.trailing_space + + self.trades_space) def assign_params(self, params_dict: Dict, category: str) -> None: """ @@ -328,6 +338,9 @@ class Hyperopt: self.backtesting.strategy.trailing_only_offset_is_reached = \ d['trailing_only_offset_is_reached'] + if HyperoptTools.has_space(self.config, 'trades'): + self.max_open_trades = params_dict['max_open_trades'] + with self.data_pickle_file.open('rb') as f: processed = load(f, mmap_mode='r') if self.analyze_per_epoch: diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py index 5bc0af42b..0627b52c3 100644 --- a/freqtrade/optimize/hyperopt_auto.py +++ b/freqtrade/optimize/hyperopt_auto.py @@ -91,5 +91,8 @@ class HyperOptAuto(IHyperOpt): def trailing_space(self) -> List['Dimension']: return self._get_func('trailing_space')() + def trades_space(self) -> List['Dimension']: + return self._get_func('trades_space')() + def generate_estimator(self, dimensions: List['Dimension'], **kwargs) -> EstimatorType: return self._get_func('generate_estimator')(dimensions=dimensions, **kwargs) diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index a7c64ffb0..b692abea4 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -191,6 +191,16 @@ class IHyperOpt(ABC): Categorical([True, False], name='trailing_only_offset_is_reached'), ] + def trades_space(self) -> List[Dimension]: + """ + Create a max open trades space. + + You may override it in your custom Hyperopt class. + """ + return [ + Integer(1, 10, name='max_open_trades'), + ] + # This is needed for proper unpickling the class attribute timeframe # which is set to the actual value by the resolver. # Why do I still need such shamanic mantras in modern python? diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 7007ec55e..6c16100d3 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -96,7 +96,7 @@ class HyperoptTools(): Tell if the space value is contained in the configuration """ # 'trailing' and 'protection spaces are not included in the 'default' set of spaces - if space in ('trailing', 'protection'): + if space in ('trailing', 'protection', 'trades'): return any(s in config['spaces'] for s in [space, 'all']) else: return any(s in config['spaces'] for s in [space, 'all', 'default']) @@ -187,7 +187,8 @@ class HyperoptTools(): if print_json: result_dict: Dict = {} - for s in ['buy', 'sell', 'protection', 'roi', 'stoploss', 'trailing']: + for s in ['buy', 'sell', 'protection', + 'roi', 'stoploss', 'trailing', 'max_open_trades']: HyperoptTools._params_update_for_json(result_dict, params, non_optimized, s) print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE)) @@ -201,6 +202,8 @@ class HyperoptTools(): HyperoptTools._params_pretty_print(params, 'roi', "ROI table:", non_optimized) HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:", non_optimized) HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:", non_optimized) + HyperoptTools._params_pretty_print( + params, 'max_open_trades', "Max Open Trades:", non_optimized) @staticmethod def _params_update_for_json(result_dict, params, non_optimized, space: str) -> None: @@ -239,7 +242,9 @@ class HyperoptTools(): if space == "stoploss": stoploss = safe_value_fallback2(space_params, no_params, space, space) result += (f"stoploss = {stoploss}{appendix}") - + elif space == "max_open_trades": + max_open_trades = safe_value_fallback2(space_params, no_params, space, space) + result += (f"max_open_trades = {max_open_trades}{appendix}") elif space == "roi": result = result[:-1] + f'{appendix}\n' minimal_roi_result = rapidjson.dumps({ diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 8ad37e7d8..3871ac64e 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -190,7 +190,7 @@ def generate_tag_metrics(tag_type: str, return [] -def generate_exit_reason_stats(max_open_trades: int, results: DataFrame) -> List[Dict]: +def generate_exit_reason_stats(max_open_trades: int | float, results: DataFrame) -> List[Dict]: """ Generate small table outlining Backtest results :param max_open_trades: Max_open_trades parameter diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 67df49dcb..859388a4e 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -76,6 +76,7 @@ class StrategyResolver(IResolver): ("ignore_buying_expired_candle_after", 0), ("position_adjustment_enable", False), ("max_entry_position_adjustment", -1), + ("max_open_trades", -1) ] for attribute, default in attributes: StrategyResolver._override_attribute_helper(strategy, config, @@ -128,6 +129,8 @@ class StrategyResolver(IResolver): key=lambda t: t[0])) if hasattr(strategy, 'stoploss'): strategy.stoploss = float(strategy.stoploss) + if hasattr(strategy, 'max_open_trades') and strategy.max_open_trades == -1: + strategy.max_open_trades = float('inf') return strategy @staticmethod diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index 6f62c9d3d..4dac4154f 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -80,6 +80,8 @@ class HyperStrategyMixin: self.stoploss = params.get('stoploss', {}).get( 'stoploss', getattr(self, 'stoploss', -0.1)) + self.max_open_trades = params.get('max_open_trades', {}).get( + 'max_open_trades', getattr(self, 'max_open_trades', -1)) trailing = params.get('trailing', {}) self.trailing_stop = trailing.get( 'trailing_stop', getattr(self, 'trailing_stop', False)) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 781ae6c5c..e39c403b5 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -54,6 +54,9 @@ class IStrategy(ABC, HyperStrategyMixin): # associated stoploss stoploss: float + # max open trades for the strategy + max_open_trades: int | float + # trailing stoploss trailing_stop: bool = False trailing_stop_positive: Optional[float] = None diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 5bce9f419..8b57883f7 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -292,6 +292,8 @@ def test_params_no_optimize_details(hyperopt) -> None: assert res['roi']['0'] == 0.04 assert "stoploss" in res assert res['stoploss']['stoploss'] == -0.1 + assert "max_open_trades" in res + assert res['max_open_trades']['max_open_trades'] == 1 def test_start_calls_optimizer(mocker, hyperopt_conf, capsys) -> None: @@ -474,6 +476,7 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: 'trailing_stop_positive': 0.02, 'trailing_stop_positive_offset_p1': 0.05, 'trailing_only_offset_is_reached': False, + 'max_open_trades': 3, } response_expected = { 'loss': 1.9147239021396234, @@ -499,7 +502,9 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: 'trailing': {'trailing_only_offset_is_reached': False, 'trailing_stop': True, 'trailing_stop_positive': 0.02, - 'trailing_stop_positive_offset': 0.07}}, + 'trailing_stop_positive_offset': 0.07}, + 'max_open_trades': {'max_open_trades': 3} + }, 'params_dict': optimizer_param, 'params_not_optimized': {'buy': {}, 'protection': {}, 'sell': {}}, 'results_metrics': ANY, @@ -548,7 +553,8 @@ def test_print_json_spaces_all(mocker, hyperopt_conf, capsys) -> None: 'buy': {'mfi-value': None}, 'sell': {'sell-mfi-value': None}, 'roi': {}, 'stoploss': {'stoploss': None}, - 'trailing': {'trailing_stop': None} + 'trailing': {'trailing_stop': None}, + 'max_open_trades': {'max_open_trades': None} }, 'results_metrics': generate_result_metrics(), }]) @@ -571,7 +577,7 @@ def test_print_json_spaces_all(mocker, hyperopt_conf, capsys) -> None: out, err = capsys.readouterr() result_str = ( '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi"' - ':{},"stoploss":null,"trailing_stop":null}' + ':{},"stoploss":null,"trailing_stop":null,"max_open_trades":null}' ) assert result_str in out # noqa: E501 # Should be called for historical candle data @@ -874,6 +880,7 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: assert hyperopt.backtesting.strategy.buy_rsi.value == 35 assert hyperopt.backtesting.strategy.sell_rsi.value == 74 assert hyperopt.backtesting.strategy.protection_cooldown_lookback.value == 30 + assert hyperopt.max_open_trades == 1 buy_rsi_range = hyperopt.backtesting.strategy.buy_rsi.range assert isinstance(buy_rsi_range, range) # Range from 0 - 50 (inclusive) @@ -884,6 +891,7 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: assert hyperopt.backtesting.strategy.protection_cooldown_lookback.value != 30 assert hyperopt.backtesting.strategy.buy_rsi.value != 35 assert hyperopt.backtesting.strategy.sell_rsi.value != 74 + assert hyperopt.max_open_trades != 1 hyperopt.custom_hyperopt.generate_estimator = lambda *args, **kwargs: 'ET1' with pytest.raises(OperationalException, match="Estimator ET1 not supported."): diff --git a/tests/optimize/test_hyperopt_tools.py b/tests/optimize/test_hyperopt_tools.py index 7d4fef3bd..eace78eee 100644 --- a/tests/optimize/test_hyperopt_tools.py +++ b/tests/optimize/test_hyperopt_tools.py @@ -66,52 +66,58 @@ def test_load_previous_results2(mocker, testdatadir, caplog) -> None: @pytest.mark.parametrize("spaces, expected_results", [ (['buy'], {'buy': True, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': False, - 'protection': False}), + 'protection': False, 'trades': False}), (['sell'], {'buy': False, 'sell': True, 'roi': False, 'stoploss': False, 'trailing': False, - 'protection': False}), + 'protection': False, 'trades': False}), (['roi'], {'buy': False, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False, - 'protection': False}), + 'protection': False, 'trades': False}), (['stoploss'], {'buy': False, 'sell': False, 'roi': False, 'stoploss': True, 'trailing': False, - 'protection': False}), + 'protection': False, 'trades': False}), (['trailing'], {'buy': False, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': True, - 'protection': False}), + 'protection': False, 'trades': False}), (['buy', 'sell', 'roi', 'stoploss'], {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False, - 'protection': False}), + 'protection': False, 'trades': False}), (['buy', 'sell', 'roi', 'stoploss', 'trailing'], {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True, - 'protection': False}), + 'protection': False, 'trades': False}), (['buy', 'roi'], {'buy': True, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False, - 'protection': False}), + 'protection': False, 'trades': False}), (['all'], {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True, - 'protection': True}), + 'protection': True, 'trades': True}), (['default'], {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False, - 'protection': False}), + 'protection': False, 'trades': False}), (['default', 'trailing'], {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True, - 'protection': False}), + 'protection': False, 'trades': False}), (['all', 'buy'], {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True, - 'protection': True}), + 'protection': True, 'trades': True}), (['default', 'buy'], {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False, - 'protection': False}), + 'protection': False, 'trades': False}), (['all'], {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True, - 'protection': True}), + 'protection': True, 'trades': True}), (['protection'], {'buy': False, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': False, - 'protection': True}), + 'protection': True, 'trades': False}), + (['trades'], + {'buy': False, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': False, + 'protection': False, 'trades': True}), + (['default', 'trades'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False, + 'protection': False, 'trades': True}), ]) def test_has_space(hyperopt_conf, spaces, expected_results): - for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing', 'protection']: + for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing', 'protection', 'trades']: hyperopt_conf.update({'spaces': spaces}) assert HyperoptTools.has_space(hyperopt_conf, s) == expected_results[s] @@ -193,6 +199,9 @@ def test_export_params(tmpdir): "346": 0.08499, "507": 0.049, "1595": 0 + }, + "max_open_trades": { + "max_open_trades": 5 } }, "params_not_optimized": { @@ -219,6 +228,7 @@ def test_export_params(tmpdir): assert "roi" in content["params"] assert "stoploss" in content["params"] assert "trailing" in content["params"] + assert "max_open_trades" in content["params"] def test_try_export_params(default_conf, tmpdir, caplog, mocker): @@ -297,6 +307,9 @@ def test_params_print(capsys): "trailing_stop_positive_offset": 0.1, "trailing_only_offset_is_reached": True }, + "max_open_trades": { + "max_open_trades": 5 + } } HyperoptTools._params_pretty_print(params, 'buy', 'No header', non_optimized) @@ -327,6 +340,13 @@ def test_params_print(capsys): assert re.search('trailing_stop_positive_offset = 0.1 # value loaded.*\n', captured.out) assert re.search('trailing_only_offset_is_reached = True # value loaded.*\n', captured.out) + HyperoptTools._params_pretty_print( + params, 'max_open_trades', "Max Open Trades:", non_optimized) + captured = capsys.readouterr() + + assert re.search("# Max Open Trades:", captured.out) + assert re.search('max_open_trades = 5 # value loaded.*\n', captured.out) + def test_hyperopt_serializer(): diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index 088ab21d4..6f5ff573b 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -30,6 +30,9 @@ class StrategyTestV3(IStrategy): "0": 0.04 } + # Optimal max_open_trades for the strategy + max_open_trades = -1 + # Optimal stoploss designed for the strategy stoploss = -0.10 diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 5fcc75026..e08193531 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -175,6 +175,18 @@ def test_strategy_override_stoploss(caplog, default_conf): assert log_has("Override strategy 'stoploss' with value in config file: -0.5.", caplog) +def test_strategy_override_max_open_trades(caplog, default_conf): + caplog.set_level(logging.INFO) + default_conf.update({ + 'strategy': CURRENT_TEST_STRATEGY, + 'max_open_trades': 7 + }) + strategy = StrategyResolver.load_strategy(default_conf) + + assert strategy.max_open_trades == 7 + assert log_has("Override strategy 'max_open_trades' with value in config file: 7.", caplog) + + def test_strategy_override_trailing_stop(caplog, default_conf): caplog.set_level(logging.INFO) default_conf.update({ @@ -349,6 +361,31 @@ def test_strategy_override_use_exit_profit_only(caplog, default_conf): assert log_has("Override strategy 'exit_profit_only' with value in config file: True.", caplog) +def test_strategy_max_open_trades_infinity_from_strategy(caplog, default_conf): + caplog.set_level(logging.INFO) + default_conf.update({ + 'strategy': CURRENT_TEST_STRATEGY, + }) + del default_conf['max_open_trades'] + + strategy = StrategyResolver.load_strategy(default_conf) + + # this test assumes -1 set to 'max_open_trades' in CURRENT_TEST_STRATEGY + assert strategy.max_open_trades == float('inf') + + +def test_strategy_max_open_trades_infinity_from_config(caplog, default_conf): + caplog.set_level(logging.INFO) + default_conf.update({ + 'strategy': CURRENT_TEST_STRATEGY, + 'max_open_trades': -1 + }) + + strategy = StrategyResolver.load_strategy(default_conf) + + assert strategy.max_open_trades == float('inf') + + @ pytest.mark.filterwarnings("ignore:deprecated") def test_missing_implements(default_conf, caplog): From 1c5e172683be8d705d41e3c87c8eff2e9788d92a Mon Sep 17 00:00:00 2001 From: Antonio Della Fortuna Date: Wed, 4 Jan 2023 12:54:35 +0100 Subject: [PATCH 02/10] docs update --- docs/advanced-hyperopt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index b958ca22e..0ef4911cd 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -75,7 +75,7 @@ This function needs to return a floating point number (`float`). Smaller numbers ## Overriding pre-defined spaces -To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_space`, `trailing_space`), define a nested class called Hyperopt and define the required spaces as follows: +To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_space`, `trailing_space`, `trades_space`), define a nested class called Hyperopt and define the required spaces as follows: ```python from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal From f2fa476dc69a9412f705fe1d27bb9a27868bea8d Mon Sep 17 00:00:00 2001 From: Antonio Della Fortuna Date: Wed, 4 Jan 2023 16:09:27 +0100 Subject: [PATCH 03/10] max_open_trades should be an integer Max open trades will be always an integer in the strategy (-1 for infinity), but in the config -1 will be parsed as infinity --- freqtrade/data/btanalysis.py | 2 +- freqtrade/freqtradebot.py | 2 +- freqtrade/optimize/backtesting.py | 9 +++++---- freqtrade/optimize/hyperopt.py | 5 ++++- freqtrade/optimize/optimize_reports.py | 2 +- freqtrade/resolvers/strategy_resolver.py | 14 ++++++++++---- freqtrade/strategy/interface.py | 2 +- tests/strategy/test_strategy_loading.py | 7 ++++--- 8 files changed, 27 insertions(+), 16 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index bc28d9de0..3102683b2 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -332,7 +332,7 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF def evaluate_result_multi(results: pd.DataFrame, timeframe: str, - max_open_trades: int | float) -> pd.DataFrame: + max_open_trades: int) -> pd.DataFrame: """ Find overlapping trades by expanding each trade once per period it was open and then counting overlaps diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4bf208176..779c134bd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -521,7 +521,7 @@ class FreqtradeBot(LoggingMixin): self.log_once(f"Pair {pair} is currently locked.", logger.info) return False stake_amount = self.wallets.get_trade_stake_amount( - pair, self.edge, self.config['max_open_trades']) + pair, self.edge) bid_check_dom = self.config.get('entry_pricing', {}).get('check_depth_of_market', {}) if ((bid_check_dom.get('enabled', False)) and diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 178b2c18d..181a5b580 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -920,7 +920,7 @@ class Backtesting: trade.close(exit_row[OPEN_IDX], show_msg=False) LocalTrade.close_bt_trade(trade) - def trade_slot_available(self, max_open_trades: int | float, open_trade_count: int) -> bool: + def trade_slot_available(self, max_open_trades: int, open_trade_count: int) -> bool: # Always allow trades when max_open_trades is enabled. if max_open_trades <= 0 or open_trade_count < max_open_trades: return True @@ -1051,7 +1051,7 @@ class Backtesting: def backtest_loop( self, row: Tuple, pair: str, current_time: datetime, end_date: datetime, - max_open_trades: int | float, + max_open_trades: int, open_trade_count_start: int, is_first: bool = True) -> int: """ NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized. @@ -1123,7 +1123,7 @@ class Backtesting: def backtest(self, processed: Dict, start_date: datetime, end_date: datetime, - max_open_trades: int | float = 0) -> Dict[str, Any]: + max_open_trades: int = 0) -> Dict[str, Any]: """ Implement backtesting functionality @@ -1228,7 +1228,8 @@ class Backtesting: # Use max_open_trades in backtesting, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): # Must come from strategy config, as the strategy may modify this setting. - max_open_trades = self.strategy.config['max_open_trades'] + max_open_trades = self.strategy.config['max_open_trades'] \ + if self.strategy.config['max_open_trades'] != float('inf') else -1 else: logger.info( 'Ignoring max_open_trades (--disable-max-market-positions was used) ...') diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index aae7802a5..4ab9b1a5d 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -119,11 +119,14 @@ class Hyperopt: # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): - self.max_open_trades = self.config['max_open_trades'] + self.max_open_trades = self.config['max_open_trades'] \ + if self.config['max_open_trades'] != float('inf') else -1 else: logger.debug('Ignoring max_open_trades (--disable-max-market-positions was used) ...') self.max_open_trades = 0 + print("Strategy max open trades", self.max_open_trades) + if HyperoptTools.has_space(self.config, 'sell'): # Make sure use_exit_signal is enabled self.config['use_exit_signal'] = True diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index e09cbf428..7de8f1a47 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -191,7 +191,7 @@ def generate_tag_metrics(tag_type: str, return [] -def generate_exit_reason_stats(max_open_trades: int | float, results: DataFrame) -> List[Dict]: +def generate_exit_reason_stats(max_open_trades: int, results: DataFrame) -> List[Dict]: """ Generate small table outlining Backtest results :param max_open_trades: Max_open_trades parameter diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 859388a4e..febda7822 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -104,14 +104,22 @@ class StrategyResolver(IResolver): if (attribute in config and not isinstance(getattr(type(strategy), attribute, None), property)): # Ensure Properties are not overwritten - setattr(strategy, attribute, config[attribute]) + val = config[attribute] + # max_open_trades set to float('inf') in the config will be copied as -1 in the strategy + if attribute == 'max_open_trades' and val == float('inf'): + val = -1 + setattr(strategy, attribute, val) logger.info("Override strategy '%s' with value in config file: %s.", attribute, config[attribute]) elif hasattr(strategy, attribute): val = getattr(strategy, attribute) # None's cannot exist in the config, so do not copy them if val is not None: - config[attribute] = val + # max_open_trades set to -1 in the strategy will be copied as infinity in the config + if attribute == 'max_open_trades' and val == -1: + config[attribute] = float('inf') + else: + config[attribute] = val # Explicitly check for None here as other "falsy" values are possible elif default is not None: setattr(strategy, attribute, default) @@ -129,8 +137,6 @@ class StrategyResolver(IResolver): key=lambda t: t[0])) if hasattr(strategy, 'stoploss'): strategy.stoploss = float(strategy.stoploss) - if hasattr(strategy, 'max_open_trades') and strategy.max_open_trades == -1: - strategy.max_open_trades = float('inf') return strategy @staticmethod diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index e39c403b5..d1b9a8498 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -55,7 +55,7 @@ class IStrategy(ABC, HyperStrategyMixin): stoploss: float # max open trades for the strategy - max_open_trades: int | float + max_open_trades: int # trailing stoploss trailing_stop: bool = False diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index e08193531..2296d4bc6 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -371,19 +371,20 @@ def test_strategy_max_open_trades_infinity_from_strategy(caplog, default_conf): strategy = StrategyResolver.load_strategy(default_conf) # this test assumes -1 set to 'max_open_trades' in CURRENT_TEST_STRATEGY - assert strategy.max_open_trades == float('inf') + assert strategy.max_open_trades == -1 + assert default_conf['max_open_trades'] == float('inf') def test_strategy_max_open_trades_infinity_from_config(caplog, default_conf): caplog.set_level(logging.INFO) default_conf.update({ 'strategy': CURRENT_TEST_STRATEGY, - 'max_open_trades': -1 + 'max_open_trades': float('inf') }) strategy = StrategyResolver.load_strategy(default_conf) - assert strategy.max_open_trades == float('inf') + assert strategy.max_open_trades == -1 @ pytest.mark.filterwarnings("ignore:deprecated") From 464cb4761c78f587b2574d8ea02f7fce0cb3380d Mon Sep 17 00:00:00 2001 From: Antonio Della Fortuna Date: Sun, 8 Jan 2023 12:39:39 +0100 Subject: [PATCH 04/10] Fixed max_open_trades update from hyperopt Fixed max_open_trades update from hyperopt + removed max_open_trades as a param to backtesting + refactoring --- freqtrade/commands/cli_options.py | 2 +- freqtrade/optimize/backtesting.py | 26 +++--- freqtrade/optimize/hyperopt.py | 32 ++++--- tests/optimize/test_backtest_detail.py | 2 +- tests/optimize/test_backtesting.py | 31 ++++--- .../test_backtesting_adjust_position.py | 2 +- tests/optimize/test_hyperopt.py | 89 ++++++++++++++++--- 7 files changed, 132 insertions(+), 52 deletions(-) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index c70073582..fcb741867 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -252,7 +252,7 @@ AVAILABLE_CLI_OPTIONS = { '--spaces', help='Specify which parameters to hyperopt. Space-separated list.', choices=['all', 'buy', 'sell', 'roi', 'stoploss', - 'trailing', 'protection', 'default', 'trades'], + 'trailing', 'protection', 'trades', 'default'], nargs='+', default='default', ), diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8b8ba0c50..518e53ef8 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -920,8 +920,9 @@ class Backtesting: trade.close(exit_row[OPEN_IDX], show_msg=False) LocalTrade.close_bt_trade(trade) - def trade_slot_available(self, max_open_trades: int, open_trade_count: int) -> bool: + def trade_slot_available(self, open_trade_count: int) -> bool: # Always allow trades when max_open_trades is enabled. + max_open_trades = self.config['max_open_trades'] if max_open_trades <= 0 or open_trade_count < max_open_trades: return True # Rejected trade @@ -1051,7 +1052,6 @@ class Backtesting: def backtest_loop( self, row: Tuple, pair: str, current_time: datetime, end_date: datetime, - max_open_trades: int, open_trade_count_start: int, is_first: bool = True) -> int: """ NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized. @@ -1075,7 +1075,7 @@ class Backtesting: if ( (self._position_stacking or len(LocalTrade.bt_trades_open_pp[pair]) == 0) and is_first - and self.trade_slot_available(max_open_trades, open_trade_count_start) + and self.trade_slot_available(open_trade_count_start) and current_time != end_date and trade_dir is not None and not PairLocks.is_pair_locked(pair, row[DATE_IDX], trade_dir) @@ -1122,8 +1122,7 @@ class Backtesting: return open_trade_count_start def backtest(self, processed: Dict, - start_date: datetime, end_date: datetime, - max_open_trades: int = 0) -> Dict[str, Any]: + start_date: datetime, end_date: datetime) -> Dict[str, Any]: """ Implement backtesting functionality @@ -1135,7 +1134,6 @@ class Backtesting: optimize memory usage! :param start_date: backtesting timerange start datetime :param end_date: backtesting timerange end datetime - :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited :return: DataFrame with trades (results of backtesting) """ self.prepare_backtest(self.enable_protections) @@ -1176,7 +1174,7 @@ class Backtesting: if len(detail_data) == 0: # Fall back to "regular" data if no detail data was found for this candle open_trade_count_start = self.backtest_loop( - row, pair, current_time, end_date, max_open_trades, + row, pair, current_time, end_date, open_trade_count_start) continue detail_data.loc[:, 'enter_long'] = row[LONG_IDX] @@ -1189,13 +1187,13 @@ class Backtesting: current_time_det = current_time for det_row in detail_data[HEADERS].values.tolist(): open_trade_count_start = self.backtest_loop( - det_row, pair, current_time_det, end_date, max_open_trades, + det_row, pair, current_time_det, end_date, open_trade_count_start, is_first) current_time_det += timedelta(minutes=self.timeframe_detail_min) is_first = False else: open_trade_count_start = self.backtest_loop( - row, pair, current_time, end_date, max_open_trades, open_trade_count_start) + row, pair, current_time, end_date, open_trade_count_start) # Move time one configured time_interval ahead. self.progress.increment() @@ -1227,14 +1225,11 @@ class Backtesting: self._set_strategy(strat) # Use max_open_trades in backtesting, except --disable-max-market-positions is set - if self.config.get('use_max_market_positions', True): - # Must come from strategy config, as the strategy may modify this setting. - max_open_trades = self.strategy.config['max_open_trades'] \ - if self.strategy.config['max_open_trades'] != float('inf') else -1 - else: + if not self.config.get('use_max_market_positions', True): logger.info( 'Ignoring max_open_trades (--disable-max-market-positions was used) ...') - max_open_trades = 0 + self.strategy.max_open_trades = -1 + self.config.update({'max_open_trades': float('inf')}) # need to reprocess data every time to populate signals preprocessed = self.strategy.advise_all_indicators(data) @@ -1257,7 +1252,6 @@ class Backtesting: processed=preprocessed, start_date=min_date, end_date=max_date, - max_open_trades=max_open_trades, ) backtest_end_time = datetime.now(timezone.utc) results.update({ diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 4ab9b1a5d..595226fca 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -118,14 +118,10 @@ class Hyperopt: self.current_best_epoch: Optional[Dict[str, Any]] = None # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set - if self.config.get('use_max_market_positions', True): - self.max_open_trades = self.config['max_open_trades'] \ - if self.config['max_open_trades'] != float('inf') else -1 - else: + if not self.config.get('use_max_market_positions', True): logger.debug('Ignoring max_open_trades (--disable-max-market-positions was used) ...') - self.max_open_trades = 0 - - print("Strategy max open trades", self.max_open_trades) + self.backtesting.strategy.max_open_trades = -1 + config.update({'max_open_trades': float('inf')}) if HyperoptTools.has_space(self.config, 'sell'): # Make sure use_exit_signal is enabled @@ -214,7 +210,8 @@ class Hyperopt: if HyperoptTools.has_space(self.config, 'trailing'): result['trailing'] = self.custom_hyperopt.generate_trailing_params(params) if HyperoptTools.has_space(self.config, 'trades'): - result['max_open_trades'] = {p.name: params.get(p.name) for p in self.trades_space} + result['max_open_trades'] = { + 'max_open_trades': self.backtesting.strategy.max_open_trades} return result @@ -342,7 +339,21 @@ class Hyperopt: d['trailing_only_offset_is_reached'] if HyperoptTools.has_space(self.config, 'trades'): - self.max_open_trades = params_dict['max_open_trades'] + if self.config["stake_amount"] == "unlimited" and \ + (params_dict['max_open_trades'] == -1 or params_dict['max_open_trades'] == 0): + # Ignore unlimited max open trades if stake amount is unlimited + params_dict.update({'max_open_trades': self.config['max_open_trades']}) + + updated_config_max_open_trades = int(params_dict['max_open_trades']) \ + if (params_dict['max_open_trades'] != -1 + and params_dict['max_open_trades'] != 0) else float('inf') + + updated_strategy_max_open_trades = int(updated_config_max_open_trades) \ + if updated_config_max_open_trades != float('inf') else -1 + + self.config.update({'max_open_trades': updated_config_max_open_trades}) + + self.backtesting.strategy.max_open_trades = updated_strategy_max_open_trades with self.data_pickle_file.open('rb') as f: processed = load(f, mmap_mode='r') @@ -353,8 +364,7 @@ class Hyperopt: bt_results = self.backtesting.backtest( processed=processed, start_date=self.min_date, - end_date=self.max_date, - max_open_trades=self.max_open_trades, + end_date=self.max_date ) backtest_end_time = datetime.now(timezone.utc) bt_results.update({ diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index a18196507..4e78fc139 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -919,6 +919,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data: BTContainer) default_conf["trailing_stop_positive"] = data.trailing_stop_positive default_conf["trailing_stop_positive_offset"] = data.trailing_stop_positive_offset default_conf["use_exit_signal"] = data.use_exit_signal + default_conf["max_open_trades"] = 10 mocker.patch("freqtrade.exchange.Exchange.get_fee", return_value=0.0) mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) @@ -951,7 +952,6 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data: BTContainer) processed=data_processed, start_date=min_date, end_date=max_date, - max_open_trades=10, ) results = result['results'] diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index fc14a0f88..b291b2fec 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -96,7 +96,6 @@ def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC'): 'processed': processed, 'start_date': min_date, 'end_date': max_date, - 'max_open_trades': 10, } @@ -684,6 +683,8 @@ def test_backtest__get_sell_trade_entry(default_conf, fee, mocker) -> None: def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: default_conf['use_exit_signal'] = False + default_conf['max_open_trades'] = 10 + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=float('inf')) @@ -701,7 +702,6 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: processed=deepcopy(processed), start_date=min_date, end_date=max_date, - max_open_trades=10, ) results = result['results'] assert not results.empty @@ -785,6 +785,8 @@ def test_backtest_one_detail(default_conf_usdt, fee, mocker, testdatadir, use_de def custom_entry_price(proposed_rate, **kwargs): return proposed_rate * 0.997 + default_conf_usdt['max_open_trades'] = 10 + backtesting = Backtesting(default_conf_usdt) backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.populate_entry_trend = advise_entry @@ -805,7 +807,6 @@ def test_backtest_one_detail(default_conf_usdt, fee, mocker, testdatadir, use_de processed=deepcopy(processed), start_date=min_date, end_date=max_date, - max_open_trades=10, ) results = result['results'] assert not results.empty @@ -859,6 +860,7 @@ def test_backtest_timedout_entry_orders(default_conf, fee, mocker, testdatadir) mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=float('inf')) patch_exchange(mocker) + default_conf['max_open_trades'] = 1 backtesting = Backtesting(default_conf) backtesting._set_strategy(backtesting.strategylist[0]) # Testing dataframe contains 11 candles. Expecting 10 timed out orders. @@ -871,7 +873,6 @@ def test_backtest_timedout_entry_orders(default_conf, fee, mocker, testdatadir) processed=deepcopy(data), start_date=min_date, end_date=max_date, - max_open_trades=1, ) assert result['timedout_entry_orders'] == 10 @@ -879,6 +880,7 @@ def test_backtest_timedout_entry_orders(default_conf, fee, mocker, testdatadir) def test_backtest_1min_timeframe(default_conf, fee, mocker, testdatadir) -> None: default_conf['use_exit_signal'] = False + default_conf['max_open_trades'] = 1 mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=float('inf')) @@ -896,7 +898,6 @@ def test_backtest_1min_timeframe(default_conf, fee, mocker, testdatadir) -> None processed=processed, start_date=min_date, end_date=max_date, - max_open_trades=1, ) assert not results['results'].empty assert len(results['results']) == 1 @@ -904,6 +905,8 @@ def test_backtest_1min_timeframe(default_conf, fee, mocker, testdatadir) -> None def test_backtest_trim_no_data_left(default_conf, fee, mocker, testdatadir) -> None: default_conf['use_exit_signal'] = False + default_conf['max_open_trades'] = 10 + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=float('inf')) @@ -927,7 +930,6 @@ def test_backtest_trim_no_data_left(default_conf, fee, mocker, testdatadir) -> N processed=deepcopy(processed), start_date=min_date, end_date=max_date, - max_open_trades=10, ) @@ -948,6 +950,7 @@ def test_processed(default_conf, mocker, testdatadir) -> None: def test_backtest_dataprovider_analyzed_df(default_conf, fee, mocker, testdatadir) -> None: default_conf['use_exit_signal'] = False + default_conf['max_open_trades'] = 10 mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=100000) @@ -981,7 +984,6 @@ def test_backtest_dataprovider_analyzed_df(default_conf, fee, mocker, testdatadi processed=deepcopy(processed), start_date=min_date, end_date=max_date, - max_open_trades=10, ) assert count == 5 @@ -998,6 +1000,7 @@ def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatad default_conf['enable_protections'] = True default_conf['timeframe'] = '1m' + default_conf['max_open_trades'] = 1 mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=float('inf')) @@ -1024,7 +1027,6 @@ def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatad processed=processed, start_date=min_date, end_date=max_date, - max_open_trades=1, ) assert len(results['results']) == numres @@ -1062,11 +1064,12 @@ def test_backtest_pricecontours(default_conf, fee, mocker, testdatadir, processed = backtesting.strategy.advise_all_indicators(data) min_date, max_date = get_timerange(processed) assert isinstance(processed, dict) + backtesting.strategy.max_open_trades = 1 + backtesting.config.update({'max_open_trades': 1}) results = backtesting.backtest( processed=processed, start_date=min_date, end_date=max_date, - max_open_trades=1, ) assert len(results['results']) == expected @@ -1077,7 +1080,7 @@ def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir): buy_value = 1 sell_value = 1 return _trend(dataframe, buy_value, sell_value) - + default_conf['max_open_trades'] = 10 backtest_conf = _make_backtest_conf(mocker, conf=default_conf, datadir=testdatadir) backtesting = Backtesting(default_conf) backtesting._set_strategy(backtesting.strategylist[0]) @@ -1094,6 +1097,7 @@ def test_backtest_only_sell(mocker, default_conf, testdatadir): sell_value = 1 return _trend(dataframe, buy_value, sell_value) + default_conf['max_open_trades'] = 10 backtest_conf = _make_backtest_conf(mocker, conf=default_conf, datadir=testdatadir) backtesting = Backtesting(default_conf) backtesting._set_strategy(backtesting.strategylist[0]) @@ -1107,6 +1111,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=float('inf')) mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + default_conf['max_open_trades'] = 10 backtest_conf = _make_backtest_conf(mocker, conf=default_conf, pair='UNITTEST/BTC', datadir=testdatadir) default_conf['timeframe'] = '1m' @@ -1165,6 +1170,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) if tres > 0: data[pair] = data[pair][tres:].reset_index() default_conf['timeframe'] = '5m' + default_conf['max_open_trades'] = 3 backtesting = Backtesting(default_conf) backtesting._set_strategy(backtesting.strategylist[0]) @@ -1173,11 +1179,11 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) processed = backtesting.strategy.advise_all_indicators(data) min_date, max_date = get_timerange(processed) + backtest_conf = { 'processed': deepcopy(processed), 'start_date': min_date, 'end_date': max_date, - 'max_open_trades': 3, } results = backtesting.backtest(**backtest_conf) @@ -1195,11 +1201,12 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) backtesting.dataprovider.get_analyzed_dataframe('NXT/BTC', '5m')[0] ) == len(data['NXT/BTC']) - 1 - backtesting.strategy.startup_candle_count + backtesting.strategy.max_open_trades = 1 + backtesting.config.update({'max_open_trades': 1}) backtest_conf = { 'processed': deepcopy(processed), 'start_date': min_date, 'end_date': max_date, - 'max_open_trades': 1, } results = backtesting.backtest(**backtest_conf) assert len(evaluate_result_multi(results['results'], '5m', 1)) == 0 diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 5c740458f..23b5eb93b 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -17,6 +17,7 @@ from tests.conftest import patch_exchange def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> None: default_conf['use_exit_signal'] = False + default_conf['max_open_trades'] = 10 mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) mocker.patch('freqtrade.optimize.backtesting.amount_to_contract_precision', lambda x, *args, **kwargs: round(x, 8)) @@ -41,7 +42,6 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> processed=deepcopy(processed), start_date=min_date, end_date=max_date, - max_open_trades=10, ) results = result['results'] assert not results.empty diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 8b57883f7..d7ab20ee5 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 from datetime import datetime, timedelta +from functools import wraps from pathlib import Path from unittest.mock import ANY, MagicMock, PropertyMock @@ -336,8 +337,7 @@ def test_start_calls_optimizer(mocker, hyperopt_conf, capsys) -> None: assert dumper2.call_count == 1 assert hasattr(hyperopt.backtesting.strategy, "advise_exit") assert hasattr(hyperopt.backtesting.strategy, "advise_entry") - assert hasattr(hyperopt, "max_open_trades") - assert hyperopt.max_open_trades == hyperopt_conf['max_open_trades'] + assert hyperopt.backtesting.strategy.max_open_trades == hyperopt_conf['max_open_trades'] assert hasattr(hyperopt.backtesting, "_position_stacking") @@ -708,8 +708,7 @@ def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> Non assert hasattr(hyperopt.backtesting.strategy, "advise_exit") assert hasattr(hyperopt.backtesting.strategy, "advise_entry") - assert hasattr(hyperopt, "max_open_trades") - assert hyperopt.max_open_trades == hyperopt_conf['max_open_trades'] + assert hyperopt.backtesting.strategy.max_open_trades == hyperopt_conf['max_open_trades'] assert hasattr(hyperopt.backtesting, "_position_stacking") @@ -782,8 +781,7 @@ def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None: assert dumper2.call_count == 1 assert hasattr(hyperopt.backtesting.strategy, "advise_exit") assert hasattr(hyperopt.backtesting.strategy, "advise_entry") - assert hasattr(hyperopt, "max_open_trades") - assert hyperopt.max_open_trades == hyperopt_conf['max_open_trades'] + assert hyperopt.backtesting.strategy.max_open_trades == hyperopt_conf['max_open_trades'] assert hasattr(hyperopt.backtesting, "_position_stacking") @@ -825,8 +823,7 @@ def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: assert dumper2.call_count == 1 assert hasattr(hyperopt.backtesting.strategy, "advise_exit") assert hasattr(hyperopt.backtesting.strategy, "advise_entry") - assert hasattr(hyperopt, "max_open_trades") - assert hyperopt.max_open_trades == hyperopt_conf['max_open_trades'] + assert hyperopt.backtesting.strategy.max_open_trades == hyperopt_conf['max_open_trades'] assert hasattr(hyperopt.backtesting, "_position_stacking") @@ -880,7 +877,7 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: assert hyperopt.backtesting.strategy.buy_rsi.value == 35 assert hyperopt.backtesting.strategy.sell_rsi.value == 74 assert hyperopt.backtesting.strategy.protection_cooldown_lookback.value == 30 - assert hyperopt.max_open_trades == 1 + assert hyperopt.backtesting.strategy.max_open_trades == 1 buy_rsi_range = hyperopt.backtesting.strategy.buy_rsi.range assert isinstance(buy_rsi_range, range) # Range from 0 - 50 (inclusive) @@ -891,7 +888,7 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: assert hyperopt.backtesting.strategy.protection_cooldown_lookback.value != 30 assert hyperopt.backtesting.strategy.buy_rsi.value != 35 assert hyperopt.backtesting.strategy.sell_rsi.value != 74 - assert hyperopt.max_open_trades != 1 + assert hyperopt.backtesting.strategy.max_open_trades != 1 hyperopt.custom_hyperopt.generate_estimator = lambda *args, **kwargs: 'ET1' with pytest.raises(OperationalException, match="Estimator ET1 not supported."): @@ -992,3 +989,75 @@ def test_SKDecimal(): assert space.transform([2.0]) == [200] assert space.transform([1.0]) == [100] assert space.transform([1.5, 1.6]) == [150, 160] + + +def test_stake_amount_unlimited_max_open_trades(mocker, hyperopt_conf, tmpdir, fee) -> None: + # This test is to ensure that unlimited max_open_trades are ignored for the backtesting + # if we have an unlimited stake amount + patch_exchange(mocker) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + (Path(tmpdir) / 'hyperopt_results').mkdir(parents=True) + hyperopt_conf.update({ + 'strategy': 'HyperoptableStrategy', + 'user_data_dir': Path(tmpdir), + 'hyperopt_random_state': 42, + 'spaces': ['trades'], + 'stake_amount': 'unlimited' + }) + hyperopt = Hyperopt(hyperopt_conf) + mocker.patch('freqtrade.optimize.hyperopt.Hyperopt._get_params_dict', + return_value={ + 'max_open_trades': -1 + }) + + assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto) + + assert hyperopt.backtesting.strategy.max_open_trades == 1 + + hyperopt.start() + + assert hyperopt.backtesting.strategy.max_open_trades == 1 + + +def test_max_open_trades_consistency(mocker, hyperopt_conf, tmpdir, fee) -> None: + # This test is to ensure that max_open_trades is the same across all functions needing it + # after it has been changed from the hyperopt + patch_exchange(mocker) + mocker.patch('freqtrade.exchange.Exchange.get_fee', return_value=0) + + (Path(tmpdir) / 'hyperopt_results').mkdir(parents=True) + hyperopt_conf.update({ + 'strategy': 'HyperoptableStrategy', + 'user_data_dir': Path(tmpdir), + 'hyperopt_random_state': 42, + 'spaces': ['trades'], + 'stake_amount': 'unlimited', + 'dry_run_wallet': 8, + 'available_capital': 8, + 'dry_run': True, + 'epochs': 1 + }) + hyperopt = Hyperopt(hyperopt_conf) + + assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto) + + first_time_evaluated = False + + def stake_amount_interceptor(func): + @wraps(func) + def wrapper(*args, **kwargs): + nonlocal first_time_evaluated + stake_amount = func(*args, **kwargs) + if first_time_evaluated is False: + assert stake_amount == 1 + first_time_evaluated = True + return stake_amount + return wrapper + + hyperopt.backtesting.wallets._calculate_unlimited_stake_amount = stake_amount_interceptor( + hyperopt.backtesting.wallets._calculate_unlimited_stake_amount) + + hyperopt.start() + + assert hyperopt.backtesting.strategy.max_open_trades == 8 + assert hyperopt.config['max_open_trades'] == 8 From f77dffc951520bcee11910b7af55bd8c5282f93e Mon Sep 17 00:00:00 2001 From: Antonio Della Fortuna Date: Sun, 8 Jan 2023 12:46:27 +0100 Subject: [PATCH 05/10] align to develop --- freqtrade/freqtradebot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f34b02585..659eb2660 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -520,8 +520,7 @@ class FreqtradeBot(LoggingMixin): else: self.log_once(f"Pair {pair} is currently locked.", logger.info) return False - stake_amount = self.wallets.get_trade_stake_amount( - pair, self.edge) + stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge) bid_check_dom = self.config.get('entry_pricing', {}).get('check_depth_of_market', {}) if ((bid_check_dom.get('enabled', False)) and From 10d8b016e434234fe2ba3fca8a744d4b8cee2db8 Mon Sep 17 00:00:00 2001 From: Antonio Della Fortuna Date: Sun, 8 Jan 2023 12:48:36 +0100 Subject: [PATCH 06/10] Changed max_open_trades default range for optimization --- docs/advanced-hyperopt.md | 2 +- freqtrade/optimize/hyperopt_interface.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 0ef4911cd..c3f286135 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -127,7 +127,7 @@ class MyAwesomeStrategy(IStrategy): # Define a custom max_open_trades space def trades_space(self) -> List[Dimension]: return [ - Integer(1, 10, name='max_open_trades'), + Integer(-1, 10, name='max_open_trades'), ] ``` diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index b692abea4..25f65cf22 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -198,7 +198,7 @@ class IHyperOpt(ABC): You may override it in your custom Hyperopt class. """ return [ - Integer(1, 10, name='max_open_trades'), + Integer(-1, 10, name='max_open_trades'), ] # This is needed for proper unpickling the class attribute timeframe From 7d27afd4b8832a70a3887727a2b007aeb41e8565 Mon Sep 17 00:00:00 2001 From: Antonio Della Fortuna Date: Sun, 8 Jan 2023 16:11:41 +0100 Subject: [PATCH 07/10] Fixed test broken due to change in trades_space range --- tests/optimize/test_hyperopt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index d7ab20ee5..cf6faa0b9 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -8,6 +8,7 @@ import pandas as pd import pytest from arrow import Arrow from filelock import Timeout +from skopt.space import Integer from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_hyperopt from freqtrade.data.history import load_data @@ -1041,6 +1042,8 @@ def test_max_open_trades_consistency(mocker, hyperopt_conf, tmpdir, fee) -> None assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto) + hyperopt.custom_hyperopt.trades_space = lambda: [Integer(1, 10, name='max_open_trades')] + first_time_evaluated = False def stake_amount_interceptor(func): From b0f1d914c8b56fe0b23a3ff658d69bb274d8e503 Mon Sep 17 00:00:00 2001 From: Antonio Della Fortuna Date: Sun, 15 Jan 2023 11:44:10 +0100 Subject: [PATCH 08/10] Changed max_open_trades type to int or inf --- freqtrade/constants.py | 1 + freqtrade/data/btanalysis.py | 4 +-- freqtrade/optimize/backtesting.py | 4 +-- freqtrade/optimize/hyperopt.py | 16 ++++----- freqtrade/optimize/optimize_reports.py | 4 +-- freqtrade/resolvers/strategy_resolver.py | 8 ++--- freqtrade/rpc/api_server/api_schemas.py | 6 ++-- freqtrade/rpc/rpc.py | 1 + freqtrade/strategy/interface.py | 4 +-- tests/optimize/test_hyperopt.py | 46 ++++++++++++++++++++++++ tests/strategy/test_strategy_loading.py | 33 ++++++++++++++--- 11 files changed, 97 insertions(+), 30 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 95efa63b8..b41a3ad9c 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -681,3 +681,4 @@ MakerTaker = Literal['maker', 'taker'] BidAsk = Literal['bid', 'ask'] Config = Dict[str, Any] +IntOrInf = float diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 3102683b2..0dcd05646 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -10,7 +10,7 @@ from typing import Any, Dict, List, Optional, Union import numpy as np import pandas as pd -from freqtrade.constants import LAST_BT_RESULT_FN +from freqtrade.constants import LAST_BT_RESULT_FN, IntOrInf from freqtrade.exceptions import OperationalException from freqtrade.misc import json_load from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename @@ -332,7 +332,7 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF def evaluate_result_multi(results: pd.DataFrame, timeframe: str, - max_open_trades: int) -> pd.DataFrame: + max_open_trades: IntOrInf) -> pd.DataFrame: """ Find overlapping trades by expanding each trade once per period it was open and then counting overlaps diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d92f834df..3d560fd2a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1237,8 +1237,8 @@ class Backtesting: if not self.config.get('use_max_market_positions', True): logger.info( 'Ignoring max_open_trades (--disable-max-market-positions was used) ...') - self.strategy.max_open_trades = -1 - self.config.update({'max_open_trades': float('inf')}) + self.strategy.max_open_trades = float('inf') + self.config.update({'max_open_trades': self.strategy.max_open_trades}) # need to reprocess data every time to populate signals preprocessed = self.strategy.advise_all_indicators(data) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 595226fca..2e0dbdd65 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -120,8 +120,8 @@ class Hyperopt: # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set if not self.config.get('use_max_market_positions', True): logger.debug('Ignoring max_open_trades (--disable-max-market-positions was used) ...') - self.backtesting.strategy.max_open_trades = -1 - config.update({'max_open_trades': float('inf')}) + self.backtesting.strategy.max_open_trades = float('inf') + config.update({'max_open_trades': self.backtesting.strategy.max_open_trades}) if HyperoptTools.has_space(self.config, 'sell'): # Make sure use_exit_signal is enabled @@ -211,7 +211,8 @@ class Hyperopt: result['trailing'] = self.custom_hyperopt.generate_trailing_params(params) if HyperoptTools.has_space(self.config, 'trades'): result['max_open_trades'] = { - 'max_open_trades': self.backtesting.strategy.max_open_trades} + 'max_open_trades': self.backtesting.strategy.max_open_trades + if self.backtesting.strategy.max_open_trades != float('inf') else -1} return result @@ -344,16 +345,13 @@ class Hyperopt: # Ignore unlimited max open trades if stake amount is unlimited params_dict.update({'max_open_trades': self.config['max_open_trades']}) - updated_config_max_open_trades = int(params_dict['max_open_trades']) \ + updated_max_open_trades = int(params_dict['max_open_trades']) \ if (params_dict['max_open_trades'] != -1 and params_dict['max_open_trades'] != 0) else float('inf') - updated_strategy_max_open_trades = int(updated_config_max_open_trades) \ - if updated_config_max_open_trades != float('inf') else -1 + self.config.update({'max_open_trades': updated_max_open_trades}) - self.config.update({'max_open_trades': updated_config_max_open_trades}) - - self.backtesting.strategy.max_open_trades = updated_strategy_max_open_trades + self.backtesting.strategy.max_open_trades = updated_max_open_trades with self.data_pickle_file.open('rb') as f: processed = load(f, mmap_mode='r') diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 7de8f1a47..83f698fbe 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -8,7 +8,7 @@ from pandas import DataFrame, to_datetime from tabulate import tabulate from freqtrade.constants import (DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT, - Config) + Config, IntOrInf) from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum, calculate_expectancy, calculate_market_change, calculate_max_drawdown, calculate_sharpe, calculate_sortino) @@ -191,7 +191,7 @@ def generate_tag_metrics(tag_type: str, return [] -def generate_exit_reason_stats(max_open_trades: int, results: DataFrame) -> List[Dict]: +def generate_exit_reason_stats(max_open_trades: IntOrInf, results: DataFrame) -> List[Dict]: """ Generate small table outlining Backtest results :param max_open_trades: Max_open_trades parameter diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index febda7822..e82aa7ac9 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -104,11 +104,7 @@ class StrategyResolver(IResolver): if (attribute in config and not isinstance(getattr(type(strategy), attribute, None), property)): # Ensure Properties are not overwritten - val = config[attribute] - # max_open_trades set to float('inf') in the config will be copied as -1 in the strategy - if attribute == 'max_open_trades' and val == float('inf'): - val = -1 - setattr(strategy, attribute, val) + setattr(strategy, attribute, config[attribute]) logger.info("Override strategy '%s' with value in config file: %s.", attribute, config[attribute]) elif hasattr(strategy, attribute): @@ -137,6 +133,8 @@ class StrategyResolver(IResolver): key=lambda t: t[0])) if hasattr(strategy, 'stoploss'): strategy.stoploss = float(strategy.stoploss) + if hasattr(strategy, 'max_open_trades') and strategy.max_open_trades < 0: + strategy.max_open_trades = float('inf') return strategy @staticmethod diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 404d64d16..d96055b69 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel -from freqtrade.constants import DATETIME_PRINT_FORMAT +from freqtrade.constants import DATETIME_PRINT_FORMAT, IntOrInf from freqtrade.enums import OrderTypeValues, SignalDirection, TradingMode @@ -165,7 +165,7 @@ class ShowConfig(BaseModel): stake_amount: str available_capital: Optional[float] stake_currency_decimals: int - max_open_trades: int + max_open_trades: IntOrInf minimal_roi: Dict[str, Any] stoploss: Optional[float] trailing_stop: Optional[bool] @@ -422,7 +422,7 @@ class BacktestRequest(BaseModel): timeframe: Optional[str] timeframe_detail: Optional[str] timerange: Optional[str] - max_open_trades: Optional[int] + max_open_trades: Optional[IntOrInf] stake_amount: Optional[str] enable_protections: bool dry_run_wallet: Optional[float] diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index ed905d844..32563376c 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -673,6 +673,7 @@ class RPC: if self._freqtrade.state == State.RUNNING: # Set 'max_open_trades' to 0 self._freqtrade.config['max_open_trades'] = 0 + self._freqtrade.strategy.max_open_trades = 0 return {'status': 'No more entries will occur from now. Run /reload_config to reset.'} diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 4d4e4f1ba..e6aed5c5a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -10,7 +10,7 @@ from typing import Dict, List, Optional, Tuple, Union import arrow from pandas import DataFrame -from freqtrade.constants import Config, ListPairsWithTimeframes +from freqtrade.constants import Config, IntOrInf, ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, RunMode, SignalDirection, SignalTagType, SignalType, TradingMode) @@ -55,7 +55,7 @@ class IStrategy(ABC, HyperStrategyMixin): stoploss: float # max open trades for the strategy - max_open_trades: int + max_open_trades: IntOrInf # trailing stoploss trailing_stop: bool = False diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index cf6faa0b9..4a8455fa7 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1020,6 +1020,52 @@ def test_stake_amount_unlimited_max_open_trades(mocker, hyperopt_conf, tmpdir, f assert hyperopt.backtesting.strategy.max_open_trades == 1 +def test_max_open_trades_dump(mocker, hyperopt_conf, tmpdir, fee, capsys) -> None: + # This test is to ensure that after hyperopting, max_open_trades is never + # saved as inf in the output json params + patch_exchange(mocker) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + (Path(tmpdir) / 'hyperopt_results').mkdir(parents=True) + hyperopt_conf.update({ + 'strategy': 'HyperoptableStrategy', + 'user_data_dir': Path(tmpdir), + 'hyperopt_random_state': 42, + 'spaces': ['trades'], + }) + hyperopt = Hyperopt(hyperopt_conf) + mocker.patch('freqtrade.optimize.hyperopt.Hyperopt._get_params_dict', + return_value={ + 'max_open_trades': -1 + }) + + assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto) + + hyperopt.start() + + out, err = capsys.readouterr() + + assert 'max_open_trades = -1' in out + assert 'max_open_trades = inf' not in out + + ############## + + hyperopt_conf.update({'print_json': True}) + + hyperopt = Hyperopt(hyperopt_conf) + mocker.patch('freqtrade.optimize.hyperopt.Hyperopt._get_params_dict', + return_value={ + 'max_open_trades': -1 + }) + + assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto) + + hyperopt.start() + + out, err = capsys.readouterr() + + assert '"max_open_trades":-1' in out + + def test_max_open_trades_consistency(mocker, hyperopt_conf, tmpdir, fee) -> None: # This test is to ensure that max_open_trades is the same across all functions needing it # after it has been changed from the hyperopt diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 2296d4bc6..d60d3ade9 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -6,6 +6,7 @@ from pathlib import Path import pytest from pandas import DataFrame +from freqtrade.configuration.configuration import Configuration from freqtrade.exceptions import OperationalException from freqtrade.resolvers import StrategyResolver from freqtrade.strategy.interface import IStrategy @@ -371,20 +372,26 @@ def test_strategy_max_open_trades_infinity_from_strategy(caplog, default_conf): strategy = StrategyResolver.load_strategy(default_conf) # this test assumes -1 set to 'max_open_trades' in CURRENT_TEST_STRATEGY - assert strategy.max_open_trades == -1 + assert strategy.max_open_trades == float('inf') assert default_conf['max_open_trades'] == float('inf') -def test_strategy_max_open_trades_infinity_from_config(caplog, default_conf): +def test_strategy_max_open_trades_infinity_from_config(caplog, default_conf, mocker): caplog.set_level(logging.INFO) default_conf.update({ 'strategy': CURRENT_TEST_STRATEGY, - 'max_open_trades': float('inf') + 'max_open_trades': -1, + 'exchange': 'binance' }) - strategy = StrategyResolver.load_strategy(default_conf) + configuration = Configuration(args=default_conf) + parsed_config = configuration.get_config() - assert strategy.max_open_trades == -1 + assert parsed_config['max_open_trades'] == float('inf') + + strategy = StrategyResolver.load_strategy(parsed_config) + + assert strategy.max_open_trades == float('inf') @ pytest.mark.filterwarnings("ignore:deprecated") @@ -476,3 +483,19 @@ def test_strategy_interface_versioning(dataframe_1m, default_conf): assert isinstance(exitdf, DataFrame) assert 'sell' not in exitdf assert 'exit_long' in exitdf + + +def test_strategy_ft_load_params_from_file(mocker, default_conf): + default_conf.update({'strategy': 'StrategyTestV2'}) + del default_conf['max_open_trades'] + mocker.patch('freqtrade.strategy.hyper.HyperStrategyMixin.load_params_from_file', + return_value={ + 'params': { + 'max_open_trades': { + 'max_open_trades': -1 + } + } + }) + strategy = StrategyResolver.load_strategy(default_conf) + assert strategy.max_open_trades == float('inf') + assert strategy.config['max_open_trades'] == float('inf') From ab12aace5f97d363d35e8fbbe859117a9a1849bc Mon Sep 17 00:00:00 2001 From: Antonio Della Fortuna Date: Sun, 15 Jan 2023 11:50:40 +0100 Subject: [PATCH 09/10] changed `trades_space` to `max_open_trades_space` --- docs/advanced-hyperopt.md | 4 ++-- docs/hyperopt.md | 2 +- freqtrade/optimize/hyperopt.py | 6 +++--- freqtrade/optimize/hyperopt_auto.py | 4 ++-- freqtrade/optimize/hyperopt_interface.py | 2 +- tests/optimize/test_hyperopt.py | 3 ++- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index c3f286135..ff0521f4f 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -75,7 +75,7 @@ This function needs to return a floating point number (`float`). Smaller numbers ## Overriding pre-defined spaces -To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_space`, `trailing_space`, `trades_space`), define a nested class called Hyperopt and define the required spaces as follows: +To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_space`, `trailing_space`, `max_open_trades_space`), define a nested class called Hyperopt and define the required spaces as follows: ```python from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal @@ -125,7 +125,7 @@ class MyAwesomeStrategy(IStrategy): ] # Define a custom max_open_trades space - def trades_space(self) -> List[Dimension]: + def max_open_trades_space(self) -> List[Dimension]: return [ Integer(-1, 10, name='max_open_trades'), ] diff --git a/docs/hyperopt.md b/docs/hyperopt.md index c105dceed..19bffd742 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -180,7 +180,7 @@ Rarely you may also need to create a [nested class](advanced-hyperopt.md#overrid * `generate_roi_table` - for custom ROI optimization (if you need the ranges for the values in the ROI table that differ from default or the number of entries (steps) in the ROI table which differs from the default 4 steps) * `stoploss_space` - for custom stoploss optimization (if you need the range for the stoploss parameter in the optimization hyperspace that differs from default) * `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default) -* `trades_space` - for custom max_open_trades optimization (if you need the ranges for the max_open_trades parameter in the optimization hyperspace that differ from default) +* `max_open_trades_space` - for custom max_open_trades optimization (if you need the ranges for the max_open_trades parameter in the optimization hyperspace that differ from default) !!! Tip "Quickly optimize ROI, stoploss and trailing stoploss" You can quickly optimize the spaces `roi`, `stoploss` and `trailing` without changing anything in your strategy. diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2e0dbdd65..96c95c4a2 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -74,7 +74,7 @@ class Hyperopt: self.roi_space: List[Dimension] = [] self.stoploss_space: List[Dimension] = [] self.trailing_space: List[Dimension] = [] - self.trades_space: List[Dimension] = [] + self.max_open_trades_space: List[Dimension] = [] self.dimensions: List[Dimension] = [] self.config = config @@ -288,11 +288,11 @@ class Hyperopt: if HyperoptTools.has_space(self.config, 'trades'): logger.debug("Hyperopt has 'trades' space") - self.trades_space = self.custom_hyperopt.trades_space() + self.max_open_trades_space = self.custom_hyperopt.max_open_trades_space() self.dimensions = (self.buy_space + self.sell_space + self.protection_space + self.roi_space + self.stoploss_space + self.trailing_space - + self.trades_space) + + self.max_open_trades_space) def assign_params(self, params_dict: Dict, category: str) -> None: """ diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py index 0627b52c3..13c036a28 100644 --- a/freqtrade/optimize/hyperopt_auto.py +++ b/freqtrade/optimize/hyperopt_auto.py @@ -91,8 +91,8 @@ class HyperOptAuto(IHyperOpt): def trailing_space(self) -> List['Dimension']: return self._get_func('trailing_space')() - def trades_space(self) -> List['Dimension']: - return self._get_func('trades_space')() + def max_open_trades_space(self) -> List['Dimension']: + return self._get_func('max_open_trades_space')() def generate_estimator(self, dimensions: List['Dimension'], **kwargs) -> EstimatorType: return self._get_func('generate_estimator')(dimensions=dimensions, **kwargs) diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 25f65cf22..65dd7ed87 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -191,7 +191,7 @@ class IHyperOpt(ABC): Categorical([True, False], name='trailing_only_offset_is_reached'), ] - def trades_space(self) -> List[Dimension]: + def max_open_trades_space(self) -> List[Dimension]: """ Create a max open trades space. diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 4a8455fa7..36ceaeab2 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1088,7 +1088,8 @@ def test_max_open_trades_consistency(mocker, hyperopt_conf, tmpdir, fee) -> None assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto) - hyperopt.custom_hyperopt.trades_space = lambda: [Integer(1, 10, name='max_open_trades')] + hyperopt.custom_hyperopt.max_open_trades_space = lambda: [ + Integer(1, 10, name='max_open_trades')] first_time_evaluated = False From c8ecedf6d56f8f3b22669401f0c3224aa26938d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Jan 2023 20:05:18 +0100 Subject: [PATCH 10/10] Clarify a variable via typehint --- freqtrade/optimize/backtesting.py | 4 ++-- tests/strategy/test_strategy_loading.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a01884ae0..2391605bc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -15,7 +15,7 @@ from pandas import DataFrame from freqtrade import constants from freqtrade.configuration import TimeRange, validate_config_consistency -from freqtrade.constants import DATETIME_PRINT_FORMAT, Config, LongShort +from freqtrade.constants import DATETIME_PRINT_FORMAT, Config, IntOrInf, LongShort from freqtrade.data import history from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe from freqtrade.data.converter import trim_dataframe, trim_dataframes @@ -924,7 +924,7 @@ class Backtesting: def trade_slot_available(self, open_trade_count: int) -> bool: # Always allow trades when max_open_trades is enabled. - max_open_trades = self.config['max_open_trades'] + max_open_trades: IntOrInf = self.config['max_open_trades'] if max_open_trades <= 0 or open_trade_count < max_open_trades: return True # Rejected trade diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index d60d3ade9..98185e152 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -6,7 +6,7 @@ from pathlib import Path import pytest from pandas import DataFrame -from freqtrade.configuration.configuration import Configuration +from freqtrade.configuration import Configuration from freqtrade.exceptions import OperationalException from freqtrade.resolvers import StrategyResolver from freqtrade.strategy.interface import IStrategy