From d96e842a2169aa60aa69a4e951e543142f3107fb Mon Sep 17 00:00:00 2001 From: orehunt Date: Mon, 24 Feb 2020 13:31:46 +0100 Subject: [PATCH] added effort as new argument --- freqtrade/commands/arguments.py | 132 ++++---- freqtrade/commands/cli_options.py | 303 +++++++++++------- freqtrade/constants.py | 406 +++++++++++++++++++------ freqtrade/optimize/hyperopt.py | 146 +++++---- freqtrade/optimize/hyperopt_backend.py | 3 +- 5 files changed, 660 insertions(+), 330 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 73e77d69d..d4866bd8c 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -15,18 +15,17 @@ ARGS_STRATEGY = ["strategy", "strategy_path"] ARGS_TRADE = ["db_url", "sd_notify", "dry_run"] -ARGS_COMMON_OPTIMIZE = ["ticker_interval", "timerange", - "max_open_trades", "stake_amount", "fee"] +ARGS_COMMON_OPTIMIZE = ["ticker_interval", "timerange", "max_open_trades", "stake_amount", "fee"] -ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", - "strategy_list", "export", "exportfilename"] +ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + [ + "position_stacking", "use_max_market_positions", "strategy_list", "export", "exportfilename" +] -ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", - "position_stacking", "epochs", "spaces", - "use_max_market_positions", "print_all", - "print_colorized", "print_json", "hyperopt_jobs", - "hyperopt_random_state", "hyperopt_min_trades", - "hyperopt_continue", "hyperopt_loss"] +ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + [ + "hyperopt", "hyperopt_path", "position_stacking", "epochs", "spaces", + "use_max_market_positions", "print_all", "print_colorized", "print_json", "hyperopt_jobs", + "hyperopt_random_state", "hyperopt_min_trades", "hyperopt_continue", "hyperopt_loss", "effort" +] ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] @@ -38,8 +37,10 @@ ARGS_LIST_EXCHANGES = ["print_one_column", "list_exchanges_all"] ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"] -ARGS_LIST_PAIRS = ["exchange", "print_list", "list_pairs_print_json", "print_one_column", - "print_csv", "base_currencies", "quote_currencies", "list_pairs_all"] +ARGS_LIST_PAIRS = [ + "exchange", "print_list", "list_pairs_print_json", "print_one_column", "print_csv", + "base_currencies", "quote_currencies", "list_pairs_all" +] ARGS_TEST_PAIRLIST = ["config", "quote_currencies", "print_one_column", "list_pairs_print_json"] @@ -54,30 +55,38 @@ ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] -ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", - "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"] +ARGS_DOWNLOAD_DATA = [ + "pairs", "pairs_file", "days", "download_trades", "exchange", "timeframes", "erase", + "dataformat_ohlcv", "dataformat_trades" +] -ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", - "db_url", "trade_source", "export", "exportfilename", - "timerange", "ticker_interval"] +ARGS_PLOT_DATAFRAME = [ + "pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", + "exportfilename", "timerange", "ticker_interval" +] -ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", - "trade_source", "ticker_interval"] +ARGS_PLOT_PROFIT = [ + "pairs", "timerange", "export", "exportfilename", "db_url", "trade_source", "ticker_interval" +] -ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", - "hyperopt_list_min_trades", "hyperopt_list_max_trades", - "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time", - "hyperopt_list_min_avg_profit", "hyperopt_list_max_avg_profit", - "hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit", - "print_colorized", "print_json", "hyperopt_list_no_details"] +ARGS_HYPEROPT_LIST = [ + "hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_list_min_trades", + "hyperopt_list_max_trades", "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time", + "hyperopt_list_min_avg_profit", "hyperopt_list_max_avg_profit", + "hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit", "print_colorized", + "print_json", "hyperopt_list_no_details" +] -ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", - "print_json", "hyperopt_show_no_header"] +ARGS_HYPEROPT_SHOW = [ + "hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", "print_json", + "hyperopt_show_no_header" +] -NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", - "list-markets", "list-pairs", "list-strategies", - "list-hyperopts", "hyperopt-list", "hyperopt-show", - "plot-dataframe", "plot-profit"] +NO_CONF_REQURIED = [ + "convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", + "list-pairs", "list-strategies", "list-hyperopts", "hyperopt-list", "hyperopt-show", + "plot-dataframe", "plot-profit" +] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"] @@ -86,7 +95,6 @@ class Arguments: """ Arguments Class. Manage the arguments received by the cli """ - def __init__(self, args: Optional[List[str]]) -> None: self.args = args self._parsed_arg: Optional[argparse.Namespace] = None @@ -155,70 +163,70 @@ class Arguments: self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot') self._build_args(optionlist=['version'], parser=self.parser) - from freqtrade.commands import (start_create_userdir, start_convert_data, - start_download_data, - start_hyperopt_list, start_hyperopt_show, - start_list_exchanges, start_list_hyperopts, - start_list_markets, start_list_strategies, - start_list_timeframes, start_new_config, - start_new_hyperopt, start_new_strategy, - start_plot_dataframe, start_plot_profit, - start_backtesting, start_hyperopt, start_edge, - start_test_pairlist, start_trading) + from freqtrade.commands import ( + start_create_userdir, start_convert_data, start_download_data, start_hyperopt_list, + start_hyperopt_show, start_list_exchanges, start_list_hyperopts, start_list_markets, + start_list_strategies, start_list_timeframes, start_new_config, start_new_hyperopt, + start_new_strategy, start_plot_dataframe, start_plot_profit, start_backtesting, + start_hyperopt, start_edge, start_test_pairlist, start_trading) - subparsers = self.parser.add_subparsers(dest='command', - # Use custom message when no subhandler is added - # shown from `main.py` - # required=True - ) + subparsers = self.parser.add_subparsers( + dest='command', + # Use custom message when no subhandler is added + # shown from `main.py` + # required=True + ) # Add trade subcommand - trade_cmd = subparsers.add_parser('trade', help='Trade module.', + trade_cmd = subparsers.add_parser('trade', + help='Trade module.', parents=[_common_parser, _strategy_parser]) trade_cmd.set_defaults(func=start_trading) self._build_args(optionlist=ARGS_TRADE, parser=trade_cmd) # Add backtesting subcommand - backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.', + backtesting_cmd = subparsers.add_parser('backtesting', + help='Backtesting module.', parents=[_common_parser, _strategy_parser]) backtesting_cmd.set_defaults(func=start_backtesting) self._build_args(optionlist=ARGS_BACKTEST, parser=backtesting_cmd) # Add edge subcommand - edge_cmd = subparsers.add_parser('edge', help='Edge module.', + edge_cmd = subparsers.add_parser('edge', + help='Edge module.', parents=[_common_parser, _strategy_parser]) edge_cmd.set_defaults(func=start_edge) self._build_args(optionlist=ARGS_EDGE, parser=edge_cmd) # Add hyperopt subcommand - hyperopt_cmd = subparsers.add_parser('hyperopt', help='Hyperopt module.', - parents=[_common_parser, _strategy_parser], - ) + hyperopt_cmd = subparsers.add_parser( + 'hyperopt', + help='Hyperopt module.', + parents=[_common_parser, _strategy_parser], + ) hyperopt_cmd.set_defaults(func=start_hyperopt) self._build_args(optionlist=ARGS_HYPEROPT, parser=hyperopt_cmd) # add create-userdir subcommand - create_userdir_cmd = subparsers.add_parser('create-userdir', - help="Create user-data directory.", - ) + create_userdir_cmd = subparsers.add_parser( + 'create-userdir', + help="Create user-data directory.", + ) create_userdir_cmd.set_defaults(func=start_create_userdir) self._build_args(optionlist=ARGS_CREATE_USERDIR, parser=create_userdir_cmd) # add new-config subcommand - build_config_cmd = subparsers.add_parser('new-config', - help="Create new config") + build_config_cmd = subparsers.add_parser('new-config', help="Create new config") build_config_cmd.set_defaults(func=start_new_config) self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd) # add new-strategy subcommand - build_strategy_cmd = subparsers.add_parser('new-strategy', - help="Create new strategy") + build_strategy_cmd = subparsers.add_parser('new-strategy', help="Create new strategy") build_strategy_cmd.set_defaults(func=start_new_strategy) self._build_args(optionlist=ARGS_BUILD_STRATEGY, parser=build_strategy_cmd) # add new-hyperopt subcommand - build_hyperopt_cmd = subparsers.add_parser('new-hyperopt', - help="Create new hyperopt") + build_hyperopt_cmd = subparsers.add_parser('new-hyperopt', help="Create new hyperopt") build_hyperopt_cmd.set_defaults(func=start_new_hyperopt) self._build_args(optionlist=ARGS_BUILD_HYPEROPT, parser=build_hyperopt_cmd) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index a8d4bc198..c5a4c10d5 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -13,8 +13,7 @@ def check_int_positive(value: str) -> int: raise ValueError except ValueError: raise ArgumentTypeError( - f"{value} is invalid for this parameter, should be a positive integer value" - ) + f"{value} is invalid for this parameter, should be a positive integer value") return uint @@ -25,8 +24,7 @@ def check_int_nonzero(value: str) -> int: raise ValueError except ValueError: raise ArgumentTypeError( - f"{value} is invalid for this parameter, should be a non-zero integer value" - ) + f"{value} is invalid for this parameter, should be a non-zero integer value") return uint @@ -40,25 +38,32 @@ class Arg: # List of available command line options AVAILABLE_CLI_OPTIONS = { # Common options - "verbosity": Arg( - '-v', '--verbose', + "verbosity": + Arg( + '-v', + '--verbose', help='Verbose mode (-vv for more, -vvv to get all messages).', action='count', default=0, ), - "logfile": Arg( + "logfile": + Arg( '--logfile', help="Log to the file specified. Special values are: 'syslog', 'journald'. " - "See the documentation for more details.", + "See the documentation for more details.", metavar='FILE', ), - "version": Arg( - '-V', '--version', + "version": + Arg( + '-V', + '--version', action='version', version=f'%(prog)s {__version__}', ), - "config": Arg( - '-c', '--config', + "config": + Arg( + '-c', + '--config', help=f'Specify configuration file (default: `userdir/{constants.DEFAULT_CONFIG}` ' f'or `config.json` whichever exists). ' f'Multiple --config options may be used. ' @@ -66,84 +71,105 @@ AVAILABLE_CLI_OPTIONS = { action='append', metavar='PATH', ), - "datadir": Arg( - '-d', '--datadir', + "datadir": + Arg( + '-d', + '--datadir', help='Path to directory with historical backtesting data.', metavar='PATH', ), - "user_data_dir": Arg( - '--userdir', '--user-data-dir', + "user_data_dir": + Arg( + '--userdir', + '--user-data-dir', help='Path to userdata directory.', metavar='PATH', ), - "reset": Arg( + "reset": + Arg( '--reset', help='Reset sample files to their original state.', action='store_true', ), # Main options - "strategy": Arg( - '-s', '--strategy', + "strategy": + Arg( + '-s', + '--strategy', help='Specify strategy class name which will be used by the bot.', metavar='NAME', ), - "strategy_path": Arg( + "strategy_path": + Arg( '--strategy-path', help='Specify additional strategy lookup path.', metavar='PATH', ), - "db_url": Arg( + "db_url": + Arg( '--db-url', help=f'Override trades database URL, this is useful in custom deployments ' f'(default: `{constants.DEFAULT_DB_PROD_URL}` for Live Run mode, ' f'`{constants.DEFAULT_DB_DRYRUN_URL}` for Dry Run).', metavar='PATH', ), - "sd_notify": Arg( + "sd_notify": + Arg( '--sd-notify', help='Notify systemd service manager.', action='store_true', ), - "dry_run": Arg( + "dry_run": + Arg( '--dry-run', help='Enforce dry-run for trading (removes Exchange secrets and simulates trades).', action='store_true', ), # Optimize common - "ticker_interval": Arg( - '-i', '--ticker-interval', + "ticker_interval": + Arg( + '-i', + '--ticker-interval', help='Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`).', ), - "timerange": Arg( + "timerange": + Arg( '--timerange', help='Specify what timerange of data to use.', ), - "max_open_trades": Arg( + "max_open_trades": + Arg( '--max-open-trades', help='Override the value of the `max_open_trades` configuration setting.', type=int, metavar='INT', ), - "stake_amount": Arg( + "stake_amount": + Arg( '--stake-amount', help='Override the value of the `stake_amount` configuration setting.', type=float, ), # Backtesting - "position_stacking": Arg( - '--eps', '--enable-position-stacking', + "position_stacking": + Arg( + '--eps', + '--enable-position-stacking', help='Allow buying the same pair multiple times (position stacking).', action='store_true', default=False, ), - "use_max_market_positions": Arg( - '--dmmp', '--disable-max-market-positions', + "use_max_market_positions": + Arg( + '--dmmp', + '--disable-max-market-positions', help='Disable applying `max_open_trades` during backtest ' '(same as setting `max_open_trades` to a very high number).', action='store_false', default=True, ), - "strategy_list": Arg( + "strategy_list": + Arg( '--strategy-list', help='Provide a space-separated list of strategies to backtest. ' 'Please note that ticker-interval needs to be set either in config ' @@ -152,77 +178,100 @@ AVAILABLE_CLI_OPTIONS = { '(so `backtest-data.json` becomes `backtest-data-DefaultStrategy.json`', nargs='+', ), - "export": Arg( + "export": + Arg( '--export', help='Export backtest results, argument are: trades. ' 'Example: `--export=trades`', ), - "exportfilename": Arg( + "exportfilename": + Arg( '--export-filename', help='Save backtest results to the file with this filename. ' 'Requires `--export` to be set as well. ' 'Example: `--export-filename=user_data/backtest_results/backtest_today.json`', metavar='PATH', ), - "fee": Arg( + "fee": + Arg( '--fee', help='Specify fee ratio. Will be applied twice (on trade entry and exit).', type=float, metavar='FLOAT', ), # Edge - "stoploss_range": Arg( + "stoploss_range": + Arg( '--stoplosses', help='Defines a range of stoploss values against which edge will assess the strategy. ' 'The format is "min,max,step" (without any space). ' 'Example: `--stoplosses=-0.01,-0.1,-0.001`', ), # Hyperopt - "hyperopt": Arg( + "hyperopt": + Arg( '--hyperopt', help='Specify hyperopt class name which will be used by the bot.', metavar='NAME', ), - "hyperopt_path": Arg( + "hyperopt_path": + Arg( '--hyperopt-path', help='Specify additional lookup path for Hyperopt and Hyperopt Loss functions.', metavar='PATH', ), - "epochs": Arg( - '-e', '--epochs', + "epochs": + Arg( + '-e', + '--epochs', help='Specify number of epochs (default: %(default)d).', type=check_int_positive, metavar='INT', default=constants.HYPEROPT_EPOCH, ), - "spaces": Arg( + "effort": + Arg( + '--effort', + help=('The higher the number, the longer will be the search if' + 'no epochs are defined (default: %(default)d).'), + type=check_int_positive, + metavar='INT', + default=constants.HYPEROPT_EFFORT, + ), + "spaces": + Arg( '--spaces', help='Specify which parameters to hyperopt. Space-separated list.', choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'default'], nargs='+', default='default', ), - "print_all": Arg( + "print_all": + Arg( '--print-all', help='Print all results, not only the best ones.', action='store_true', default=False, ), - "print_colorized": Arg( + "print_colorized": + Arg( '--no-color', help='Disable colorization of hyperopt results. May be useful if you are ' 'redirecting output to a file.', action='store_false', default=True, ), - "print_json": Arg( + "print_json": + Arg( '--print-json', help='Print best result detailization in JSON format.', action='store_true', default=False, ), - "hyperopt_jobs": Arg( - '-j', '--job-workers', + "hyperopt_jobs": + Arg( + '-j', + '--job-workers', help='The number of concurrently running jobs for hyperoptimization ' '(hyperopt worker processes). ' 'If -1 (default), all CPUs are used, for -2, all CPUs but one are used, etc. ' @@ -231,13 +280,15 @@ AVAILABLE_CLI_OPTIONS = { metavar='JOBS', default=-1, ), - "hyperopt_random_state": Arg( + "hyperopt_random_state": + Arg( '--random-state', help='Set random state to some positive integer for reproducible hyperopt results.', type=check_int_positive, metavar='INT', ), - "hyperopt_min_trades": Arg( + "hyperopt_min_trades": + Arg( '--min-trades', help="Set minimal desired number of trades for evaluations in the hyperopt " "optimization path (default: 1).", @@ -245,14 +296,16 @@ AVAILABLE_CLI_OPTIONS = { metavar='INT', default=1, ), - "hyperopt_continue": Arg( + "hyperopt_continue": + Arg( "--continue", help="Continue hyperopt from previous runs. " "By default, temporary files will be removed and hyperopt will start from scratch.", default=False, action='store_true', ), - "hyperopt_loss": Arg( + "hyperopt_loss": + Arg( '--hyperopt-loss', help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). ' 'Different functions can generate completely different results, ' @@ -263,121 +316,143 @@ AVAILABLE_CLI_OPTIONS = { default=constants.DEFAULT_HYPEROPT_LOSS, ), # List exchanges - "print_one_column": Arg( - '-1', '--one-column', + "print_one_column": + Arg( + '-1', + '--one-column', help='Print output in one column.', action='store_true', ), - "list_exchanges_all": Arg( - '-a', '--all', + "list_exchanges_all": + Arg( + '-a', + '--all', help='Print all exchanges known to the ccxt library.', action='store_true', ), # List pairs / markets - "list_pairs_all": Arg( - '-a', '--all', + "list_pairs_all": + Arg( + '-a', + '--all', help='Print all pairs or market symbols. By default only active ' - 'ones are shown.', + 'ones are shown.', action='store_true', ), - "print_list": Arg( + "print_list": + Arg( '--print-list', help='Print list of pairs or market symbols. By default data is ' - 'printed in the tabular format.', + 'printed in the tabular format.', action='store_true', ), - "list_pairs_print_json": Arg( + "list_pairs_print_json": + Arg( '--print-json', help='Print list of pairs or market symbols in JSON format.', action='store_true', default=False, ), - "print_csv": Arg( + "print_csv": + Arg( '--print-csv', help='Print exchange pair or market data in the csv format.', action='store_true', ), - "quote_currencies": Arg( + "quote_currencies": + Arg( '--quote', help='Specify quote currency(-ies). Space-separated list.', nargs='+', metavar='QUOTE_CURRENCY', ), - "base_currencies": Arg( + "base_currencies": + Arg( '--base', help='Specify base currency(-ies). Space-separated list.', nargs='+', metavar='BASE_CURRENCY', ), # Script options - "pairs": Arg( - '-p', '--pairs', + "pairs": + Arg( + '-p', + '--pairs', help='Show profits for only these pairs. Pairs are space-separated.', nargs='+', ), # Download data - "pairs_file": Arg( + "pairs_file": + Arg( '--pairs-file', help='File containing a list of pairs to download.', metavar='FILE', ), - "days": Arg( + "days": + Arg( '--days', help='Download data for given number of days.', type=check_int_positive, metavar='INT', ), - "download_trades": Arg( + "download_trades": + Arg( '--dl-trades', help='Download trades instead of OHLCV data. The bot will resample trades to the ' - 'desired timeframe as specified as --timeframes/-t.', + 'desired timeframe as specified as --timeframes/-t.', action='store_true', ), - "format_from": Arg( + "format_from": + Arg( '--format-from', help='Source format for data conversion.', choices=constants.AVAILABLE_DATAHANDLERS, required=True, ), - "format_to": Arg( + "format_to": + Arg( '--format-to', help='Destination format for data conversion.', choices=constants.AVAILABLE_DATAHANDLERS, required=True, ), - "dataformat_ohlcv": Arg( - '--data-format-ohlcv', + "dataformat_ohlcv": + Arg('--data-format-ohlcv', help='Storage format for downloaded ohlcv data. (default: `%(default)s`).', choices=constants.AVAILABLE_DATAHANDLERS, - default='json' - ), - "dataformat_trades": Arg( - '--data-format-trades', + default='json'), + "dataformat_trades": + Arg('--data-format-trades', help='Storage format for downloaded trades data. (default: `%(default)s`).', choices=constants.AVAILABLE_DATAHANDLERS, - default='jsongz' - ), - "exchange": Arg( + default='jsongz'), + "exchange": + Arg( '--exchange', help=f'Exchange name (default: `{constants.DEFAULT_EXCHANGE}`). ' f'Only valid if no config is provided.', ), - "timeframes": Arg( - '-t', '--timeframes', + "timeframes": + Arg( + '-t', + '--timeframes', help=f'Specify which tickers to download. Space-separated list. ' f'Default: `1m 5m`.', - choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', - '6h', '8h', '12h', '1d', '3d', '1w'], + choices=[ + '1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h', '1d', '3d', '1w' + ], default=['1m', '5m'], nargs='+', ), - "erase": Arg( + "erase": + Arg( '--erase', help='Clean all existing data for the selected exchange/pairs/timeframes.', action='store_true', ), # Templating options - "template": Arg( + "template": + Arg( '--template', help='Use a template which is either `minimal` or ' '`full` (containing multiple sample indicators). Default: `%(default)s`.', @@ -385,19 +460,22 @@ AVAILABLE_CLI_OPTIONS = { default='full', ), # Plot dataframe - "indicators1": Arg( + "indicators1": + Arg( '--indicators1', help='Set indicators from your strategy you want in the first row of the graph. ' "Space-separated list. Example: `ema3 ema5`. Default: `['sma', 'ema3', 'ema5']`.", nargs='+', ), - "indicators2": Arg( + "indicators2": + Arg( '--indicators2', help='Set indicators from your strategy you want in the third row of the graph. ' "Space-separated list. Example: `fastd fastk`. Default: `['macd', 'macdsignal']`.", nargs='+', ), - "plot_limit": Arg( + "plot_limit": + Arg( '--plot-limit', help='Specify tick limit for plotting. Notice: too high values cause huge files. ' 'Default: %(default)s.', @@ -405,7 +483,8 @@ AVAILABLE_CLI_OPTIONS = { metavar='INT', default=750, ), - "trade_source": Arg( + "trade_source": + Arg( '--trade-source', help='Specify the source for trades (Can be DB or file (backtest file)) ' 'Default: %(default)s', @@ -413,76 +492,90 @@ AVAILABLE_CLI_OPTIONS = { default="file", ), # hyperopt-list, hyperopt-show - "hyperopt_list_profitable": Arg( + "hyperopt_list_profitable": + Arg( '--profitable', help='Select only profitable epochs.', action='store_true', ), - "hyperopt_list_best": Arg( + "hyperopt_list_best": + Arg( '--best', help='Select only best epochs.', action='store_true', ), - "hyperopt_list_min_trades": Arg( + "hyperopt_list_min_trades": + Arg( '--min-trades', help='Select epochs with more than INT trades.', type=check_int_positive, metavar='INT', ), - "hyperopt_list_max_trades": Arg( + "hyperopt_list_max_trades": + Arg( '--max-trades', help='Select epochs with less than INT trades.', type=check_int_positive, metavar='INT', ), - "hyperopt_list_min_avg_time": Arg( + "hyperopt_list_min_avg_time": + Arg( '--min-avg-time', help='Select epochs on above average time.', type=float, metavar='FLOAT', ), - "hyperopt_list_max_avg_time": Arg( + "hyperopt_list_max_avg_time": + Arg( '--max-avg-time', help='Select epochs on under average time.', type=float, metavar='FLOAT', ), - "hyperopt_list_min_avg_profit": Arg( + "hyperopt_list_min_avg_profit": + Arg( '--min-avg-profit', help='Select epochs on above average profit.', type=float, metavar='FLOAT', ), - "hyperopt_list_max_avg_profit": Arg( + "hyperopt_list_max_avg_profit": + Arg( '--max-avg-profit', help='Select epochs on below average profit.', type=float, metavar='FLOAT', ), - "hyperopt_list_min_total_profit": Arg( + "hyperopt_list_min_total_profit": + Arg( '--min-total-profit', help='Select epochs on above total profit.', type=float, metavar='FLOAT', ), - "hyperopt_list_max_total_profit": Arg( + "hyperopt_list_max_total_profit": + Arg( '--max-total-profit', help='Select epochs on below total profit.', type=float, metavar='FLOAT', ), - "hyperopt_list_no_details": Arg( + "hyperopt_list_no_details": + Arg( '--no-details', help='Do not print best epoch details.', action='store_true', ), - "hyperopt_show_index": Arg( - '-n', '--index', + "hyperopt_show_index": + Arg( + '-n', + '--index', help='Specify the index of the epoch to print details for.', type=check_int_nonzero, metavar='INT', ), - "hyperopt_show_no_header": Arg( + "hyperopt_show_no_header": + Arg( '--no-header', help='Do not print epoch details header.', action='store_true', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 105cd6b53..f8fa7bc70 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -1,12 +1,12 @@ # pragma pylint: disable=too-few-public-methods - """ bot constants """ DEFAULT_CONFIG = 'config.json' DEFAULT_EXCHANGE = 'bittrex' PROCESS_THROTTLE_SECS = 5 # sec -HYPEROPT_EPOCH = 100 # epochs +HYPEROPT_EPOCH = 0 # epochs +HYPEROPT_EFFORT = 0 # /10 RETRY_TIMEOUT = 30 # sec DEFAULT_HYPEROPT_LOSS = 'DefaultHyperOptLoss' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' @@ -17,8 +17,9 @@ REQUIRED_ORDERTIF = ['buy', 'sell'] REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', - 'PrecisionFilter', 'PriceFilter', 'SpreadFilter'] +AVAILABLE_PAIRLISTS = [ + 'StaticPairList', 'VolumePairList', 'PrecisionFilter', 'PriceFilter', 'SpreadFilter' +] AVAILABLE_DATAHANDLERS = ['json', 'jsongz'] DRY_RUN_WALLET = 1000 MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons @@ -38,11 +39,9 @@ USER_DATA_FILES = { } SUPPORTED_FIAT = [ - "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", - "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", - "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", - "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD", - "BTC", "XBT", "ETH", "XRP", "LTC", "BCH", "USDT" + "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", + "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", + "SGD", "THB", "TRY", "TWD", "ZAR", "USD", "BTC", "XBT", "ETH", "XRP", "LTC", "BCH", "USDT" ] MINIMAL_CONFIG = { @@ -63,9 +62,16 @@ MINIMAL_CONFIG = { CONF_SCHEMA = { 'type': 'object', 'properties': { - 'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1}, - 'ticker_interval': {'type': 'string'}, - 'stake_currency': {'type': 'string'}, + 'max_open_trades': { + 'type': ['integer', 'number'], + 'minimum': -1 + }, + 'ticker_interval': { + 'type': 'string' + }, + 'stake_currency': { + 'type': 'string' + }, 'stake_amount': { 'type': ['number', 'string'], 'minimum': 0.0001, @@ -77,32 +83,76 @@ CONF_SCHEMA = { 'maximum': 1, 'default': 0.99 }, - 'amend_last_stake_amount': {'type': 'boolean', 'default': False}, - 'last_stake_amount_min_ratio': { - 'type': 'number', 'minimum': 0.0, 'maximum': 1.0, 'default': 0.5 + 'amend_last_stake_amount': { + 'type': 'boolean', + 'default': False + }, + 'last_stake_amount_min_ratio': { + 'type': 'number', + 'minimum': 0.0, + 'maximum': 1.0, + 'default': 0.5 + }, + 'fiat_display_currency': { + 'type': 'string', + 'enum': SUPPORTED_FIAT + }, + 'dry_run': { + 'type': 'boolean' + }, + 'dry_run_wallet': { + 'type': 'number', + 'default': DRY_RUN_WALLET + }, + 'process_only_new_candles': { + 'type': 'boolean' }, - 'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT}, - 'dry_run': {'type': 'boolean'}, - 'dry_run_wallet': {'type': 'number', 'default': DRY_RUN_WALLET}, - 'process_only_new_candles': {'type': 'boolean'}, 'minimal_roi': { 'type': 'object', 'patternProperties': { - '^[0-9.]+$': {'type': 'number'} + '^[0-9.]+$': { + 'type': 'number' + } }, 'minProperties': 1 }, - 'amount_reserve_percent': {'type': 'number', 'minimum': 0.0, 'maximum': 0.5}, - 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, - 'trailing_stop': {'type': 'boolean'}, - 'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1}, - 'trailing_stop_positive_offset': {'type': 'number', 'minimum': 0, 'maximum': 1}, - 'trailing_only_offset_is_reached': {'type': 'boolean'}, + 'amount_reserve_percent': { + 'type': 'number', + 'minimum': 0.0, + 'maximum': 0.5 + }, + 'stoploss': { + 'type': 'number', + 'maximum': 0, + 'exclusiveMaximum': True + }, + 'trailing_stop': { + 'type': 'boolean' + }, + 'trailing_stop_positive': { + 'type': 'number', + 'minimum': 0, + 'maximum': 1 + }, + 'trailing_stop_positive_offset': { + 'type': 'number', + 'minimum': 0, + 'maximum': 1 + }, + 'trailing_only_offset_is_reached': { + 'type': 'boolean' + }, 'unfilledtimeout': { 'type': 'object', 'properties': { - 'buy': {'type': 'number', 'minimum': 1}, - 'sell': {'type': 'number', 'minimum': 1} + 'buy': { + 'type': 'number', + 'minimum': 1 + }, + 'sell': { + 'type': 'number', + 'minimum': 1 + } } }, 'bid_strategy': { @@ -113,13 +163,24 @@ CONF_SCHEMA = { 'minimum': 0, 'maximum': 1, 'exclusiveMaximum': False, - 'use_order_book': {'type': 'boolean'}, - 'order_book_top': {'type': 'integer', 'maximum': 20, 'minimum': 1}, + 'use_order_book': { + 'type': 'boolean' + }, + 'order_book_top': { + 'type': 'integer', + 'maximum': 20, + 'minimum': 1 + }, 'check_depth_of_market': { 'type': 'object', 'properties': { - 'enabled': {'type': 'boolean'}, - 'bids_to_ask_delta': {'type': 'number', 'minimum': 0}, + 'enabled': { + 'type': 'boolean' + }, + 'bids_to_ask_delta': { + 'type': 'number', + 'minimum': 0 + }, } }, }, @@ -129,43 +190,92 @@ CONF_SCHEMA = { 'ask_strategy': { 'type': 'object', 'properties': { - 'use_order_book': {'type': 'boolean'}, - 'order_book_min': {'type': 'integer', 'minimum': 1}, - 'order_book_max': {'type': 'integer', 'minimum': 1, 'maximum': 50}, - 'use_sell_signal': {'type': 'boolean'}, - 'sell_profit_only': {'type': 'boolean'}, - 'ignore_roi_if_buy_signal': {'type': 'boolean'} + 'use_order_book': { + 'type': 'boolean' + }, + 'order_book_min': { + 'type': 'integer', + 'minimum': 1 + }, + 'order_book_max': { + 'type': 'integer', + 'minimum': 1, + 'maximum': 50 + }, + 'use_sell_signal': { + 'type': 'boolean' + }, + 'sell_profit_only': { + 'type': 'boolean' + }, + 'ignore_roi_if_buy_signal': { + 'type': 'boolean' + } } }, 'order_types': { 'type': 'object', 'properties': { - 'buy': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, - 'sell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, - 'emergencysell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, - 'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, - 'stoploss_on_exchange': {'type': 'boolean'}, - 'stoploss_on_exchange_interval': {'type': 'number'} + 'buy': { + 'type': 'string', + 'enum': ORDERTYPE_POSSIBILITIES + }, + 'sell': { + 'type': 'string', + 'enum': ORDERTYPE_POSSIBILITIES + }, + 'emergencysell': { + 'type': 'string', + 'enum': ORDERTYPE_POSSIBILITIES + }, + 'stoploss': { + 'type': 'string', + 'enum': ORDERTYPE_POSSIBILITIES + }, + 'stoploss_on_exchange': { + 'type': 'boolean' + }, + 'stoploss_on_exchange_interval': { + 'type': 'number' + } }, 'required': ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] }, 'order_time_in_force': { 'type': 'object', 'properties': { - 'buy': {'type': 'string', 'enum': ORDERTIF_POSSIBILITIES}, - 'sell': {'type': 'string', 'enum': ORDERTIF_POSSIBILITIES} + 'buy': { + 'type': 'string', + 'enum': ORDERTIF_POSSIBILITIES + }, + 'sell': { + 'type': 'string', + 'enum': ORDERTIF_POSSIBILITIES + } }, 'required': ['buy', 'sell'] }, - 'exchange': {'$ref': '#/definitions/exchange'}, - 'edge': {'$ref': '#/definitions/edge'}, + 'exchange': { + '$ref': '#/definitions/exchange' + }, + 'edge': { + '$ref': '#/definitions/edge' + }, 'experimental': { 'type': 'object', 'properties': { - 'use_sell_signal': {'type': 'boolean'}, - 'sell_profit_only': {'type': 'boolean'}, - 'ignore_roi_if_buy_signal': {'type': 'boolean'}, - 'block_bad_exchanges': {'type': 'boolean'} + 'use_sell_signal': { + 'type': 'boolean' + }, + 'sell_profit_only': { + 'type': 'boolean' + }, + 'ignore_roi_if_buy_signal': { + 'type': 'boolean' + }, + 'block_bad_exchanges': { + 'type': 'boolean' + } } }, 'pairlists': { @@ -173,8 +283,13 @@ CONF_SCHEMA = { 'items': { 'type': 'object', 'properties': { - 'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS}, - 'config': {'type': 'object'} + 'method': { + 'type': 'string', + 'enum': AVAILABLE_PAIRLISTS + }, + 'config': { + 'type': 'object' + } }, 'required': ['method'], } @@ -182,71 +297,126 @@ CONF_SCHEMA = { 'telegram': { 'type': 'object', 'properties': { - 'enabled': {'type': 'boolean'}, - 'token': {'type': 'string'}, - 'chat_id': {'type': 'string'}, + 'enabled': { + 'type': 'boolean' + }, + 'token': { + 'type': 'string' + }, + 'chat_id': { + 'type': 'string' + }, }, 'required': ['enabled', 'token', 'chat_id'] }, 'webhook': { 'type': 'object', 'properties': { - 'enabled': {'type': 'boolean'}, - 'webhookbuy': {'type': 'object'}, - 'webhookbuycancel': {'type': 'object'}, - 'webhooksell': {'type': 'object'}, - 'webhooksellcancel': {'type': 'object'}, - 'webhookstatus': {'type': 'object'}, + 'enabled': { + 'type': 'boolean' + }, + 'webhookbuy': { + 'type': 'object' + }, + 'webhookbuycancel': { + 'type': 'object' + }, + 'webhooksell': { + 'type': 'object' + }, + 'webhooksellcancel': { + 'type': 'object' + }, + 'webhookstatus': { + 'type': 'object' + }, }, }, 'api_server': { 'type': 'object', 'properties': { - 'enabled': {'type': 'boolean'}, - 'listen_ip_address': {'format': 'ipv4'}, + 'enabled': { + 'type': 'boolean' + }, + 'listen_ip_address': { + 'format': 'ipv4' + }, 'listen_port': { 'type': 'integer', 'minimum': 1024, 'maximum': 65535 }, - 'username': {'type': 'string'}, - 'password': {'type': 'string'}, + 'username': { + 'type': 'string' + }, + 'password': { + 'type': 'string' + }, }, 'required': ['enabled', 'listen_ip_address', 'listen_port', 'username', 'password'] }, - 'db_url': {'type': 'string'}, - 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, - 'forcebuy_enable': {'type': 'boolean'}, + 'db_url': { + 'type': 'string' + }, + 'initial_state': { + 'type': 'string', + 'enum': ['running', 'stopped'] + }, + 'forcebuy_enable': { + 'type': 'boolean' + }, 'internals': { 'type': 'object', 'default': {}, 'properties': { - 'process_throttle_secs': {'type': 'integer'}, - 'interval': {'type': 'integer'}, - 'sd_notify': {'type': 'boolean'}, + 'process_throttle_secs': { + 'type': 'integer' + }, + 'interval': { + 'type': 'integer' + }, + 'sd_notify': { + 'type': 'boolean' + }, } }, 'dataformat_ohlcv': { 'type': 'string', - 'enum': AVAILABLE_DATAHANDLERS, - 'default': 'json' + 'enum': AVAILABLE_DATAHANDLERS, + 'default': 'json' }, 'dataformat_trades': { 'type': 'string', - 'enum': AVAILABLE_DATAHANDLERS, - 'default': 'jsongz' + 'enum': AVAILABLE_DATAHANDLERS, + 'default': 'jsongz' } }, 'definitions': { 'exchange': { 'type': 'object', 'properties': { - 'name': {'type': 'string'}, - 'sandbox': {'type': 'boolean', 'default': False}, - 'key': {'type': 'string', 'default': ''}, - 'secret': {'type': 'string', 'default': ''}, - 'password': {'type': 'string', 'default': ''}, - 'uid': {'type': 'string'}, + 'name': { + 'type': 'string' + }, + 'sandbox': { + 'type': 'boolean', + 'default': False + }, + 'key': { + 'type': 'string', + 'default': '' + }, + 'secret': { + 'type': 'string', + 'default': '' + }, + 'password': { + 'type': 'string', + 'default': '' + }, + 'uid': { + 'type': 'string' + }, 'pair_whitelist': { 'type': 'array', 'items': { @@ -263,29 +433,65 @@ CONF_SCHEMA = { }, 'uniqueItems': True }, - 'outdated_offset': {'type': 'integer', 'minimum': 1}, - 'markets_refresh_interval': {'type': 'integer'}, - 'ccxt_config': {'type': 'object'}, - 'ccxt_async_config': {'type': 'object'} + 'outdated_offset': { + 'type': 'integer', + 'minimum': 1 + }, + 'markets_refresh_interval': { + 'type': 'integer' + }, + 'ccxt_config': { + 'type': 'object' + }, + 'ccxt_async_config': { + 'type': 'object' + } }, 'required': ['name'] }, 'edge': { 'type': 'object', 'properties': { - 'enabled': {'type': 'boolean'}, - 'process_throttle_secs': {'type': 'integer', 'minimum': 600}, - 'calculate_since_number_of_days': {'type': 'integer'}, - 'allowed_risk': {'type': 'number'}, - 'capital_available_percentage': {'type': 'number'}, - 'stoploss_range_min': {'type': 'number'}, - 'stoploss_range_max': {'type': 'number'}, - 'stoploss_range_step': {'type': 'number'}, - 'minimum_winrate': {'type': 'number'}, - 'minimum_expectancy': {'type': 'number'}, - 'min_trade_number': {'type': 'number'}, - 'max_trade_duration_minute': {'type': 'integer'}, - 'remove_pumps': {'type': 'boolean'} + 'enabled': { + 'type': 'boolean' + }, + 'process_throttle_secs': { + 'type': 'integer', + 'minimum': 600 + }, + 'calculate_since_number_of_days': { + 'type': 'integer' + }, + 'allowed_risk': { + 'type': 'number' + }, + 'capital_available_percentage': { + 'type': 'number' + }, + 'stoploss_range_min': { + 'type': 'number' + }, + 'stoploss_range_max': { + 'type': 'number' + }, + 'stoploss_range_step': { + 'type': 'number' + }, + 'minimum_winrate': { + 'type': 'number' + }, + 'minimum_expectancy': { + 'type': 'number' + }, + 'min_trade_number': { + 'type': 'number' + }, + 'max_trade_duration_minute': { + 'type': 'integer' + }, + 'remove_pumps': { + 'type': 'boolean' + } }, 'required': ['process_throttle_secs', 'allowed_risk'] } diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 14a77cdf5..6b9a7a559 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -2,7 +2,6 @@ """ This module contains the hyperopt logic """ - import os import functools import locale @@ -10,12 +9,12 @@ import logging import random import sys import warnings -from collections import OrderedDict +from collections import OrderedDict, deque from math import factorial, log from operator import itemgetter from pathlib import Path from pprint import pprint -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Callable import rapidjson from colorama import Fore, Style @@ -32,7 +31,6 @@ from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver, HyperOptResolver) from joblib import (Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects) -from joblib._parallel_backends import LokyBackend from joblib import register_parallel_backend, parallel_backend from pandas import DataFrame @@ -62,6 +60,7 @@ class Hyperopt: hyperopt.start() """ def __init__(self, config: Dict[str, Any]) -> None: + self.config = config self.backtesting = Backtesting(self.config) @@ -75,16 +74,16 @@ class Hyperopt: 'hyperopt_results.pickle') self.tickerdata_pickle = (self.config['user_data_dir'] / 'hyperopt_results' / 'hyperopt_tickerdata.pkl') - self.effort = config.get('epochs', 0) or 1 - self.total_epochs = 9999 - self.max_epoch = 9999 + self.total_epochs = config['epochs'] if 'epochs' in config else 0 + self.effort = config['effort'] if 'effort' in config else -1 + self.max_epoch = 0 self.search_space_size = 0 self.max_epoch_reached = False self.min_epochs = INITIAL_POINTS self.current_best_loss = 100 self.current_best_epoch = 0 - self.epochs_since_last_best = [] + self.epochs_since_last_best: List = [] self.avg_best_occurrence = 0 if not self.config.get('hyperopt_continue'): @@ -100,6 +99,10 @@ class Hyperopt: self.opt: Optimizer self.opt = None self.f_val: List = [] + self.to_ask: deque + self.to_ask = deque() + self.tell: Callable + self.tell = None # Populate functions here (hasattr is slow so should not be run during "regular" operations) if hasattr(self.custom_hyperopt, 'populate_indicators'): @@ -163,6 +166,7 @@ class Hyperopt: Save hyperopt trials to file """ num_trials = len(self.trials) + print() if num_trials > self.num_trials_saved: logger.info(f"Saving {num_trials} {plural(num_trials, 'epoch')}.") dump(self.trials, self.trials_file) @@ -276,8 +280,8 @@ class Hyperopt: """ is_best = results['is_best'] if self.print_all or is_best: - self.print_results_explanation(results, self.total_epochs, self.print_all, - self.print_colorized) + self.print_results_explanation(results, self.total_epochs or self.max_epoch, + self.print_all, self.print_colorized) @staticmethod def print_results_explanation(results, total_epochs, highlight_best: bool, @@ -386,10 +390,10 @@ class Hyperopt: position_stacking=self.position_stacking, ) return self._get_results_dict(backtesting_results, min_date, max_date, params_dict, - params_details) + params_details, raw_params) def _get_results_dict(self, backtesting_results, min_date, max_date, params_dict, - params_details): + params_details, raw_params): results_metrics = self._calculate_results_metrics(backtesting_results) results_explanation = self._format_results_explanation_string(results_metrics) @@ -413,6 +417,7 @@ class Hyperopt: 'results_metrics': results_metrics, 'results_explanation': results_explanation, 'total_profit': total_profit, + 'asked': raw_params, } def _calculate_results_metrics(self, backtesting_results: DataFrame) -> Dict: @@ -448,38 +453,51 @@ class Hyperopt: random_state=self.random_state, ) - def run_optimizer_parallel(self, parallel, tries: int, first_try: int) -> List: + def run_optimizer_parallel(self, parallel: Parallel, tries: int, first_try: int, + jobs: int) -> List: result = parallel( delayed(wrap_non_picklable_objects(self.parallel_objective))(asked, i) - for asked, i in zip(self.opt_generator(), range(first_try, first_try + tries))) + for asked, i in zip(self.opt_generator(jobs, tries), range( + first_try, first_try + tries))) return result - def opt_generator(self): + def opt_generator(self, jobs: int, tries: int): while True: if self.f_val: - # print("opt.tell(): ", - # [v['params_dict'] for v in self.f_val], [v['loss'] for v in self.f_val]) - functools.partial(self.opt.tell, - ([v['params_dict'] - for v in self.f_val], [v['loss'] for v in self.f_val])) + # print("opt.tell(): ", [v['asked'] for v in self.f_val], + # [v['loss'] for v in self.f_val]) + self.tell = functools.partial(self.opt.tell, [v['asked'] for v in self.f_val], + [v['loss'] for v in self.f_val]) self.f_val = [] - yield self.opt.ask() + + if not self.to_ask: + self.opt.update_next() + self.to_ask.extend(self.opt.ask(n_points=tries)) + self.fit = True + yield self.to_ask.popleft() + # yield self.opt.ask() def parallel_objective(self, asked, n): self.log_results_immediate(n) return self.generate_optimizer(asked) def parallel_callback(self, f_val): + if self.tell: + self.tell(fit=self.fit) + self.tell = None + self.fit = False self.f_val.extend(f_val) def log_results_immediate(self, n) -> None: print('.', end='') sys.stdout.flush() - def log_results(self, f_val, frame_start, max_epoch) -> None: + def log_results(self, f_val, frame_start, total_epochs: int) -> None: """ Log results if it is better than any previous evaluation """ + print() + current = frame_start + 1 for i, v in enumerate(f_val): is_best = self.is_best_loss(v, self.current_best_loss) current = frame_start + i + 1 @@ -493,15 +511,10 @@ class Hyperopt: self.print_results(v) self.trials.append(v) # Save results after every batch - print('\n') self.save_trials() # give up if no best since max epochs - if current > self.max_epoch: + if current + 1 > (total_epochs or self.max_epoch): self.max_epoch_reached = True - # testing trapdoor - if os.getenv('FQT_HYPEROPT_TRAP'): - logger.debug('bypassing hyperopt loop') - self.max_epoch = 1 @staticmethod def load_previous_results(trials_file: Path) -> List: @@ -522,7 +535,7 @@ class Hyperopt: return random_state or random.randint(1, 2**16 - 1) @staticmethod - def calc_epochs(dimensions: List[Dimension], config_jobs: int, effort: int): + def calc_epochs(dimensions: List[Dimension], config_jobs: int, effort: int, total_epochs: int): """ Compute a reasonable number of initial points and a minimum number of epochs to evaluate """ n_dimensions = len(dimensions) @@ -543,16 +556,18 @@ class Hyperopt: if search_space_size < config_jobs: # don't waste if the space is small n_initial_points = config_jobs + elif total_epochs > 0: + n_initial_points = total_epochs // 3 if total_epochs > config_jobs * 3 else config_jobs + min_epochs = n_initial_points else: # extract coefficients from the search space and the jobs count log_sss = int(log(search_space_size, 10)) - log_jobs = int(log(config_jobs, 2)) - log_jobs = 2 if log_jobs < 0 else log_jobs + log_jobs = int(log(config_jobs, 2)) if config_jobs > 4 else 2 jobs_ip = log_jobs * log_sss # never waste n_initial_points = log_sss if jobs_ip > search_space_size else jobs_ip - # it shall run for this much, I say - min_epochs = max(2 * n_initial_points, 3 * config_jobs) * effort + # it shall run for this much, I say + min_epochs = int(max(2 * n_initial_points, 3 * config_jobs) * (1 + effort / 10)) return n_initial_points, min_epochs, search_space_size def update_max_epoch(self, val: Dict, current: int): @@ -563,11 +578,12 @@ class Hyperopt: self.avg_best_occurrence = (sum(self.epochs_since_last_best) // len(self.epochs_since_last_best)) self.current_best_epoch = current - self.max_epoch = (self.current_best_epoch + self.avg_best_occurrence + - self.min_epochs) * self.effort + self.max_epoch = int( + (self.current_best_epoch + self.avg_best_occurrence + self.min_epochs) * + (1 + self.effort / 10)) if self.max_epoch > self.search_space_size: self.max_epoch = self.search_space_size - print('\n') + print() logger.info(f'Max epochs set to: {self.max_epoch}') def start(self) -> None: @@ -599,47 +615,53 @@ class Hyperopt: self.dimensions: List[Dimension] = self.hyperopt_space() self.n_initial_points, self.min_epochs, self.search_space_size = self.calc_epochs( - self.dimensions, config_jobs, self.effort) + self.dimensions, config_jobs, self.effort, self.total_epochs) logger.info(f"Min epochs set to: {self.min_epochs}") - self.max_epoch = self.min_epochs - self.avg_best_occurrence = self.max_epoch + if self.total_epochs < 1: + self.max_epoch = int(self.min_epochs + len(self.trials)) + else: + self.max_epoch = self.n_initial_points + self.avg_best_occurrence = self.min_epochs logger.info(f'Initial points: {self.n_initial_points}') self.opt = self.get_optimizer(self.dimensions, config_jobs, self.n_initial_points) - # last_frame_len = (self.total_epochs - 1) % self.avg_best_occurrence - if self.print_colorized: colorama_init(autoreset=True) - try: - register_parallel_backend('custom', CustomImmediateResultBackend) - with parallel_backend('custom'): - with Parallel(n_jobs=config_jobs, verbose=0) as parallel: - for frame in range(self.total_epochs): - epochs_so_far = len(self.trials) - # pad the frame length to the number of jobs to avoid desaturation - frame_len = (self.avg_best_occurrence + config_jobs - - self.avg_best_occurrence % config_jobs) - print( - f"{epochs_so_far+1}-{epochs_so_far+self.avg_best_occurrence}" - f"/{self.total_epochs}: ", - end='') - f_val = self.run_optimizer_parallel(parallel, frame_len, epochs_so_far) - self.log_results(f_val, epochs_so_far, self.total_epochs) - if self.max_epoch_reached: - logger.info("Max epoch reached, terminating.") - break + try: + register_parallel_backend('custom', CustomImmediateResultBackend) + with parallel_backend('custom'): + with Parallel(n_jobs=config_jobs, verbose=0) as parallel: + while True: + # update epochs count + epochs_so_far = len(self.trials) + # pad the frame length to the number of jobs to avoid desaturation + frame_len = (self.avg_best_occurrence + config_jobs - + self.avg_best_occurrence % config_jobs) + # don't go over the limit + if epochs_so_far + frame_len > (self.total_epochs or self.max_epoch): + frame_len = (self.total_epochs or self.max_epoch) - epochs_so_far + print( + f"{epochs_so_far+1}-{epochs_so_far+frame_len}" + f"/{self.total_epochs}: ", + end='') + f_val = self.run_optimizer_parallel(parallel, frame_len, epochs_so_far, + config_jobs) + self.log_results(f_val, epochs_so_far, self.total_epochs or self.max_epoch) + if self.max_epoch_reached: + logger.info("Max epoch reached, terminating.") + break - except KeyboardInterrupt: - print("User interrupted..") + except KeyboardInterrupt: + print("User interrupted..") self.save_trials(final=True) if self.trials: sorted_trials = sorted(self.trials, key=itemgetter('loss')) results = sorted_trials[0] - self.print_epoch_details(results, self.total_epochs, self.print_json) + self.print_epoch_details(results, self.max_epoch, self.print_json) else: # This is printed when Ctrl+C is pressed quickly, before first epochs have # a chance to be evaluated. diff --git a/freqtrade/optimize/hyperopt_backend.py b/freqtrade/optimize/hyperopt_backend.py index d7a8544cc..4d75ec88b 100644 --- a/freqtrade/optimize/hyperopt_backend.py +++ b/freqtrade/optimize/hyperopt_backend.py @@ -1,6 +1,7 @@ from joblib._parallel_backends import LokyBackend +from typing import Any -hyperopt = None +hyperopt: Any = None class MultiCallback: