From dfa6f67e43d3914491b804084b68efb884374f74 Mon Sep 17 00:00:00 2001 From: Bo van Hasselt Date: Sat, 13 Feb 2021 17:21:01 +0100 Subject: [PATCH 1/6] start of extract-strategy --- freqtrade/commands/automation_commands.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/commands/automation_commands.py b/freqtrade/commands/automation_commands.py index a639a8fa2..d6bd8da36 100644 --- a/freqtrade/commands/automation_commands.py +++ b/freqtrade/commands/automation_commands.py @@ -11,6 +11,10 @@ from freqtrade.misc import render_template logger = logging.getLogger(__name__) + + + +#-------------------------build-hyperopt----------------------------- ''' TODO -make the code below more dynamic with a large list of indicators and aims From 1fe63f358341d2dd2a1dc852ab6c6abacaaed63c Mon Sep 17 00:00:00 2001 From: Bo van Hasselt Date: Sun, 14 Feb 2021 01:13:29 +0100 Subject: [PATCH 2/6] finished extract-strategy prototype and improved custom-hyperopt(build-hyperopt before) --- freqtrade/commands/__init__.py | 2 +- freqtrade/commands/arguments.py | 15 ++- freqtrade/commands/automation_commands.py | 146 ++++++++++++++++++---- freqtrade/commands/cli_options.py | 5 + freqtrade/constants.py | 16 +++ 5 files changed, 156 insertions(+), 28 deletions(-) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 0ce9b577b..9f4a057c5 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -12,7 +12,7 @@ from freqtrade.commands.data_commands import (start_convert_data, start_download start_list_data) from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, start_new_hyperopt, start_new_strategy) -from freqtrade.commands.automation_commands import start_build_hyperopt +from freqtrade.commands.automation_commands import (start_custom_hyperopt, start_extract_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 2393a39b0..265b38dda 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -55,8 +55,11 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] +# Automation ARGS_BUILD_CUSTOM_HYPEROPT = ["buy_indicators", "sell_indicators", "hyperopt"] +ARGS_EXTRACT_STRATEGY = ["strategy", "extract_name"] + ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] @@ -175,7 +178,7 @@ class Arguments: start_list_data, start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, start_list_timeframes, start_new_config, start_new_hyperopt, - start_build_hyperopt, + start_custom_hyperopt, start_extract_strategy, start_new_strategy, start_plot_dataframe, start_plot_profit, start_show_trades, start_test_pairlist, start_trading) @@ -211,11 +214,17 @@ class Arguments: self._build_args(optionlist=ARGS_BUILD_HYPEROPT, parser=build_hyperopt_cmd) # add build-hyperopt subcommand - build_custom_hyperopt_cmd = subparsers.add_parser('build-hyperopt', + build_custom_hyperopt_cmd = subparsers.add_parser('custom-hyperopt', help="Build a custom hyperopt") - build_custom_hyperopt_cmd.set_defaults(func=start_build_hyperopt) + build_custom_hyperopt_cmd.set_defaults(func=start_custom_hyperopt) self._build_args(optionlist=ARGS_BUILD_CUSTOM_HYPEROPT, parser=build_custom_hyperopt_cmd) + # add build-hyperopt subcommand + extract_strategy_cmd = subparsers.add_parser('extract-strategy', + help="Extract data dictionaries for build-hyperopt from strategy") + extract_strategy_cmd.set_defaults(func=start_extract_strategy) + self._build_args(optionlist=ARGS_EXTRACT_STRATEGY, parser=extract_strategy_cmd) + # add new-strategy subcommand build_strategy_cmd = subparsers.add_parser('new-strategy', help="Create new strategy") diff --git a/freqtrade/commands/automation_commands.py b/freqtrade/commands/automation_commands.py index d6bd8da36..1ef91c08f 100644 --- a/freqtrade/commands/automation_commands.py +++ b/freqtrade/commands/automation_commands.py @@ -3,7 +3,11 @@ import logging from pathlib import Path from typing import Any, Dict -from freqtrade.constants import USERPATH_HYPEROPTS +from freqtrade.constants import (USERPATH_HYPEROPTS, + USERPATH_STRATEGIES, + POSSIBLE_GUARDS, + POSSIBLE_TRIGGERS, + POSSIBLE_AIMS) from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode from freqtrade.configuration import setup_utils_configuration @@ -12,23 +16,116 @@ from freqtrade.misc import render_template logger = logging.getLogger(__name__) +# ---------------------------------------------------extract-strategy------------------------------------------------------ +def extract_dicts(strategypath: Path): + # store the file in a list for reference + stored_file = [] + with open(strategypath) as file: + for line in file: + stored_file.append(line) + + # find the start and end of buy trend + for position, line in enumerate(stored_file): + if "populate_buy_trend(" in line: + start_buy_number = position + elif "populate_sell_trend(" in line: + end_buy_number = position + + # list the numbers between the start and end of buy trend + buy_lines = [] + for i in range(start_buy_number, end_buy_number): + buy_lines.append(i) + + # populate the indicators dictionaries with indicators attached to the line they are on + buyindicators = {} + sellindicators = {} + for position, line in enumerate(stored_file): + # check the lines in buy trend for indicator and add them + if position in buy_lines and "(dataframe['" in line: + # use split twice to remove the context around the indicator + back_of_line = line.split("(dataframe['", 1)[1] + buyindicator = back_of_line.split("'] ", 1)[0] + + buyindicators[buyindicator] = position + + # check the lines in sell trend for indicator and add them + elif position > end_buy_number and "(dataframe['" in line: + # use split twice to remove the context around the indicator + back_of_line = line.split("(dataframe['", 1)[1] + sellindicator = back_of_line.split("'] ", 1)[0] + + sellindicators[sellindicator] = position + + # build the final buy dictionary + buy_dict = {} + for indicator in buyindicators: + # find the corrosponding aim + for position, line in enumerate(stored_file): + if position == buyindicators[indicator]: + # use split twice to remove the context around the indicator + back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1] + aim = back_of_line.split()[0] + buy_dict[indicator] = aim + + # build the final sell dictionary + sell_dict = {} + for indicator in sellindicators: + # find the corrosponding aim + for position, line in enumerate(stored_file): + if position == sellindicators[indicator]: + # use split twice to remove the context around the indicator + back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1] + aim = back_of_line.split()[0] + sell_dict[indicator] = aim + + # put the final dicts into a tuple + final_dicts = (buy_dict, sell_dict) + + return final_dicts -#-------------------------build-hyperopt----------------------------- +def start_extract_strategy(args: Dict[str, Any]) -> None: + """ + Check if the right subcommands where passed and start extracting the strategy data + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + # check if all required options are filled in + if not 'strategy' in args or not args['strategy']: + raise OperationalException("`extract-strategy` requires --strategy to be set.") + else: + # if the name is not specified use (strategy)_extract + if not 'extract_name' in args or not args['extract_name']: + args['extract_name'] = args['strategy'] + "_extract" + + new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['extract_name'] + '.txt') + if new_path.exists(): + raise OperationalException(f"`{new_path}` already exists. " + "Please choose another name.") + # the path of the chosen strategy + strategy_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py') + + # extract the buy and sell indicators as dicts + extracted_dicts = str(extract_dicts(strategy_path)) + + # save the dicts in a file + logger.info(f"Writing custom hyperopt to `{new_path}`.") + new_path.write_text(extracted_dicts) + + +# --------------------------------------------------custom-hyperopt------------------------------------------------------ ''' TODO -make the code below more dynamic with a large list of indicators and aims -buy_space integer values variation based on aim(later deep learning) -add --mode , see notes -when making the strategy reading tool, make sure that the populate indicators gets copied to here + -Custom stoploss and roi + -cli option to read extracted strategies files (--extraction) ''' -POSSIBLE_GUARDS = ["rsi", "mfi", "fastd"] -POSSIBLE_TRIGGERS = ["bb_lowerband", "bb_upperband"] -POSSIBLE_VALUES = {"above": ">", "below": "<"} - -def build_hyperopt_buyelements(buy_indicators: Dict[str, str]): +def custom_hyperopt_buyelements(buy_indicators: Dict[str, str]): """ Build the arguments with the placefillers for the buygenerator """ @@ -41,13 +138,13 @@ def build_hyperopt_buyelements(buy_indicators: Dict[str, str]): if not indicator in POSSIBLE_GUARDS and not indicator in POSSIBLE_TRIGGERS: raise OperationalException( f"`{indicator}` is not part of the available indicators. The current options are {POSSIBLE_GUARDS + POSSIBLE_TRIGGERS}.") - elif not buy_indicators[indicator] in POSSIBLE_VALUES: + elif not buy_indicators[indicator] in POSSIBLE_AIMS: raise OperationalException( - f"`{buy_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_VALUES}.") + f"`{buy_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_AIMS}.") # If the indicator is a guard elif indicator in POSSIBLE_GUARDS: # get the symbol corrosponding to the value - aim = POSSIBLE_VALUES[buy_indicators[indicator]] + aim = POSSIBLE_AIMS[buy_indicators[indicator]] # add the guard to its argument buy_guards += f"if '{indicator}-enabled' in params and params['{indicator}-enabled']: conditions.append(dataframe['{indicator}'] {aim} params['{indicator}-value'])" @@ -57,13 +154,13 @@ def build_hyperopt_buyelements(buy_indicators: Dict[str, str]): # If the indicator is a trigger elif indicator in POSSIBLE_TRIGGERS: # get the symbol corrosponding to the value - aim = POSSIBLE_VALUES[buy_indicators[indicator]] + aim = POSSIBLE_AIMS[buy_indicators[indicator]] # add the trigger to its argument buy_triggers += f"if params['trigger'] == '{indicator}': conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])" # Final line of indicator space makes all triggers - + buy_space += "Categorical([" # adding all triggers to the list @@ -78,7 +175,7 @@ def build_hyperopt_buyelements(buy_indicators: Dict[str, str]): return {"buy_guards": buy_guards, "buy_triggers": buy_triggers, "buy_space": buy_space} -def build_hyperopt_sellelements(sell_indicators: Dict[str, str]): +def custom_hyperopt_sellelements(sell_indicators: Dict[str, str]): """ Build the arguments with the placefillers for the sellgenerator """ @@ -91,13 +188,13 @@ def build_hyperopt_sellelements(sell_indicators: Dict[str, str]): if not indicator in POSSIBLE_GUARDS and not indicator in POSSIBLE_TRIGGERS: raise OperationalException( f"`{indicator}` is not part of the available indicators. The current options are {POSSIBLE_GUARDS + POSSIBLE_TRIGGERS}.") - elif not sell_indicators[indicator] in POSSIBLE_VALUES: + elif not sell_indicators[indicator] in POSSIBLE_AIMS: raise OperationalException( - f"`{sell_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_VALUES}.") + f"`{sell_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_AIMS}.") # If indicator is a guard elif indicator in POSSIBLE_GUARDS: # get the symbol corrosponding to the value - aim = POSSIBLE_VALUES[sell_indicators[indicator]] + aim = POSSIBLE_AIMS[sell_indicators[indicator]] # add the guard to its argument sell_guards += f"if '{indicator}-enabled' in params and params['sell-{indicator}-enabled']: conditions.append(dataframe['{indicator}'] {aim} params['sell-{indicator}-value'])" @@ -107,7 +204,7 @@ def build_hyperopt_sellelements(sell_indicators: Dict[str, str]): # If the indicator is a trigger elif indicator in POSSIBLE_TRIGGERS: # get the symbol corrosponding to the value - aim = POSSIBLE_VALUES[sell_indicators[indicator]] + aim = POSSIBLE_AIMS[sell_indicators[indicator]] # add the trigger to its argument sell_triggers += f"if params['sell-trigger'] == 'sell-{indicator}': conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])" @@ -134,8 +231,8 @@ def deploy_custom_hyperopt(hyperopt_name: str, hyperopt_path: Path, buy_indicato """ # Build the arguments for the buy and sell generators - buy_args = build_hyperopt_buyelements(buy_indicators) - sell_args = build_hyperopt_sellelements(sell_indicators) + buy_args = custom_hyperopt_buyelements(buy_indicators) + sell_args = custom_hyperopt_sellelements(sell_indicators) # Build the final template strategy_text = render_template(templatefile='base_hyperopt.py.j2', @@ -152,19 +249,18 @@ def deploy_custom_hyperopt(hyperopt_name: str, hyperopt_path: Path, buy_indicato hyperopt_path.write_text(strategy_text) -def start_build_hyperopt(args: Dict[str, Any]) -> None: +def start_custom_hyperopt(args: Dict[str, Any]) -> None: """ Check if the right subcommands where passed and start building the hyperopt """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - # check what the name of the hyperopt should if not 'hyperopt' in args or not args['hyperopt']: - raise OperationalException("`build-hyperopt` requires --hyperopt to be set.") + raise OperationalException("`custom-hyperopt` requires --hyperopt to be set.") elif not 'buy_indicators' in args or not args['buy_indicators']: - raise OperationalException("`build-hyperopt` requires --buy-indicators to be set.") + raise OperationalException("`custom-hyperopt` requires --buy-indicators to be set.") elif not 'sell_indicators' in args or not args['sell_indicators']: - raise OperationalException("`build-hyperopt` requires --sell-indicators to be set.") + raise OperationalException("`custom-hyperopt` requires --sell-indicators to be set.") else: if args['hyperopt'] == 'DefaultHyperopt': raise OperationalException("DefaultHyperopt is not allowed as name.") @@ -179,3 +275,5 @@ def start_build_hyperopt(args: Dict[str, Any]) -> None: deploy_custom_hyperopt(args['hyperopt'], new_path, buy_indicators, sell_indicators) + +# --------------------------------------------------build-hyperopt------------------------------------------------------ \ No newline at end of file diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 1130319b0..7ad21cc6e 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -288,6 +288,11 @@ AVAILABLE_CLI_OPTIONS = { 'Example: --sell-indicators `{"rsi":"above","bb_lowerband":"below"}`', metavar='DICT', ), + "extract_name": Arg( + '--extract-name', + help='Specify the name of the file to which the data should be extracted. ', + metavar='FILENAME', + ), # List exchanges "print_one_column": Arg( '-1', '--one-column', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 69301ca0e..64c80dbe7 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -383,3 +383,19 @@ ListPairsWithTimeframes = List[PairWithTimeframe] # Type for trades list TradeList = List[List] + +#Build-hyperopt options +POSSIBLE_GUARDS = [ + "rsi", + "mfi", + "fastd" +] +POSSIBLE_TRIGGERS = [ + "bb_lowerband", + "bb_upperband" +] +POSSIBLE_AIMS = { + "above": ">", + "below": "<", + "equal": "==" +} From cdd0168608f09ad5d2a9820bd1f6c91a239a2146 Mon Sep 17 00:00:00 2001 From: Bo van Hasselt Date: Sun, 14 Feb 2021 02:07:52 +0100 Subject: [PATCH 3/6] full automation(build-hyperopt) prototype of strategy->hyperopt done --- freqtrade/commands/__init__.py | 2 +- freqtrade/commands/arguments.py | 16 +++-- freqtrade/commands/automation_commands.py | 85 ++++++++++++++++++----- freqtrade/constants.py | 16 +++-- 4 files changed, 89 insertions(+), 30 deletions(-) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 9f4a057c5..83c18a205 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -12,7 +12,7 @@ from freqtrade.commands.data_commands import (start_convert_data, start_download start_list_data) from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, start_new_hyperopt, start_new_strategy) -from freqtrade.commands.automation_commands import (start_custom_hyperopt, start_extract_strategy) +from freqtrade.commands.automation_commands import (start_build_hyperopt, start_custom_hyperopt, start_extract_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 265b38dda..5b9babccc 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -60,6 +60,8 @@ ARGS_BUILD_CUSTOM_HYPEROPT = ["buy_indicators", "sell_indicators", "hyperopt"] ARGS_EXTRACT_STRATEGY = ["strategy", "extract_name"] +ARGS_BUILD_BUILD_HYPEROPT = ["strategy", "hyperopt"] + ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] @@ -178,7 +180,7 @@ class Arguments: start_list_data, start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, start_list_timeframes, start_new_config, start_new_hyperopt, - start_custom_hyperopt, start_extract_strategy, + start_build_hyperopt, start_custom_hyperopt, start_extract_strategy, start_new_strategy, start_plot_dataframe, start_plot_profit, start_show_trades, start_test_pairlist, start_trading) @@ -213,18 +215,24 @@ class Arguments: build_hyperopt_cmd.set_defaults(func=start_new_hyperopt) self._build_args(optionlist=ARGS_BUILD_HYPEROPT, parser=build_hyperopt_cmd) - # add build-hyperopt subcommand + # add custom-hyperopt subcommand build_custom_hyperopt_cmd = subparsers.add_parser('custom-hyperopt', help="Build a custom hyperopt") build_custom_hyperopt_cmd.set_defaults(func=start_custom_hyperopt) self._build_args(optionlist=ARGS_BUILD_CUSTOM_HYPEROPT, parser=build_custom_hyperopt_cmd) - # add build-hyperopt subcommand + # add extract-strategy subcommand extract_strategy_cmd = subparsers.add_parser('extract-strategy', - help="Extract data dictionaries for build-hyperopt from strategy") + help="Extract data dictionaries for custom-hyperopt from strategy") extract_strategy_cmd.set_defaults(func=start_extract_strategy) self._build_args(optionlist=ARGS_EXTRACT_STRATEGY, parser=extract_strategy_cmd) + # add build-hyperopt subcommand + build_extracted_hyperopt_cmd = subparsers.add_parser('build-hyperopt', + help="Create a hyperopt for a strategy") + build_extracted_hyperopt_cmd.set_defaults(func=start_build_hyperopt) + self._build_args(optionlist=ARGS_BUILD_BUILD_HYPEROPT, parser=build_extracted_hyperopt_cmd) + # add new-strategy subcommand build_strategy_cmd = subparsers.add_parser('new-strategy', help="Create new strategy") diff --git a/freqtrade/commands/automation_commands.py b/freqtrade/commands/automation_commands.py index 1ef91c08f..d930c4851 100644 --- a/freqtrade/commands/automation_commands.py +++ b/freqtrade/commands/automation_commands.py @@ -17,6 +17,13 @@ logger = logging.getLogger(__name__) # ---------------------------------------------------extract-strategy------------------------------------------------------ +''' + TODO + - get the stoploss value to optimize + - get the values from the guards to specify the optimzation range (cooperation with custom-hyperopt) +''' + + def extract_dicts(strategypath: Path): # store the file in a list for reference stored_file = [] @@ -40,7 +47,7 @@ def extract_dicts(strategypath: Path): buyindicators = {} sellindicators = {} for position, line in enumerate(stored_file): - # check the lines in buy trend for indicator and add them + # check the lines in buy trend for indicator and add them if position in buy_lines and "(dataframe['" in line: # use split twice to remove the context around the indicator back_of_line = line.split("(dataframe['", 1)[1] @@ -48,14 +55,14 @@ def extract_dicts(strategypath: Path): buyindicators[buyindicator] = position - # check the lines in sell trend for indicator and add them + # check the lines in sell trend for indicator and add them elif position > end_buy_number and "(dataframe['" in line: # use split twice to remove the context around the indicator back_of_line = line.split("(dataframe['", 1)[1] sellindicator = back_of_line.split("'] ", 1)[0] sellindicators[sellindicator] = position - + # build the final buy dictionary buy_dict = {} for indicator in buyindicators: @@ -115,13 +122,13 @@ def start_extract_strategy(args: Dict[str, Any]) -> None: # --------------------------------------------------custom-hyperopt------------------------------------------------------ ''' - TODO + TODO -make the code below more dynamic with a large list of indicators and aims - -buy_space integer values variation based on aim(later deep learning) - -add --mode , see notes - -when making the strategy reading tool, make sure that the populate indicators gets copied to here - -Custom stoploss and roi + -buy_space integer values variation based on aim and input value form extract-strategy(later deep learning) + -add --mode , see notes (denk hierbij ook aan value range bij guards) + -Custom stoploss and roi (based on input from extract-strategy) -cli option to read extracted strategies files (--extraction) + -code cleanup (maybe the two elements functions can be combined) ''' @@ -144,20 +151,20 @@ def custom_hyperopt_buyelements(buy_indicators: Dict[str, str]): # If the indicator is a guard elif indicator in POSSIBLE_GUARDS: # get the symbol corrosponding to the value - aim = POSSIBLE_AIMS[buy_indicators[indicator]] + aim = buy_indicators[indicator] # add the guard to its argument - buy_guards += f"if '{indicator}-enabled' in params and params['{indicator}-enabled']: conditions.append(dataframe['{indicator}'] {aim} params['{indicator}-value'])" + buy_guards += f"if '{indicator}-enabled' in params and params['{indicator}-enabled']:\n conditions.append(dataframe['{indicator}'] {aim} params['{indicator}-value'])\n" # add the space to its argument - buy_space += f"Integer(10, 90, name='{indicator}-value'), Categorical([True, False], name='{indicator}-enabled')," + buy_space += f"Integer(10, 90, name='{indicator}-value'),\nCategorical([True, False], name='{indicator}-enabled'),\n" # If the indicator is a trigger elif indicator in POSSIBLE_TRIGGERS: # get the symbol corrosponding to the value - aim = POSSIBLE_AIMS[buy_indicators[indicator]] + aim = buy_indicators[indicator] # add the trigger to its argument - buy_triggers += f"if params['trigger'] == '{indicator}': conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])" + buy_triggers += f"if params['trigger'] == '{indicator}':\n conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])\n" # Final line of indicator space makes all triggers @@ -194,20 +201,20 @@ def custom_hyperopt_sellelements(sell_indicators: Dict[str, str]): # If indicator is a guard elif indicator in POSSIBLE_GUARDS: # get the symbol corrosponding to the value - aim = POSSIBLE_AIMS[sell_indicators[indicator]] + aim = sell_indicators[indicator] # add the guard to its argument - sell_guards += f"if '{indicator}-enabled' in params and params['sell-{indicator}-enabled']: conditions.append(dataframe['{indicator}'] {aim} params['sell-{indicator}-value'])" + sell_guards += f"if '{indicator}-enabled' in params and params['sell-{indicator}-enabled']:\n conditions.append(dataframe['{indicator}'] {aim} params['sell-{indicator}-value'])\n" # add the space to its argument - sell_space += f"Integer(10, 90, name='sell-{indicator}-value'), Categorical([True, False], name='sell-{indicator}-enabled')," + sell_space += f"Integer(10, 90, name='sell-{indicator}-value'),\nCategorical([True, False], name='sell-{indicator}-enabled'),\n" # If the indicator is a trigger elif indicator in POSSIBLE_TRIGGERS: # get the symbol corrosponding to the value - aim = POSSIBLE_AIMS[sell_indicators[indicator]] + aim = sell_indicators[indicator] # add the trigger to its argument - sell_triggers += f"if params['sell-trigger'] == 'sell-{indicator}': conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])" + sell_triggers += f"if params['sell-trigger'] == 'sell-{indicator}':\n conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])\n" # Final line of indicator space makes all triggers @@ -276,4 +283,44 @@ def start_custom_hyperopt(args: Dict[str, Any]) -> None: deploy_custom_hyperopt(args['hyperopt'], new_path, buy_indicators, sell_indicators) -# --------------------------------------------------build-hyperopt------------------------------------------------------ \ No newline at end of file + +# --------------------------------------------------build-hyperopt------------------------------------------------------ +''' + TODO + -hyperopt optional (door bv standaard naming toe te passen) +''' + + +def start_build_hyperopt(args: Dict[str, Any]) -> None: + """ + Check if the right subcommands where passed and start building the hyperopt + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + # strategy and hyperopt need to be defined + if not 'strategy' in args or not args['strategy']: + raise OperationalException("`build-hyperopt` requires --strategy to be set.") + if not 'hyperopt' in args or not args['hyperopt']: + raise OperationalException("`build-hyperopt` requires --hyperopt to be set.") + else: + if args['hyperopt'] == 'DefaultHyperopt': + raise OperationalException("DefaultHyperopt is not allowed as name.") + + # the path of the chosen strategy + strategy_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py') + + # the path where the hyperopt should be written + new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py') + if new_path.exists(): + raise OperationalException(f"`{new_path}` already exists. " + "Please choose another Hyperopt Name.") + + # extract the buy and sell indicators as dicts + extracted_dicts = extract_dicts(strategy_path) + + buy_indicators = extracted_dicts[0] + sell_indicators = extracted_dicts[1] + + # use the dicts to write the hyperopt + deploy_custom_hyperopt(args['hyperopt'], new_path, + buy_indicators, sell_indicators) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 64c80dbe7..d8aafee19 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -388,14 +388,18 @@ TradeList = List[List] POSSIBLE_GUARDS = [ "rsi", "mfi", - "fastd" + "fastd", + "slowk", + "CDLHAMMER", + "fisher_rsi" ] POSSIBLE_TRIGGERS = [ + "sar", "bb_lowerband", "bb_upperband" ] -POSSIBLE_AIMS = { - "above": ">", - "below": "<", - "equal": "==" -} +POSSIBLE_AIMS = [ + ">", + "<", + "==" +] From 970b0028c1cdf6e9ee5bb2e35d08b91fe9410082 Mon Sep 17 00:00:00 2001 From: Bo van Hasselt Date: Sat, 20 Feb 2021 00:58:25 +0100 Subject: [PATCH 4/6] naming updates --- freqtrade/commands/automation_commands.py | 3 +++ freqtrade/commands/cli_options.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/automation_commands.py b/freqtrade/commands/automation_commands.py index d930c4851..74cb2cca6 100644 --- a/freqtrade/commands/automation_commands.py +++ b/freqtrade/commands/automation_commands.py @@ -25,6 +25,9 @@ logger = logging.getLogger(__name__) def extract_dicts(strategypath: Path): + """ + Get the indicators, their aims and the stoploss and format them into dictionaries + """ # store the file in a list for reference stored_file = [] with open(strategypath) as file: diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 7ad21cc6e..461ce8d32 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -279,13 +279,13 @@ AVAILABLE_CLI_OPTIONS = { "buy_indicators": Arg( '-b', '--buy-indicators', help='Specify the buy indicators the hyperopt should build. ' - 'Example: --buy-indicators `{"rsi":"above","bb_lowerband":"below"}`', + 'Example: --buy-indicators `{"rsi":"<","bb_lowerband":">"}`', metavar='DICT', ), "sell_indicators": Arg( '-s', '--sell-indicators', help='Specify the sell indicators the hyperopt should build. ' - 'Example: --sell-indicators `{"rsi":"above","bb_lowerband":"below"}`', + 'Example: --sell-indicators `{"rsi":">","bb_lowerband":"<"}`', metavar='DICT', ), "extract_name": Arg( From a215dc52120033e8e34eeab42e9f5ccba1345ded Mon Sep 17 00:00:00 2001 From: Bo van Hasselt Date: Sat, 20 Feb 2021 02:59:18 +0100 Subject: [PATCH 5/6] generalized the build-hyperopt command for all indicators possible for the strategies --- freqtrade/commands/automation_commands.py | 163 +++++++++++++--------- freqtrade/constants.py | 22 +-- 2 files changed, 96 insertions(+), 89 deletions(-) diff --git a/freqtrade/commands/automation_commands.py b/freqtrade/commands/automation_commands.py index 74cb2cca6..0e3fce0c7 100644 --- a/freqtrade/commands/automation_commands.py +++ b/freqtrade/commands/automation_commands.py @@ -1,13 +1,11 @@ import ast import logging +from os import EX_USAGE from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, List from freqtrade.constants import (USERPATH_HYPEROPTS, - USERPATH_STRATEGIES, - POSSIBLE_GUARDS, - POSSIBLE_TRIGGERS, - POSSIBLE_AIMS) + USERPATH_STRATEGIES) from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode from freqtrade.configuration import setup_utils_configuration @@ -24,10 +22,11 @@ logger = logging.getLogger(__name__) ''' -def extract_dicts(strategypath: Path): +def extract_lists(strategypath: Path): """ - Get the indicators, their aims and the stoploss and format them into dictionaries + Get the indicators, their aims and the stoploss and format them into lists """ + # store the file in a list for reference stored_file = [] with open(strategypath) as file: @@ -49,13 +48,13 @@ def extract_dicts(strategypath: Path): # populate the indicators dictionaries with indicators attached to the line they are on buyindicators = {} sellindicators = {} + for position, line in enumerate(stored_file): # check the lines in buy trend for indicator and add them if position in buy_lines and "(dataframe['" in line: # use split twice to remove the context around the indicator back_of_line = line.split("(dataframe['", 1)[1] buyindicator = back_of_line.split("'] ", 1)[0] - buyindicators[buyindicator] = position # check the lines in sell trend for indicator and add them @@ -63,35 +62,70 @@ def extract_dicts(strategypath: Path): # use split twice to remove the context around the indicator back_of_line = line.split("(dataframe['", 1)[1] sellindicator = back_of_line.split("'] ", 1)[0] - sellindicators[sellindicator] = position - # build the final buy dictionary - buy_dict = {} + # build the final buy list + buy_list = [] for indicator in buyindicators: + indicator_info = [] + # find the corrosponding aim for position, line in enumerate(stored_file): if position == buyindicators[indicator]: # use split twice to remove the context around the indicator back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1] aim = back_of_line.split()[0] - buy_dict[indicator] = aim - # build the final sell dictionary - sell_dict = {} + # add the indicator and aim to the info + indicator_info.append(indicator) + indicator_info.append(aim) + + # check if first character after aim is a d in which case the indicator is a trigger + if back_of_line.split()[1][0] == "d": + indicator_info.append("trigger") + + # add the second indicator of the guard to the info list + back_of_line = back_of_line.split("dataframe['")[1] + second_indicator = back_of_line.split("'])")[0] + indicator_info.append(second_indicator) + # else it is a guard + else: + indicator_info.append("guard") + buy_list.append(indicator_info) + + # build the final sell list + sell_list = [] for indicator in sellindicators: - # find the corrosponding aim + indicator_info = [] + + # find the corrosponding aim and whether a guard or trigger for position, line in enumerate(stored_file): if position == sellindicators[indicator]: # use split twice to remove the context around the indicator back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1] aim = back_of_line.split()[0] - sell_dict[indicator] = aim - # put the final dicts into a tuple - final_dicts = (buy_dict, sell_dict) + # add the indicator and aim to the info + indicator_info.append(indicator) + indicator_info.append(aim) - return final_dicts + # check if first character after aim is a d in which case the indicator is a trigger + if back_of_line.split()[1][0] == "d": + indicator_info.append("trigger") + + # add the second indicator of the guard to the info list + back_of_line = back_of_line.split("dataframe['")[1] + second_indicator = back_of_line.split("'])")[0] + indicator_info.append(second_indicator) + # else it is a guard + else: + indicator_info.append("guard") + sell_list.append(indicator_info) + + # put the final lists into a tuple + final_lists = (buy_list, sell_list) + + return final_lists def start_extract_strategy(args: Dict[str, Any]) -> None: @@ -116,11 +150,11 @@ def start_extract_strategy(args: Dict[str, Any]) -> None: strategy_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py') # extract the buy and sell indicators as dicts - extracted_dicts = str(extract_dicts(strategy_path)) + extracted_lists = str(extract_lists(strategy_path)) # save the dicts in a file logger.info(f"Writing custom hyperopt to `{new_path}`.") - new_path.write_text(extracted_dicts) + new_path.write_text(extracted_lists) # --------------------------------------------------custom-hyperopt------------------------------------------------------ @@ -135,7 +169,7 @@ def start_extract_strategy(args: Dict[str, Any]) -> None: ''' -def custom_hyperopt_buyelements(buy_indicators: Dict[str, str]): +def custom_hyperopt_buyelements(buy_indicators: List): """ Build the arguments with the placefillers for the buygenerator """ @@ -143,39 +177,35 @@ def custom_hyperopt_buyelements(buy_indicators: Dict[str, str]): buy_triggers = "" buy_space = "" - for indicator in buy_indicators: - # Error handling - if not indicator in POSSIBLE_GUARDS and not indicator in POSSIBLE_TRIGGERS: - raise OperationalException( - f"`{indicator}` is not part of the available indicators. The current options are {POSSIBLE_GUARDS + POSSIBLE_TRIGGERS}.") - elif not buy_indicators[indicator] in POSSIBLE_AIMS: - raise OperationalException( - f"`{buy_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_AIMS}.") - # If the indicator is a guard - elif indicator in POSSIBLE_GUARDS: - # get the symbol corrosponding to the value - aim = buy_indicators[indicator] + for indicator_info in buy_indicators: + indicator = indicator_info[0] + aim = indicator_info[1] + usage = indicator_info[2] + # If the indicator is a guard + if usage == "guard": # add the guard to its argument - buy_guards += f"if '{indicator}-enabled' in params and params['{indicator}-enabled']:\n conditions.append(dataframe['{indicator}'] {aim} params['{indicator}-value'])\n" + buy_guards += f"if params.get('{indicator}-enabled'):\n conditions.append(dataframe['{indicator}'] {aim} params['{indicator}-value'])\n" # add the space to its argument buy_space += f"Integer(10, 90, name='{indicator}-value'),\nCategorical([True, False], name='{indicator}-enabled'),\n" - # If the indicator is a trigger - elif indicator in POSSIBLE_TRIGGERS: - # get the symbol corrosponding to the value - aim = buy_indicators[indicator] + # If the indicator is a trigger + elif usage == "trigger": + secondindicator = indicator_info[3] # add the trigger to its argument - buy_triggers += f"if params['trigger'] == '{indicator}':\n conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])\n" + buy_triggers += f"if params['trigger'] == '{indicator}':\n conditions.append(dataframe['{indicator}'] {aim} dataframe['{secondindicator}'])\n" # Final line of indicator space makes all triggers buy_space += "Categorical([" # adding all triggers to the list - for indicator in buy_indicators: - if indicator in POSSIBLE_TRIGGERS: + for indicator_info in buy_indicators: + indicator = indicator_info[0] + usage = indicator_info[2] + + if usage == "trigger": buy_space += f"'{indicator}', " # Deleting the last ", " @@ -192,45 +222,42 @@ def custom_hyperopt_sellelements(sell_indicators: Dict[str, str]): sell_guards = "" sell_triggers = "" sell_space = "" + print(sell_indicators) + for indicator_info in sell_indicators: + indicator = indicator_info[0] + aim = indicator_info[1] + usage = indicator_info[2] - for indicator in sell_indicators: - # Error handling - if not indicator in POSSIBLE_GUARDS and not indicator in POSSIBLE_TRIGGERS: - raise OperationalException( - f"`{indicator}` is not part of the available indicators. The current options are {POSSIBLE_GUARDS + POSSIBLE_TRIGGERS}.") - elif not sell_indicators[indicator] in POSSIBLE_AIMS: - raise OperationalException( - f"`{sell_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_AIMS}.") - # If indicator is a guard - elif indicator in POSSIBLE_GUARDS: - # get the symbol corrosponding to the value - aim = sell_indicators[indicator] - + # If the indicator is a guard + if usage == "guard": # add the guard to its argument - sell_guards += f"if '{indicator}-enabled' in params and params['sell-{indicator}-enabled']:\n conditions.append(dataframe['{indicator}'] {aim} params['sell-{indicator}-value'])\n" + sell_guards += f"if params.get('sell-{indicator}-enabled'):\n conditions.append(dataframe['{indicator}'] {aim} params['sell-{indicator}-value'])\n" # add the space to its argument sell_space += f"Integer(10, 90, name='sell-{indicator}-value'),\nCategorical([True, False], name='sell-{indicator}-enabled'),\n" + # If the indicator is a trigger - elif indicator in POSSIBLE_TRIGGERS: - # get the symbol corrosponding to the value - aim = sell_indicators[indicator] + elif usage == "trigger": + secondindicator = indicator_info[3] # add the trigger to its argument - sell_triggers += f"if params['sell-trigger'] == 'sell-{indicator}':\n conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])\n" + sell_triggers += f"if params['sell-trigger'] == 'sell-{indicator}':\n conditions.append(dataframe['{indicator}'] {aim} dataframe['{secondindicator}'])\n" # Final line of indicator space makes all triggers sell_space += "Categorical([" - # Adding all triggers to the list - for indicator in sell_indicators: - if indicator in POSSIBLE_TRIGGERS: + # adding all triggers to the list + for indicator_info in sell_indicators: + indicator = indicator_info[0] + usage = indicator_info[2] + + if usage == "trigger": sell_space += f"'sell-{indicator}', " # Deleting the last ", " sell_space = sell_space[:-2] - sell_space += "], name='trigger')" + sell_space += "], name='sell-trigger')" return {"sell_guards": sell_guards, "sell_triggers": sell_triggers, "sell_space": sell_space} @@ -319,10 +346,10 @@ def start_build_hyperopt(args: Dict[str, Any]) -> None: "Please choose another Hyperopt Name.") # extract the buy and sell indicators as dicts - extracted_dicts = extract_dicts(strategy_path) - - buy_indicators = extracted_dicts[0] - sell_indicators = extracted_dicts[1] + extracted_lists = extract_lists(strategy_path) + print(extracted_lists) + buy_indicators = extracted_lists[0] + sell_indicators = extracted_lists[1] # use the dicts to write the hyperopt deploy_custom_hyperopt(args['hyperopt'], new_path, diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d8aafee19..782b69982 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -382,24 +382,4 @@ PairWithTimeframe = Tuple[str, str] ListPairsWithTimeframes = List[PairWithTimeframe] # Type for trades list -TradeList = List[List] - -#Build-hyperopt options -POSSIBLE_GUARDS = [ - "rsi", - "mfi", - "fastd", - "slowk", - "CDLHAMMER", - "fisher_rsi" -] -POSSIBLE_TRIGGERS = [ - "sar", - "bb_lowerband", - "bb_upperband" -] -POSSIBLE_AIMS = [ - ">", - "<", - "==" -] +TradeList = List[List] \ No newline at end of file From ee378a2a5467e6c7bb69effffa305f36d8442b1d Mon Sep 17 00:00:00 2001 From: Bo van Hasselt Date: Tue, 23 Feb 2021 00:27:29 +0100 Subject: [PATCH 6/6] version 1 ready for testing phase --- freqtrade/commands/automation_commands.py | 164 ++++++++---------- freqtrade/commands/cli_options.py | 10 +- .../templates/base_custom_hyperopt.py.j2 | 164 ++++++++++++++++++ freqtrade/templates/base_hyperopt.py.j2 | 24 ++- 4 files changed, 267 insertions(+), 95 deletions(-) create mode 100644 freqtrade/templates/base_custom_hyperopt.py.j2 diff --git a/freqtrade/commands/automation_commands.py b/freqtrade/commands/automation_commands.py index 0e3fce0c7..2e65e2076 100644 --- a/freqtrade/commands/automation_commands.py +++ b/freqtrade/commands/automation_commands.py @@ -1,6 +1,5 @@ import ast import logging -from os import EX_USAGE from pathlib import Path from typing import Any, Dict, List @@ -15,14 +14,54 @@ logger = logging.getLogger(__name__) # ---------------------------------------------------extract-strategy------------------------------------------------------ -''' - TODO - - get the stoploss value to optimize - - get the values from the guards to specify the optimzation range (cooperation with custom-hyperopt) -''' + +def get_indicator_info(file: List, indicators: Dict) -> None: + """ + Get all necessary information to build a custom hyperopt space using + the file and a dictionary filled with the indicators and their corropsonding line numbers. + """ + info_list = [] + for indicator in indicators: + indicator_info = [] + + # find the corrosponding aim + for position, line in enumerate(file): + if position == indicators[indicator]: + # use split twice to remove the context around the indicator + back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1] + aim = back_of_line.split()[0] + + # add the indicator and aim to the info + indicator_info.append(indicator) + indicator_info.append(aim) + + # check if first character after aim is a d in which case the indicator is a trigger + if back_of_line.split()[1][0] == "d": + indicator_info.append("trigger") + + # add the second indicator of the guard to the info list + back_of_line = back_of_line.split("dataframe['")[1] + second_indicator = back_of_line.split("'])")[0] + indicator_info.append(second_indicator) + + # elif indicator[0:3] == "CDL": + # indicator_info.append("guard") + + # else it is a regular guard + else: + indicator_info.append("guard") + + value = back_of_line.split()[1] + value = value[:-1] + value = float(value) + + indicator_info.append(value) + info_list.append(indicator_info) + + return info_list -def extract_lists(strategypath: Path): +def extract_lists(strategypath: Path) -> None: """ Get the indicators, their aims and the stoploss and format them into lists """ @@ -64,71 +103,17 @@ def extract_lists(strategypath: Path): sellindicator = back_of_line.split("'] ", 1)[0] sellindicators[sellindicator] = position - # build the final buy list - buy_list = [] - for indicator in buyindicators: - indicator_info = [] - - # find the corrosponding aim - for position, line in enumerate(stored_file): - if position == buyindicators[indicator]: - # use split twice to remove the context around the indicator - back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1] - aim = back_of_line.split()[0] - - # add the indicator and aim to the info - indicator_info.append(indicator) - indicator_info.append(aim) - - # check if first character after aim is a d in which case the indicator is a trigger - if back_of_line.split()[1][0] == "d": - indicator_info.append("trigger") - - # add the second indicator of the guard to the info list - back_of_line = back_of_line.split("dataframe['")[1] - second_indicator = back_of_line.split("'])")[0] - indicator_info.append(second_indicator) - # else it is a guard - else: - indicator_info.append("guard") - buy_list.append(indicator_info) - - # build the final sell list - sell_list = [] - for indicator in sellindicators: - indicator_info = [] - - # find the corrosponding aim and whether a guard or trigger - for position, line in enumerate(stored_file): - if position == sellindicators[indicator]: - # use split twice to remove the context around the indicator - back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1] - aim = back_of_line.split()[0] - - # add the indicator and aim to the info - indicator_info.append(indicator) - indicator_info.append(aim) - - # check if first character after aim is a d in which case the indicator is a trigger - if back_of_line.split()[1][0] == "d": - indicator_info.append("trigger") - - # add the second indicator of the guard to the info list - back_of_line = back_of_line.split("dataframe['")[1] - second_indicator = back_of_line.split("'])")[0] - indicator_info.append(second_indicator) - # else it is a guard - else: - indicator_info.append("guard") - sell_list.append(indicator_info) - + # build the final lists + buy_info_list = get_indicator_info(stored_file, buyindicators) + sell_info_list = get_indicator_info(stored_file, sellindicators) + # put the final lists into a tuple - final_lists = (buy_list, sell_list) + final_lists = (buy_info_list, sell_info_list) return final_lists -def start_extract_strategy(args: Dict[str, Any]) -> None: +def start_extract_strategy(args: Dict) -> None: """ Check if the right subcommands where passed and start extracting the strategy data """ @@ -158,16 +143,6 @@ def start_extract_strategy(args: Dict[str, Any]) -> None: # --------------------------------------------------custom-hyperopt------------------------------------------------------ -''' - TODO - -make the code below more dynamic with a large list of indicators and aims - -buy_space integer values variation based on aim and input value form extract-strategy(later deep learning) - -add --mode , see notes (denk hierbij ook aan value range bij guards) - -Custom stoploss and roi (based on input from extract-strategy) - -cli option to read extracted strategies files (--extraction) - -code cleanup (maybe the two elements functions can be combined) -''' - def custom_hyperopt_buyelements(buy_indicators: List): """ @@ -184,11 +159,20 @@ def custom_hyperopt_buyelements(buy_indicators: List): # If the indicator is a guard if usage == "guard": + value = indicator_info[3] + + if value >= -1.0 and value <= 1.0: + lower_bound = value - 0.3 + upper_bound = value + 0.3 + else: + lower_bound = value - 30.0 + upper_bound = value + 30.0 + # add the guard to its argument buy_guards += f"if params.get('{indicator}-enabled'):\n conditions.append(dataframe['{indicator}'] {aim} params['{indicator}-value'])\n" # add the space to its argument - buy_space += f"Integer(10, 90, name='{indicator}-value'),\nCategorical([True, False], name='{indicator}-enabled'),\n" + buy_space += f"Integer({lower_bound}, {upper_bound}, name='{indicator}-value'),\nCategorical([True, False], name='{indicator}-enabled'),\n" # If the indicator is a trigger elif usage == "trigger": @@ -222,7 +206,7 @@ def custom_hyperopt_sellelements(sell_indicators: Dict[str, str]): sell_guards = "" sell_triggers = "" sell_space = "" - print(sell_indicators) + for indicator_info in sell_indicators: indicator = indicator_info[0] aim = indicator_info[1] @@ -230,11 +214,20 @@ def custom_hyperopt_sellelements(sell_indicators: Dict[str, str]): # If the indicator is a guard if usage == "guard": + value = indicator_info[3] + + if value >= -1 and value <= 1: + lower_bound = value - 0.3 + upper_bound = value + 0.3 + else: + lower_bound = value - 30 + upper_bound = value + 30 + # add the guard to its argument sell_guards += f"if params.get('sell-{indicator}-enabled'):\n conditions.append(dataframe['{indicator}'] {aim} params['sell-{indicator}-value'])\n" # add the space to its argument - sell_space += f"Integer(10, 90, name='sell-{indicator}-value'),\nCategorical([True, False], name='sell-{indicator}-enabled'),\n" + sell_space += f"Integer({lower_bound}, {upper_bound}, name='sell-{indicator}-value'),\nCategorical([True, False], name='sell-{indicator}-enabled'),\n" # If the indicator is a trigger elif usage == "trigger": @@ -272,7 +265,7 @@ def deploy_custom_hyperopt(hyperopt_name: str, hyperopt_path: Path, buy_indicato sell_args = custom_hyperopt_sellelements(sell_indicators) # Build the final template - strategy_text = render_template(templatefile='base_hyperopt.py.j2', + strategy_text = render_template(templatefile='base_custom_hyperopt.py.j2', arguments={"hyperopt": hyperopt_name, "buy_guards": buy_args["buy_guards"], "buy_triggers": buy_args["buy_triggers"], @@ -315,11 +308,6 @@ def start_custom_hyperopt(args: Dict[str, Any]) -> None: # --------------------------------------------------build-hyperopt------------------------------------------------------ -''' - TODO - -hyperopt optional (door bv standaard naming toe te passen) -''' - def start_build_hyperopt(args: Dict[str, Any]) -> None: """ @@ -331,7 +319,7 @@ def start_build_hyperopt(args: Dict[str, Any]) -> None: if not 'strategy' in args or not args['strategy']: raise OperationalException("`build-hyperopt` requires --strategy to be set.") if not 'hyperopt' in args or not args['hyperopt']: - raise OperationalException("`build-hyperopt` requires --hyperopt to be set.") + args['hyperopt'] = args['strategy'] + "opt" else: if args['hyperopt'] == 'DefaultHyperopt': raise OperationalException("DefaultHyperopt is not allowed as name.") @@ -347,7 +335,7 @@ def start_build_hyperopt(args: Dict[str, Any]) -> None: # extract the buy and sell indicators as dicts extracted_lists = extract_lists(strategy_path) - print(extracted_lists) + buy_indicators = extracted_lists[0] sell_indicators = extracted_lists[1] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 461ce8d32..4704c31c3 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -279,14 +279,16 @@ AVAILABLE_CLI_OPTIONS = { "buy_indicators": Arg( '-b', '--buy-indicators', help='Specify the buy indicators the hyperopt should build. ' - 'Example: --buy-indicators `{"rsi":"<","bb_lowerband":">"}`', - metavar='DICT', + 'Example: --buy-indicators `[["rsi","<","trigger",30.0],["bb_lowerband",">","guard","close"]]`' + 'Check the documentation for specific requirements for the lists.', + metavar='LIST', ), "sell_indicators": Arg( '-s', '--sell-indicators', help='Specify the sell indicators the hyperopt should build. ' - 'Example: --sell-indicators `{"rsi":">","bb_lowerband":"<"}`', - metavar='DICT', + 'Example: --sell-indicators [["rsi",">","trigger",70.0],["bb_lowerband","<","guard","close"]]' + 'Check the documentation for specific requirements for the lists.', + metavar='LIST', ), "extract_name": Arg( '--extract-name', diff --git a/freqtrade/templates/base_custom_hyperopt.py.j2 b/freqtrade/templates/base_custom_hyperopt.py.j2 new file mode 100644 index 000000000..672b0b119 --- /dev/null +++ b/freqtrade/templates/base_custom_hyperopt.py.j2 @@ -0,0 +1,164 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +# --- Do not remove these libs --- +from functools import reduce +from typing import Any, Callable, Dict, List + +import numpy as np # noqa +import pandas as pd # noqa +from pandas import DataFrame +from skopt.space import Categorical, Dimension, Integer, Real # noqa + +from freqtrade.optimize.hyperopt_interface import IHyperOpt + +# -------------------------------- +# Add your lib to import here +import talib.abstract as ta # noqa +import freqtrade.vendor.qtpylib.indicators as qtpylib + + +class {{ hyperopt }}(IHyperOpt): + """ + This is a Hyperopt template to get you started. + + More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ + + You should: + - Add any lib you need to build your hyperopt. + + You must keep: + - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. + + The methods roi_space, generate_roi_table and stoploss_space are not required + and are provided by default. + However, you may override them if you need 'roi' and 'stoploss' spaces that + differ from the defaults offered by Freqtrade. + Sample implementation of these methods will be copied to `user_data/hyperopts` when + creating the user-data directory using `freqtrade create-userdir --userdir user_data`, + or is available online under the following URL: + https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py. + """ + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by Hyperopt. + """ + def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Buy strategy Hyperopt will build and use. + """ + conditions = [] + + # GUARDS AND TRENDS + {{ buy_guards | indent(12) }} + + # TRIGGERS + if 'trigger' in params: + {{ buy_triggers | indent(16) }} + + # Check that the candle had volume + conditions.append(dataframe['volume'] > 0) + + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching buy strategy parameters. + """ + return [ + {{ buy_space | indent(12) }} + ] + + @staticmethod + def sell_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the sell strategy parameters to be used by Hyperopt. + """ + def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Sell strategy Hyperopt will build and use. + """ + conditions = [] + + # GUARDS AND TRENDS + {{ sell_guards | indent(12) }} + + # TRIGGERS + if 'sell-trigger' in params: + {{ sell_triggers | indent(16) }} + + # Check that the candle had volume + conditions.append(dataframe['volume'] > 0) + + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'sell'] = 1 + + return dataframe + + return populate_sell_trend + + @staticmethod + def sell_indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching sell strategy parameters. + """ + return [ + {{ sell_space | indent(12) }} + ] + @ staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + This implementation generates the default legacy Freqtrade ROI tables. + Change it if you need different number of steps in the generated + ROI tables or other structure of the ROI tables. + Please keep it aligned with parameters in the 'roi' optimization + hyperspace defined by the roi_space method. + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @ staticmethod + def roi_space() -> List[Dimension]: + """ + Values to search for each ROI steps + Override it if you need some different ranges for the parameters in the + 'roi' optimization hyperspace. + Please keep it aligned with the implementation of the + generate_roi_table method. + """ + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + Real(0.01, 0.04, name='roi_p1'), + Real(0.01, 0.07, name='roi_p2'), + Real(0.01, 0.20, name='roi_p3'), + ] + + @ staticmethod + def stoploss_space() -> List[Dimension]: + """ + Stoploss Value to search + Override it if you need some different range for the parameter in the + 'stoploss' optimization hyperspace. + """ + return [ + Real(-0.35, -0.02, name='stoploss'), + ] diff --git a/freqtrade/templates/base_hyperopt.py.j2 b/freqtrade/templates/base_hyperopt.py.j2 index 38e2f4172..2bdfdba16 100644 --- a/freqtrade/templates/base_hyperopt.py.j2 +++ b/freqtrade/templates/base_hyperopt.py.j2 @@ -55,7 +55,16 @@ class {{ hyperopt }}(IHyperOpt): # TRIGGERS if 'trigger' in params: - {{ buy_triggers | indent(16) }} + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) # Check that the candle had volume conditions.append(dataframe['volume'] > 0) @@ -94,7 +103,16 @@ class {{ hyperopt }}(IHyperOpt): # TRIGGERS if 'sell-trigger' in params: - {{ sell_triggers | indent(16) }} + if params['sell-trigger'] == 'sell-bb_upper': + conditions.append(dataframe['close'] > dataframe['bb_upperband']) + if params['sell-trigger'] == 'sell-macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macdsignal'], dataframe['macd'] + )) + if params['sell-trigger'] == 'sell-sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['sar'], dataframe['close'] + )) # Check that the candle had volume conditions.append(dataframe['volume'] > 0) @@ -115,4 +133,4 @@ class {{ hyperopt }}(IHyperOpt): """ return [ {{ sell_space | indent(12) }} - ] + ] \ No newline at end of file