diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 0ce9b577b..83c18a205 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -12,7 +12,7 @@ from freqtrade.commands.data_commands import (start_convert_data, start_download start_list_data) from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, start_new_hyperopt, start_new_strategy) -from freqtrade.commands.automation_commands import start_build_hyperopt +from freqtrade.commands.automation_commands import (start_build_hyperopt, start_custom_hyperopt, start_extract_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 2393a39b0..5b9babccc 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -55,8 +55,13 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] +# Automation ARGS_BUILD_CUSTOM_HYPEROPT = ["buy_indicators", "sell_indicators", "hyperopt"] +ARGS_EXTRACT_STRATEGY = ["strategy", "extract_name"] + +ARGS_BUILD_BUILD_HYPEROPT = ["strategy", "hyperopt"] + ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] @@ -175,7 +180,7 @@ class Arguments: start_list_data, start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, start_list_timeframes, start_new_config, start_new_hyperopt, - start_build_hyperopt, + start_build_hyperopt, start_custom_hyperopt, start_extract_strategy, start_new_strategy, start_plot_dataframe, start_plot_profit, start_show_trades, start_test_pairlist, start_trading) @@ -210,12 +215,24 @@ class Arguments: build_hyperopt_cmd.set_defaults(func=start_new_hyperopt) self._build_args(optionlist=ARGS_BUILD_HYPEROPT, parser=build_hyperopt_cmd) - # add build-hyperopt subcommand - build_custom_hyperopt_cmd = subparsers.add_parser('build-hyperopt', + # add custom-hyperopt subcommand + build_custom_hyperopt_cmd = subparsers.add_parser('custom-hyperopt', help="Build a custom hyperopt") - build_custom_hyperopt_cmd.set_defaults(func=start_build_hyperopt) + build_custom_hyperopt_cmd.set_defaults(func=start_custom_hyperopt) self._build_args(optionlist=ARGS_BUILD_CUSTOM_HYPEROPT, parser=build_custom_hyperopt_cmd) + # add extract-strategy subcommand + extract_strategy_cmd = subparsers.add_parser('extract-strategy', + help="Extract data dictionaries for custom-hyperopt from strategy") + extract_strategy_cmd.set_defaults(func=start_extract_strategy) + self._build_args(optionlist=ARGS_EXTRACT_STRATEGY, parser=extract_strategy_cmd) + + # add build-hyperopt subcommand + build_extracted_hyperopt_cmd = subparsers.add_parser('build-hyperopt', + help="Create a hyperopt for a strategy") + build_extracted_hyperopt_cmd.set_defaults(func=start_build_hyperopt) + self._build_args(optionlist=ARGS_BUILD_BUILD_HYPEROPT, parser=build_extracted_hyperopt_cmd) + # add new-strategy subcommand build_strategy_cmd = subparsers.add_parser('new-strategy', help="Create new strategy") diff --git a/freqtrade/commands/automation_commands.py b/freqtrade/commands/automation_commands.py index a639a8fa2..d930c4851 100644 --- a/freqtrade/commands/automation_commands.py +++ b/freqtrade/commands/automation_commands.py @@ -3,7 +3,11 @@ import logging from pathlib import Path from typing import Any, Dict -from freqtrade.constants import USERPATH_HYPEROPTS +from freqtrade.constants import (USERPATH_HYPEROPTS, + USERPATH_STRATEGIES, + POSSIBLE_GUARDS, + POSSIBLE_TRIGGERS, + POSSIBLE_AIMS) from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode from freqtrade.configuration import setup_utils_configuration @@ -11,20 +15,124 @@ from freqtrade.misc import render_template logger = logging.getLogger(__name__) + +# ---------------------------------------------------extract-strategy------------------------------------------------------ ''' - TODO + TODO + - get the stoploss value to optimize + - get the values from the guards to specify the optimzation range (cooperation with custom-hyperopt) +''' + + +def extract_dicts(strategypath: Path): + # store the file in a list for reference + stored_file = [] + with open(strategypath) as file: + for line in file: + stored_file.append(line) + + # find the start and end of buy trend + for position, line in enumerate(stored_file): + if "populate_buy_trend(" in line: + start_buy_number = position + elif "populate_sell_trend(" in line: + end_buy_number = position + + # list the numbers between the start and end of buy trend + buy_lines = [] + for i in range(start_buy_number, end_buy_number): + buy_lines.append(i) + + # populate the indicators dictionaries with indicators attached to the line they are on + buyindicators = {} + sellindicators = {} + for position, line in enumerate(stored_file): + # check the lines in buy trend for indicator and add them + if position in buy_lines and "(dataframe['" in line: + # use split twice to remove the context around the indicator + back_of_line = line.split("(dataframe['", 1)[1] + buyindicator = back_of_line.split("'] ", 1)[0] + + buyindicators[buyindicator] = position + + # check the lines in sell trend for indicator and add them + elif position > end_buy_number and "(dataframe['" in line: + # use split twice to remove the context around the indicator + back_of_line = line.split("(dataframe['", 1)[1] + sellindicator = back_of_line.split("'] ", 1)[0] + + sellindicators[sellindicator] = position + + # build the final buy dictionary + buy_dict = {} + for indicator in buyindicators: + # find the corrosponding aim + for position, line in enumerate(stored_file): + if position == buyindicators[indicator]: + # use split twice to remove the context around the indicator + back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1] + aim = back_of_line.split()[0] + buy_dict[indicator] = aim + + # build the final sell dictionary + sell_dict = {} + for indicator in sellindicators: + # find the corrosponding aim + for position, line in enumerate(stored_file): + if position == sellindicators[indicator]: + # use split twice to remove the context around the indicator + back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1] + aim = back_of_line.split()[0] + sell_dict[indicator] = aim + + # put the final dicts into a tuple + final_dicts = (buy_dict, sell_dict) + + return final_dicts + + +def start_extract_strategy(args: Dict[str, Any]) -> None: + """ + Check if the right subcommands where passed and start extracting the strategy data + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + # check if all required options are filled in + if not 'strategy' in args or not args['strategy']: + raise OperationalException("`extract-strategy` requires --strategy to be set.") + else: + # if the name is not specified use (strategy)_extract + if not 'extract_name' in args or not args['extract_name']: + args['extract_name'] = args['strategy'] + "_extract" + + new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['extract_name'] + '.txt') + if new_path.exists(): + raise OperationalException(f"`{new_path}` already exists. " + "Please choose another name.") + # the path of the chosen strategy + strategy_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py') + + # extract the buy and sell indicators as dicts + extracted_dicts = str(extract_dicts(strategy_path)) + + # save the dicts in a file + logger.info(f"Writing custom hyperopt to `{new_path}`.") + new_path.write_text(extracted_dicts) + + +# --------------------------------------------------custom-hyperopt------------------------------------------------------ +''' + TODO -make the code below more dynamic with a large list of indicators and aims - -buy_space integer values variation based on aim(later deep learning) - -add --mode , see notes - -when making the strategy reading tool, make sure that the populate indicators gets copied to here + -buy_space integer values variation based on aim and input value form extract-strategy(later deep learning) + -add --mode , see notes (denk hierbij ook aan value range bij guards) + -Custom stoploss and roi (based on input from extract-strategy) + -cli option to read extracted strategies files (--extraction) + -code cleanup (maybe the two elements functions can be combined) ''' -POSSIBLE_GUARDS = ["rsi", "mfi", "fastd"] -POSSIBLE_TRIGGERS = ["bb_lowerband", "bb_upperband"] -POSSIBLE_VALUES = {"above": ">", "below": "<"} - -def build_hyperopt_buyelements(buy_indicators: Dict[str, str]): +def custom_hyperopt_buyelements(buy_indicators: Dict[str, str]): """ Build the arguments with the placefillers for the buygenerator """ @@ -37,29 +145,29 @@ def build_hyperopt_buyelements(buy_indicators: Dict[str, str]): if not indicator in POSSIBLE_GUARDS and not indicator in POSSIBLE_TRIGGERS: raise OperationalException( f"`{indicator}` is not part of the available indicators. The current options are {POSSIBLE_GUARDS + POSSIBLE_TRIGGERS}.") - elif not buy_indicators[indicator] in POSSIBLE_VALUES: + elif not buy_indicators[indicator] in POSSIBLE_AIMS: raise OperationalException( - f"`{buy_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_VALUES}.") + f"`{buy_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_AIMS}.") # If the indicator is a guard elif indicator in POSSIBLE_GUARDS: # get the symbol corrosponding to the value - aim = POSSIBLE_VALUES[buy_indicators[indicator]] + aim = buy_indicators[indicator] # add the guard to its argument - buy_guards += f"if '{indicator}-enabled' in params and params['{indicator}-enabled']: conditions.append(dataframe['{indicator}'] {aim} params['{indicator}-value'])" + buy_guards += f"if '{indicator}-enabled' in params and params['{indicator}-enabled']:\n conditions.append(dataframe['{indicator}'] {aim} params['{indicator}-value'])\n" # add the space to its argument - buy_space += f"Integer(10, 90, name='{indicator}-value'), Categorical([True, False], name='{indicator}-enabled')," + buy_space += f"Integer(10, 90, name='{indicator}-value'),\nCategorical([True, False], name='{indicator}-enabled'),\n" # If the indicator is a trigger elif indicator in POSSIBLE_TRIGGERS: # get the symbol corrosponding to the value - aim = POSSIBLE_VALUES[buy_indicators[indicator]] + aim = buy_indicators[indicator] # add the trigger to its argument - buy_triggers += f"if params['trigger'] == '{indicator}': conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])" + buy_triggers += f"if params['trigger'] == '{indicator}':\n conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])\n" # Final line of indicator space makes all triggers - + buy_space += "Categorical([" # adding all triggers to the list @@ -74,7 +182,7 @@ def build_hyperopt_buyelements(buy_indicators: Dict[str, str]): return {"buy_guards": buy_guards, "buy_triggers": buy_triggers, "buy_space": buy_space} -def build_hyperopt_sellelements(sell_indicators: Dict[str, str]): +def custom_hyperopt_sellelements(sell_indicators: Dict[str, str]): """ Build the arguments with the placefillers for the sellgenerator """ @@ -87,26 +195,26 @@ def build_hyperopt_sellelements(sell_indicators: Dict[str, str]): if not indicator in POSSIBLE_GUARDS and not indicator in POSSIBLE_TRIGGERS: raise OperationalException( f"`{indicator}` is not part of the available indicators. The current options are {POSSIBLE_GUARDS + POSSIBLE_TRIGGERS}.") - elif not sell_indicators[indicator] in POSSIBLE_VALUES: + elif not sell_indicators[indicator] in POSSIBLE_AIMS: raise OperationalException( - f"`{sell_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_VALUES}.") + f"`{sell_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_AIMS}.") # If indicator is a guard elif indicator in POSSIBLE_GUARDS: # get the symbol corrosponding to the value - aim = POSSIBLE_VALUES[sell_indicators[indicator]] + aim = sell_indicators[indicator] # add the guard to its argument - sell_guards += f"if '{indicator}-enabled' in params and params['sell-{indicator}-enabled']: conditions.append(dataframe['{indicator}'] {aim} params['sell-{indicator}-value'])" + sell_guards += f"if '{indicator}-enabled' in params and params['sell-{indicator}-enabled']:\n conditions.append(dataframe['{indicator}'] {aim} params['sell-{indicator}-value'])\n" # add the space to its argument - sell_space += f"Integer(10, 90, name='sell-{indicator}-value'), Categorical([True, False], name='sell-{indicator}-enabled')," + sell_space += f"Integer(10, 90, name='sell-{indicator}-value'),\nCategorical([True, False], name='sell-{indicator}-enabled'),\n" # If the indicator is a trigger elif indicator in POSSIBLE_TRIGGERS: # get the symbol corrosponding to the value - aim = POSSIBLE_VALUES[sell_indicators[indicator]] + aim = sell_indicators[indicator] # add the trigger to its argument - sell_triggers += f"if params['sell-trigger'] == 'sell-{indicator}': conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])" + sell_triggers += f"if params['sell-trigger'] == 'sell-{indicator}':\n conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])\n" # Final line of indicator space makes all triggers @@ -130,8 +238,8 @@ def deploy_custom_hyperopt(hyperopt_name: str, hyperopt_path: Path, buy_indicato """ # Build the arguments for the buy and sell generators - buy_args = build_hyperopt_buyelements(buy_indicators) - sell_args = build_hyperopt_sellelements(sell_indicators) + buy_args = custom_hyperopt_buyelements(buy_indicators) + sell_args = custom_hyperopt_sellelements(sell_indicators) # Build the final template strategy_text = render_template(templatefile='base_hyperopt.py.j2', @@ -148,19 +256,18 @@ def deploy_custom_hyperopt(hyperopt_name: str, hyperopt_path: Path, buy_indicato hyperopt_path.write_text(strategy_text) -def start_build_hyperopt(args: Dict[str, Any]) -> None: +def start_custom_hyperopt(args: Dict[str, Any]) -> None: """ Check if the right subcommands where passed and start building the hyperopt """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - # check what the name of the hyperopt should if not 'hyperopt' in args or not args['hyperopt']: - raise OperationalException("`build-hyperopt` requires --hyperopt to be set.") + raise OperationalException("`custom-hyperopt` requires --hyperopt to be set.") elif not 'buy_indicators' in args or not args['buy_indicators']: - raise OperationalException("`build-hyperopt` requires --buy-indicators to be set.") + raise OperationalException("`custom-hyperopt` requires --buy-indicators to be set.") elif not 'sell_indicators' in args or not args['sell_indicators']: - raise OperationalException("`build-hyperopt` requires --sell-indicators to be set.") + raise OperationalException("`custom-hyperopt` requires --sell-indicators to be set.") else: if args['hyperopt'] == 'DefaultHyperopt': raise OperationalException("DefaultHyperopt is not allowed as name.") @@ -175,3 +282,45 @@ def start_build_hyperopt(args: Dict[str, Any]) -> None: deploy_custom_hyperopt(args['hyperopt'], new_path, buy_indicators, sell_indicators) + + +# --------------------------------------------------build-hyperopt------------------------------------------------------ +''' + TODO + -hyperopt optional (door bv standaard naming toe te passen) +''' + + +def start_build_hyperopt(args: Dict[str, Any]) -> None: + """ + Check if the right subcommands where passed and start building the hyperopt + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + # strategy and hyperopt need to be defined + if not 'strategy' in args or not args['strategy']: + raise OperationalException("`build-hyperopt` requires --strategy to be set.") + if not 'hyperopt' in args or not args['hyperopt']: + raise OperationalException("`build-hyperopt` requires --hyperopt to be set.") + else: + if args['hyperopt'] == 'DefaultHyperopt': + raise OperationalException("DefaultHyperopt is not allowed as name.") + + # the path of the chosen strategy + strategy_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py') + + # the path where the hyperopt should be written + new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py') + if new_path.exists(): + raise OperationalException(f"`{new_path}` already exists. " + "Please choose another Hyperopt Name.") + + # extract the buy and sell indicators as dicts + extracted_dicts = extract_dicts(strategy_path) + + buy_indicators = extracted_dicts[0] + sell_indicators = extracted_dicts[1] + + # use the dicts to write the hyperopt + deploy_custom_hyperopt(args['hyperopt'], new_path, + buy_indicators, sell_indicators) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 1130319b0..7ad21cc6e 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -288,6 +288,11 @@ AVAILABLE_CLI_OPTIONS = { 'Example: --sell-indicators `{"rsi":"above","bb_lowerband":"below"}`', metavar='DICT', ), + "extract_name": Arg( + '--extract-name', + help='Specify the name of the file to which the data should be extracted. ', + metavar='FILENAME', + ), # List exchanges "print_one_column": Arg( '-1', '--one-column', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 69301ca0e..d8aafee19 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -383,3 +383,23 @@ ListPairsWithTimeframes = List[PairWithTimeframe] # Type for trades list TradeList = List[List] + +#Build-hyperopt options +POSSIBLE_GUARDS = [ + "rsi", + "mfi", + "fastd", + "slowk", + "CDLHAMMER", + "fisher_rsi" +] +POSSIBLE_TRIGGERS = [ + "sar", + "bb_lowerband", + "bb_upperband" +] +POSSIBLE_AIMS = [ + ">", + "<", + "==" +]