Merge pull request #2 from Bovhasselt/Bo

Bo

freqtrade/commands/__init__.py

@@ -12,7 +12,7 @@ from freqtrade.commands.data_commands import (start_convert_data, start_download
                                               start_list_data)
 from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui,
                                                 start_new_hyperopt, start_new_strategy)
-from freqtrade.commands.automation_commands import start_build_hyperopt
+from freqtrade.commands.automation_commands import (start_build_hyperopt, start_custom_hyperopt, start_extract_strategy)
 from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show
 from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts,
                                               start_list_markets, start_list_strategies,

freqtrade/commands/arguments.py

@@ -55,8 +55,13 @@ ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"]
 
 ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"]
 
+# Automation
 ARGS_BUILD_CUSTOM_HYPEROPT = ["buy_indicators", "sell_indicators", "hyperopt"]
 
+ARGS_EXTRACT_STRATEGY = ["strategy", "extract_name"]
+
+ARGS_BUILD_BUILD_HYPEROPT = ["strategy", "hyperopt"]
+
 ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"]
 ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
 
@@ -175,7 +180,7 @@ class Arguments:
                                         start_list_data, start_list_exchanges, start_list_hyperopts,
                                         start_list_markets, start_list_strategies,
                                         start_list_timeframes, start_new_config, start_new_hyperopt,
-                                        start_build_hyperopt,
+                                        start_build_hyperopt, start_custom_hyperopt, start_extract_strategy,
                                         start_new_strategy, start_plot_dataframe, start_plot_profit,
                                         start_show_trades, start_test_pairlist, start_trading)
 
@@ -210,12 +215,24 @@ class Arguments:
         build_hyperopt_cmd.set_defaults(func=start_new_hyperopt)
         self._build_args(optionlist=ARGS_BUILD_HYPEROPT, parser=build_hyperopt_cmd)
 
-        # add build-hyperopt subcommand
+        # add custom-hyperopt subcommand
-        build_custom_hyperopt_cmd = subparsers.add_parser('build-hyperopt',
+        build_custom_hyperopt_cmd = subparsers.add_parser('custom-hyperopt',
                                                           help="Build a custom hyperopt")
-        build_custom_hyperopt_cmd.set_defaults(func=start_build_hyperopt)
+        build_custom_hyperopt_cmd.set_defaults(func=start_custom_hyperopt)
         self._build_args(optionlist=ARGS_BUILD_CUSTOM_HYPEROPT, parser=build_custom_hyperopt_cmd)
 
+        # add extract-strategy subcommand
+        extract_strategy_cmd = subparsers.add_parser('extract-strategy',
+                                                     help="Extract data dictionaries for custom-hyperopt from strategy")
+        extract_strategy_cmd.set_defaults(func=start_extract_strategy)
+        self._build_args(optionlist=ARGS_EXTRACT_STRATEGY, parser=extract_strategy_cmd)
+
+        # add build-hyperopt subcommand
+        build_extracted_hyperopt_cmd = subparsers.add_parser('build-hyperopt',
+                                                     help="Create a hyperopt for a strategy")
+        build_extracted_hyperopt_cmd.set_defaults(func=start_build_hyperopt)
+        self._build_args(optionlist=ARGS_BUILD_BUILD_HYPEROPT, parser=build_extracted_hyperopt_cmd)
+
         # add new-strategy subcommand
         build_strategy_cmd = subparsers.add_parser('new-strategy',
                                                    help="Create new strategy")

freqtrade/commands/automation_commands.py

@@ -3,7 +3,11 @@ import logging
 from pathlib import Path
 from typing import Any, Dict
 
-from freqtrade.constants import USERPATH_HYPEROPTS
+from freqtrade.constants import (USERPATH_HYPEROPTS,
+                                 USERPATH_STRATEGIES,
+                                 POSSIBLE_GUARDS,
+                                 POSSIBLE_TRIGGERS,
+                                 POSSIBLE_AIMS)
 from freqtrade.exceptions import OperationalException
 from freqtrade.state import RunMode
 from freqtrade.configuration import setup_utils_configuration
@@ -11,20 +15,124 @@ from freqtrade.misc import render_template
 
 logger = logging.getLogger(__name__)
 
+
+# ---------------------------------------------------extract-strategy------------------------------------------------------
+'''
+    TODO
+    - get the stoploss value to optimize
+    - get the values from the guards to specify the optimzation range (cooperation with custom-hyperopt)
+'''
+
+
+def extract_dicts(strategypath: Path):
+    # store the file in a list for reference
+    stored_file = []
+    with open(strategypath) as file:
+        for line in file:
+            stored_file.append(line)
+
+    # find the start and end of buy trend
+    for position, line in enumerate(stored_file):
+        if "populate_buy_trend(" in line:
+            start_buy_number = position
+        elif "populate_sell_trend(" in line:
+            end_buy_number = position
+
+    # list the numbers between the start and end of buy trend
+    buy_lines = []
+    for i in range(start_buy_number, end_buy_number):
+        buy_lines.append(i)
+
+    # populate the indicators dictionaries with indicators attached to the line they are on
+    buyindicators = {}
+    sellindicators = {}
+    for position, line in enumerate(stored_file):
+        # check the lines in buy trend for indicator and add them
+        if position in buy_lines and "(dataframe['" in line:
+            # use split twice to remove the context around the indicator
+            back_of_line = line.split("(dataframe['", 1)[1]
+            buyindicator = back_of_line.split("'] ", 1)[0]
+
+            buyindicators[buyindicator] = position
+
+        # check the lines in sell trend for indicator and add them
+        elif position > end_buy_number and "(dataframe['" in line:
+            # use split twice to remove the context around the indicator
+            back_of_line = line.split("(dataframe['", 1)[1]
+            sellindicator = back_of_line.split("'] ", 1)[0]
+
+            sellindicators[sellindicator] = position
+
+    # build the final buy dictionary
+    buy_dict = {}
+    for indicator in buyindicators:
+        # find the corrosponding aim
+        for position, line in enumerate(stored_file):
+            if position == buyindicators[indicator]:
+                # use split twice to remove the context around the indicator
+                back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1]
+                aim = back_of_line.split()[0]
+        buy_dict[indicator] = aim
+
+    # build the final sell dictionary
+    sell_dict = {}
+    for indicator in sellindicators:
+        # find the corrosponding aim
+        for position, line in enumerate(stored_file):
+            if position == sellindicators[indicator]:
+                # use split twice to remove the context around the indicator
+                back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1]
+                aim = back_of_line.split()[0]
+        sell_dict[indicator] = aim
+
+    # put the final dicts into a tuple
+    final_dicts = (buy_dict, sell_dict)
+
+    return final_dicts
+
+
+def start_extract_strategy(args: Dict[str, Any]) -> None:
+    """
+    Check if the right subcommands where passed and start extracting the strategy data
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    # check if all required options are filled in
+    if not 'strategy' in args or not args['strategy']:
+        raise OperationalException("`extract-strategy` requires --strategy to be set.")
+    else:
+        # if the name is not specified use (strategy)_extract
+        if not 'extract_name' in args or not args['extract_name']:
+            args['extract_name'] = args['strategy'] + "_extract"
+
+        new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['extract_name'] + '.txt')
+        if new_path.exists():
+            raise OperationalException(f"`{new_path}` already exists. "
+                                       "Please choose another name.")
+        # the path of the chosen strategy
+        strategy_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py')
+
+        # extract the buy and sell indicators as dicts
+        extracted_dicts = str(extract_dicts(strategy_path))
+
+        # save the dicts in a file
+        logger.info(f"Writing custom hyperopt to `{new_path}`.")
+        new_path.write_text(extracted_dicts)
+
+
+# --------------------------------------------------custom-hyperopt------------------------------------------------------
 '''
     TODO
     -make the code below more dynamic with a large list of indicators and aims
-    -buy_space integer values variation based on aim(later deep learning)
+    -buy_space integer values variation based on aim and input value form extract-strategy(later deep learning)
-    -add --mode , see notes
+    -add --mode , see notes (denk hierbij ook aan value range bij guards)
-    -when making the strategy reading tool, make sure that the populate indicators gets copied to here
+    -Custom stoploss and roi (based on input from extract-strategy)
+    -cli option to read extracted strategies files (--extraction)
+    -code cleanup (maybe the two elements functions can be combined)
 '''
 
-POSSIBLE_GUARDS = ["rsi", "mfi", "fastd"]
-POSSIBLE_TRIGGERS = ["bb_lowerband", "bb_upperband"]
-POSSIBLE_VALUES = {"above": ">", "below": "<"}
 
-
+def custom_hyperopt_buyelements(buy_indicators: Dict[str, str]):
-def build_hyperopt_buyelements(buy_indicators: Dict[str, str]):
     """
     Build the arguments with the placefillers for the buygenerator
     """
@@ -37,26 +145,26 @@ def build_hyperopt_buyelements(buy_indicators: Dict[str, str]):
         if not indicator in POSSIBLE_GUARDS and not indicator in POSSIBLE_TRIGGERS:
             raise OperationalException(
                 f"`{indicator}` is not part of the available indicators. The current options are {POSSIBLE_GUARDS + POSSIBLE_TRIGGERS}.")
-        elif not buy_indicators[indicator] in POSSIBLE_VALUES:
+        elif not buy_indicators[indicator] in POSSIBLE_AIMS:
             raise OperationalException(
-                f"`{buy_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_VALUES}.")
+                f"`{buy_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_AIMS}.")
         # If the indicator is a guard
         elif indicator in POSSIBLE_GUARDS:
             # get the symbol corrosponding to the value
-            aim = POSSIBLE_VALUES[buy_indicators[indicator]]
+            aim = buy_indicators[indicator]
 
             # add the guard to its argument
-            buy_guards += f"if '{indicator}-enabled' in params and params['{indicator}-enabled']: conditions.append(dataframe['{indicator}'] {aim} params['{indicator}-value'])"
+            buy_guards += f"if '{indicator}-enabled' in params and params['{indicator}-enabled']:\n    conditions.append(dataframe['{indicator}'] {aim} params['{indicator}-value'])\n"
 
             # add the space to its argument
-            buy_space += f"Integer(10, 90, name='{indicator}-value'), Categorical([True, False], name='{indicator}-enabled'),"
+            buy_space += f"Integer(10, 90, name='{indicator}-value'),\nCategorical([True, False], name='{indicator}-enabled'),\n"
         # If the indicator is a trigger
         elif indicator in POSSIBLE_TRIGGERS:
             # get the symbol corrosponding to the value
-            aim = POSSIBLE_VALUES[buy_indicators[indicator]]
+            aim = buy_indicators[indicator]
 
             # add the trigger to its argument
-            buy_triggers += f"if params['trigger'] == '{indicator}': conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])"
+            buy_triggers += f"if params['trigger'] == '{indicator}':\n    conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])\n"
 
     # Final line of indicator space makes all triggers
 
@@ -74,7 +182,7 @@ def build_hyperopt_buyelements(buy_indicators: Dict[str, str]):
     return {"buy_guards": buy_guards, "buy_triggers": buy_triggers, "buy_space": buy_space}
 
 
-def build_hyperopt_sellelements(sell_indicators: Dict[str, str]):
+def custom_hyperopt_sellelements(sell_indicators: Dict[str, str]):
     """
     Build the arguments with the placefillers for the sellgenerator
     """
@@ -87,26 +195,26 @@ def build_hyperopt_sellelements(sell_indicators: Dict[str, str]):
         if not indicator in POSSIBLE_GUARDS and not indicator in POSSIBLE_TRIGGERS:
             raise OperationalException(
                 f"`{indicator}` is not part of the available indicators. The current options are {POSSIBLE_GUARDS + POSSIBLE_TRIGGERS}.")
-        elif not sell_indicators[indicator] in POSSIBLE_VALUES:
+        elif not sell_indicators[indicator] in POSSIBLE_AIMS:
             raise OperationalException(
-                f"`{sell_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_VALUES}.")
+                f"`{sell_indicators[indicator]}` is not part of the available indicator options. The current options are {POSSIBLE_AIMS}.")
         # If indicator is a guard
         elif indicator in POSSIBLE_GUARDS:
             # get the symbol corrosponding to the value
-            aim = POSSIBLE_VALUES[sell_indicators[indicator]]
+            aim = sell_indicators[indicator]
 
             # add the guard to its argument
-            sell_guards += f"if '{indicator}-enabled' in params and params['sell-{indicator}-enabled']: conditions.append(dataframe['{indicator}'] {aim} params['sell-{indicator}-value'])"
+            sell_guards += f"if '{indicator}-enabled' in params and params['sell-{indicator}-enabled']:\n    conditions.append(dataframe['{indicator}'] {aim} params['sell-{indicator}-value'])\n"
 
             # add the space to its argument
-            sell_space += f"Integer(10, 90, name='sell-{indicator}-value'), Categorical([True, False], name='sell-{indicator}-enabled'),"
+            sell_space += f"Integer(10, 90, name='sell-{indicator}-value'),\nCategorical([True, False], name='sell-{indicator}-enabled'),\n"
         # If the indicator is a trigger
         elif indicator in POSSIBLE_TRIGGERS:
             # get the symbol corrosponding to the value
-            aim = POSSIBLE_VALUES[sell_indicators[indicator]]
+            aim = sell_indicators[indicator]
 
             # add the trigger to its argument
-            sell_triggers += f"if params['sell-trigger'] == 'sell-{indicator}': conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])"
+            sell_triggers += f"if params['sell-trigger'] == 'sell-{indicator}':\n    conditions.append(dataframe['{indicator}'] {aim} dataframe['close'])\n"
 
     # Final line of indicator space makes all triggers
 
@@ -130,8 +238,8 @@ def deploy_custom_hyperopt(hyperopt_name: str, hyperopt_path: Path, buy_indicato
     """
 
     # Build the arguments for the buy and sell generators
-    buy_args = build_hyperopt_buyelements(buy_indicators)
+    buy_args = custom_hyperopt_buyelements(buy_indicators)
-    sell_args = build_hyperopt_sellelements(sell_indicators)
+    sell_args = custom_hyperopt_sellelements(sell_indicators)
 
     # Build the final template
     strategy_text = render_template(templatefile='base_hyperopt.py.j2',
@@ -148,19 +256,18 @@ def deploy_custom_hyperopt(hyperopt_name: str, hyperopt_path: Path, buy_indicato
     hyperopt_path.write_text(strategy_text)
 
 
-def start_build_hyperopt(args: Dict[str, Any]) -> None:
+def start_custom_hyperopt(args: Dict[str, Any]) -> None:
     """
     Check if the right subcommands where passed and start building the hyperopt
     """
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
-    # check what the name of the hyperopt should
     if not 'hyperopt' in args or not args['hyperopt']:
-        raise OperationalException("`build-hyperopt` requires --hyperopt to be set.")
+        raise OperationalException("`custom-hyperopt` requires --hyperopt to be set.")
     elif not 'buy_indicators' in args or not args['buy_indicators']:
-        raise OperationalException("`build-hyperopt` requires --buy-indicators to be set.")
+        raise OperationalException("`custom-hyperopt` requires --buy-indicators to be set.")
     elif not 'sell_indicators' in args or not args['sell_indicators']:
-        raise OperationalException("`build-hyperopt` requires --sell-indicators to be set.")
+        raise OperationalException("`custom-hyperopt` requires --sell-indicators to be set.")
     else:
         if args['hyperopt'] == 'DefaultHyperopt':
             raise OperationalException("DefaultHyperopt is not allowed as name.")
@@ -175,3 +282,45 @@ def start_build_hyperopt(args: Dict[str, Any]) -> None:
 
         deploy_custom_hyperopt(args['hyperopt'], new_path,
                                buy_indicators, sell_indicators)
+
+
+# --------------------------------------------------build-hyperopt------------------------------------------------------
+'''
+    TODO
+    -hyperopt optional (door bv standaard naming toe te passen)
+'''
+
+
+def start_build_hyperopt(args: Dict[str, Any]) -> None:
+    """
+    Check if the right subcommands where passed and start building the hyperopt
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    # strategy and hyperopt need to be defined
+    if not 'strategy' in args or not args['strategy']:
+        raise OperationalException("`build-hyperopt` requires --strategy to be set.")
+    if not 'hyperopt' in args or not args['hyperopt']:
+        raise OperationalException("`build-hyperopt` requires --hyperopt to be set.")
+    else:
+        if args['hyperopt'] == 'DefaultHyperopt':
+            raise OperationalException("DefaultHyperopt is not allowed as name.")
+
+        # the path of the chosen strategy
+        strategy_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py')
+
+        # the path where the hyperopt should be written
+        new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py')
+        if new_path.exists():
+            raise OperationalException(f"`{new_path}` already exists. "
+                                       "Please choose another Hyperopt Name.")
+
+        # extract the buy and sell indicators as dicts
+        extracted_dicts = extract_dicts(strategy_path)
+
+        buy_indicators = extracted_dicts[0]
+        sell_indicators = extracted_dicts[1]
+
+        # use the dicts to write the hyperopt
+        deploy_custom_hyperopt(args['hyperopt'], new_path,
+                               buy_indicators, sell_indicators)

freqtrade/commands/cli_options.py

@@ -288,6 +288,11 @@ AVAILABLE_CLI_OPTIONS = {
         'Example: --sell-indicators `{"rsi":"above","bb_lowerband":"below"}`',
         metavar='DICT',
     ),
+    "extract_name": Arg(
+        '--extract-name',
+        help='Specify the name of the file to which the data should be extracted. ',
+        metavar='FILENAME',
+    ),
     # List exchanges
     "print_one_column": Arg(
         '-1', '--one-column',

freqtrade/constants.py

@@ -383,3 +383,23 @@ ListPairsWithTimeframes = List[PairWithTimeframe]
 
 # Type for trades list
 TradeList = List[List]
+
+#Build-hyperopt options
+POSSIBLE_GUARDS = [
+    "rsi", 
+    "mfi", 
+    "fastd",
+    "slowk",
+    "CDLHAMMER",
+    "fisher_rsi"
+]
+POSSIBLE_TRIGGERS = [
+    "sar",
+    "bb_lowerband", 
+    "bb_upperband"
+]
+POSSIBLE_AIMS = [
+    ">", 
+    "<",
+    "=="
+]