diff --git a/freqtrade/commands/automation_commands.py b/freqtrade/commands/automation_commands.py index 0e3fce0c7..2e65e2076 100644 --- a/freqtrade/commands/automation_commands.py +++ b/freqtrade/commands/automation_commands.py @@ -1,6 +1,5 @@ import ast import logging -from os import EX_USAGE from pathlib import Path from typing import Any, Dict, List @@ -15,14 +14,54 @@ logger = logging.getLogger(__name__) # ---------------------------------------------------extract-strategy------------------------------------------------------ -''' - TODO - - get the stoploss value to optimize - - get the values from the guards to specify the optimzation range (cooperation with custom-hyperopt) -''' + +def get_indicator_info(file: List, indicators: Dict) -> None: + """ + Get all necessary information to build a custom hyperopt space using + the file and a dictionary filled with the indicators and their corropsonding line numbers. + """ + info_list = [] + for indicator in indicators: + indicator_info = [] + + # find the corrosponding aim + for position, line in enumerate(file): + if position == indicators[indicator]: + # use split twice to remove the context around the indicator + back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1] + aim = back_of_line.split()[0] + + # add the indicator and aim to the info + indicator_info.append(indicator) + indicator_info.append(aim) + + # check if first character after aim is a d in which case the indicator is a trigger + if back_of_line.split()[1][0] == "d": + indicator_info.append("trigger") + + # add the second indicator of the guard to the info list + back_of_line = back_of_line.split("dataframe['")[1] + second_indicator = back_of_line.split("'])")[0] + indicator_info.append(second_indicator) + + # elif indicator[0:3] == "CDL": + # indicator_info.append("guard") + + # else it is a regular guard + else: + indicator_info.append("guard") + + value = back_of_line.split()[1] + value = value[:-1] + value = float(value) + + indicator_info.append(value) + info_list.append(indicator_info) + + return info_list -def extract_lists(strategypath: Path): +def extract_lists(strategypath: Path) -> None: """ Get the indicators, their aims and the stoploss and format them into lists """ @@ -64,71 +103,17 @@ def extract_lists(strategypath: Path): sellindicator = back_of_line.split("'] ", 1)[0] sellindicators[sellindicator] = position - # build the final buy list - buy_list = [] - for indicator in buyindicators: - indicator_info = [] - - # find the corrosponding aim - for position, line in enumerate(stored_file): - if position == buyindicators[indicator]: - # use split twice to remove the context around the indicator - back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1] - aim = back_of_line.split()[0] - - # add the indicator and aim to the info - indicator_info.append(indicator) - indicator_info.append(aim) - - # check if first character after aim is a d in which case the indicator is a trigger - if back_of_line.split()[1][0] == "d": - indicator_info.append("trigger") - - # add the second indicator of the guard to the info list - back_of_line = back_of_line.split("dataframe['")[1] - second_indicator = back_of_line.split("'])")[0] - indicator_info.append(second_indicator) - # else it is a guard - else: - indicator_info.append("guard") - buy_list.append(indicator_info) - - # build the final sell list - sell_list = [] - for indicator in sellindicators: - indicator_info = [] - - # find the corrosponding aim and whether a guard or trigger - for position, line in enumerate(stored_file): - if position == sellindicators[indicator]: - # use split twice to remove the context around the indicator - back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1] - aim = back_of_line.split()[0] - - # add the indicator and aim to the info - indicator_info.append(indicator) - indicator_info.append(aim) - - # check if first character after aim is a d in which case the indicator is a trigger - if back_of_line.split()[1][0] == "d": - indicator_info.append("trigger") - - # add the second indicator of the guard to the info list - back_of_line = back_of_line.split("dataframe['")[1] - second_indicator = back_of_line.split("'])")[0] - indicator_info.append(second_indicator) - # else it is a guard - else: - indicator_info.append("guard") - sell_list.append(indicator_info) - + # build the final lists + buy_info_list = get_indicator_info(stored_file, buyindicators) + sell_info_list = get_indicator_info(stored_file, sellindicators) + # put the final lists into a tuple - final_lists = (buy_list, sell_list) + final_lists = (buy_info_list, sell_info_list) return final_lists -def start_extract_strategy(args: Dict[str, Any]) -> None: +def start_extract_strategy(args: Dict) -> None: """ Check if the right subcommands where passed and start extracting the strategy data """ @@ -158,16 +143,6 @@ def start_extract_strategy(args: Dict[str, Any]) -> None: # --------------------------------------------------custom-hyperopt------------------------------------------------------ -''' - TODO - -make the code below more dynamic with a large list of indicators and aims - -buy_space integer values variation based on aim and input value form extract-strategy(later deep learning) - -add --mode , see notes (denk hierbij ook aan value range bij guards) - -Custom stoploss and roi (based on input from extract-strategy) - -cli option to read extracted strategies files (--extraction) - -code cleanup (maybe the two elements functions can be combined) -''' - def custom_hyperopt_buyelements(buy_indicators: List): """ @@ -184,11 +159,20 @@ def custom_hyperopt_buyelements(buy_indicators: List): # If the indicator is a guard if usage == "guard": + value = indicator_info[3] + + if value >= -1.0 and value <= 1.0: + lower_bound = value - 0.3 + upper_bound = value + 0.3 + else: + lower_bound = value - 30.0 + upper_bound = value + 30.0 + # add the guard to its argument buy_guards += f"if params.get('{indicator}-enabled'):\n conditions.append(dataframe['{indicator}'] {aim} params['{indicator}-value'])\n" # add the space to its argument - buy_space += f"Integer(10, 90, name='{indicator}-value'),\nCategorical([True, False], name='{indicator}-enabled'),\n" + buy_space += f"Integer({lower_bound}, {upper_bound}, name='{indicator}-value'),\nCategorical([True, False], name='{indicator}-enabled'),\n" # If the indicator is a trigger elif usage == "trigger": @@ -222,7 +206,7 @@ def custom_hyperopt_sellelements(sell_indicators: Dict[str, str]): sell_guards = "" sell_triggers = "" sell_space = "" - print(sell_indicators) + for indicator_info in sell_indicators: indicator = indicator_info[0] aim = indicator_info[1] @@ -230,11 +214,20 @@ def custom_hyperopt_sellelements(sell_indicators: Dict[str, str]): # If the indicator is a guard if usage == "guard": + value = indicator_info[3] + + if value >= -1 and value <= 1: + lower_bound = value - 0.3 + upper_bound = value + 0.3 + else: + lower_bound = value - 30 + upper_bound = value + 30 + # add the guard to its argument sell_guards += f"if params.get('sell-{indicator}-enabled'):\n conditions.append(dataframe['{indicator}'] {aim} params['sell-{indicator}-value'])\n" # add the space to its argument - sell_space += f"Integer(10, 90, name='sell-{indicator}-value'),\nCategorical([True, False], name='sell-{indicator}-enabled'),\n" + sell_space += f"Integer({lower_bound}, {upper_bound}, name='sell-{indicator}-value'),\nCategorical([True, False], name='sell-{indicator}-enabled'),\n" # If the indicator is a trigger elif usage == "trigger": @@ -272,7 +265,7 @@ def deploy_custom_hyperopt(hyperopt_name: str, hyperopt_path: Path, buy_indicato sell_args = custom_hyperopt_sellelements(sell_indicators) # Build the final template - strategy_text = render_template(templatefile='base_hyperopt.py.j2', + strategy_text = render_template(templatefile='base_custom_hyperopt.py.j2', arguments={"hyperopt": hyperopt_name, "buy_guards": buy_args["buy_guards"], "buy_triggers": buy_args["buy_triggers"], @@ -315,11 +308,6 @@ def start_custom_hyperopt(args: Dict[str, Any]) -> None: # --------------------------------------------------build-hyperopt------------------------------------------------------ -''' - TODO - -hyperopt optional (door bv standaard naming toe te passen) -''' - def start_build_hyperopt(args: Dict[str, Any]) -> None: """ @@ -331,7 +319,7 @@ def start_build_hyperopt(args: Dict[str, Any]) -> None: if not 'strategy' in args or not args['strategy']: raise OperationalException("`build-hyperopt` requires --strategy to be set.") if not 'hyperopt' in args or not args['hyperopt']: - raise OperationalException("`build-hyperopt` requires --hyperopt to be set.") + args['hyperopt'] = args['strategy'] + "opt" else: if args['hyperopt'] == 'DefaultHyperopt': raise OperationalException("DefaultHyperopt is not allowed as name.") @@ -347,7 +335,7 @@ def start_build_hyperopt(args: Dict[str, Any]) -> None: # extract the buy and sell indicators as dicts extracted_lists = extract_lists(strategy_path) - print(extracted_lists) + buy_indicators = extracted_lists[0] sell_indicators = extracted_lists[1] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 461ce8d32..4704c31c3 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -279,14 +279,16 @@ AVAILABLE_CLI_OPTIONS = { "buy_indicators": Arg( '-b', '--buy-indicators', help='Specify the buy indicators the hyperopt should build. ' - 'Example: --buy-indicators `{"rsi":"<","bb_lowerband":">"}`', - metavar='DICT', + 'Example: --buy-indicators `[["rsi","<","trigger",30.0],["bb_lowerband",">","guard","close"]]`' + 'Check the documentation for specific requirements for the lists.', + metavar='LIST', ), "sell_indicators": Arg( '-s', '--sell-indicators', help='Specify the sell indicators the hyperopt should build. ' - 'Example: --sell-indicators `{"rsi":">","bb_lowerband":"<"}`', - metavar='DICT', + 'Example: --sell-indicators [["rsi",">","trigger",70.0],["bb_lowerband","<","guard","close"]]' + 'Check the documentation for specific requirements for the lists.', + metavar='LIST', ), "extract_name": Arg( '--extract-name', diff --git a/freqtrade/templates/base_custom_hyperopt.py.j2 b/freqtrade/templates/base_custom_hyperopt.py.j2 new file mode 100644 index 000000000..672b0b119 --- /dev/null +++ b/freqtrade/templates/base_custom_hyperopt.py.j2 @@ -0,0 +1,164 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +# --- Do not remove these libs --- +from functools import reduce +from typing import Any, Callable, Dict, List + +import numpy as np # noqa +import pandas as pd # noqa +from pandas import DataFrame +from skopt.space import Categorical, Dimension, Integer, Real # noqa + +from freqtrade.optimize.hyperopt_interface import IHyperOpt + +# -------------------------------- +# Add your lib to import here +import talib.abstract as ta # noqa +import freqtrade.vendor.qtpylib.indicators as qtpylib + + +class {{ hyperopt }}(IHyperOpt): + """ + This is a Hyperopt template to get you started. + + More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ + + You should: + - Add any lib you need to build your hyperopt. + + You must keep: + - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. + + The methods roi_space, generate_roi_table and stoploss_space are not required + and are provided by default. + However, you may override them if you need 'roi' and 'stoploss' spaces that + differ from the defaults offered by Freqtrade. + Sample implementation of these methods will be copied to `user_data/hyperopts` when + creating the user-data directory using `freqtrade create-userdir --userdir user_data`, + or is available online under the following URL: + https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py. + """ + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by Hyperopt. + """ + def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Buy strategy Hyperopt will build and use. + """ + conditions = [] + + # GUARDS AND TRENDS + {{ buy_guards | indent(12) }} + + # TRIGGERS + if 'trigger' in params: + {{ buy_triggers | indent(16) }} + + # Check that the candle had volume + conditions.append(dataframe['volume'] > 0) + + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching buy strategy parameters. + """ + return [ + {{ buy_space | indent(12) }} + ] + + @staticmethod + def sell_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the sell strategy parameters to be used by Hyperopt. + """ + def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Sell strategy Hyperopt will build and use. + """ + conditions = [] + + # GUARDS AND TRENDS + {{ sell_guards | indent(12) }} + + # TRIGGERS + if 'sell-trigger' in params: + {{ sell_triggers | indent(16) }} + + # Check that the candle had volume + conditions.append(dataframe['volume'] > 0) + + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'sell'] = 1 + + return dataframe + + return populate_sell_trend + + @staticmethod + def sell_indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching sell strategy parameters. + """ + return [ + {{ sell_space | indent(12) }} + ] + @ staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + This implementation generates the default legacy Freqtrade ROI tables. + Change it if you need different number of steps in the generated + ROI tables or other structure of the ROI tables. + Please keep it aligned with parameters in the 'roi' optimization + hyperspace defined by the roi_space method. + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @ staticmethod + def roi_space() -> List[Dimension]: + """ + Values to search for each ROI steps + Override it if you need some different ranges for the parameters in the + 'roi' optimization hyperspace. + Please keep it aligned with the implementation of the + generate_roi_table method. + """ + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + Real(0.01, 0.04, name='roi_p1'), + Real(0.01, 0.07, name='roi_p2'), + Real(0.01, 0.20, name='roi_p3'), + ] + + @ staticmethod + def stoploss_space() -> List[Dimension]: + """ + Stoploss Value to search + Override it if you need some different range for the parameter in the + 'stoploss' optimization hyperspace. + """ + return [ + Real(-0.35, -0.02, name='stoploss'), + ] diff --git a/freqtrade/templates/base_hyperopt.py.j2 b/freqtrade/templates/base_hyperopt.py.j2 index 38e2f4172..2bdfdba16 100644 --- a/freqtrade/templates/base_hyperopt.py.j2 +++ b/freqtrade/templates/base_hyperopt.py.j2 @@ -55,7 +55,16 @@ class {{ hyperopt }}(IHyperOpt): # TRIGGERS if 'trigger' in params: - {{ buy_triggers | indent(16) }} + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) # Check that the candle had volume conditions.append(dataframe['volume'] > 0) @@ -94,7 +103,16 @@ class {{ hyperopt }}(IHyperOpt): # TRIGGERS if 'sell-trigger' in params: - {{ sell_triggers | indent(16) }} + if params['sell-trigger'] == 'sell-bb_upper': + conditions.append(dataframe['close'] > dataframe['bb_upperband']) + if params['sell-trigger'] == 'sell-macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macdsignal'], dataframe['macd'] + )) + if params['sell-trigger'] == 'sell-sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['sar'], dataframe['close'] + )) # Check that the candle had volume conditions.append(dataframe['volume'] > 0) @@ -115,4 +133,4 @@ class {{ hyperopt }}(IHyperOpt): """ return [ {{ sell_space | indent(12) }} - ] + ] \ No newline at end of file