version 1 ready for testing phase

freqtrade/commands/automation_commands.py

@@ -1,6 +1,5 @@
 import ast
 import logging
-from os import EX_USAGE
 from pathlib import Path
 from typing import Any, Dict, List
 
@@ -15,14 +14,54 @@ logger = logging.getLogger(__name__)
 
 
 # ---------------------------------------------------extract-strategy------------------------------------------------------
-'''
+
-    TODO
+def get_indicator_info(file: List, indicators: Dict) -> None:
-    - get the stoploss value to optimize
+    """
-    - get the values from the guards to specify the optimzation range (cooperation with custom-hyperopt)
+    Get all necessary information to build a custom hyperopt space using
-'''
+    the file and a dictionary filled with the indicators and their corropsonding line numbers.
+    """
+    info_list = []
+    for indicator in indicators:
+        indicator_info = []
+
+        # find the corrosponding aim
+        for position, line in enumerate(file):
+            if position == indicators[indicator]:
+                # use split twice to remove the context around the indicator
+                back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1]
+                aim = back_of_line.split()[0]
+
+                # add the indicator and aim to the info
+                indicator_info.append(indicator)
+                indicator_info.append(aim)
+
+                # check if first character after aim is a d in which case the indicator is a trigger
+                if back_of_line.split()[1][0] == "d":
+                    indicator_info.append("trigger")
+
+                    # add the second indicator of the guard to the info list
+                    back_of_line = back_of_line.split("dataframe['")[1]
+                    second_indicator = back_of_line.split("'])")[0]
+                    indicator_info.append(second_indicator)
+
+                # elif indicator[0:3] == "CDL":
+                    # indicator_info.append("guard")
+
+                # else it is a regular guard
+                else:
+                    indicator_info.append("guard")
+
+                    value = back_of_line.split()[1]
+                    value = value[:-1]
+                    value = float(value)
+
+                    indicator_info.append(value)
+        info_list.append(indicator_info)
+
+    return info_list
 
 
-def extract_lists(strategypath: Path):
+def extract_lists(strategypath: Path) -> None:
     """
     Get the indicators, their aims and the stoploss and format them into lists
     """
@@ -64,71 +103,17 @@ def extract_lists(strategypath: Path):
             sellindicator = back_of_line.split("'] ", 1)[0]
             sellindicators[sellindicator] = position
 
-    # build the final buy list
+    # build the final lists
-    buy_list = []
+    buy_info_list = get_indicator_info(stored_file, buyindicators)
-    for indicator in buyindicators:
+    sell_info_list = get_indicator_info(stored_file, sellindicators)
-        indicator_info = []
+    
-
-        # find the corrosponding aim
-        for position, line in enumerate(stored_file):
-            if position == buyindicators[indicator]:
-                # use split twice to remove the context around the indicator
-                back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1]
-                aim = back_of_line.split()[0]
-
-                # add the indicator and aim to the info
-                indicator_info.append(indicator)
-                indicator_info.append(aim)
-
-                # check if first character after aim is a d in which case the indicator is a trigger
-                if back_of_line.split()[1][0] == "d":
-                    indicator_info.append("trigger")
-
-                    # add the second indicator of the guard to the info list
-                    back_of_line = back_of_line.split("dataframe['")[1]
-                    second_indicator = back_of_line.split("'])")[0]
-                    indicator_info.append(second_indicator)
-                # else it is a guard
-                else:
-                    indicator_info.append("guard")
-        buy_list.append(indicator_info)
-
-    # build the final sell list
-    sell_list = []
-    for indicator in sellindicators:
-        indicator_info = []
-
-        # find the corrosponding aim and whether a guard or trigger
-        for position, line in enumerate(stored_file):
-            if position == sellindicators[indicator]:
-                # use split twice to remove the context around the indicator
-                back_of_line = line.split(f"(dataframe['{indicator}'] ", 1)[1]
-                aim = back_of_line.split()[0]
-
-                # add the indicator and aim to the info
-                indicator_info.append(indicator)
-                indicator_info.append(aim)
-
-                # check if first character after aim is a d in which case the indicator is a trigger
-                if back_of_line.split()[1][0] == "d":
-                    indicator_info.append("trigger")
-
-                    # add the second indicator of the guard to the info list
-                    back_of_line = back_of_line.split("dataframe['")[1]
-                    second_indicator = back_of_line.split("'])")[0]
-                    indicator_info.append(second_indicator)
-                # else it is a guard
-                else:
-                    indicator_info.append("guard")
-        sell_list.append(indicator_info)
-
     # put the final lists into a tuple
-    final_lists = (buy_list, sell_list)
+    final_lists = (buy_info_list, sell_info_list)
 
     return final_lists
 
 
-def start_extract_strategy(args: Dict[str, Any]) -> None:
+def start_extract_strategy(args: Dict) -> None:
     """
     Check if the right subcommands where passed and start extracting the strategy data
     """
@@ -158,16 +143,6 @@ def start_extract_strategy(args: Dict[str, Any]) -> None:
 
 
 # --------------------------------------------------custom-hyperopt------------------------------------------------------
-'''
-    TODO
-    -make the code below more dynamic with a large list of indicators and aims
-    -buy_space integer values variation based on aim and input value form extract-strategy(later deep learning)
-    -add --mode , see notes (denk hierbij ook aan value range bij guards)
-    -Custom stoploss and roi (based on input from extract-strategy)
-    -cli option to read extracted strategies files (--extraction)
-    -code cleanup (maybe the two elements functions can be combined)
-'''
-
 
 def custom_hyperopt_buyelements(buy_indicators: List):
     """
@@ -184,11 +159,20 @@ def custom_hyperopt_buyelements(buy_indicators: List):
 
         # If the indicator is a guard
         if usage == "guard":
+            value = indicator_info[3]
+            
+            if value >= -1.0 and value <= 1.0:
+                lower_bound = value - 0.3
+                upper_bound = value + 0.3
+            else:
+                lower_bound = value - 30.0
+                upper_bound = value + 30.0
+
             # add the guard to its argument
             buy_guards += f"if params.get('{indicator}-enabled'):\n    conditions.append(dataframe['{indicator}'] {aim} params['{indicator}-value'])\n"
 
             # add the space to its argument
-            buy_space += f"Integer(10, 90, name='{indicator}-value'),\nCategorical([True, False], name='{indicator}-enabled'),\n"
+            buy_space += f"Integer({lower_bound}, {upper_bound}, name='{indicator}-value'),\nCategorical([True, False], name='{indicator}-enabled'),\n"
 
         # If the indicator is a trigger
         elif usage == "trigger":
@@ -222,7 +206,7 @@ def custom_hyperopt_sellelements(sell_indicators: Dict[str, str]):
     sell_guards = ""
     sell_triggers = ""
     sell_space = ""
-    print(sell_indicators)
+
     for indicator_info in sell_indicators:
         indicator = indicator_info[0]
         aim = indicator_info[1]
@@ -230,11 +214,20 @@ def custom_hyperopt_sellelements(sell_indicators: Dict[str, str]):
 
         # If the indicator is a guard
         if usage == "guard":
+            value = indicator_info[3]
+            
+            if value >= -1 and value <= 1:
+                lower_bound = value - 0.3
+                upper_bound = value + 0.3
+            else:
+                lower_bound = value - 30
+                upper_bound = value + 30
+
             # add the guard to its argument
             sell_guards += f"if params.get('sell-{indicator}-enabled'):\n    conditions.append(dataframe['{indicator}'] {aim} params['sell-{indicator}-value'])\n"
 
             # add the space to its argument
-            sell_space += f"Integer(10, 90, name='sell-{indicator}-value'),\nCategorical([True, False], name='sell-{indicator}-enabled'),\n"
+            sell_space += f"Integer({lower_bound}, {upper_bound}, name='sell-{indicator}-value'),\nCategorical([True, False], name='sell-{indicator}-enabled'),\n"
 
         # If the indicator is a trigger
         elif usage == "trigger":
@@ -272,7 +265,7 @@ def deploy_custom_hyperopt(hyperopt_name: str, hyperopt_path: Path, buy_indicato
     sell_args = custom_hyperopt_sellelements(sell_indicators)
 
     # Build the final template
-    strategy_text = render_template(templatefile='base_hyperopt.py.j2',
+    strategy_text = render_template(templatefile='base_custom_hyperopt.py.j2',
                                     arguments={"hyperopt": hyperopt_name,
                                                "buy_guards": buy_args["buy_guards"],
                                                "buy_triggers": buy_args["buy_triggers"],
@@ -315,11 +308,6 @@ def start_custom_hyperopt(args: Dict[str, Any]) -> None:
 
 
 # --------------------------------------------------build-hyperopt------------------------------------------------------
-'''
-    TODO
-    -hyperopt optional (door bv standaard naming toe te passen)
-'''
-
 
 def start_build_hyperopt(args: Dict[str, Any]) -> None:
     """
@@ -331,7 +319,7 @@ def start_build_hyperopt(args: Dict[str, Any]) -> None:
     if not 'strategy' in args or not args['strategy']:
         raise OperationalException("`build-hyperopt` requires --strategy to be set.")
     if not 'hyperopt' in args or not args['hyperopt']:
-        raise OperationalException("`build-hyperopt` requires --hyperopt to be set.")
+        args['hyperopt'] = args['strategy'] + "opt"
     else:
         if args['hyperopt'] == 'DefaultHyperopt':
             raise OperationalException("DefaultHyperopt is not allowed as name.")
@@ -347,7 +335,7 @@ def start_build_hyperopt(args: Dict[str, Any]) -> None:
 
         # extract the buy and sell indicators as dicts
         extracted_lists = extract_lists(strategy_path)
-        print(extracted_lists)
+
         buy_indicators = extracted_lists[0]
         sell_indicators = extracted_lists[1]
 

freqtrade/commands/cli_options.py

@@ -279,14 +279,16 @@ AVAILABLE_CLI_OPTIONS = {
     "buy_indicators": Arg(
         '-b', '--buy-indicators',
         help='Specify the buy indicators the hyperopt should build. '
-        'Example: --buy-indicators `{"rsi":"<","bb_lowerband":">"}`',
+        'Example: --buy-indicators `[["rsi","<","trigger",30.0],["bb_lowerband",">","guard","close"]]`'
-        metavar='DICT',
+        'Check the documentation for specific requirements for the lists.',
+        metavar='LIST',
     ),
     "sell_indicators": Arg(
         '-s', '--sell-indicators',
         help='Specify the sell indicators the hyperopt should build. '
-        'Example: --sell-indicators `{"rsi":">","bb_lowerband":"<"}`',
+        'Example: --sell-indicators [["rsi",">","trigger",70.0],["bb_lowerband","<","guard","close"]]'
-        metavar='DICT',
+        'Check the documentation for specific requirements for the lists.',
+        metavar='LIST',
     ),
     "extract_name": Arg(
         '--extract-name',

freqtrade/templates/base_custom_hyperopt.py.j2

@@ -0,0 +1,164 @@
+# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement
+
+# --- Do not remove these libs ---
+from functools import reduce
+from typing import Any, Callable, Dict, List
+
+import numpy as np  # noqa
+import pandas as pd  # noqa
+from pandas import DataFrame
+from skopt.space import Categorical, Dimension, Integer, Real  # noqa
+
+from freqtrade.optimize.hyperopt_interface import IHyperOpt
+
+# --------------------------------
+# Add your lib to import here
+import talib.abstract as ta  # noqa
+import freqtrade.vendor.qtpylib.indicators as qtpylib
+
+
+class {{ hyperopt }}(IHyperOpt):
+    """
+    This is a Hyperopt template to get you started.
+
+    More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/
+
+    You should:
+    - Add any lib you need to build your hyperopt.
+
+    You must keep:
+    - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator.
+
+    The methods roi_space, generate_roi_table and stoploss_space are not required
+    and are provided by default.
+    However, you may override them if you need 'roi' and 'stoploss' spaces that
+    differ from the defaults offered by Freqtrade.
+    Sample implementation of these methods will be copied to `user_data/hyperopts` when
+    creating the user-data directory using `freqtrade create-userdir --userdir user_data`,
+    or is available online under the following URL:
+    https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py.
+    """
+
+    @staticmethod
+    def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
+        """
+        Define the buy strategy parameters to be used by Hyperopt.
+        """
+        def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
+            """
+            Buy strategy Hyperopt will build and use.
+            """
+            conditions = []
+
+            # GUARDS AND TRENDS
+            {{ buy_guards | indent(12) }}
+
+            # TRIGGERS
+            if 'trigger' in params:
+                {{ buy_triggers | indent(16) }}
+
+            # Check that the candle had volume
+            conditions.append(dataframe['volume'] > 0)
+
+            if conditions:
+                dataframe.loc[
+                    reduce(lambda x, y: x & y, conditions),
+                    'buy'] = 1
+
+            return dataframe
+
+        return populate_buy_trend
+
+    @staticmethod
+    def indicator_space() -> List[Dimension]:
+        """
+        Define your Hyperopt space for searching buy strategy parameters.
+        """
+        return [
+            {{ buy_space | indent(12) }}
+        ]
+
+    @staticmethod
+    def sell_strategy_generator(params: Dict[str, Any]) -> Callable:
+        """
+        Define the sell strategy parameters to be used by Hyperopt.
+        """
+        def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
+            """
+            Sell strategy Hyperopt will build and use.
+            """
+            conditions = []
+
+            # GUARDS AND TRENDS
+            {{ sell_guards | indent(12) }}
+
+            # TRIGGERS
+            if 'sell-trigger' in params:
+                {{ sell_triggers | indent(16) }}
+
+            # Check that the candle had volume
+            conditions.append(dataframe['volume'] > 0)
+
+            if conditions:
+                dataframe.loc[
+                    reduce(lambda x, y: x & y, conditions),
+                    'sell'] = 1
+
+            return dataframe
+
+        return populate_sell_trend
+
+    @staticmethod
+    def sell_indicator_space() -> List[Dimension]:
+        """
+        Define your Hyperopt space for searching sell strategy parameters.
+        """
+        return [
+            {{ sell_space | indent(12) }}
+        ]
+    @ staticmethod
+    def generate_roi_table(params: Dict) -> Dict[int, float]:
+        """
+        Generate the ROI table that will be used by Hyperopt
+        This implementation generates the default legacy Freqtrade ROI tables.
+        Change it if you need different number of steps in the generated
+        ROI tables or other structure of the ROI tables.
+        Please keep it aligned with parameters in the 'roi' optimization
+        hyperspace defined by the roi_space method.
+        """
+        roi_table = {}
+        roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3']
+        roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2']
+        roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1']
+        roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0
+
+        return roi_table
+
+    @ staticmethod
+    def roi_space() -> List[Dimension]:
+        """
+        Values to search for each ROI steps
+        Override it if you need some different ranges for the parameters in the
+        'roi' optimization hyperspace.
+        Please keep it aligned with the implementation of the
+        generate_roi_table method.
+        """
+        return [
+            Integer(10, 120, name='roi_t1'),
+            Integer(10, 60, name='roi_t2'),
+            Integer(10, 40, name='roi_t3'),
+            Real(0.01, 0.04, name='roi_p1'),
+            Real(0.01, 0.07, name='roi_p2'),
+            Real(0.01, 0.20, name='roi_p3'),
+        ]
+
+    @ staticmethod
+    def stoploss_space() -> List[Dimension]:
+        """
+        Stoploss Value to search
+        Override it if you need some different range for the parameter in the
+        'stoploss' optimization hyperspace.
+        """
+        return [
+            Real(-0.35, -0.02, name='stoploss'),
+        ]

freqtrade/templates/base_hyperopt.py.j2

@@ -55,7 +55,16 @@ class {{ hyperopt }}(IHyperOpt):
 
             # TRIGGERS
             if 'trigger' in params:
-                {{ buy_triggers | indent(16) }}
+                if params['trigger'] == 'bb_lower':
+                    conditions.append(dataframe['close'] < dataframe['bb_lowerband'])
+                if params['trigger'] == 'macd_cross_signal':
+                    conditions.append(qtpylib.crossed_above(
+                        dataframe['macd'], dataframe['macdsignal']
+                    ))
+                if params['trigger'] == 'sar_reversal':
+                    conditions.append(qtpylib.crossed_above(
+                        dataframe['close'], dataframe['sar']
+                    ))
 
             # Check that the candle had volume
             conditions.append(dataframe['volume'] > 0)
@@ -94,7 +103,16 @@ class {{ hyperopt }}(IHyperOpt):
 
             # TRIGGERS
             if 'sell-trigger' in params:
-                {{ sell_triggers | indent(16) }}
+                if params['sell-trigger'] == 'sell-bb_upper':
+                    conditions.append(dataframe['close'] > dataframe['bb_upperband'])
+                if params['sell-trigger'] == 'sell-macd_cross_signal':
+                    conditions.append(qtpylib.crossed_above(
+                        dataframe['macdsignal'], dataframe['macd']
+                    ))
+                if params['sell-trigger'] == 'sell-sar_reversal':
+                    conditions.append(qtpylib.crossed_above(
+                        dataframe['sar'], dataframe['close']
+                    ))
 
             # Check that the candle had volume
             conditions.append(dataframe['volume'] > 0)
@@ -115,4 +133,4 @@ class {{ hyperopt }}(IHyperOpt):
         """
         return [
             {{ sell_space | indent(12) }}
-        ]
+        ]