Merge pull request #2465 from freqtrade/hyperopt_populate_from_strategy

Hyperopt populate from strategy

.travis.yml

@@ -32,7 +32,7 @@ jobs:
       name: backtest
     - script:
       - cp config.json.example config.json
-      - freqtrade --datadir tests/testdata hyperopt -e 5
+      - freqtrade --datadir tests/testdata --strategy SampleStrategy hyperopt --customhyperopt SampleHyperOpts -e 5
       name: hyperopt
     - script: flake8
       name: flake8

docs/hyperopt.md

@@ -23,17 +23,23 @@ Configuring hyperopt is similar to writing your own strategy, and many tasks wil
 
 Depending on the space you want to optimize, only some of the below are required:
 
-* fill `populate_indicators` - probably a copy from your strategy
 * fill `buy_strategy_generator` - for buy signal optimization
 * fill `indicator_space` - for buy signal optimzation
 * fill `sell_strategy_generator` - for sell signal optimization
 * fill `sell_indicator_space` - for sell signal optimzation
 
-Optional, but recommended:
+!!! Note
+    `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work.
 
+Optional - can also be loaded from a strategy:
+
+* copy `populate_indicators` from your strategy - otherwise default-strategy will be used
 * copy `populate_buy_trend` from your strategy - otherwise default-strategy will be used
 * copy `populate_sell_trend` from your strategy - otherwise default-strategy will be used
 
+!!! Note
+    Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead.
+
 Rarely you may also need to override:
 
 * `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default)
@@ -156,7 +162,7 @@ that minimizes the value of the [loss function](#loss-functions).
 
 The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators.
 When you want to test an indicator that isn't used by the bot currently, remember to
-add it to the `populate_indicators()` method in `hyperopt.py`.
+add it to the `populate_indicators()` method in your custom hyperopt file.
 
 ## Loss-functions
 
@@ -270,6 +276,14 @@ For example, to use one month of data, pass the following parameter to the hyper
 freqtrade hyperopt --timerange 20180401-20180501
 ```
 
+### Running Hyperopt using methods from a strategy
+
+Hyperopt can reuse `populate_indicators`, `populate_buy_trend`, `populate_sell_trend` from your strategy, assuming these methods are **not** in your custom hyperopt file, and a strategy is provided.
+
+```bash
+freqtrade --strategy SampleStrategy hyperopt --customhyperopt SampleHyperopt
+```
+
 ### Running Hyperopt with Smaller Search Space
 
 Use the `--spaces` argument to limit the search space used by hyperopt.
@@ -341,8 +355,7 @@ So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that t
 (dataframe['rsi'] < 29.0)
 ```
 
-Translating your whole hyperopt result as the new buy-signal
-would then look like:
+Translating your whole hyperopt result as the new buy-signal would then look like:
 
 ```python
 def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:

freqtrade/optimize/hyperopt_interface.py

@@ -5,10 +5,9 @@ This module defines the interface to apply for hyperopts
 import logging
 import math
 
-from abc import ABC, abstractmethod
+from abc import ABC
 from typing import Dict, Any, Callable, List
 
-from pandas import DataFrame
 from skopt.space import Dimension, Integer, Real
 
 from freqtrade import OperationalException
@@ -42,15 +41,6 @@ class IHyperOpt(ABC):
         # Assign ticker_interval to be used in hyperopt
         IHyperOpt.ticker_interval = str(config['ticker_interval'])
 
-    @staticmethod
-    @abstractmethod
-    def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame:
-        """
-        Populate indicators that will be used in the Buy and Sell strategy.
-        :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe().
-        :return: A Dataframe with all mandatory indicators for the strategies.
-        """
-
     @staticmethod
     def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
         """

freqtrade/resolvers/hyperopt_resolver.py

@@ -34,6 +34,9 @@ class HyperOptResolver(IResolver):
         self.hyperopt = self._load_hyperopt(hyperopt_name, config,
                                             extra_dir=config.get('hyperopt_path'))
 
+        if not hasattr(self.hyperopt, 'populate_indicators'):
+            logger.warning("Hyperopt class does not provide populate_indicators() method. "
+                           "Using populate_indicators from the strategy.")
         if not hasattr(self.hyperopt, 'populate_buy_trend'):
             logger.warning("Hyperopt class does not provide populate_buy_trend() method. "
                            "Using populate_buy_trend from the strategy.")

tests/optimize/test_hyperopt.py

@@ -149,6 +149,7 @@ def test_hyperoptresolver(mocker, default_conf, caplog) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
 
     hyperopt = DefaultHyperOpt
+    delattr(hyperopt, 'populate_indicators')
     delattr(hyperopt, 'populate_buy_trend')
     delattr(hyperopt, 'populate_sell_trend')
     mocker.patch(
@@ -156,8 +157,11 @@ def test_hyperoptresolver(mocker, default_conf, caplog) -> None:
         MagicMock(return_value=hyperopt(default_conf))
     )
     x = HyperOptResolver(default_conf, ).hyperopt
+    assert not hasattr(x, 'populate_indicators')
     assert not hasattr(x, 'populate_buy_trend')
     assert not hasattr(x, 'populate_sell_trend')
+    assert log_has("Hyperopt class does not provide populate_indicators() method. "
+                   "Using populate_indicators from the strategy.", caplog)
     assert log_has("Hyperopt class does not provide populate_sell_trend() method. "
                    "Using populate_sell_trend from the strategy.", caplog)
     assert log_has("Hyperopt class does not provide populate_buy_trend() method. "

user_data/hyperopts/sample_hyperopt.py

@@ -2,12 +2,11 @@
 
 from functools import reduce
 from typing import Any, Callable, Dict, List
-from datetime import datetime
 
-import numpy as np
+import numpy as np  # noqa
 import talib.abstract as ta
 from pandas import DataFrame
-from skopt.space import Categorical, Dimension, Integer, Real
+from skopt.space import Categorical, Dimension, Integer, Real  # noqa
 
 import freqtrade.vendor.qtpylib.indicators as qtpylib
 from freqtrade.optimize.hyperopt_interface import IHyperOpt
@@ -34,34 +33,6 @@ class SampleHyperOpts(IHyperOpt):
     Sample implementation of these methods can be found in
     https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py
     """
-    @staticmethod
-    def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame:
-        """
-        Add several indicators needed for buy and sell strategies defined below.
-        """
-        # ADX
-        dataframe['adx'] = ta.ADX(dataframe)
-        # MACD
-        macd = ta.MACD(dataframe)
-        dataframe['macd'] = macd['macd']
-        dataframe['macdsignal'] = macd['macdsignal']
-        # MFI
-        dataframe['mfi'] = ta.MFI(dataframe)
-        # RSI
-        dataframe['rsi'] = ta.RSI(dataframe)
-        # Stochastic Fast
-        stoch_fast = ta.STOCHF(dataframe)
-        dataframe['fastd'] = stoch_fast['fastd']
-        # Minus-DI
-        dataframe['minus_di'] = ta.MINUS_DI(dataframe)
-        # Bollinger bands
-        bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2)
-        dataframe['bb_lowerband'] = bollinger['lower']
-        dataframe['bb_upperband'] = bollinger['upper']
-        # SAR
-        dataframe['sar'] = ta.SAR(dataframe)
-
-        return dataframe
 
     @staticmethod
     def buy_strategy_generator(params: Dict[str, Any]) -> Callable:

user_data/hyperopts/sample_hyperopt_advanced.py

@@ -37,6 +37,9 @@ class AdvancedSampleHyperOpts(IHyperOpt):
     """
     @staticmethod
     def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame:
+        """
+        This method can also be loaded from the strategy, if it doesn't exist in the hyperopt class.
+        """
         dataframe['adx'] = ta.ADX(dataframe)
         macd = ta.MACD(dataframe)
         dataframe['macd'] = macd['macd']
@@ -229,8 +232,10 @@ class AdvancedSampleHyperOpts(IHyperOpt):
 
     def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
-        Based on TA indicators. Should be a copy of from strategy
-        must align to populate_indicators in this file
+        Based on TA indicators.
+        Can be a copy of the corresponding method from the strategy,
+        or will be loaded from the strategy.
+        Must align to populate_indicators used (either from this File, or from the strategy)
         Only used when --spaces does not include buy
         """
         dataframe.loc[
@@ -246,8 +251,10 @@ class AdvancedSampleHyperOpts(IHyperOpt):
 
     def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
-        Based on TA indicators. Should be a copy of from strategy
-        must align to populate_indicators in this file
+        Based on TA indicators.
+        Can be a copy of the corresponding method from the strategy,
+        or will be loaded from the strategy.
+        Must align to populate_indicators used (either from this File, or from the strategy)
         Only used when --spaces does not include sell
         """
         dataframe.loc[

user_data/strategies/sample_strategy.py

@@ -107,16 +107,16 @@ class SampleStrategy(IStrategy):
         # RSI
         dataframe['rsi'] = ta.RSI(dataframe)
 
-        """
+
         # ADX
         dataframe['adx'] = ta.ADX(dataframe)
-
+        """
         # Awesome oscillator
         dataframe['ao'] = qtpylib.awesome_oscillator(dataframe)
 
         # Commodity Channel Index: values Oversold:<-100, Overbought:>100
         dataframe['cci'] = ta.CCI(dataframe)
-
+        """
         # MACD
         macd = ta.MACD(dataframe)
         dataframe['macd'] = macd['macd']
@@ -126,6 +126,7 @@ class SampleStrategy(IStrategy):
         # MFI
         dataframe['mfi'] = ta.MFI(dataframe)
 
+        """
         # Minus Directional Indicator / Movement
         dataframe['minus_dm'] = ta.MINUS_DM(dataframe)
         dataframe['minus_di'] = ta.MINUS_DI(dataframe)
@@ -149,12 +150,13 @@ class SampleStrategy(IStrategy):
         stoch = ta.STOCH(dataframe)
         dataframe['slowd'] = stoch['slowd']
         dataframe['slowk'] = stoch['slowk']
-
+        """
         # Stoch fast
         stoch_fast = ta.STOCHF(dataframe)
         dataframe['fastd'] = stoch_fast['fastd']
         dataframe['fastk'] = stoch_fast['fastk']
 
+        """
         # Stoch RSI
         stoch_rsi = ta.STOCHRSI(dataframe)
         dataframe['fastd_rsi'] = stoch_rsi['fastd']
@@ -178,12 +180,11 @@ class SampleStrategy(IStrategy):
         dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50)
         dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100)
 
-        # SAR Parabol
-        dataframe['sar'] = ta.SAR(dataframe)
-
         # SMA - Simple Moving Average
         dataframe['sma'] = ta.SMA(dataframe, timeperiod=40)
         """
+        # SAR Parabol
+        dataframe['sar'] = ta.SAR(dataframe)
 
         # TEMA - Triple Exponential Moving Average
         dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9)