From 861f10dca65b64a76a3fd83459161b95e2a737ef Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Nov 2019 11:10:33 +0100 Subject: [PATCH 1/6] Allow populate-indicators to come from strategy --- freqtrade/optimize/hyperopt_interface.py | 12 +----------- freqtrade/resolvers/hyperopt_resolver.py | 3 +++ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 4208b29d3..142f305df 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -5,10 +5,9 @@ This module defines the interface to apply for hyperopts import logging import math -from abc import ABC, abstractmethod +from abc import ABC from typing import Dict, Any, Callable, List -from pandas import DataFrame from skopt.space import Dimension, Integer, Real from freqtrade import OperationalException @@ -42,15 +41,6 @@ class IHyperOpt(ABC): # Assign ticker_interval to be used in hyperopt IHyperOpt.ticker_interval = str(config['ticker_interval']) - @staticmethod - @abstractmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Populate indicators that will be used in the Buy and Sell strategy. - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe(). - :return: A Dataframe with all mandatory indicators for the strategies. - """ - @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index db51c3ca5..1ad53fe33 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -34,6 +34,9 @@ class HyperOptResolver(IResolver): self.hyperopt = self._load_hyperopt(hyperopt_name, config, extra_dir=config.get('hyperopt_path')) + if not hasattr(self.hyperopt, 'populate_indicators'): + logger.warning("Hyperopt class does not provide populate_indicators() method. " + "Using populate_indicators from the strategy.") if not hasattr(self.hyperopt, 'populate_buy_trend'): logger.warning("Hyperopt class does not provide populate_buy_trend() method. " "Using populate_buy_trend from the strategy.") From 97d0f93d3ce0d4ab3decaa75f7b0432fc320efd4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Nov 2019 11:11:13 +0100 Subject: [PATCH 2/6] Align samples (hyperopt and strategy) to work together --- user_data/hyperopts/sample_hyperopt.py | 33 ++----------------------- user_data/strategies/sample_strategy.py | 15 +++++------ 2 files changed, 10 insertions(+), 38 deletions(-) diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index fabfdb23e..2721ab405 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -2,12 +2,11 @@ from functools import reduce from typing import Any, Callable, Dict, List -from datetime import datetime -import numpy as np +import numpy as np # noqa import talib.abstract as ta from pandas import DataFrame -from skopt.space import Categorical, Dimension, Integer, Real +from skopt.space import Categorical, Dimension, Integer, Real # noqa import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.optimize.hyperopt_interface import IHyperOpt @@ -34,34 +33,6 @@ class SampleHyperOpts(IHyperOpt): Sample implementation of these methods can be found in https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py """ - @staticmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Add several indicators needed for buy and sell strategies defined below. - """ - # ADX - dataframe['adx'] = ta.ADX(dataframe) - # MACD - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - # MFI - dataframe['mfi'] = ta.MFI(dataframe) - # RSI - dataframe['rsi'] = ta.RSI(dataframe) - # Stochastic Fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - # Minus-DI - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_upperband'] = bollinger['upper'] - # SAR - dataframe['sar'] = ta.SAR(dataframe) - - return dataframe @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: diff --git a/user_data/strategies/sample_strategy.py b/user_data/strategies/sample_strategy.py index c2fd681d2..36dea65c9 100644 --- a/user_data/strategies/sample_strategy.py +++ b/user_data/strategies/sample_strategy.py @@ -107,16 +107,16 @@ class SampleStrategy(IStrategy): # RSI dataframe['rsi'] = ta.RSI(dataframe) - """ + # ADX dataframe['adx'] = ta.ADX(dataframe) - + """ # Awesome oscillator dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) # Commodity Channel Index: values Oversold:<-100, Overbought:>100 dataframe['cci'] = ta.CCI(dataframe) - + """ # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -126,6 +126,7 @@ class SampleStrategy(IStrategy): # MFI dataframe['mfi'] = ta.MFI(dataframe) + """ # Minus Directional Indicator / Movement dataframe['minus_dm'] = ta.MINUS_DM(dataframe) dataframe['minus_di'] = ta.MINUS_DI(dataframe) @@ -149,12 +150,13 @@ class SampleStrategy(IStrategy): stoch = ta.STOCH(dataframe) dataframe['slowd'] = stoch['slowd'] dataframe['slowk'] = stoch['slowk'] - + """ # Stoch fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] dataframe['fastk'] = stoch_fast['fastk'] + """ # Stoch RSI stoch_rsi = ta.STOCHRSI(dataframe) dataframe['fastd_rsi'] = stoch_rsi['fastd'] @@ -178,12 +180,11 @@ class SampleStrategy(IStrategy): dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - # SAR Parabol - dataframe['sar'] = ta.SAR(dataframe) - # SMA - Simple Moving Average dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) """ + # SAR Parabol + dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) From 12e86ee4bd48300cf5057ebecc11f7b8b257089c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Nov 2019 11:12:08 +0100 Subject: [PATCH 3/6] Make travis test-hyperopt the sample strategy --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index eb171521d..1cc22dfbd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ jobs: name: backtest - script: - cp config.json.example config.json - - freqtrade --datadir tests/testdata hyperopt -e 5 + - freqtrade --datadir tests/testdata --strategy SampleStrategy hyperopt --customhyperopt SampleHyperOpts -e 5 name: hyperopt - script: flake8 name: flake8 From 3287cdd47ad5fa042f7f82f85c66a47666aa1530 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Nov 2019 13:01:36 +0100 Subject: [PATCH 4/6] Improve documentation regarding loading methods from hyperopt --- docs/hyperopt.md | 23 +++++++++++++++---- .../hyperopts/sample_hyperopt_advanced.py | 13 +++++++---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 99331707f..3c42a0428 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -23,17 +23,23 @@ Configuring hyperopt is similar to writing your own strategy, and many tasks wil Depending on the space you want to optimize, only some of the below are required: -* fill `populate_indicators` - probably a copy from your strategy * fill `buy_strategy_generator` - for buy signal optimization * fill `indicator_space` - for buy signal optimzation * fill `sell_strategy_generator` - for sell signal optimization * fill `sell_indicator_space` - for sell signal optimzation -Optional, but recommended: +!!! Note + `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work. +Optional - can also be loaded from a strategy: + +* copy `populate_indicators` from your strategy - otherwise default-strategy will be used * copy `populate_buy_trend` from your strategy - otherwise default-strategy will be used * copy `populate_sell_trend` from your strategy - otherwise default-strategy will be used +!!! Note + Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead. + Rarely you may also need to override: * `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default) @@ -156,7 +162,7 @@ that minimizes the value of the [loss function](#loss-functions). The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. When you want to test an indicator that isn't used by the bot currently, remember to -add it to the `populate_indicators()` method in `hyperopt.py`. +add it to the `populate_indicators()` method in your custom hyperopt file. ## Loss-functions @@ -270,6 +276,14 @@ For example, to use one month of data, pass the following parameter to the hyper freqtrade hyperopt --timerange 20180401-20180501 ``` +### Running Hyperopt using methods from a strategy + +Hyperopt can reuse `populate_indicators`, `populate_buy_trend`, `populate_sell_trend` from your strategy, assuming these methods are **not** in your custom hyperopt file, and a strategy is provided. + +```bash +freqtrade --strategy SampleStrategy hyperopt --customhyperopt SampleHyperopt +``` + ### Running Hyperopt with Smaller Search Space Use the `--spaces` argument to limit the search space used by hyperopt. @@ -341,8 +355,7 @@ So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that t (dataframe['rsi'] < 29.0) ``` -Translating your whole hyperopt result as the new buy-signal -would then look like: +Translating your whole hyperopt result as the new buy-signal would then look like: ```python def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: diff --git a/user_data/hyperopts/sample_hyperopt_advanced.py b/user_data/hyperopts/sample_hyperopt_advanced.py index 00062a58d..6986854ee 100644 --- a/user_data/hyperopts/sample_hyperopt_advanced.py +++ b/user_data/hyperopts/sample_hyperopt_advanced.py @@ -37,6 +37,9 @@ class AdvancedSampleHyperOpts(IHyperOpt): """ @staticmethod def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + This method can also be loaded from the strategy, if it doesn't exist in the hyperopt class. + """ dataframe['adx'] = ta.ADX(dataframe) macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -229,8 +232,9 @@ class AdvancedSampleHyperOpts(IHyperOpt): def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators. Should be a copy of from strategy - must align to populate_indicators in this file + Based on TA indicators. + Can be a copy of from the strategy, or will be loaded from the strategy. + must align to populate_indicators used (either from this File, or from the strategy) Only used when --spaces does not include buy """ dataframe.loc[ @@ -246,8 +250,9 @@ class AdvancedSampleHyperOpts(IHyperOpt): def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators. Should be a copy of from strategy - must align to populate_indicators in this file + Based on TA indicators. + Can be a copy of from the strategy, or will be loaded from the strategy. + must align to populate_indicators used (either from this File, or from the strategy) Only used when --spaces does not include sell """ dataframe.loc[ From 80ad37ad93761b8ab69df1a2f10a3a14b4e6ad85 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Nov 2019 14:17:15 +0100 Subject: [PATCH 5/6] Updated plot_indicators test --- tests/optimize/test_hyperopt.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 675bbd62e..36902dcfc 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -149,6 +149,7 @@ def test_hyperoptresolver(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) hyperopt = DefaultHyperOpt + delattr(hyperopt, 'populate_indicators') delattr(hyperopt, 'populate_buy_trend') delattr(hyperopt, 'populate_sell_trend') mocker.patch( @@ -156,8 +157,11 @@ def test_hyperoptresolver(mocker, default_conf, caplog) -> None: MagicMock(return_value=hyperopt(default_conf)) ) x = HyperOptResolver(default_conf, ).hyperopt + assert not hasattr(x, 'populate_indicators') assert not hasattr(x, 'populate_buy_trend') assert not hasattr(x, 'populate_sell_trend') + assert log_has("Hyperopt class does not provide populate_indicators() method. " + "Using populate_indicators from the strategy.", caplog) assert log_has("Hyperopt class does not provide populate_sell_trend() method. " "Using populate_sell_trend from the strategy.", caplog) assert log_has("Hyperopt class does not provide populate_buy_trend() method. " From 6550e1fa992bef966fcffe1703ede0a20851da04 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Nov 2019 09:55:38 +0100 Subject: [PATCH 6/6] Change docstring in sampleHyperopt --- user_data/hyperopts/sample_hyperopt_advanced.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/user_data/hyperopts/sample_hyperopt_advanced.py b/user_data/hyperopts/sample_hyperopt_advanced.py index 6986854ee..c5d28878c 100644 --- a/user_data/hyperopts/sample_hyperopt_advanced.py +++ b/user_data/hyperopts/sample_hyperopt_advanced.py @@ -233,8 +233,9 @@ class AdvancedSampleHyperOpts(IHyperOpt): def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators. - Can be a copy of from the strategy, or will be loaded from the strategy. - must align to populate_indicators used (either from this File, or from the strategy) + Can be a copy of the corresponding method from the strategy, + or will be loaded from the strategy. + Must align to populate_indicators used (either from this File, or from the strategy) Only used when --spaces does not include buy """ dataframe.loc[ @@ -251,8 +252,9 @@ class AdvancedSampleHyperOpts(IHyperOpt): def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators. - Can be a copy of from the strategy, or will be loaded from the strategy. - must align to populate_indicators used (either from this File, or from the strategy) + Can be a copy of the corresponding method from the strategy, + or will be loaded from the strategy. + Must align to populate_indicators used (either from this File, or from the strategy) Only used when --spaces does not include sell """ dataframe.loc[