From 469db0d434e86965d2d85e7d37097740de3c4e24 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Thu, 22 Mar 2018 19:27:13 +1100 Subject: [PATCH] Decoupled custom hyperopts from hyperopt.py --- freqtrade/arguments.py | 8 + freqtrade/configuration.py | 3 + freqtrade/constants.py | 1 + freqtrade/optimize/custom_hyperopt.py | 152 +++++++++++ freqtrade/optimize/default_hyperopt.py | 314 ++++++++++++++++++++++ freqtrade/optimize/hyperopt.py | 138 +--------- freqtrade/optimize/interface.py | 59 ++++ freqtrade/tests/optimize/test_hyperopt.py | 10 +- user_data/hyperopts/__init__.py | 0 user_data/hyperopts/test_hyperopt.py | 283 +++++++++++++++++++ 10 files changed, 839 insertions(+), 129 deletions(-) create mode 100644 freqtrade/optimize/custom_hyperopt.py create mode 100644 freqtrade/optimize/default_hyperopt.py create mode 100644 freqtrade/optimize/interface.py create mode 100644 user_data/hyperopts/__init__.py create mode 100644 user_data/hyperopts/test_hyperopt.py diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index bb571b4ea..a36b3e66a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -104,6 +104,14 @@ class Arguments(object): type=str, metavar='PATH', ) + self.parser.add_argument( + '--hyperopt', + help='specify hyperopt file (default: %(default)s)', + dest='hyperopt', + default=Constants.DEFAULT_HYPEROPT, + type=str, + metavar='PATH', + ) self.parser.add_argument( '--dynamic-whitelist', help='dynamically generate and update whitelist' diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index e043525a7..7b43ff2f8 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -52,6 +52,9 @@ class Configuration(object): if self.args.strategy_path: config.update({'strategy_path': self.args.strategy_path}) + # Add the hyperopt file to use + config.update({'hyperopt': self.args.hyperopt}) + # Load Common configuration config = self._load_common_config(config) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 2b09aa6c9..a03efdc4b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -9,6 +9,7 @@ TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_STRATEGY = 'DefaultStrategy' +DEFAULT_HYPEROPT = 'default_hyperopt' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' diff --git a/freqtrade/optimize/custom_hyperopt.py b/freqtrade/optimize/custom_hyperopt.py new file mode 100644 index 000000000..688d58df8 --- /dev/null +++ b/freqtrade/optimize/custom_hyperopt.py @@ -0,0 +1,152 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import importlib +import os +import sys +from typing import Dict, Any, Callable + +from pandas import DataFrame + +from freqtrade.constants import Constants +from freqtrade.logger import Logger +from freqtrade.optimize.interface import IHyperOpt + +sys.path.insert(0, r'../../user_data/hyperopts') + + +class CustomHyperOpt(object): + """ + This class contains all the logic to load custom hyperopt class + """ + def __init__(self, config: dict = {}) -> None: + """ + Load the custom class from config parameter + :param config: + :return: + """ + self.logger = Logger(name=__name__).get_logger() + + # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt + if 'hyperopt' in config: + hyperopt = config['hyperopt'] + else: + hyperopt = Constants.DEFAULT_HYPEROPT + + # Load the hyperopt + self._load_hyperopt(hyperopt) + + def _load_hyperopt(self, hyperopt_name: str) -> None: + """ + Search and load the custom hyperopt. If no hyperopt found, fallback on the default hyperopt + Set the object into self.custom_hyperopt + :param hyperopt_name: name of the module to import + :return: None + """ + + try: + # Start by sanitizing the file name (remove any extensions) + hyperopt_name = self._sanitize_module_name(filename=hyperopt_name) + + # Search where can be the hyperopt file + path = self._search_hyperopt(filename=hyperopt_name) + + # Load the hyperopt + self.custom_hyperopt = self._load_class(path + hyperopt_name) + + # Fallback to the default hyperopt + except (ImportError, TypeError) as error: + self.logger.error( + "Impossible to load Hyperopt 'user_data/hyperopts/%s.py'. This file does not exist" + " or contains Python code errors", + hyperopt_name + ) + self.logger.error( + "The error is:\n%s.", + error + ) + + def _load_class(self, filename: str) -> IHyperOpt: + """ + Import a hyperopt as a module + :param filename: path to the hyperopt (path from freqtrade/optimize/) + :return: return the hyperopt class + """ + module = importlib.import_module(filename, __package__) + custom_hyperopt = getattr(module, module.class_name) + + self.logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) + return custom_hyperopt() + + @staticmethod + def _sanitize_module_name(filename: str) -> str: + """ + Remove any extension from filename + :param filename: filename to sanatize + :return: return the filename without extensions + """ + filename = os.path.basename(filename) + filename = os.path.splitext(filename)[0] + return filename + + @staticmethod + def _search_hyperopt(filename: str) -> str: + """ + Search for the hyperopt file in different folder + 1. search into the user_data/hyperopts folder + 2. search into the freqtrade/optimize folder + 3. if nothing found, return None + :param hyperopt_name: module name to search + :return: module path where is the hyperopt + """ + pwd = os.path.dirname(os.path.realpath(__file__)) + '/' + user_data = os.path.join(pwd, '..', '..', 'user_data', 'hyperopts', filename + '.py') + hyperopt_folder = os.path.join(pwd, filename + '.py') + + path = None + if os.path.isfile(user_data): + path = 'user_data.hyperopts.' + elif os.path.isfile(hyperopt_folder): + path = '.' + + return path + + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + """ + Populate indicators that will be used in the Buy and Sell hyperopt + :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :return: a Dataframe with all mandatory indicators for the strategies + """ + return self.custom_hyperopt.populate_indicators(dataframe) + + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + """ + Create a buy strategy generator + """ + return self.custom_hyperopt.buy_strategy_generator(params) + + def indicator_space(self) -> Dict[str, Any]: + """ + Create an indicator space + """ + return self.custom_hyperopt.indicator_space() + + def generate_roi_table(self, params: Dict) -> Dict[int, float]: + """ + Create an roi table + """ + return self.custom_hyperopt.generate_roi_table(params) + + def stoploss_space(self) -> Dict[str, Any]: + """ + Create a stoploss space + """ + return self.custom_hyperopt.stoploss_space() + + def roi_space(self) -> Dict[str, Any]: + """ + Create a roi space + """ + return self.custom_hyperopt.roi_space() diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py new file mode 100644 index 000000000..6ef95bb49 --- /dev/null +++ b/freqtrade/optimize/default_hyperopt.py @@ -0,0 +1,314 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +import talib.abstract as ta +from pandas import DataFrame +from typing import Dict, Any, Callable +from functools import reduce + +import numpy +from hyperopt import hp + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.optimize.interface import IHyperOpt + +class_name = 'DefaultHyperOpts' + + +class DefaultHyperOpts(IHyperOpt): + """ + Default hyperopt provided by freqtrade bot. + You can override it with your own hyperopt + """ + + @staticmethod + def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + """ + dataframe['adx'] = ta.ADX(dataframe) + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + dataframe['cci'] = ta.CCI(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['roc'] = ta.ROC(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + # SAR Parabolic + dataframe['sar'] = ta.SAR(dataframe) + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] + + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ + + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ + + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ + + # Chart type + # ------------------------------------ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] + + return dataframe + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + """ + Buy strategy Hyperopt will build and use + """ + conditions = [] + # GUARDS AND TRENDS + if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in params and params['mfi']['enabled']: + conditions.append(dataframe['mfi'] < params['mfi']['value']) + if 'fastd' in params and params['fastd']['enabled']: + conditions.append(dataframe['fastd'] < params['fastd']['value']) + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'rsi' in params and params['rsi']['enabled']: + conditions.append(dataframe['rsi'] < params['rsi']['value']) + if 'over_sar' in params and params['over_sar']['enabled']: + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in params and params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + + # TRIGGERS + triggers = { + 'lower_bb': ( + dataframe['close'] < dataframe['bb_lowerband'] + ), + 'lower_bb_tema': ( + dataframe['tema'] < dataframe['bb_lowerband'] + ), + 'faststoch10': (qtpylib.crossed_above( + dataframe['fastd'], 10.0 + )), + 'ao_cross_zero': (qtpylib.crossed_above( + dataframe['ao'], 0.0 + )), + 'ema3_cross_ema10': (qtpylib.crossed_above( + dataframe['ema3'], dataframe['ema10'] + )), + 'macd_cross_signal': (qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )), + 'sar_reversal': (qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )), + 'ht_sine': (qtpylib.crossed_above( + dataframe['htleadsine'], dataframe['htsine'] + )), + 'heiken_reversal_bull': ( + (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & + (dataframe['ha_low'] == dataframe['ha_open']) + ), + 'di_cross': (qtpylib.crossed_above( + dataframe['plus_di'], dataframe['minus_di'] + )), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def indicator_space() -> Dict[str, Any]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return { + 'macd_below_zero': hp.choice('macd_below_zero', [ + {'enabled': False}, + {'enabled': True} + ]), + 'mfi': hp.choice('mfi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} + ]), + 'fastd': hp.choice('fastd', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} + ]), + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} + ]), + 'rsi': hp.choice('rsi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} + ]), + 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'over_sar': hp.choice('over_sar', [ + {'enabled': False}, + {'enabled': True} + ]), + 'green_candle': hp.choice('green_candle', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_sma': hp.choice('uptrend_sma', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'lower_bb'}, + {'type': 'lower_bb_tema'}, + {'type': 'faststoch10'}, + {'type': 'ao_cross_zero'}, + {'type': 'ema3_cross_ema10'}, + {'type': 'macd_cross_signal'}, + {'type': 'sar_reversal'}, + {'type': 'ht_sine'}, + {'type': 'heiken_reversal_bull'}, + {'type': 'di_cross'}, + ]), + } + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def stoploss_space() -> Dict[str, Any]: + """ + Stoploss Value to search + """ + return { + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), + } + + @staticmethod + def roi_space() -> Dict[str, Any]: + """ + Values to search for each ROI steps + """ + return { + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), + } diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b2d05d603..5e23458d7 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -9,22 +9,20 @@ import multiprocessing import os import sys from argparse import Namespace -from functools import reduce from math import exp from operator import itemgetter -from typing import Any, Callable, Dict, List +from typing import Any, Dict, List -import talib.abstract as ta from pandas import DataFrame from sklearn.externals.joblib import Parallel, delayed, dump, load from skopt import Optimizer -from skopt.space import Categorical, Dimension, Integer, Real +from skopt.space import Dimension -import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting +from freqtrade.optimize.custom_hyperopt import CustomHyperOpt logger = logging.getLogger(__name__) @@ -42,6 +40,9 @@ class Hyperopt(Backtesting): """ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) + + self.custom_hyperopt = CustomHyperOpt(self.config) + # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days self.target_trades = 600 @@ -74,24 +75,6 @@ class Hyperopt(Backtesting): arg_dict = {dim.name: value for dim, value in zip(dimensions, params)} return arg_dict - @staticmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe['adx'] = ta.ADX(dataframe) - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['mfi'] = ta.MFI(dataframe) - dataframe['rsi'] = ta.RSI(dataframe) - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['sar'] = ta.SAR(dataframe) - - return dataframe - def save_trials(self) -> None: """ Save hyperopt trials to file @@ -149,59 +132,6 @@ class Hyperopt(Backtesting): result = trade_loss + profit_loss + duration_loss return result - @staticmethod - def generate_roi_table(params: Dict) -> Dict[int, float]: - """ - Generate the ROI table that will be used by Hyperopt - """ - roi_table = {} - roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] - roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] - roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 - - return roi_table - - @staticmethod - def roi_space() -> List[Dimension]: - """ - Values to search for each ROI steps - """ - return [ - Integer(10, 120, name='roi_t1'), - Integer(10, 60, name='roi_t2'), - Integer(10, 40, name='roi_t3'), - Real(0.01, 0.04, name='roi_p1'), - Real(0.01, 0.07, name='roi_p2'), - Real(0.01, 0.20, name='roi_p3'), - ] - - @staticmethod - def stoploss_space() -> List[Dimension]: - """ - Stoploss search space - """ - return [ - Real(-0.5, -0.02, name='stoploss'), - ] - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - def has_space(self, space: str) -> bool: """ Tell if a space value is contained in the configuration @@ -216,61 +146,19 @@ class Hyperopt(Backtesting): """ spaces: List[Dimension] = [] if self.has_space('buy'): - spaces += Hyperopt.indicator_space() + spaces = {**spaces, **self.custom_hyperopt.indicator_space()} if self.has_space('roi'): - spaces += Hyperopt.roi_space() + spaces = {**spaces, **self.custom_hyperopt.roi_space()} if self.has_space('stoploss'): - spaces += Hyperopt.stoploss_space() + spaces = {**spaces, **self.custom_hyperopt.stoploss_space()} return spaces - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use - """ - conditions = [] - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - def generate_optimizer(self, _params) -> Dict: - params = self.get_args(_params) - + def generate_optimizer(self, params: Dict) -> Dict: if self.has_space('roi'): - self.strategy.minimal_roi = self.generate_roi_table(params) + self.analyze.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) if self.has_space('buy'): - self.advise_buy = self.buy_strategy_generator(params) + self.populate_buy_trend = self.custom_hyperopt.buy_strategy_generator(params) if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] @@ -351,7 +239,7 @@ class Hyperopt(Backtesting): ) if self.has_space('buy'): - self.strategy.advise_indicators = Hyperopt.populate_indicators # type: ignore + self.strategy.advise_indicators = self.custom_hyperopt.populate_indicators # type: ignore dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() diff --git a/freqtrade/optimize/interface.py b/freqtrade/optimize/interface.py new file mode 100644 index 000000000..8bc3866b2 --- /dev/null +++ b/freqtrade/optimize/interface.py @@ -0,0 +1,59 @@ +""" +IHyperOpt interface +This module defines the interface to apply for hyperopts +""" + +from abc import ABC, abstractmethod +from typing import Dict, Any, Callable + +from pandas import DataFrame + + +class IHyperOpt(ABC): + """ + Interface for freqtrade hyperopts + Defines the mandatory structure must follow any custom strategies + + Attributes you can use: + minimal_roi -> Dict: Minimal ROI designed for the strategy + stoploss -> float: optimal stoploss designed for the strategy + ticker_interval -> int: value of the ticker interval to use for the strategy + """ + + @abstractmethod + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + """ + Populate indicators that will be used in the Buy and Sell strategy + :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :return: a Dataframe with all mandatory indicators for the strategies + """ + + @abstractmethod + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + """ + Create a buy strategy generator + """ + + @abstractmethod + def indicator_space(self) -> Dict[str, Any]: + """ + Create an indicator space + """ + + @abstractmethod + def generate_roi_table(self, params: Dict) -> Dict[int, float]: + """ + Create an roi table + """ + + @abstractmethod + def stoploss_space(self) -> Dict[str, Any]: + """ + Create a stoploss space + """ + + @abstractmethod + def roi_space(self) -> Dict[str, Any]: + """ + Create a roi space + """ diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index c93f2d316..85d140b6d 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -175,7 +175,7 @@ def test_roi_table_generation(hyperopt) -> None: 'roi_p3': 3, } - assert hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} + assert hyperopt.custom_hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: @@ -243,7 +243,8 @@ def test_populate_indicators(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) - dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) + dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], + {'pair': 'UNITTEST/BTC'}) # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe @@ -255,9 +256,10 @@ def test_buy_strategy_generator(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) - dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) + dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], + {'pair': 'UNITTEST/BTC'}) - populate_buy_trend = hyperopt.buy_strategy_generator( + populate_buy_trend = hyperopt.custom_hyperopt.buy_strategy_generator( { 'adx-value': 20, 'fastd-value': 20, diff --git a/user_data/hyperopts/__init__.py b/user_data/hyperopts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/user_data/hyperopts/test_hyperopt.py b/user_data/hyperopts/test_hyperopt.py new file mode 100644 index 000000000..0ee43c64f --- /dev/null +++ b/user_data/hyperopts/test_hyperopt.py @@ -0,0 +1,283 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +import talib.abstract as ta +from pandas import DataFrame +from typing import Dict, Any, Callable +from functools import reduce +from math import exp + +import numpy +import talib.abstract as ta +from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.indicator_helpers import fishers_inverse +from freqtrade.optimize.interface import IHyperOpt + + +# Update this variable if you change the class name +class_name = 'TestHyperOpt' + + +# This class is a sample. Feel free to customize it. +class TestHyperOpt(IHyperOpt): + """ + This is a test hyperopt to inspire you. + More information in https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md + + You can: + - Rename the class name (Do not forget to update class_name) + - Add any methods you want to build your hyperopt + - Add any lib you need to build your hyperopt + + You must keep: + - the prototype for the methods: populate_indicators, indicator_space, buy_strategy_generator, + roi_space, generate_roi_table, stoploss_space + """ + + @staticmethod + def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + + Performance Note: For the best performance be frugal on the number of indicators + you are using. Let uncomment only the indicator you are using in your strategies + or your hyperopt configuration, otherwise you will waste your memory and CPU usage. + """ + + # Momentum Indicator + # ------------------------------------ + + # ADX + dataframe['adx'] = ta.ADX(dataframe) + + """ + # Awesome oscillator + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + + # Commodity Channel Index: values Oversold:<-100, Overbought:>100 + dataframe['cci'] = ta.CCI(dataframe) + + # MACD + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + + # MFI + dataframe['mfi'] = ta.MFI(dataframe) + + # Minus Directional Indicator / Movement + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # Plus Directional Indicator / Movement + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # ROC + dataframe['roc'] = ta.ROC(dataframe) + + # RSI + dataframe['rsi'] = ta.RSI(dataframe) + + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + """ + + # Overlap Studies + # ------------------------------------ + + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + + """ + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + + # SAR Parabol + dataframe['sar'] = ta.SAR(dataframe) + + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + """ + + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + + # Cycle Indicator + # ------------------------------------ + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] + + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ + + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ + + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ + + # Chart type + # ------------------------------------ + """ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] + """ + + return dataframe + + @staticmethod + def indicator_space() -> Dict[str, Any]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return { + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 50, 80, 5)} + ]), + 'uptrend_tema': hp.choice('uptrend_tema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'middle_bb_tema'}, + ]), + } + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + conditions = [] + # GUARDS AND TRENDS + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'uptrend_tema' in params and params['uptrend_tema']['enabled']: + prevtema = dataframe['tema'].shift(1) + conditions.append(dataframe['tema'] > prevtema) + + # TRIGGERS + triggers = { + 'middle_bb_tema': ( + dataframe['tema'] > dataframe['bb_middleband'] + ), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def roi_space() -> Dict[str, Any]: + return { + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), + } + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def stoploss_space() -> Dict[str, Any]: + return { + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), + } \ No newline at end of file