From 469db0d434e86965d2d85e7d37097740de3c4e24 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Thu, 22 Mar 2018 19:27:13 +1100 Subject: [PATCH 01/14] Decoupled custom hyperopts from hyperopt.py --- freqtrade/arguments.py | 8 + freqtrade/configuration.py | 3 + freqtrade/constants.py | 1 + freqtrade/optimize/custom_hyperopt.py | 152 +++++++++++ freqtrade/optimize/default_hyperopt.py | 314 ++++++++++++++++++++++ freqtrade/optimize/hyperopt.py | 138 +--------- freqtrade/optimize/interface.py | 59 ++++ freqtrade/tests/optimize/test_hyperopt.py | 10 +- user_data/hyperopts/__init__.py | 0 user_data/hyperopts/test_hyperopt.py | 283 +++++++++++++++++++ 10 files changed, 839 insertions(+), 129 deletions(-) create mode 100644 freqtrade/optimize/custom_hyperopt.py create mode 100644 freqtrade/optimize/default_hyperopt.py create mode 100644 freqtrade/optimize/interface.py create mode 100644 user_data/hyperopts/__init__.py create mode 100644 user_data/hyperopts/test_hyperopt.py diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index bb571b4ea..a36b3e66a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -104,6 +104,14 @@ class Arguments(object): type=str, metavar='PATH', ) + self.parser.add_argument( + '--hyperopt', + help='specify hyperopt file (default: %(default)s)', + dest='hyperopt', + default=Constants.DEFAULT_HYPEROPT, + type=str, + metavar='PATH', + ) self.parser.add_argument( '--dynamic-whitelist', help='dynamically generate and update whitelist' diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index e043525a7..7b43ff2f8 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -52,6 +52,9 @@ class Configuration(object): if self.args.strategy_path: config.update({'strategy_path': self.args.strategy_path}) + # Add the hyperopt file to use + config.update({'hyperopt': self.args.hyperopt}) + # Load Common configuration config = self._load_common_config(config) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 2b09aa6c9..a03efdc4b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -9,6 +9,7 @@ TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_STRATEGY = 'DefaultStrategy' +DEFAULT_HYPEROPT = 'default_hyperopt' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' diff --git a/freqtrade/optimize/custom_hyperopt.py b/freqtrade/optimize/custom_hyperopt.py new file mode 100644 index 000000000..688d58df8 --- /dev/null +++ b/freqtrade/optimize/custom_hyperopt.py @@ -0,0 +1,152 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import importlib +import os +import sys +from typing import Dict, Any, Callable + +from pandas import DataFrame + +from freqtrade.constants import Constants +from freqtrade.logger import Logger +from freqtrade.optimize.interface import IHyperOpt + +sys.path.insert(0, r'../../user_data/hyperopts') + + +class CustomHyperOpt(object): + """ + This class contains all the logic to load custom hyperopt class + """ + def __init__(self, config: dict = {}) -> None: + """ + Load the custom class from config parameter + :param config: + :return: + """ + self.logger = Logger(name=__name__).get_logger() + + # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt + if 'hyperopt' in config: + hyperopt = config['hyperopt'] + else: + hyperopt = Constants.DEFAULT_HYPEROPT + + # Load the hyperopt + self._load_hyperopt(hyperopt) + + def _load_hyperopt(self, hyperopt_name: str) -> None: + """ + Search and load the custom hyperopt. If no hyperopt found, fallback on the default hyperopt + Set the object into self.custom_hyperopt + :param hyperopt_name: name of the module to import + :return: None + """ + + try: + # Start by sanitizing the file name (remove any extensions) + hyperopt_name = self._sanitize_module_name(filename=hyperopt_name) + + # Search where can be the hyperopt file + path = self._search_hyperopt(filename=hyperopt_name) + + # Load the hyperopt + self.custom_hyperopt = self._load_class(path + hyperopt_name) + + # Fallback to the default hyperopt + except (ImportError, TypeError) as error: + self.logger.error( + "Impossible to load Hyperopt 'user_data/hyperopts/%s.py'. This file does not exist" + " or contains Python code errors", + hyperopt_name + ) + self.logger.error( + "The error is:\n%s.", + error + ) + + def _load_class(self, filename: str) -> IHyperOpt: + """ + Import a hyperopt as a module + :param filename: path to the hyperopt (path from freqtrade/optimize/) + :return: return the hyperopt class + """ + module = importlib.import_module(filename, __package__) + custom_hyperopt = getattr(module, module.class_name) + + self.logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) + return custom_hyperopt() + + @staticmethod + def _sanitize_module_name(filename: str) -> str: + """ + Remove any extension from filename + :param filename: filename to sanatize + :return: return the filename without extensions + """ + filename = os.path.basename(filename) + filename = os.path.splitext(filename)[0] + return filename + + @staticmethod + def _search_hyperopt(filename: str) -> str: + """ + Search for the hyperopt file in different folder + 1. search into the user_data/hyperopts folder + 2. search into the freqtrade/optimize folder + 3. if nothing found, return None + :param hyperopt_name: module name to search + :return: module path where is the hyperopt + """ + pwd = os.path.dirname(os.path.realpath(__file__)) + '/' + user_data = os.path.join(pwd, '..', '..', 'user_data', 'hyperopts', filename + '.py') + hyperopt_folder = os.path.join(pwd, filename + '.py') + + path = None + if os.path.isfile(user_data): + path = 'user_data.hyperopts.' + elif os.path.isfile(hyperopt_folder): + path = '.' + + return path + + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + """ + Populate indicators that will be used in the Buy and Sell hyperopt + :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :return: a Dataframe with all mandatory indicators for the strategies + """ + return self.custom_hyperopt.populate_indicators(dataframe) + + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + """ + Create a buy strategy generator + """ + return self.custom_hyperopt.buy_strategy_generator(params) + + def indicator_space(self) -> Dict[str, Any]: + """ + Create an indicator space + """ + return self.custom_hyperopt.indicator_space() + + def generate_roi_table(self, params: Dict) -> Dict[int, float]: + """ + Create an roi table + """ + return self.custom_hyperopt.generate_roi_table(params) + + def stoploss_space(self) -> Dict[str, Any]: + """ + Create a stoploss space + """ + return self.custom_hyperopt.stoploss_space() + + def roi_space(self) -> Dict[str, Any]: + """ + Create a roi space + """ + return self.custom_hyperopt.roi_space() diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py new file mode 100644 index 000000000..6ef95bb49 --- /dev/null +++ b/freqtrade/optimize/default_hyperopt.py @@ -0,0 +1,314 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +import talib.abstract as ta +from pandas import DataFrame +from typing import Dict, Any, Callable +from functools import reduce + +import numpy +from hyperopt import hp + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.optimize.interface import IHyperOpt + +class_name = 'DefaultHyperOpts' + + +class DefaultHyperOpts(IHyperOpt): + """ + Default hyperopt provided by freqtrade bot. + You can override it with your own hyperopt + """ + + @staticmethod + def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + """ + dataframe['adx'] = ta.ADX(dataframe) + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + dataframe['cci'] = ta.CCI(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['roc'] = ta.ROC(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + # SAR Parabolic + dataframe['sar'] = ta.SAR(dataframe) + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] + + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ + + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ + + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ + + # Chart type + # ------------------------------------ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] + + return dataframe + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + """ + Buy strategy Hyperopt will build and use + """ + conditions = [] + # GUARDS AND TRENDS + if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in params and params['mfi']['enabled']: + conditions.append(dataframe['mfi'] < params['mfi']['value']) + if 'fastd' in params and params['fastd']['enabled']: + conditions.append(dataframe['fastd'] < params['fastd']['value']) + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'rsi' in params and params['rsi']['enabled']: + conditions.append(dataframe['rsi'] < params['rsi']['value']) + if 'over_sar' in params and params['over_sar']['enabled']: + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in params and params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + + # TRIGGERS + triggers = { + 'lower_bb': ( + dataframe['close'] < dataframe['bb_lowerband'] + ), + 'lower_bb_tema': ( + dataframe['tema'] < dataframe['bb_lowerband'] + ), + 'faststoch10': (qtpylib.crossed_above( + dataframe['fastd'], 10.0 + )), + 'ao_cross_zero': (qtpylib.crossed_above( + dataframe['ao'], 0.0 + )), + 'ema3_cross_ema10': (qtpylib.crossed_above( + dataframe['ema3'], dataframe['ema10'] + )), + 'macd_cross_signal': (qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )), + 'sar_reversal': (qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )), + 'ht_sine': (qtpylib.crossed_above( + dataframe['htleadsine'], dataframe['htsine'] + )), + 'heiken_reversal_bull': ( + (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & + (dataframe['ha_low'] == dataframe['ha_open']) + ), + 'di_cross': (qtpylib.crossed_above( + dataframe['plus_di'], dataframe['minus_di'] + )), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def indicator_space() -> Dict[str, Any]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return { + 'macd_below_zero': hp.choice('macd_below_zero', [ + {'enabled': False}, + {'enabled': True} + ]), + 'mfi': hp.choice('mfi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} + ]), + 'fastd': hp.choice('fastd', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} + ]), + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} + ]), + 'rsi': hp.choice('rsi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} + ]), + 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'over_sar': hp.choice('over_sar', [ + {'enabled': False}, + {'enabled': True} + ]), + 'green_candle': hp.choice('green_candle', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_sma': hp.choice('uptrend_sma', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'lower_bb'}, + {'type': 'lower_bb_tema'}, + {'type': 'faststoch10'}, + {'type': 'ao_cross_zero'}, + {'type': 'ema3_cross_ema10'}, + {'type': 'macd_cross_signal'}, + {'type': 'sar_reversal'}, + {'type': 'ht_sine'}, + {'type': 'heiken_reversal_bull'}, + {'type': 'di_cross'}, + ]), + } + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def stoploss_space() -> Dict[str, Any]: + """ + Stoploss Value to search + """ + return { + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), + } + + @staticmethod + def roi_space() -> Dict[str, Any]: + """ + Values to search for each ROI steps + """ + return { + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), + } diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b2d05d603..5e23458d7 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -9,22 +9,20 @@ import multiprocessing import os import sys from argparse import Namespace -from functools import reduce from math import exp from operator import itemgetter -from typing import Any, Callable, Dict, List +from typing import Any, Dict, List -import talib.abstract as ta from pandas import DataFrame from sklearn.externals.joblib import Parallel, delayed, dump, load from skopt import Optimizer -from skopt.space import Categorical, Dimension, Integer, Real +from skopt.space import Dimension -import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting +from freqtrade.optimize.custom_hyperopt import CustomHyperOpt logger = logging.getLogger(__name__) @@ -42,6 +40,9 @@ class Hyperopt(Backtesting): """ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) + + self.custom_hyperopt = CustomHyperOpt(self.config) + # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days self.target_trades = 600 @@ -74,24 +75,6 @@ class Hyperopt(Backtesting): arg_dict = {dim.name: value for dim, value in zip(dimensions, params)} return arg_dict - @staticmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe['adx'] = ta.ADX(dataframe) - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['mfi'] = ta.MFI(dataframe) - dataframe['rsi'] = ta.RSI(dataframe) - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['sar'] = ta.SAR(dataframe) - - return dataframe - def save_trials(self) -> None: """ Save hyperopt trials to file @@ -149,59 +132,6 @@ class Hyperopt(Backtesting): result = trade_loss + profit_loss + duration_loss return result - @staticmethod - def generate_roi_table(params: Dict) -> Dict[int, float]: - """ - Generate the ROI table that will be used by Hyperopt - """ - roi_table = {} - roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] - roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] - roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 - - return roi_table - - @staticmethod - def roi_space() -> List[Dimension]: - """ - Values to search for each ROI steps - """ - return [ - Integer(10, 120, name='roi_t1'), - Integer(10, 60, name='roi_t2'), - Integer(10, 40, name='roi_t3'), - Real(0.01, 0.04, name='roi_p1'), - Real(0.01, 0.07, name='roi_p2'), - Real(0.01, 0.20, name='roi_p3'), - ] - - @staticmethod - def stoploss_space() -> List[Dimension]: - """ - Stoploss search space - """ - return [ - Real(-0.5, -0.02, name='stoploss'), - ] - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - def has_space(self, space: str) -> bool: """ Tell if a space value is contained in the configuration @@ -216,61 +146,19 @@ class Hyperopt(Backtesting): """ spaces: List[Dimension] = [] if self.has_space('buy'): - spaces += Hyperopt.indicator_space() + spaces = {**spaces, **self.custom_hyperopt.indicator_space()} if self.has_space('roi'): - spaces += Hyperopt.roi_space() + spaces = {**spaces, **self.custom_hyperopt.roi_space()} if self.has_space('stoploss'): - spaces += Hyperopt.stoploss_space() + spaces = {**spaces, **self.custom_hyperopt.stoploss_space()} return spaces - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use - """ - conditions = [] - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - def generate_optimizer(self, _params) -> Dict: - params = self.get_args(_params) - + def generate_optimizer(self, params: Dict) -> Dict: if self.has_space('roi'): - self.strategy.minimal_roi = self.generate_roi_table(params) + self.analyze.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) if self.has_space('buy'): - self.advise_buy = self.buy_strategy_generator(params) + self.populate_buy_trend = self.custom_hyperopt.buy_strategy_generator(params) if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] @@ -351,7 +239,7 @@ class Hyperopt(Backtesting): ) if self.has_space('buy'): - self.strategy.advise_indicators = Hyperopt.populate_indicators # type: ignore + self.strategy.advise_indicators = self.custom_hyperopt.populate_indicators # type: ignore dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() diff --git a/freqtrade/optimize/interface.py b/freqtrade/optimize/interface.py new file mode 100644 index 000000000..8bc3866b2 --- /dev/null +++ b/freqtrade/optimize/interface.py @@ -0,0 +1,59 @@ +""" +IHyperOpt interface +This module defines the interface to apply for hyperopts +""" + +from abc import ABC, abstractmethod +from typing import Dict, Any, Callable + +from pandas import DataFrame + + +class IHyperOpt(ABC): + """ + Interface for freqtrade hyperopts + Defines the mandatory structure must follow any custom strategies + + Attributes you can use: + minimal_roi -> Dict: Minimal ROI designed for the strategy + stoploss -> float: optimal stoploss designed for the strategy + ticker_interval -> int: value of the ticker interval to use for the strategy + """ + + @abstractmethod + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + """ + Populate indicators that will be used in the Buy and Sell strategy + :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :return: a Dataframe with all mandatory indicators for the strategies + """ + + @abstractmethod + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + """ + Create a buy strategy generator + """ + + @abstractmethod + def indicator_space(self) -> Dict[str, Any]: + """ + Create an indicator space + """ + + @abstractmethod + def generate_roi_table(self, params: Dict) -> Dict[int, float]: + """ + Create an roi table + """ + + @abstractmethod + def stoploss_space(self) -> Dict[str, Any]: + """ + Create a stoploss space + """ + + @abstractmethod + def roi_space(self) -> Dict[str, Any]: + """ + Create a roi space + """ diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index c93f2d316..85d140b6d 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -175,7 +175,7 @@ def test_roi_table_generation(hyperopt) -> None: 'roi_p3': 3, } - assert hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} + assert hyperopt.custom_hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: @@ -243,7 +243,8 @@ def test_populate_indicators(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) - dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) + dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], + {'pair': 'UNITTEST/BTC'}) # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe @@ -255,9 +256,10 @@ def test_buy_strategy_generator(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) - dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) + dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], + {'pair': 'UNITTEST/BTC'}) - populate_buy_trend = hyperopt.buy_strategy_generator( + populate_buy_trend = hyperopt.custom_hyperopt.buy_strategy_generator( { 'adx-value': 20, 'fastd-value': 20, diff --git a/user_data/hyperopts/__init__.py b/user_data/hyperopts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/user_data/hyperopts/test_hyperopt.py b/user_data/hyperopts/test_hyperopt.py new file mode 100644 index 000000000..0ee43c64f --- /dev/null +++ b/user_data/hyperopts/test_hyperopt.py @@ -0,0 +1,283 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +import talib.abstract as ta +from pandas import DataFrame +from typing import Dict, Any, Callable +from functools import reduce +from math import exp + +import numpy +import talib.abstract as ta +from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.indicator_helpers import fishers_inverse +from freqtrade.optimize.interface import IHyperOpt + + +# Update this variable if you change the class name +class_name = 'TestHyperOpt' + + +# This class is a sample. Feel free to customize it. +class TestHyperOpt(IHyperOpt): + """ + This is a test hyperopt to inspire you. + More information in https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md + + You can: + - Rename the class name (Do not forget to update class_name) + - Add any methods you want to build your hyperopt + - Add any lib you need to build your hyperopt + + You must keep: + - the prototype for the methods: populate_indicators, indicator_space, buy_strategy_generator, + roi_space, generate_roi_table, stoploss_space + """ + + @staticmethod + def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + + Performance Note: For the best performance be frugal on the number of indicators + you are using. Let uncomment only the indicator you are using in your strategies + or your hyperopt configuration, otherwise you will waste your memory and CPU usage. + """ + + # Momentum Indicator + # ------------------------------------ + + # ADX + dataframe['adx'] = ta.ADX(dataframe) + + """ + # Awesome oscillator + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + + # Commodity Channel Index: values Oversold:<-100, Overbought:>100 + dataframe['cci'] = ta.CCI(dataframe) + + # MACD + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + + # MFI + dataframe['mfi'] = ta.MFI(dataframe) + + # Minus Directional Indicator / Movement + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # Plus Directional Indicator / Movement + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # ROC + dataframe['roc'] = ta.ROC(dataframe) + + # RSI + dataframe['rsi'] = ta.RSI(dataframe) + + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + """ + + # Overlap Studies + # ------------------------------------ + + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + + """ + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + + # SAR Parabol + dataframe['sar'] = ta.SAR(dataframe) + + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + """ + + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + + # Cycle Indicator + # ------------------------------------ + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] + + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ + + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ + + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ + + # Chart type + # ------------------------------------ + """ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] + """ + + return dataframe + + @staticmethod + def indicator_space() -> Dict[str, Any]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return { + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 50, 80, 5)} + ]), + 'uptrend_tema': hp.choice('uptrend_tema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'middle_bb_tema'}, + ]), + } + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + conditions = [] + # GUARDS AND TRENDS + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'uptrend_tema' in params and params['uptrend_tema']['enabled']: + prevtema = dataframe['tema'].shift(1) + conditions.append(dataframe['tema'] > prevtema) + + # TRIGGERS + triggers = { + 'middle_bb_tema': ( + dataframe['tema'] > dataframe['bb_middleband'] + ), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def roi_space() -> Dict[str, Any]: + return { + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), + } + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def stoploss_space() -> Dict[str, Any]: + return { + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), + } \ No newline at end of file From 5816d1c1bdf54031b407574a1ff1198d4dc1562e Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Thu, 22 Mar 2018 21:07:22 +1100 Subject: [PATCH 02/14] Updated documentation for new hyperopt --- docs/hyperopt.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index e2dcf3e95..36e5d2d74 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -21,16 +21,19 @@ and still take a long time. We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) -### Configure your Guards and Triggers +### 1. Configure your Guards and Triggers -There are two places you need to change to add a new buy strategy for testing: -- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L231-L264). -- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L213-L224) -and the associated methods `indicator_space`, `roi_space`, `stoploss_space`. +There are two places you need to change in your strategy file to add a +new buy strategy for testing: -There you have two different type of indicators: 1. `guards` and 2. `triggers`. -1. Guards are conditions like "never buy if ADX < 10", or "never buy if -current price is over EMA10". +- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). +- Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). + +There you have two different type of indicators: 1. `guards` and 2. +`triggers`. + +1. Guards are conditions like "never buy if ADX < 10", or never buy if +current price is over EMA10. 2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower bollinger band". @@ -124,9 +127,11 @@ Because hyperopt tries a lot of combinations to find the best parameters it will We strongly recommend to use `screen` or `tmux` to prevent any connection loss. ```bash -python3 ./freqtrade/main.py -c config.json hyperopt -e 5000 +python3 ./freqtrade/main.py -s --hyperopt -c config.json hyperopt -e 5000 ``` +Use `` and `` as the names of the custom strategy and custom hyperopt used. + The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. From 40368bd1b20655a7c77768e82fb194a9d0155958 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Sat, 31 Mar 2018 16:49:06 +1100 Subject: [PATCH 03/14] Added more hyperopt documentation --- docs/bot-usage.md | 2 ++ docs/hyperopt.md | 27 ++++++++++++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 83a8ee833..f6c15f26d 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -204,6 +204,8 @@ optional arguments: number) --timerange TIMERANGE specify what timerange of data to use. + --hyperopt PATH specify hyperopt file (default: + freqtrade/optimize/default_hyperopt.py) -e INT, --epochs INT specify number of epochs (default: 100) -s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...], --spaces {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...] Specify which parameters to hyperopt. Space separate diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 36e5d2d74..0484613c2 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -19,15 +19,23 @@ and still take a long time. ## Prepare Hyperopting -We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) +## Prepare Hyperopt +Before we start digging in Hyperopt, we recommend you to take a look at +an example hyperopt file located into [user_data/strategies/](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) + +### 1. Install a Custom Hyperopt File +This is very simple. Put your hyperopt file into the folder +`user_data/hyperopts`. + +Let assume you want a hyperopt file `awesome_hyperopt.py`: +1. Copy the file `user_data/hyperopts/test_hyperopt.py` into `user_data/hyperopts/awesome_hyperopt.py` + -### 1. Configure your Guards and Triggers - -There are two places you need to change in your strategy file to add a -new buy strategy for testing: - -- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). -- Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). +### 2. Configure your Guards and Triggers +There are two places you need to change in your hyperopt file to add a +new buy hyperopt for testing: +- Inside [populate_buy_trend()](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). +- Inside [indicator_space()](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). There you have two different type of indicators: 1. `guards` and 2. `triggers`. @@ -130,7 +138,8 @@ We strongly recommend to use `screen` or `tmux` to prevent any connection loss. python3 ./freqtrade/main.py -s --hyperopt -c config.json hyperopt -e 5000 ``` -Use `` and `` as the names of the custom strategy and custom hyperopt used. +Use `` and `` as the names of the custom strategy +(only required for generating sells) and the custom hyperopt used. The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. From e0f420983e5b9b628a3af0437bea61efda4e27b0 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Sat, 31 Mar 2018 17:21:15 +1100 Subject: [PATCH 04/14] Updated logger in custom_hyperopt --- freqtrade/optimize/custom_hyperopt.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/custom_hyperopt.py b/freqtrade/optimize/custom_hyperopt.py index 688d58df8..80168c46e 100644 --- a/freqtrade/optimize/custom_hyperopt.py +++ b/freqtrade/optimize/custom_hyperopt.py @@ -4,6 +4,7 @@ This module load custom hyperopts """ import importlib +import logging import os import sys from typing import Dict, Any, Callable @@ -11,11 +12,12 @@ from typing import Dict, Any, Callable from pandas import DataFrame from freqtrade.constants import Constants -from freqtrade.logger import Logger from freqtrade.optimize.interface import IHyperOpt sys.path.insert(0, r'../../user_data/hyperopts') +logger = logging.getLogger(__name__) + class CustomHyperOpt(object): """ @@ -27,7 +29,6 @@ class CustomHyperOpt(object): :param config: :return: """ - self.logger = Logger(name=__name__).get_logger() # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt if 'hyperopt' in config: @@ -58,12 +59,12 @@ class CustomHyperOpt(object): # Fallback to the default hyperopt except (ImportError, TypeError) as error: - self.logger.error( + logger.error( "Impossible to load Hyperopt 'user_data/hyperopts/%s.py'. This file does not exist" " or contains Python code errors", hyperopt_name ) - self.logger.error( + logger.error( "The error is:\n%s.", error ) @@ -77,7 +78,7 @@ class CustomHyperOpt(object): module = importlib.import_module(filename, __package__) custom_hyperopt = getattr(module, module.class_name) - self.logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) + logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) return custom_hyperopt() @staticmethod From 477515c4b5904b81cd9d6a6c99aa93f59daf39a8 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Sun, 1 Apr 2018 21:06:50 +1000 Subject: [PATCH 05/14] Now using resolver for custom hyperopts --- freqtrade/arguments.py | 6 +- freqtrade/constants.py | 2 +- freqtrade/optimize/custom_hyperopt.py | 153 -------------------------- freqtrade/optimize/hyperopt.py | 7 +- freqtrade/optimize/resolver.py | 104 +++++++++++++++++ user_data/hyperopts/test_hyperopt.py | 4 - 6 files changed, 112 insertions(+), 164 deletions(-) delete mode 100644 freqtrade/optimize/custom_hyperopt.py create mode 100644 freqtrade/optimize/resolver.py diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index a36b3e66a..429a7ca5a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -105,12 +105,12 @@ class Arguments(object): metavar='PATH', ) self.parser.add_argument( - '--hyperopt', - help='specify hyperopt file (default: %(default)s)', + '--customhyperopt', + help='specify hyperopt class name (default: %(default)s)', dest='hyperopt', default=Constants.DEFAULT_HYPEROPT, type=str, - metavar='PATH', + metavar='NAME', ) self.parser.add_argument( '--dynamic-whitelist', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index a03efdc4b..e76c54ba2 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -9,7 +9,7 @@ TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_STRATEGY = 'DefaultStrategy' -DEFAULT_HYPEROPT = 'default_hyperopt' +DEFAULT_HYPEROPT = 'DefaultHyperOpts' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' diff --git a/freqtrade/optimize/custom_hyperopt.py b/freqtrade/optimize/custom_hyperopt.py deleted file mode 100644 index 80168c46e..000000000 --- a/freqtrade/optimize/custom_hyperopt.py +++ /dev/null @@ -1,153 +0,0 @@ -# pragma pylint: disable=attribute-defined-outside-init - -""" -This module load custom hyperopts -""" -import importlib -import logging -import os -import sys -from typing import Dict, Any, Callable - -from pandas import DataFrame - -from freqtrade.constants import Constants -from freqtrade.optimize.interface import IHyperOpt - -sys.path.insert(0, r'../../user_data/hyperopts') - -logger = logging.getLogger(__name__) - - -class CustomHyperOpt(object): - """ - This class contains all the logic to load custom hyperopt class - """ - def __init__(self, config: dict = {}) -> None: - """ - Load the custom class from config parameter - :param config: - :return: - """ - - # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt - if 'hyperopt' in config: - hyperopt = config['hyperopt'] - else: - hyperopt = Constants.DEFAULT_HYPEROPT - - # Load the hyperopt - self._load_hyperopt(hyperopt) - - def _load_hyperopt(self, hyperopt_name: str) -> None: - """ - Search and load the custom hyperopt. If no hyperopt found, fallback on the default hyperopt - Set the object into self.custom_hyperopt - :param hyperopt_name: name of the module to import - :return: None - """ - - try: - # Start by sanitizing the file name (remove any extensions) - hyperopt_name = self._sanitize_module_name(filename=hyperopt_name) - - # Search where can be the hyperopt file - path = self._search_hyperopt(filename=hyperopt_name) - - # Load the hyperopt - self.custom_hyperopt = self._load_class(path + hyperopt_name) - - # Fallback to the default hyperopt - except (ImportError, TypeError) as error: - logger.error( - "Impossible to load Hyperopt 'user_data/hyperopts/%s.py'. This file does not exist" - " or contains Python code errors", - hyperopt_name - ) - logger.error( - "The error is:\n%s.", - error - ) - - def _load_class(self, filename: str) -> IHyperOpt: - """ - Import a hyperopt as a module - :param filename: path to the hyperopt (path from freqtrade/optimize/) - :return: return the hyperopt class - """ - module = importlib.import_module(filename, __package__) - custom_hyperopt = getattr(module, module.class_name) - - logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) - return custom_hyperopt() - - @staticmethod - def _sanitize_module_name(filename: str) -> str: - """ - Remove any extension from filename - :param filename: filename to sanatize - :return: return the filename without extensions - """ - filename = os.path.basename(filename) - filename = os.path.splitext(filename)[0] - return filename - - @staticmethod - def _search_hyperopt(filename: str) -> str: - """ - Search for the hyperopt file in different folder - 1. search into the user_data/hyperopts folder - 2. search into the freqtrade/optimize folder - 3. if nothing found, return None - :param hyperopt_name: module name to search - :return: module path where is the hyperopt - """ - pwd = os.path.dirname(os.path.realpath(__file__)) + '/' - user_data = os.path.join(pwd, '..', '..', 'user_data', 'hyperopts', filename + '.py') - hyperopt_folder = os.path.join(pwd, filename + '.py') - - path = None - if os.path.isfile(user_data): - path = 'user_data.hyperopts.' - elif os.path.isfile(hyperopt_folder): - path = '.' - - return path - - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: - """ - Populate indicators that will be used in the Buy and Sell hyperopt - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() - :return: a Dataframe with all mandatory indicators for the strategies - """ - return self.custom_hyperopt.populate_indicators(dataframe) - - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: - """ - Create a buy strategy generator - """ - return self.custom_hyperopt.buy_strategy_generator(params) - - def indicator_space(self) -> Dict[str, Any]: - """ - Create an indicator space - """ - return self.custom_hyperopt.indicator_space() - - def generate_roi_table(self, params: Dict) -> Dict[int, float]: - """ - Create an roi table - """ - return self.custom_hyperopt.generate_roi_table(params) - - def stoploss_space(self) -> Dict[str, Any]: - """ - Create a stoploss space - """ - return self.custom_hyperopt.stoploss_space() - - def roi_space(self) -> Dict[str, Any]: - """ - Create a roi space - """ - return self.custom_hyperopt.roi_space() diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5e23458d7..07c24ff18 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,7 +22,8 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from freqtrade.optimize.custom_hyperopt import CustomHyperOpt +from freqtrade.optimize.resolver import HyperOptResolver + logger = logging.getLogger(__name__) @@ -40,8 +41,8 @@ class Hyperopt(Backtesting): """ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) - - self.custom_hyperopt = CustomHyperOpt(self.config) + self.config = config + self.custom_hyperopt = HyperOptResolver(self.config).hyperopt # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days diff --git a/freqtrade/optimize/resolver.py b/freqtrade/optimize/resolver.py new file mode 100644 index 000000000..78637483b --- /dev/null +++ b/freqtrade/optimize/resolver.py @@ -0,0 +1,104 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import importlib.util +import inspect +import logging +import os +from typing import Optional, Dict, Type + +from freqtrade.constants import Constants +from freqtrade.optimize.interface import IHyperOpt + + +logger = logging.getLogger(__name__) + + +class HyperOptResolver(object): + """ + This class contains all the logic to load custom hyperopt class + """ + + __slots__ = ['hyperopt'] + + def __init__(self, config: Optional[Dict] = None) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary or None + """ + config = config or {} + + # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt + hyperopt_name = config.get('hyperopt') or Constants.DEFAULT_HYPEROPT + self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) + + def _load_hyperopt( + self, hyperopt_name: str, extra_dir: Optional[str] = None) -> Optional[IHyperOpt]: + """ + Search and loads the specified hyperopt. + :param hyperopt_name: name of the module to import + :param extra_dir: additional directory to search for the given hyperopt + :return: HyperOpt instance or None + """ + current_path = os.path.dirname(os.path.realpath(__file__)) + abs_paths = [ + os.path.join(current_path, '..', '..', 'user_data', 'hyperopts'), + current_path, + ] + + if extra_dir: + # Add extra hyperopt directory on top of search paths + abs_paths.insert(0, extra_dir) + + for path in abs_paths: + hyperopt = self._search_hyperopt(path, hyperopt_name) + if hyperopt: + logger.info('Using resolved hyperopt %s from \'%s\'', hyperopt_name, path) + return hyperopt + + raise ImportError( + "Impossible to load Hyperopt '{}'. This class does not exist" + " or contains Python code errors".format(hyperopt_name) + ) + + @staticmethod + def _get_valid_hyperopts(module_path: str, hyperopt_name: str) -> Optional[Type[IHyperOpt]]: + """ + Returns a list of all possible hyperopts for the given module_path + :param module_path: absolute path to the module + :param hyperopt_name: Class name of the hyperopt + :return: Tuple with (name, class) or None + """ + + # Generate spec based on absolute path + spec = importlib.util.spec_from_file_location('user_data.hyperopts', module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + valid_hyperopts_gen = ( + obj for name, obj in inspect.getmembers(module, inspect.isclass) + if hyperopt_name == name and IHyperOpt in obj.__bases__ + ) + return next(valid_hyperopts_gen, None) + + @staticmethod + def _search_hyperopt(directory: str, hyperopt_name: str) -> Optional[IHyperOpt]: + """ + Search for the hyperopt_name in the given directory + :param directory: relative or absolute directory path + :return: name of the hyperopt class + """ + logger.debug('Searching for hyperopt %s in \'%s\'', hyperopt_name, directory) + for entry in os.listdir(directory): + # Only consider python files + if not entry.endswith('.py'): + logger.debug('Ignoring %s', entry) + continue + hyperopt = HyperOptResolver._get_valid_hyperopts( + os.path.abspath(os.path.join(directory, entry)), hyperopt_name + ) + if hyperopt: + return hyperopt() + return None diff --git a/user_data/hyperopts/test_hyperopt.py b/user_data/hyperopts/test_hyperopt.py index 0ee43c64f..b742af6c9 100644 --- a/user_data/hyperopts/test_hyperopt.py +++ b/user_data/hyperopts/test_hyperopt.py @@ -15,10 +15,6 @@ from freqtrade.indicator_helpers import fishers_inverse from freqtrade.optimize.interface import IHyperOpt -# Update this variable if you change the class name -class_name = 'TestHyperOpt' - - # This class is a sample. Feel free to customize it. class TestHyperOpt(IHyperOpt): """ From 8044846d3741408731967410692f7cf51d916671 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 7 Nov 2018 07:05:40 +0100 Subject: [PATCH 06/14] Fix some refactoring problems --- freqtrade/arguments.py | 2 +- freqtrade/optimize/resolver.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 429a7ca5a..ceaf3e7e5 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -108,7 +108,7 @@ class Arguments(object): '--customhyperopt', help='specify hyperopt class name (default: %(default)s)', dest='hyperopt', - default=Constants.DEFAULT_HYPEROPT, + default=constants.DEFAULT_HYPEROPT, type=str, metavar='NAME', ) diff --git a/freqtrade/optimize/resolver.py b/freqtrade/optimize/resolver.py index 78637483b..1e0316c12 100644 --- a/freqtrade/optimize/resolver.py +++ b/freqtrade/optimize/resolver.py @@ -9,7 +9,7 @@ import logging import os from typing import Optional, Dict, Type -from freqtrade.constants import Constants +from freqtrade.constants import DEFAULT_HYPEROPT from freqtrade.optimize.interface import IHyperOpt @@ -31,7 +31,7 @@ class HyperOptResolver(object): config = config or {} # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt - hyperopt_name = config.get('hyperopt') or Constants.DEFAULT_HYPEROPT + hyperopt_name = config.get('hyperopt') or DEFAULT_HYPEROPT self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) def _load_hyperopt( From 7b62e71f235e2e4a5240fc1bf0b348db37b95040 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 7 Nov 2018 19:46:04 +0100 Subject: [PATCH 07/14] Fix some tests and rebase issues --- freqtrade/optimize/__init__.py | 1 + freqtrade/optimize/default_hyperopt.py | 278 +++++------------------- freqtrade/optimize/hyperopt.py | 15 +- user_data/hyperopts/sample_hyperopt.py | 138 ++++++++++++ user_data/hyperopts/test_hyperopt.py | 279 ------------------------- 5 files changed, 194 insertions(+), 517 deletions(-) create mode 100644 user_data/hyperopts/sample_hyperopt.py delete mode 100644 user_data/hyperopts/test_hyperopt.py diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 52766f78e..b1407de18 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -20,6 +20,7 @@ from pandas import DataFrame from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange +from freqtrade.optimize.default_hyperopt import DefaultHyperOpts # noqa: F401 logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 6ef95bb49..e127fd6d8 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -2,11 +2,11 @@ import talib.abstract as ta from pandas import DataFrame -from typing import Dict, Any, Callable +from typing import Dict, Any, Callable, List from functools import reduce import numpy -from hyperopt import hp +from skopt.space import Categorical, Dimension, Integer, Real import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.optimize.interface import IHyperOpt @@ -21,193 +21,52 @@ class DefaultHyperOpts(IHyperOpt): """ @staticmethod - def populate_indicators(dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - """ + def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe['adx'] = ta.ADX(dataframe) - dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - dataframe['cci'] = ta.CCI(dataframe) macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] - dataframe['macdhist'] = macd['macdhist'] dataframe['mfi'] = ta.MFI(dataframe) - dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - dataframe['plus_di'] = ta.PLUS_DI(dataframe) - dataframe['roc'] = ta.ROC(dataframe) dataframe['rsi'] = ta.RSI(dataframe) - # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - rsi = 0.1 * (dataframe['rsi'] - 50) - dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) - # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - # Stoch - stoch = ta.STOCH(dataframe) - dataframe['slowd'] = stoch['slowd'] - dataframe['slowk'] = stoch['slowk'] - # Stoch fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - # Stoch RSI - stoch_rsi = ta.STOCHRSI(dataframe) - dataframe['fastd_rsi'] = stoch_rsi['fastd'] - dataframe['fastk_rsi'] = stoch_rsi['fastk'] + dataframe['minus_di'] = ta.MINUS_DI(dataframe) # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_middleband'] = bollinger['mid'] - dataframe['bb_upperband'] = bollinger['upper'] - # EMA - Exponential Moving Average - dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) - dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) - dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - # SAR Parabolic dataframe['sar'] = ta.SAR(dataframe) - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] - - # Pattern Recognition - Bullish candlestick patterns - # ------------------------------------ - """ - # Hammer: values [0, 100] - dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) - # Inverted Hammer: values [0, 100] - dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) - # Dragonfly Doji: values [0, 100] - dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) - # Piercing Line: values [0, 100] - dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] - # Morningstar: values [0, 100] - dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] - # Three White Soldiers: values [0, 100] - dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ - - # Pattern Recognition - Bearish candlestick patterns - # ------------------------------------ - """ - # Hanging Man: values [0, 100] - dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) - # Shooting Star: values [0, 100] - dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) - # Gravestone Doji: values [0, 100] - dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) - # Dark Cloud Cover: values [0, 100] - dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) - # Evening Doji Star: values [0, 100] - dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) - # Evening Star: values [0, 100] - dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - - # Pattern Recognition - Bullish/Bearish candlestick patterns - # ------------------------------------ - """ - # Three Line Strike: values [0, -100, 100] - dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) - # Spinning Top: values [0, -100, 100] - dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] - # Engulfing: values [0, -100, 100] - dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] - # Harami: values [0, -100, 100] - dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] - # Three Outside Up/Down: values [0, -100, 100] - dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] - # Three Inside Up/Down: values [0, -100, 100] - dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ - - # Chart type - # ------------------------------------ - # Heikinashi stategy - heikinashi = qtpylib.heikinashi(dataframe) - dataframe['ha_open'] = heikinashi['open'] - dataframe['ha_close'] = heikinashi['close'] - dataframe['ha_high'] = heikinashi['high'] - dataframe['ha_low'] = heikinashi['low'] - return dataframe - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: """ Define the buy strategy parameters to be used by hyperopt """ - def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: """ Buy strategy Hyperopt will build and use """ conditions = [] # GUARDS AND TRENDS - if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: - conditions.append(dataframe['ema50'] > dataframe['ema100']) - if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: - conditions.append(dataframe['macd'] < 0) - if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: - conditions.append(dataframe['ema5'] > dataframe['ema10']) - if 'mfi' in params and params['mfi']['enabled']: - conditions.append(dataframe['mfi'] < params['mfi']['value']) - if 'fastd' in params and params['fastd']['enabled']: - conditions.append(dataframe['fastd'] < params['fastd']['value']) - if 'adx' in params and params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if 'rsi' in params and params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) - if 'over_sar' in params and params['over_sar']['enabled']: - conditions.append(dataframe['close'] > dataframe['sar']) - if 'green_candle' in params and params['green_candle']['enabled']: - conditions.append(dataframe['close'] > dataframe['open']) - if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: - prevsma = dataframe['sma'].shift(1) - conditions.append(dataframe['sma'] > prevsma) + if 'mfi-enabled' in params and params['mfi-enabled']: + conditions.append(dataframe['mfi'] < params['mfi-value']) + if 'fastd-enabled' in params and params['fastd-enabled']: + conditions.append(dataframe['fastd'] < params['fastd-value']) + if 'adx-enabled' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi-enabled' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS - triggers = { - 'lower_bb': ( - dataframe['close'] < dataframe['bb_lowerband'] - ), - 'lower_bb_tema': ( - dataframe['tema'] < dataframe['bb_lowerband'] - ), - 'faststoch10': (qtpylib.crossed_above( - dataframe['fastd'], 10.0 - )), - 'ao_cross_zero': (qtpylib.crossed_above( - dataframe['ao'], 0.0 - )), - 'ema3_cross_ema10': (qtpylib.crossed_above( - dataframe['ema3'], dataframe['ema10'] - )), - 'macd_cross_signal': (qtpylib.crossed_above( + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( dataframe['macd'], dataframe['macdsignal'] - )), - 'sar_reversal': (qtpylib.crossed_above( + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( dataframe['close'], dataframe['sar'] - )), - 'ht_sine': (qtpylib.crossed_above( - dataframe['htleadsine'], dataframe['htsine'] - )), - 'heiken_reversal_bull': ( - (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & - (dataframe['ha_low'] == dataframe['ha_open']) - ), - 'di_cross': (qtpylib.crossed_above( - dataframe['plus_di'], dataframe['minus_di'] - )), - } - conditions.append(triggers.get(params['trigger']['type'])) + )) dataframe.loc[ reduce(lambda x, y: x & y, conditions), @@ -218,64 +77,21 @@ class DefaultHyperOpts(IHyperOpt): return populate_buy_trend @staticmethod - def indicator_space() -> Dict[str, Any]: + def indicator_space() -> List[Dimension]: """ Define your Hyperopt space for searching strategy parameters """ - return { - 'macd_below_zero': hp.choice('macd_below_zero', [ - {'enabled': False}, - {'enabled': True} - ]), - 'mfi': hp.choice('mfi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} - ]), - 'fastd': hp.choice('fastd', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} - ]), - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} - ]), - 'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} - ]), - 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'over_sar': hp.choice('over_sar', [ - {'enabled': False}, - {'enabled': True} - ]), - 'green_candle': hp.choice('green_candle', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_sma': hp.choice('uptrend_sma', [ - {'enabled': False}, - {'enabled': True} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'lower_bb'}, - {'type': 'lower_bb_tema'}, - {'type': 'faststoch10'}, - {'type': 'ao_cross_zero'}, - {'type': 'ema3_cross_ema10'}, - {'type': 'macd_cross_signal'}, - {'type': 'sar_reversal'}, - {'type': 'ht_sine'}, - {'type': 'heiken_reversal_bull'}, - {'type': 'di_cross'}, - ]), - } + return [ + Integer(10, 25, name='mfi-value'), + Integer(15, 45, name='fastd-value'), + Integer(20, 50, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='mfi-enabled'), + Categorical([True, False], name='fastd-enabled'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') + ] @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: @@ -291,24 +107,24 @@ class DefaultHyperOpts(IHyperOpt): return roi_table @staticmethod - def stoploss_space() -> Dict[str, Any]: + def stoploss_space() -> List[Dimension]: """ Stoploss Value to search """ - return { - 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), - } + return [ + Real(-0.5, -0.02, name='stoploss'), + ] @staticmethod - def roi_space() -> Dict[str, Any]: + def roi_space() -> List[Dimension]: """ Values to search for each ROI steps """ - return { - 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), - 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), - 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), - 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), - 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), - 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), - } + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + Real(0.01, 0.04, name='roi_p1'), + Real(0.01, 0.07, name='roi_p2'), + Real(0.01, 0.20, name='roi_p3'), + ] diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 07c24ff18..f5f4222d6 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -105,7 +105,7 @@ class Hyperopt(Backtesting): best_result['params'] ) if 'roi_t1' in best_result['params']: - logger.info('ROI table:\n%s', self.generate_roi_table(best_result['params'])) + logger.info('ROI table:\n%s', self.custom_hyperopt.generate_roi_table(best_result['params'])) def log_results(self, results) -> None: """ @@ -147,19 +147,20 @@ class Hyperopt(Backtesting): """ spaces: List[Dimension] = [] if self.has_space('buy'): - spaces = {**spaces, **self.custom_hyperopt.indicator_space()} + spaces += self.custom_hyperopt.indicator_space() if self.has_space('roi'): - spaces = {**spaces, **self.custom_hyperopt.roi_space()} + spaces += self.custom_hyperopt.roi_space() if self.has_space('stoploss'): - spaces = {**spaces, **self.custom_hyperopt.stoploss_space()} + spaces += self.custom_hyperopt.stoploss_space() return spaces - def generate_optimizer(self, params: Dict) -> Dict: + def generate_optimizer(self, _params: Dict) -> Dict: + params = self.get_args(_params) if self.has_space('roi'): - self.analyze.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) + self.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) if self.has_space('buy'): - self.populate_buy_trend = self.custom_hyperopt.buy_strategy_generator(params) + self.advise_buy = self.custom_hyperopt.buy_strategy_generator(params) if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py new file mode 100644 index 000000000..77e3b1696 --- /dev/null +++ b/user_data/hyperopts/sample_hyperopt.py @@ -0,0 +1,138 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +import talib.abstract as ta +from pandas import DataFrame +from typing import Dict, Any, Callable, List +from functools import reduce + +import numpy +from skopt.space import Categorical, Dimension, Integer, Real + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.optimize.interface import IHyperOpt + +class_name = 'SampleHyperOpts' + + +# This class is a sample. Feel free to customize it. +class SampleHyperOpts(IHyperOpt): + """ + This is a test hyperopt to inspire you. + More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md + You can: + - Rename the class name (Do not forget to update class_name) + - Add any methods you want to build your hyperopt + - Add any lib you need to build your hyperopt + You must keep: + - the prototype for the methods: populate_indicators, indicator_space, buy_strategy_generator, + roi_space, generate_roi_table, stoploss_space + """ + + @staticmethod + def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['adx'] = ta.ADX(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['sar'] = ta.SAR(dataframe) + return dataframe + + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Buy strategy Hyperopt will build and use + """ + conditions = [] + # GUARDS AND TRENDS + if 'mfi-enabled' in params and params['mfi-enabled']: + conditions.append(dataframe['mfi'] < params['mfi-value']) + if 'fastd-enabled' in params and params['fastd-enabled']: + conditions.append(dataframe['fastd'] < params['fastd-value']) + if 'adx-enabled' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi-enabled' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) + + # TRIGGERS + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return [ + Integer(10, 25, name='mfi-value'), + Integer(15, 45, name='fastd-value'), + Integer(20, 50, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='mfi-enabled'), + Categorical([True, False], name='fastd-enabled'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') + ] + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def stoploss_space() -> List[Dimension]: + """ + Stoploss Value to search + """ + return [ + Real(-0.5, -0.02, name='stoploss'), + ] + + @staticmethod + def roi_space() -> List[Dimension]: + """ + Values to search for each ROI steps + """ + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + Real(0.01, 0.04, name='roi_p1'), + Real(0.01, 0.07, name='roi_p2'), + Real(0.01, 0.20, name='roi_p3'), + ] diff --git a/user_data/hyperopts/test_hyperopt.py b/user_data/hyperopts/test_hyperopt.py deleted file mode 100644 index b742af6c9..000000000 --- a/user_data/hyperopts/test_hyperopt.py +++ /dev/null @@ -1,279 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement - -import talib.abstract as ta -from pandas import DataFrame -from typing import Dict, Any, Callable -from functools import reduce -from math import exp - -import numpy -import talib.abstract as ta -from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe - -import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.indicator_helpers import fishers_inverse -from freqtrade.optimize.interface import IHyperOpt - - -# This class is a sample. Feel free to customize it. -class TestHyperOpt(IHyperOpt): - """ - This is a test hyperopt to inspire you. - More information in https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md - - You can: - - Rename the class name (Do not forget to update class_name) - - Add any methods you want to build your hyperopt - - Add any lib you need to build your hyperopt - - You must keep: - - the prototype for the methods: populate_indicators, indicator_space, buy_strategy_generator, - roi_space, generate_roi_table, stoploss_space - """ - - @staticmethod - def populate_indicators(dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - - Performance Note: For the best performance be frugal on the number of indicators - you are using. Let uncomment only the indicator you are using in your strategies - or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - """ - - # Momentum Indicator - # ------------------------------------ - - # ADX - dataframe['adx'] = ta.ADX(dataframe) - - """ - # Awesome oscillator - dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - - # Commodity Channel Index: values Oversold:<-100, Overbought:>100 - dataframe['cci'] = ta.CCI(dataframe) - - # MACD - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['macdhist'] = macd['macdhist'] - - # MFI - dataframe['mfi'] = ta.MFI(dataframe) - - # Minus Directional Indicator / Movement - dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - - # Plus Directional Indicator / Movement - dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - dataframe['plus_di'] = ta.PLUS_DI(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - - # ROC - dataframe['roc'] = ta.ROC(dataframe) - - # RSI - dataframe['rsi'] = ta.RSI(dataframe) - - # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - rsi = 0.1 * (dataframe['rsi'] - 50) - dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) - - # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - - # Stoch - stoch = ta.STOCH(dataframe) - dataframe['slowd'] = stoch['slowd'] - dataframe['slowk'] = stoch['slowk'] - - # Stoch fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - - # Stoch RSI - stoch_rsi = ta.STOCHRSI(dataframe) - dataframe['fastd_rsi'] = stoch_rsi['fastd'] - dataframe['fastk_rsi'] = stoch_rsi['fastk'] - """ - - # Overlap Studies - # ------------------------------------ - - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_middleband'] = bollinger['mid'] - dataframe['bb_upperband'] = bollinger['upper'] - - """ - # EMA - Exponential Moving Average - dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) - dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) - dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - - # SAR Parabol - dataframe['sar'] = ta.SAR(dataframe) - - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - """ - - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - - # Cycle Indicator - # ------------------------------------ - # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] - - # Pattern Recognition - Bullish candlestick patterns - # ------------------------------------ - """ - # Hammer: values [0, 100] - dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) - # Inverted Hammer: values [0, 100] - dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) - # Dragonfly Doji: values [0, 100] - dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) - # Piercing Line: values [0, 100] - dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] - # Morningstar: values [0, 100] - dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] - # Three White Soldiers: values [0, 100] - dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ - - # Pattern Recognition - Bearish candlestick patterns - # ------------------------------------ - """ - # Hanging Man: values [0, 100] - dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) - # Shooting Star: values [0, 100] - dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) - # Gravestone Doji: values [0, 100] - dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) - # Dark Cloud Cover: values [0, 100] - dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) - # Evening Doji Star: values [0, 100] - dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) - # Evening Star: values [0, 100] - dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - - # Pattern Recognition - Bullish/Bearish candlestick patterns - # ------------------------------------ - """ - # Three Line Strike: values [0, -100, 100] - dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) - # Spinning Top: values [0, -100, 100] - dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] - # Engulfing: values [0, -100, 100] - dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] - # Harami: values [0, -100, 100] - dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] - # Three Outside Up/Down: values [0, -100, 100] - dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] - # Three Inside Up/Down: values [0, -100, 100] - dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ - - # Chart type - # ------------------------------------ - """ - # Heikinashi stategy - heikinashi = qtpylib.heikinashi(dataframe) - dataframe['ha_open'] = heikinashi['open'] - dataframe['ha_close'] = heikinashi['close'] - dataframe['ha_high'] = heikinashi['high'] - dataframe['ha_low'] = heikinashi['low'] - """ - - return dataframe - - @staticmethod - def indicator_space() -> Dict[str, Any]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return { - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 50, 80, 5)} - ]), - 'uptrend_tema': hp.choice('uptrend_tema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'middle_bb_tema'}, - ]), - } - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame) -> DataFrame: - conditions = [] - # GUARDS AND TRENDS - if 'adx' in params and params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if 'uptrend_tema' in params and params['uptrend_tema']['enabled']: - prevtema = dataframe['tema'].shift(1) - conditions.append(dataframe['tema'] > prevtema) - - # TRIGGERS - triggers = { - 'middle_bb_tema': ( - dataframe['tema'] > dataframe['bb_middleband'] - ), - } - conditions.append(triggers.get(params['trigger']['type'])) - - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def roi_space() -> Dict[str, Any]: - return { - 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), - 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), - 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), - 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), - 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), - 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), - } - - @staticmethod - def generate_roi_table(params: Dict) -> Dict[int, float]: - """ - Generate the ROI table that will be used by Hyperopt - """ - roi_table = {} - roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] - roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] - roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 - - return roi_table - - @staticmethod - def stoploss_space() -> Dict[str, Any]: - return { - 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), - } \ No newline at end of file From 4f800bfbc88335df5047292c00cc5b19c7d2b08c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Nov 2018 20:25:43 +0100 Subject: [PATCH 08/14] Fix pickling-error --- freqtrade/optimize/hyperopt.py | 4 ++-- requirements.txt | 1 + setup.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f5f4222d6..f9a2924a4 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -14,7 +14,7 @@ from operator import itemgetter from typing import Any, Dict, List from pandas import DataFrame -from sklearn.externals.joblib import Parallel, delayed, dump, load +from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects from skopt import Optimizer from skopt.space import Dimension @@ -219,7 +219,7 @@ class Hyperopt(Backtesting): ) def run_optimizer_parallel(self, parallel, asked) -> List: - return parallel(delayed(self.generate_optimizer)(v) for v in asked) + return parallel(delayed(wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) def load_previous_results(self): """ read trials file if we have one """ diff --git a/requirements.txt b/requirements.txt index 2671660ea..11d1bea6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ urllib3==1.24.1 wrapt==1.10.11 pandas==0.23.4 scikit-learn==0.20.0 +joblib==0.13.0 scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.4 diff --git a/setup.py b/setup.py index 8853ef7f8..119ad03db 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ setup(name='freqtrade', 'wrapt', 'pandas', 'scikit-learn', + 'joblib' 'scipy', 'jsonschema', 'TA-Lib', From dcf9930858d92ffed54671b1e864a3bd6fd6f27f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Nov 2018 19:36:04 +0100 Subject: [PATCH 09/14] improve hyperopt documentation (links) --- docs/hyperopt.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 0484613c2..8df864c29 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -21,21 +21,21 @@ and still take a long time. ## Prepare Hyperopt Before we start digging in Hyperopt, we recommend you to take a look at -an example hyperopt file located into [user_data/strategies/](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) +an example hyperopt file located into [user_data/hyperopts/](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) ### 1. Install a Custom Hyperopt File This is very simple. Put your hyperopt file into the folder `user_data/hyperopts`. Let assume you want a hyperopt file `awesome_hyperopt.py`: -1. Copy the file `user_data/hyperopts/test_hyperopt.py` into `user_data/hyperopts/awesome_hyperopt.py` +1. Copy the file `user_data/hyperopts/sample_hyperopt.py` into `user_data/hyperopts/awesome_hyperopt.py` ### 2. Configure your Guards and Triggers There are two places you need to change in your hyperopt file to add a new buy hyperopt for testing: -- Inside [populate_buy_trend()](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). -- Inside [indicator_space()](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). +- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). +- Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). There you have two different type of indicators: 1. `guards` and 2. `triggers`. From e69f9439118c07a9abb2f5b1467ec36181cc1bf2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Nov 2018 19:53:16 +0100 Subject: [PATCH 10/14] Add missing semicolon --- docs/hyperopt.md | 1 - setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 8df864c29..9fd11e5de 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -19,7 +19,6 @@ and still take a long time. ## Prepare Hyperopting -## Prepare Hyperopt Before we start digging in Hyperopt, we recommend you to take a look at an example hyperopt file located into [user_data/hyperopts/](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) diff --git a/setup.py b/setup.py index 119ad03db..d2a9b97f9 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ setup(name='freqtrade', 'wrapt', 'pandas', 'scikit-learn', - 'joblib' + 'joblib', 'scipy', 'jsonschema', 'TA-Lib', From 5a550ef2af5227ae217a6fa5f57ff05a4697b2ec Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Nov 2018 17:36:17 +0100 Subject: [PATCH 11/14] Fix docs typo in hyperopt --- docs/hyperopt.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 9fd11e5de..dffe84d1d 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -36,8 +36,7 @@ new buy hyperopt for testing: - Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). - Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). -There you have two different type of indicators: 1. `guards` and 2. -`triggers`. +There you have two different types of indicators: 1. `guards` and 2. `triggers`. 1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. From 5dd013c3b19dbb1fa1680d391aa9c068526418c8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Nov 2018 17:40:45 +0100 Subject: [PATCH 12/14] Rename hyperopt interface and resolver --- freqtrade/optimize/default_hyperopt.py | 3 +-- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/optimize/{interface.py => hyperopt_interface.py} | 0 freqtrade/optimize/{resolver.py => hyperopt_resolver.py} | 2 +- user_data/hyperopts/sample_hyperopt.py | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) rename freqtrade/optimize/{interface.py => hyperopt_interface.py} (100%) rename freqtrade/optimize/{resolver.py => hyperopt_resolver.py} (98%) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index e127fd6d8..723dfde17 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -5,11 +5,10 @@ from pandas import DataFrame from typing import Dict, Any, Callable, List from functools import reduce -import numpy from skopt.space import Categorical, Dimension, Integer, Real import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.optimize.interface import IHyperOpt +from freqtrade.optimize.hyperopt_interface import IHyperOpt class_name = 'DefaultHyperOpts' diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f9a2924a4..bd6e43daa 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,7 +22,7 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from freqtrade.optimize.resolver import HyperOptResolver +from freqtrade.optimize.hyperopt_resolver import HyperOptResolver logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/interface.py b/freqtrade/optimize/hyperopt_interface.py similarity index 100% rename from freqtrade/optimize/interface.py rename to freqtrade/optimize/hyperopt_interface.py diff --git a/freqtrade/optimize/resolver.py b/freqtrade/optimize/hyperopt_resolver.py similarity index 98% rename from freqtrade/optimize/resolver.py rename to freqtrade/optimize/hyperopt_resolver.py index 1e0316c12..1c3a9d577 100644 --- a/freqtrade/optimize/resolver.py +++ b/freqtrade/optimize/hyperopt_resolver.py @@ -10,7 +10,7 @@ import os from typing import Optional, Dict, Type from freqtrade.constants import DEFAULT_HYPEROPT -from freqtrade.optimize.interface import IHyperOpt +from freqtrade.optimize.hyperopt_interface import IHyperOpt logger = logging.getLogger(__name__) diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index 77e3b1696..83dc5f71a 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -9,7 +9,7 @@ import numpy from skopt.space import Categorical, Dimension, Integer, Real import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.optimize.interface import IHyperOpt +from freqtrade.optimize.hyperopt_interface import IHyperOpt class_name = 'SampleHyperOpts' From 7757c53b0668d53fac0c9b178e2def511c4101da Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Nov 2018 17:43:49 +0100 Subject: [PATCH 13/14] Small fixes --- freqtrade/optimize/hyperopt.py | 9 ++++++--- setup.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index bd6e43daa..70d20673c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -105,7 +105,8 @@ class Hyperopt(Backtesting): best_result['params'] ) if 'roi_t1' in best_result['params']: - logger.info('ROI table:\n%s', self.custom_hyperopt.generate_roi_table(best_result['params'])) + logger.info('ROI table:\n%s', + self.custom_hyperopt.generate_roi_table(best_result['params'])) def log_results(self, results) -> None: """ @@ -219,7 +220,8 @@ class Hyperopt(Backtesting): ) def run_optimizer_parallel(self, parallel, asked) -> List: - return parallel(delayed(wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) + return parallel(delayed( + wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) def load_previous_results(self): """ read trials file if we have one """ @@ -241,7 +243,8 @@ class Hyperopt(Backtesting): ) if self.has_space('buy'): - self.strategy.advise_indicators = self.custom_hyperopt.populate_indicators # type: ignore + self.strategy.advise_indicators = \ + self.custom_hyperopt.populate_indicators # type: ignore dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() diff --git a/setup.py b/setup.py index d2a9b97f9..34d4c8755 100644 --- a/setup.py +++ b/setup.py @@ -30,8 +30,8 @@ setup(name='freqtrade', 'wrapt', 'pandas', 'scikit-learn', - 'joblib', 'scipy', + 'joblib', 'jsonschema', 'TA-Lib', 'tabulate', From a3b600411570eaf23443c19bb3322b2c36d14cc6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Nov 2018 19:41:07 +0100 Subject: [PATCH 14/14] IHyperopt: all methods static, somef ixes for mypy --- freqtrade/optimize/default_hyperopt.py | 3 ++- freqtrade/optimize/hyperopt_interface.py | 21 ++++++++++++++------- freqtrade/optimize/hyperopt_resolver.py | 4 ++-- user_data/hyperopts/sample_hyperopt.py | 3 ++- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 723dfde17..6139f8140 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -36,7 +36,8 @@ class DefaultHyperOpts(IHyperOpt): dataframe['sar'] = ta.SAR(dataframe) return dataframe - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ Define the buy strategy parameters to be used by hyperopt """ diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 8bc3866b2..d42206658 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -4,9 +4,10 @@ This module defines the interface to apply for hyperopts """ from abc import ABC, abstractmethod -from typing import Dict, Any, Callable +from typing import Dict, Any, Callable, List from pandas import DataFrame +from skopt.space import Dimension class IHyperOpt(ABC): @@ -20,40 +21,46 @@ class IHyperOpt(ABC): ticker_interval -> int: value of the ticker interval to use for the strategy """ + @staticmethod @abstractmethod - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: """ Populate indicators that will be used in the Buy and Sell strategy :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() :return: a Dataframe with all mandatory indicators for the strategies """ + @staticmethod @abstractmethod - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ Create a buy strategy generator """ + @staticmethod @abstractmethod - def indicator_space(self) -> Dict[str, Any]: + def indicator_space() -> List[Dimension]: """ Create an indicator space """ + @staticmethod @abstractmethod - def generate_roi_table(self, params: Dict) -> Dict[int, float]: + def generate_roi_table(params: Dict) -> Dict[int, float]: """ Create an roi table """ + @staticmethod @abstractmethod - def stoploss_space(self) -> Dict[str, Any]: + def stoploss_space() -> List[Dimension]: """ Create a stoploss space """ + @staticmethod @abstractmethod - def roi_space(self) -> Dict[str, Any]: + def roi_space() -> List[Dimension]: """ Create a roi space """ diff --git a/freqtrade/optimize/hyperopt_resolver.py b/freqtrade/optimize/hyperopt_resolver.py index 1c3a9d577..3d019e8df 100644 --- a/freqtrade/optimize/hyperopt_resolver.py +++ b/freqtrade/optimize/hyperopt_resolver.py @@ -35,7 +35,7 @@ class HyperOptResolver(object): self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) def _load_hyperopt( - self, hyperopt_name: str, extra_dir: Optional[str] = None) -> Optional[IHyperOpt]: + self, hyperopt_name: str, extra_dir: Optional[str] = None) -> IHyperOpt: """ Search and loads the specified hyperopt. :param hyperopt_name: name of the module to import @@ -75,7 +75,7 @@ class HyperOptResolver(object): # Generate spec based on absolute path spec = importlib.util.spec_from_file_location('user_data.hyperopts', module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # type: ignore # importlib does not use typehints valid_hyperopts_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index 83dc5f71a..f11236a82 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -45,7 +45,8 @@ class SampleHyperOpts(IHyperOpt): dataframe['sar'] = ta.SAR(dataframe) return dataframe - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ Define the buy strategy parameters to be used by hyperopt """