From e49e361da5481418caf626d1088217bd8103aae5 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Thu, 22 Mar 2018 23:00:47 -0500 Subject: [PATCH 01/30] Update hyperopt.py --- freqtrade/optimize/hyperopt.py | 47 +++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7dcd46fd2..c290245f6 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -21,7 +21,7 @@ import talib.abstract as ta from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from hyperopt.mongoexp import MongoTrials from pandas import DataFrame - +import fire import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration @@ -31,6 +31,18 @@ from freqtrade.optimize.backtesting import Backtesting from user_data.hyperopt_conf import hyperopt_optimize_conf + +class Testz(): + db_name = 'freqtrade_hyperopt' + MongoTrials( + arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), + exp_key='exp1' + ) + async = 'Null' + attachments = 'Null' + + + class Hyperopt(Backtesting): """ Hyperopt class, this class contains all the logic to run a hyperopt simulation @@ -68,7 +80,7 @@ class Hyperopt(Backtesting): # Hyperopt Trials self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') - self.trials = Trials() + self.trials = Testz() @staticmethod def populate_indicators(dataframe: DataFrame) -> DataFrame: @@ -515,12 +527,6 @@ class Hyperopt(Backtesting): self.logger.info( 'Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!' ) - - db_name = 'freqtrade_hyperopt' - self.trials = MongoTrials( - arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), - exp_key='exp1' - ) else: self.logger.info('Preparing Trials..') signal.signal(signal.SIGINT, self.signal_handler) @@ -536,26 +542,24 @@ class Hyperopt(Backtesting): self.total_tries ) - try: - # change the Logging format - self.logging.set_format('\n%(message)s') + A = self.generate_optimizer + B = self.hyperopt_space() + C = self.total_tries + def Tests(): best_parameters = fmin( fn=self.generate_optimizer, space=self.hyperopt_space(), algo=tpe.suggest, max_evals=self.total_tries, - trials=self.trials + trials=MongoTrials('mongo://127.0.0.1:1234/freqtrade_hyperopt/jobs') ) - - results = sorted(self.trials.results, key=itemgetter('loss')) - best_result = results[0]['result'] - - except ValueError: - best_parameters = {} - best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ - 'try with more epochs (param: -e).' - + # change the Logging format + self.logging.set_format('\n%(message)s') + # best_parameters = Tests(self) + best_parameters = Tests() + results = sorted(self.trials.results, key=itemgetter('loss')) + best_result = results[0]['result'] # Improve best parameter logging display if best_parameters: best_parameters = space_eval( @@ -586,6 +590,7 @@ class Hyperopt(Backtesting): sys.exit(0) + def start(args: Namespace) -> None: """ Start Backtesting script From 717f4eefcb2f53db34d1c321428965994cd864e4 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Thu, 22 Mar 2018 23:01:34 -0500 Subject: [PATCH 02/30] Update hyperopt.py --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c290245f6..f5a6d5e22 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -21,7 +21,7 @@ import talib.abstract as ta from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from hyperopt.mongoexp import MongoTrials from pandas import DataFrame -import fire + import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration From c6cee3ebe4bcd9d173481cde9590546b4c6f431b Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Thu, 22 Mar 2018 23:03:32 -0500 Subject: [PATCH 03/30] Update hyperopt.py --- freqtrade/optimize/hyperopt.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f5a6d5e22..ab5b91c3d 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -33,11 +33,7 @@ from user_data.hyperopt_conf import hyperopt_optimize_conf class Testz(): - db_name = 'freqtrade_hyperopt' - MongoTrials( - arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), - exp_key='exp1' - ) + MongoTrials('mongo://127.0.0.1:1234/freqtrade_hyperopt/jobs') async = 'Null' attachments = 'Null' @@ -542,10 +538,6 @@ class Hyperopt(Backtesting): self.total_tries ) - A = self.generate_optimizer - B = self.hyperopt_space() - C = self.total_tries - def Tests(): best_parameters = fmin( fn=self.generate_optimizer, @@ -556,7 +548,6 @@ class Hyperopt(Backtesting): ) # change the Logging format self.logging.set_format('\n%(message)s') - # best_parameters = Tests(self) best_parameters = Tests() results = sorted(self.trials.results, key=itemgetter('loss')) best_result = results[0]['result'] From 6280d9408e53937a385e0eddb191e81c902851b0 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Thu, 22 Mar 2018 23:05:37 -0500 Subject: [PATCH 04/30] Create hyperoptNote.md --- hyperoptNote.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 hyperoptNote.md diff --git a/hyperoptNote.md b/hyperoptNote.md new file mode 100644 index 000000000..d3ae83862 --- /dev/null +++ b/hyperoptNote.md @@ -0,0 +1,9 @@ +You will need to add: + +import dill as pickle + +INTO: + +/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py + +At the very top. From b147ece554fd284a680c90f4da88471bf5603e5b Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Thu, 22 Mar 2018 23:29:23 -0500 Subject: [PATCH 05/30] Removed Unneeded things --- freqtrade/optimize/hyperopt.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index ab5b91c3d..a90974694 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -34,8 +34,6 @@ from user_data.hyperopt_conf import hyperopt_optimize_conf class Testz(): MongoTrials('mongo://127.0.0.1:1234/freqtrade_hyperopt/jobs') - async = 'Null' - attachments = 'Null' From c396cf0dcb0c256a8c2721f4f18d100f4b453834 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 00:22:17 -0500 Subject: [PATCH 06/30] Readded things --- freqtrade/optimize/hyperopt.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a90974694..5e13ad47a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -33,8 +33,11 @@ from user_data.hyperopt_conf import hyperopt_optimize_conf class Testz(): - MongoTrials('mongo://127.0.0.1:1234/freqtrade_hyperopt/jobs') - + db_name = 'freqtrade_hyperopt' + MongoTrials( + arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), + exp_key='exp1' + ) class Hyperopt(Backtesting): @@ -542,14 +545,20 @@ class Hyperopt(Backtesting): space=self.hyperopt_space(), algo=tpe.suggest, max_evals=self.total_tries, - trials=MongoTrials('mongo://127.0.0.1:1234/freqtrade_hyperopt/jobs') + trials=MongoTrials(arg='mongo://127.0.0.1:1234/freqtrade_hyperopt/jobs', exp_key='exp1') ) - # change the Logging format - self.logging.set_format('\n%(message)s') - best_parameters = Tests() - results = sorted(self.trials.results, key=itemgetter('loss')) - best_result = results[0]['result'] + return best_parameters + try: + # change the Logging format + self.logging.set_format('\n%(message)s') + best_parameters = Tests() + results = sorted(self.trials.results, key=itemgetter('loss')) + best_result = results[0]['result'] # Improve best parameter logging display + except ValueError: + best_parameters = {} + best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ + 'try with more epochs (param: -e).' if best_parameters: best_parameters = space_eval( self.hyperopt_space(), From 12750cb7655ff6de9c0916c6da77e2e4821cff0f Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 00:33:59 -0500 Subject: [PATCH 07/30] Update hyperopt.py --- freqtrade/optimize/hyperopt.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5e13ad47a..2a4131efe 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -538,8 +538,7 @@ class Hyperopt(Backtesting): self.current_tries, self.total_tries ) - - def Tests(): + try: best_parameters = fmin( fn=self.generate_optimizer, space=self.hyperopt_space(), @@ -547,11 +546,8 @@ class Hyperopt(Backtesting): max_evals=self.total_tries, trials=MongoTrials(arg='mongo://127.0.0.1:1234/freqtrade_hyperopt/jobs', exp_key='exp1') ) - return best_parameters - try: # change the Logging format self.logging.set_format('\n%(message)s') - best_parameters = Tests() results = sorted(self.trials.results, key=itemgetter('loss')) best_result = results[0]['result'] # Improve best parameter logging display From 1c7daa713b912c5c5eeca0a2cc905846e43589e8 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 16:41:06 -0500 Subject: [PATCH 08/30] Updated --- user_data/strategies/test_strategy.py | 272 +++++++++++++++++++++++--- 1 file changed, 249 insertions(+), 23 deletions(-) diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index a164812c4..6f26c7b48 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -1,6 +1,9 @@ # --- Do not remove these libs --- from freqtrade.strategy.interface import IStrategy +from typing import Dict, List +from hyperopt import hp +from functools import reduce from pandas import DataFrame # -------------------------------- @@ -10,12 +13,40 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib import numpy # noqa -# Update this variable if you change the class name -class_name = 'TestStrategy' +# Make a backup of default_strategy then move this file over your default strategy in freqtrade/strategy and to run: +# 'python3.6 freqtrade/main.py -c config.json hyperopt -s all --realistic-simulation -i 5' +# Make sure to take note of the if statements at the bottom of the file, and the pattern of > < and >= <= and the triggers configuration. +# Then, take note of the results: There is one trigger in the results, map the trigger as an example: +# 'Trigger: 7' +# The 7th trigger in this file's if statement such as: +# 'sar_reversal': (qtpylib.crossed_above( +# dataframe['close'], dataframe['sar'] +# +# Drop the trigger into 'def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:' into your default_strategy backup as: +# dataframe['close'], dataframe['sar'] & +# Then for your other values such as: 'Mfi-Value: 15.00' +# The if statement looks like: conditions.append(dataframe['mfi'] < params['mfi']['value'] +# Drop this in as: +# (dataframe['mfi'] < 15.00) & +# The last condition in 'def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:' will have no & -# This class is a sample. Feel free to customize it. -class TestStrategy(IStrategy): +### As an example the sell/buy trend looks like this: +# dataframe.loc[ +# ( +# (dataframe['adx'] > 70) & +# (dataframe['tema'] < dataframe['tema'].shift(1)) +# ), +# 'sell'] = 1 +# + +# Once done, move your backup strategy back over this one and backtest. + + +class_name = 'DefaultStrategy' + + +class DefaultStrategy(IStrategy): """ This is a test strategy to inspire you. More information in https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md @@ -62,7 +93,7 @@ class TestStrategy(IStrategy): # ADX dataframe['adx'] = ta.ADX(dataframe) - """ + # Awesome oscillator dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) @@ -114,18 +145,28 @@ class TestStrategy(IStrategy): stoch_rsi = ta.STOCHRSI(dataframe) dataframe['fastd_rsi'] = stoch_rsi['fastd'] dataframe['fastk_rsi'] = stoch_rsi['fastk'] - """ + # Overlap Studies # ------------------------------------ + + # Previous Bollinger bands + # Because ta.BBANDS implementation is broken with small numbers, it actually + # returns middle band for all the three bands. Switch to qtpylib.bollinger_bands + # and use middle band instead. + # Is broken + """ + dataframe['blower'] = ta.BBANDS(dataframe, nbdevup=2, nbdevdn=2)['lowerband'] + """ + # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_middleband'] = bollinger['mid'] dataframe['bb_upperband'] = bollinger['upper'] - """ + # EMA - Exponential Moving Average dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) @@ -138,7 +179,6 @@ class TestStrategy(IStrategy): # SMA - Simple Moving Average dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - """ # TEMA - Triple Exponential Moving Average dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) @@ -152,7 +192,7 @@ class TestStrategy(IStrategy): # Pattern Recognition - Bullish candlestick patterns # ------------------------------------ - """ + # Hammer: values [0, 100] dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) # Inverted Hammer: values [0, 100] @@ -165,11 +205,11 @@ class TestStrategy(IStrategy): dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] # Three White Soldiers: values [0, 100] dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ + # Pattern Recognition - Bearish candlestick patterns # ------------------------------------ - """ + # Hanging Man: values [0, 100] dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) # Shooting Star: values [0, 100] @@ -182,11 +222,10 @@ class TestStrategy(IStrategy): dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) # Evening Star: values [0, 100] dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - + # Pattern Recognition - Bullish/Bearish candlestick patterns # ------------------------------------ - """ + # Three Line Strike: values [0, -100, 100] dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) # Spinning Top: values [0, -100, 100] @@ -199,18 +238,18 @@ class TestStrategy(IStrategy): dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] # Three Inside Up/Down: values [0, -100, 100] dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ + # Chart type # ------------------------------------ - """ + # Heikinashi stategy heikinashi = qtpylib.heikinashi(dataframe) dataframe['ha_open'] = heikinashi['open'] dataframe['ha_close'] = heikinashi['close'] dataframe['ha_high'] = heikinashi['high'] dataframe['ha_low'] = heikinashi['low'] - """ + return dataframe @@ -220,12 +259,65 @@ class TestStrategy(IStrategy): :param dataframe: DataFrame :return: DataFrame with buy column """ - dataframe.loc[ - ( - (dataframe['adx'] > 30) & - (dataframe['tema'] <= dataframe['bb_middleband']) & - (dataframe['tema'] > dataframe['tema'].shift(1)) + if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in params and params['mfi']['enabled']: + conditions.append(dataframe['mfi'] < params['mfi']['value']) + if 'fastd' in params and params['fastd']['enabled']: + conditions.append(dataframe['fastd'] < params['fastd']['value']) + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'rsi' in params and params['rsi']['enabled']: + conditions.append(dataframe['rsi'] < params['rsi']['value']) + if 'over_sar' in params and params['over_sar']['enabled']: + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in params and params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + # TRIGGERS + triggers = { + 'lower_bb': ( + dataframe['close'] < dataframe['bb_lowerband'] ), + 'lower_bb_tema': ( + dataframe['tema'] < dataframe['bb_lowerband'] + ), + 'faststoch10': (qtpylib.crossed_above( + dataframe['fastd'], 10.0 + )), + 'ao_cross_zero': (qtpylib.crossed_above( + dataframe['ao'], 0.0 + )), + 'ema3_cross_ema10': (qtpylib.crossed_above( + dataframe['ema3'], dataframe['ema10'] + )), + 'macd_cross_signal': (qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )), + 'sar_reversal': (qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )), + 'ht_sine': (qtpylib.crossed_above( + dataframe['htleadsine'], dataframe['htsine'] + )), + 'heiken_reversal_bull': ( + (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & + (dataframe['ha_low'] == dataframe['ha_open']) + ), + 'di_cross': (qtpylib.crossed_above( + dataframe['plus_di'], dataframe['minus_di'] + )), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), 'buy'] = 1 return dataframe @@ -239,8 +331,142 @@ class TestStrategy(IStrategy): dataframe.loc[ ( (dataframe['adx'] > 70) & - (dataframe['tema'] > dataframe['bb_middleband']) & (dataframe['tema'] < dataframe['tema'].shift(1)) ), 'sell'] = 1 return dataframe + + def hyperopt_space(self) -> List[Dict]: + """ + Define your Hyperopt space for the strategy + :return: Dict + """ + space = { + 'macd_below_zero': hp.choice('macd_below_zero', [ + {'enabled': False}, + {'enabled': True} + ]), + 'mfi': hp.choice('mfi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('mfi-value', 5, 25, 1)} + ]), + 'fastd': hp.choice('fastd', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('fastd-value', 10, 50, 1)} + ]), + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 15, 50, 1)} + ]), + 'rsi': hp.choice('rsi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)} + ]), + 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'over_sar': hp.choice('over_sar', [ + {'enabled': False}, + {'enabled': True} + ]), + 'green_candle': hp.choice('green_candle', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_sma': hp.choice('uptrend_sma', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'lower_bb'}, + {'type': 'lower_bb_tema'}, + {'type': 'faststoch10'}, + {'type': 'ao_cross_zero'}, + {'type': 'ema3_cross_ema10'}, + {'type': 'macd_cross_signal'}, + {'type': 'sar_reversal'}, + {'type': 'ht_sine'}, + {'type': 'heiken_reversal_bull'}, + {'type': 'di_cross'}, + ]), + 'stoploss': hp.uniform('stoploss', -0.5, -0.02), + } + return space + + def buy_strategy_generator(self, params) -> None: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + conditions = [] + # GUARDS AND TRENDS + if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in params and params['mfi']['enabled']: + conditions.append(dataframe['mfi'] < params['mfi']['value']) + if 'fastd' in params and params['fastd']['enabled']: + conditions.append(dataframe['fastd'] < params['fastd']['value']) + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'rsi' in params and params['rsi']['enabled']: + conditions.append(dataframe['rsi'] < params['rsi']['value']) + if 'over_sar' in params and params['over_sar']['enabled']: + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in params and params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + + # TRIGGERS + triggers = { + 'lower_bb': ( + dataframe['close'] < dataframe['bb_lowerband'] + ), + 'lower_bb_tema': ( + dataframe['tema'] < dataframe['bb_lowerband'] + ), + 'faststoch10': (qtpylib.crossed_above( + dataframe['fastd'], 10.0 + )), + 'ao_cross_zero': (qtpylib.crossed_above( + dataframe['ao'], 0.0 + )), + 'ema3_cross_ema10': (qtpylib.crossed_above( + dataframe['ema3'], dataframe['ema10'] + )), + 'macd_cross_signal': (qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )), + 'sar_reversal': (qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )), + 'ht_sine': (qtpylib.crossed_above( + dataframe['htleadsine'], dataframe['htsine'] + )), + 'heiken_reversal_bull': ( + (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & + (dataframe['ha_low'] == dataframe['ha_open']) + ), + 'di_cross': (qtpylib.crossed_above( + dataframe['plus_di'], dataframe['minus_di'] + )), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend From c352c9b293b84326eeda462e95acbd8ac770a290 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 16:44:27 -0500 Subject: [PATCH 09/30] API key error handling. --- freqtrade/exchange/bittrex.py | 102 ++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 36 deletions(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 0cba621af..002aa2ddf 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -62,49 +62,64 @@ class Bittrex(Exchange): def buy(self, pair: str, rate: float, amount: float) -> str: data = _API.buy_limit(pair.replace('_', '-'), amount, rate) if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message} params=({pair}, {rate}, {amount})'.format( - message=data['message'], - pair=pair, - rate=rate, - amount=amount)) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({pair}, {rate}, {amount})'.format( + message=data['message'], + pair=pair, + rate=rate, + amount=amount)) return data['result']['uuid'] def sell(self, pair: str, rate: float, amount: float) -> str: data = _API.sell_limit(pair.replace('_', '-'), amount, rate) if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message} params=({pair}, {rate}, {amount})'.format( - message=data['message'], - pair=pair, - rate=rate, - amount=amount)) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({pair}, {rate}, {amount})'.format( + message=data['message'], + pair=pair, + rate=rate, + amount=amount)) return data['result']['uuid'] def get_balance(self, currency: str) -> float: data = _API.get_balance(currency) if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message} params=({currency})'.format( - message=data['message'], - currency=currency)) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({currency})'.format( + message=data['message'], + currency=currency)) return float(data['result']['Balance'] or 0.0) def get_balances(self): data = _API.get_balances() if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message}'.format(message=data['message'])) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message}'.format(message=data['message'])) return data['result'] def get_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: if refresh or pair not in self.cached_ticker.keys(): data = _API.get_ticker(pair.replace('_', '-')) if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message} params=({pair})'.format( - message=data['message'], - pair=pair)) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({pair})'.format( + message=data['message'], + pair=pair)) keys = ['Bid', 'Ask', 'Last'] if not data.get('result') or\ not all(key in data.get('result', {}) for key in keys) or\ @@ -147,10 +162,13 @@ class Bittrex(Exchange): 'in response params=({})'.format(prop, pair)) if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message} params=({pair})'.format( - message=data['message'], - pair=pair)) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({pair})'.format( + message=data['message'], + pair=pair)) return data['result'] @@ -176,10 +194,13 @@ class Bittrex(Exchange): def cancel_order(self, order_id: str) -> None: data = _API.cancel(order_id) if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message} params=({order_id})'.format( - message=data['message'], - order_id=order_id)) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({order_id})'.format( + message=data['message'], + order_id=order_id)) def get_pair_detail_url(self, pair: str) -> str: return self.PAIR_DETAIL_METHOD + '?MarketName={}'.format(pair.replace('_', '-')) @@ -187,22 +208,31 @@ class Bittrex(Exchange): def get_markets(self) -> List[str]: data = _API.get_markets() if not data['success']: - Bittrex._validate_response(data) - raise OperationalException(data['message']) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException(data['message']) return [m['MarketName'].replace('-', '_') for m in data['result']] def get_market_summaries(self) -> List[Dict]: data = _API.get_market_summaries() if not data['success']: - Bittrex._validate_response(data) - raise OperationalException(data['message']) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException(data['message']) return data['result'] def get_wallet_health(self) -> List[Dict]: data = _API_V2.get_wallet_health() if not data['success']: - Bittrex._validate_response(data) - raise OperationalException(data['message']) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException(data['message']) return [{ 'Currency': entry['Health']['Currency'], 'IsActive': entry['Health']['IsActive'], From b8c5f3eeeaf699940418e237cef71fdea55f09c3 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 16:52:50 -0500 Subject: [PATCH 10/30] Reverted back to non-mongodb hyperopt. --- freqtrade/optimize/hyperopt.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2a4131efe..7dcd46fd2 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -31,15 +31,6 @@ from freqtrade.optimize.backtesting import Backtesting from user_data.hyperopt_conf import hyperopt_optimize_conf - -class Testz(): - db_name = 'freqtrade_hyperopt' - MongoTrials( - arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), - exp_key='exp1' - ) - - class Hyperopt(Backtesting): """ Hyperopt class, this class contains all the logic to run a hyperopt simulation @@ -77,7 +68,7 @@ class Hyperopt(Backtesting): # Hyperopt Trials self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') - self.trials = Testz() + self.trials = Trials() @staticmethod def populate_indicators(dataframe: DataFrame) -> DataFrame: @@ -524,6 +515,12 @@ class Hyperopt(Backtesting): self.logger.info( 'Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!' ) + + db_name = 'freqtrade_hyperopt' + self.trials = MongoTrials( + arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), + exp_key='exp1' + ) else: self.logger.info('Preparing Trials..') signal.signal(signal.SIGINT, self.signal_handler) @@ -538,23 +535,28 @@ class Hyperopt(Backtesting): self.current_tries, self.total_tries ) + try: + # change the Logging format + self.logging.set_format('\n%(message)s') + best_parameters = fmin( fn=self.generate_optimizer, space=self.hyperopt_space(), algo=tpe.suggest, max_evals=self.total_tries, - trials=MongoTrials(arg='mongo://127.0.0.1:1234/freqtrade_hyperopt/jobs', exp_key='exp1') + trials=self.trials ) - # change the Logging format - self.logging.set_format('\n%(message)s') + results = sorted(self.trials.results, key=itemgetter('loss')) best_result = results[0]['result'] - # Improve best parameter logging display + except ValueError: best_parameters = {} best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ 'try with more epochs (param: -e).' + + # Improve best parameter logging display if best_parameters: best_parameters = space_eval( self.hyperopt_space(), @@ -584,7 +586,6 @@ class Hyperopt(Backtesting): sys.exit(0) - def start(args: Namespace) -> None: """ Start Backtesting script From ccf583127506f3053e9de482accc37fca06d70d6 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 17:33:17 -0500 Subject: [PATCH 11/30] Config for hyperopt --- config.json | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 config.json diff --git a/config.json b/config.json new file mode 100644 index 000000000..960e9e7e0 --- /dev/null +++ b/config.json @@ -0,0 +1,73 @@ +{ + "max_open_trades": 15, + "stake_currency": "BTC", + "stake_amount": 0.00075, + "fiat_display_currency": "USD", + "dry_run": false, + "unfilledtimeout": 600, + "ticker_interval": 5, + "bid_strategy": { + "ask_last_balance": 0.0 + }, + "exchange": { + "name": "bittrex", + "key": "XXXXXXXXXXXX", + "secret": "XXXXXXXX", + "pair_whitelist": [ + "BTC_ETH", + "BTC_XRP", + "BTC_BCC", + "BTC_LTC", + "BTC_ADA", + "BTC_XMR", + "BTC_DASH", + "BTC_TRX", + "BTC_ETC", + "BTC_ZEC", + "BTC_WAVES", + "BTC_STEEM", + "BTC_STRAT", + "BTC_DCR", + "BTC_REP", + "BTC_SNT", + "BTC_KMD", + "BTC_ARK", + "BTC_ARDR", + "BTC_MONA", + "BTC_DGB", + "BTC_PIVX", + "BTC_SYS", + "BTC_FCT", + "BTC_BAT", + "BTC_GNT", + "BTC_XZC", + "BTC_EMC", + "BTC_NXT", + "BTC_SALT", + "BTC_PAY", + "BTC_PART", + "BTC_GBYTE", + "BTC_BNT", + "BTC_POWR", + "BTC_NXS", + "BTC_SRN" + ], + "pair_blacklist": [ + "BTC_DOGE" + ] + }, + "experimental": { + "use_sell_signal": false, + "sell_profit_only": false + }, + "telegram": { + "enabled": true, + "token": "XXXXXXXXXXXXXX", + "chat_id": "XXXXXXXXXXX" + }, + "initial_state": "running", + "internals": { + "process_throttle_secs": 5 + } +} + From 9fde3556a4605333b7b961432b662d4aa432f109 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 17:34:31 -0500 Subject: [PATCH 12/30] Removed profit roi updated with config.json --- user_data/hyperopt_conf.py | 59 ++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/user_data/hyperopt_conf.py b/user_data/hyperopt_conf.py index 8e044e549..9c31b3b84 100644 --- a/user_data/hyperopt_conf.py +++ b/user_data/hyperopt_conf.py @@ -11,31 +11,52 @@ def hyperopt_optimize_conf() -> dict: :return: """ return { - 'max_open_trades': 3, + 'max_open_trades': 15, 'stake_currency': 'BTC', - 'stake_amount': 0.01, - "minimal_roi": { - '40': 0.0, - '30': 0.01, - '20': 0.02, - '0': 0.04, - }, - 'stoploss': -0.10, + 'stake_amount': 0.00075, + 'ticker_interval': 5, "bid_strategy": { "ask_last_balance": 0.0 }, "exchange": { "pair_whitelist": [ - "BTC_ETH", - "BTC_LTC", - "BTC_ETC", - "BTC_DASH", - "BTC_ZEC", - "BTC_XLM", - "BTC_NXT", - "BTC_POWR", - "BTC_ADA", - "BTC_XMR" + 'BTC_ETH', + 'BTC_XRP', + 'BTC_BCC', + 'BTC_LTC', + 'BTC_ADA', + 'BTC_XMR', + 'BTC_DASH', + 'BTC_TRX', + 'BTC_ETC', + 'BTC_ZEC', + 'BTC_WAVES', + 'BTC_STEEM', + 'BTC_STRAT', + 'BTC_DCR', + 'BTC_REP', + 'BTC_SNT', + 'BTC_KMD', + 'BTC_ARK', + 'BTC_ARDR', + 'BTC_MONA', + 'BTC_DGB', + 'BTC_PIVX', + 'BTC_SYS', + 'BTC_FCT', + 'BTC_BAT', + 'BTC_GNT', + 'BTC_XZC', + 'BTC_EMC', + 'BTC_NXT', + 'BTC_SALT', + 'BTC_PAY', + 'BTC_PART', + 'BTC_GBYTE', + 'BTC_BNT', + 'BTC_POWR', + 'BTC_NXS', + 'BTC_SRN' ] } } From 075bac77d0b5fcf09a7e75d89c4b181d2b810aa4 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 17:53:22 -0500 Subject: [PATCH 13/30] For mongodb --- hyperopy-mongodb.py | 615 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 615 insertions(+) create mode 100644 hyperopy-mongodb.py diff --git a/hyperopy-mongodb.py b/hyperopy-mongodb.py new file mode 100644 index 000000000..2a4131efe --- /dev/null +++ b/hyperopy-mongodb.py @@ -0,0 +1,615 @@ +# pragma pylint: disable=too-many-instance-attributes, pointless-string-statement + +""" +This module contains the hyperopt logic +""" + +import json +import logging +import os +import pickle +import signal +import sys +from argparse import Namespace +from functools import reduce +from math import exp +from operator import itemgetter +from typing import Dict, Any, Callable + +import numpy +import talib.abstract as ta +from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe +from hyperopt.mongoexp import MongoTrials +from pandas import DataFrame + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.logger import Logger +from freqtrade.optimize import load_data +from freqtrade.optimize.backtesting import Backtesting +from user_data.hyperopt_conf import hyperopt_optimize_conf + + + +class Testz(): + db_name = 'freqtrade_hyperopt' + MongoTrials( + arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), + exp_key='exp1' + ) + + +class Hyperopt(Backtesting): + """ + Hyperopt class, this class contains all the logic to run a hyperopt simulation + + To run a backtest: + hyperopt = Hyperopt(config) + hyperopt.start() + """ + def __init__(self, config: Dict[str, Any]) -> None: + + super().__init__(config) + + # Rename the logging to display Hyperopt file instead of Backtesting + self.logging = Logger(name=__name__, level=config['loglevel']) + self.logger = self.logging.get_logger() + + # set TARGET_TRADES to suit your number concurrent trades so its realistic + # to the number of days + self.target_trades = 600 + self.total_tries = config.get('epochs', 0) + self.current_tries = 0 + self.current_best_loss = 100 + + # max average trade duration in minutes + # if eval ends with higher value, we consider it a failed eval + self.max_accepted_trade_duration = 300 + + # this is expexted avg profit * expected trade count + # for example 3.5%, 1100 trades, self.expected_max_profit = 3.85 + # check that the reported Σ% values do not exceed this! + self.expected_max_profit = 3.0 + + # Configuration and data used by hyperopt + self.processed = None + + # Hyperopt Trials + self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') + self.trials = Testz() + + @staticmethod + def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + """ + dataframe['adx'] = ta.ADX(dataframe) + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + dataframe['cci'] = ta.CCI(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['roc'] = ta.ROC(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + # SAR Parabolic + dataframe['sar'] = ta.SAR(dataframe) + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] + + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ + + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ + + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ + + # Chart type + # ------------------------------------ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] + + return dataframe + + def save_trials(self) -> None: + """ + Save hyperopt trials to file + """ + self.logger.info('Saving Trials to \'%s\'', self.trials_file) + pickle.dump(self.trials, open(self.trials_file, 'wb')) + + def read_trials(self) -> Trials: + """ + Read hyperopt trials file + """ + self.logger.info('Reading Trials from \'%s\'', self.trials_file) + trials = pickle.load(open(self.trials_file, 'rb')) + os.remove(self.trials_file) + return trials + + def log_trials_result(self) -> None: + """ + Display Best hyperopt result + """ + vals = json.dumps(self.trials.best_trial['misc']['vals'], indent=4) + results = self.trials.best_trial['result']['result'] + self.logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + + def log_results(self, results) -> None: + """ + Log results if it is better than any previous evaluation + """ + if results['loss'] < self.current_best_loss: + self.current_best_loss = results['loss'] + log_msg = '{:5d}/{}: {}. Loss {:.5f}'.format( + results['current_tries'], + results['total_tries'], + results['result'], + results['loss'] + ) + self.logger.info(log_msg) + else: + print('.', end='') + sys.stdout.flush() + + def calculate_loss(self, total_profit: float, trade_count: int, trade_duration: float) -> float: + """ + Objective function, returns smaller number for more optimal results + """ + trade_loss = 1 - 0.25 * exp(-(trade_count - self.target_trades) ** 2 / 10 ** 5.8) + profit_loss = max(0, 1 - total_profit / self.expected_max_profit) + duration_loss = 0.4 * min(trade_duration / self.max_accepted_trade_duration, 1) + return trade_loss + profit_loss + duration_loss + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table thqt will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def roi_space() -> Dict[str, Any]: + """ + Values to search for each ROI steps + """ + return { + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), + } + + @staticmethod + def stoploss_space() -> Dict[str, Any]: + """ + Stoploss Value to search + """ + return { + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), + } + + @staticmethod + def indicator_space() -> Dict[str, Any]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return { + 'macd_below_zero': hp.choice('macd_below_zero', [ + {'enabled': False}, + {'enabled': True} + ]), + 'mfi': hp.choice('mfi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} + ]), + 'fastd': hp.choice('fastd', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} + ]), + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} + ]), + 'rsi': hp.choice('rsi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} + ]), + 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'over_sar': hp.choice('over_sar', [ + {'enabled': False}, + {'enabled': True} + ]), + 'green_candle': hp.choice('green_candle', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_sma': hp.choice('uptrend_sma', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'lower_bb'}, + {'type': 'lower_bb_tema'}, + {'type': 'faststoch10'}, + {'type': 'ao_cross_zero'}, + {'type': 'ema3_cross_ema10'}, + {'type': 'macd_cross_signal'}, + {'type': 'sar_reversal'}, + {'type': 'ht_sine'}, + {'type': 'heiken_reversal_bull'}, + {'type': 'di_cross'}, + ]), + } + + def has_space(self, space: str) -> bool: + """ + Tell if a space value is contained in the configuration + """ + if space in self.config['spaces'] or 'all' in self.config['spaces']: + return True + return False + + def hyperopt_space(self) -> Dict[str, Any]: + """ + Return the space to use during Hyperopt + """ + spaces = {} + if self.has_space('buy'): + spaces = {**spaces, **Hyperopt.indicator_space()} + if self.has_space('roi'): + spaces = {**spaces, **Hyperopt.roi_space()} + if self.has_space('stoploss'): + spaces = {**spaces, **Hyperopt.stoploss_space()} + return spaces + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + """ + Buy strategy Hyperopt will build and use + """ + conditions = [] + # GUARDS AND TRENDS + if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in params and params['mfi']['enabled']: + conditions.append(dataframe['mfi'] < params['mfi']['value']) + if 'fastd' in params and params['fastd']['enabled']: + conditions.append(dataframe['fastd'] < params['fastd']['value']) + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'rsi' in params and params['rsi']['enabled']: + conditions.append(dataframe['rsi'] < params['rsi']['value']) + if 'over_sar' in params and params['over_sar']['enabled']: + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in params and params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + + # TRIGGERS + triggers = { + 'lower_bb': ( + dataframe['close'] < dataframe['bb_lowerband'] + ), + 'lower_bb_tema': ( + dataframe['tema'] < dataframe['bb_lowerband'] + ), + 'faststoch10': (qtpylib.crossed_above( + dataframe['fastd'], 10.0 + )), + 'ao_cross_zero': (qtpylib.crossed_above( + dataframe['ao'], 0.0 + )), + 'ema3_cross_ema10': (qtpylib.crossed_above( + dataframe['ema3'], dataframe['ema10'] + )), + 'macd_cross_signal': (qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )), + 'sar_reversal': (qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )), + 'ht_sine': (qtpylib.crossed_above( + dataframe['htleadsine'], dataframe['htsine'] + )), + 'heiken_reversal_bull': ( + (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & + (dataframe['ha_low'] == dataframe['ha_open']) + ), + 'di_cross': (qtpylib.crossed_above( + dataframe['plus_di'], dataframe['minus_di'] + )), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + def generate_optimizer(self, params: Dict) -> Dict: + if self.has_space('roi'): + self.analyze.strategy.minimal_roi = self.generate_roi_table(params) + + if self.has_space('buy'): + self.populate_buy_trend = self.buy_strategy_generator(params) + + if self.has_space('stoploss'): + self.analyze.strategy.stoploss = params['stoploss'] + + results = self.backtest( + { + 'stake_amount': self.config['stake_amount'], + 'processed': self.processed, + 'realistic': self.config.get('realistic_simulation', False), + } + ) + result_explanation = self.format_results(results) + + total_profit = results.profit_percent.sum() + trade_count = len(results.index) + trade_duration = results.duration.mean() + + if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: + print('.', end='') + return { + 'status': STATUS_FAIL, + 'loss': float('inf') + } + + loss = self.calculate_loss(total_profit, trade_count, trade_duration) + + self.current_tries += 1 + + self.log_results( + { + 'loss': loss, + 'current_tries': self.current_tries, + 'total_tries': self.total_tries, + 'result': result_explanation, + } + ) + + return { + 'loss': loss, + 'status': STATUS_OK, + 'result': result_explanation, + } + + @staticmethod + def format_results(results: DataFrame) -> str: + """ + Return the format result in a string + """ + return ('{:6d} trades. Avg profit {: 5.2f}%. ' + 'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_BTC.sum(), + results.profit_percent.sum(), + results.duration.mean(), + ) + + def start(self) -> None: + timerange = Arguments.parse_timerange(self.config.get('timerange')) + data = load_data( + datadir=self.config.get('datadir'), + pairs=self.config['exchange']['pair_whitelist'], + ticker_interval=self.ticker_interval, + timerange=timerange + ) + + if self.has_space('buy'): + self.analyze.populate_indicators = Hyperopt.populate_indicators + self.processed = self.tickerdata_to_dataframe(data) + + if self.config.get('mongodb'): + self.logger.info('Using mongodb ...') + self.logger.info( + 'Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!' + ) + else: + self.logger.info('Preparing Trials..') + signal.signal(signal.SIGINT, self.signal_handler) + # read trials file if we have one + if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: + self.trials = self.read_trials() + + self.current_tries = len(self.trials.results) + self.total_tries += self.current_tries + self.logger.info( + 'Continuing with trials. Current: %d, Total: %d', + self.current_tries, + self.total_tries + ) + try: + best_parameters = fmin( + fn=self.generate_optimizer, + space=self.hyperopt_space(), + algo=tpe.suggest, + max_evals=self.total_tries, + trials=MongoTrials(arg='mongo://127.0.0.1:1234/freqtrade_hyperopt/jobs', exp_key='exp1') + ) + # change the Logging format + self.logging.set_format('\n%(message)s') + results = sorted(self.trials.results, key=itemgetter('loss')) + best_result = results[0]['result'] + # Improve best parameter logging display + except ValueError: + best_parameters = {} + best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ + 'try with more epochs (param: -e).' + if best_parameters: + best_parameters = space_eval( + self.hyperopt_space(), + best_parameters + ) + + self.logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) + if 'roi_t1' in best_parameters: + self.logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) + + self.logger.info('Best Result:\n%s', best_result) + + # Store trials result to file to resume next time + self.save_trials() + + def signal_handler(self, sig, frame) -> None: + """ + Hyperopt SIGINT handler + """ + self.logger.info( + 'Hyperopt received %s', + signal.Signals(sig).name + ) + + self.save_trials() + self.log_trials_result() + sys.exit(0) + + + +def start(args: Namespace) -> None: + """ + Start Backtesting script + :param args: Cli args from Arguments() + :return: None + """ + + # Remove noisy log messages + logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING) + logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) + + # Initialize logger + logger = Logger(name=__name__).get_logger() + logger.info('Starting freqtrade in Hyperopt mode') + + # Initialize configuration + # Monkey patch the configuration with hyperopt_conf.py + configuration = Configuration(args) + optimize_config = hyperopt_optimize_conf() + config = configuration._load_common_config(optimize_config) + config = configuration._load_backtesting_config(config) + config = configuration._load_hyperopt_config(config) + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + + # Initialize backtesting object + hyperopt = Hyperopt(config) + hyperopt.start() From f027d20880c58e6b1744cf4c9a77929deceb816a Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 17:55:15 -0500 Subject: [PATCH 14/30] Update hyperoptNote.md --- hyperoptNote.md | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/hyperoptNote.md b/hyperoptNote.md index d3ae83862..038b61563 100644 --- a/hyperoptNote.md +++ b/hyperoptNote.md @@ -1,9 +1,37 @@ -You will need to add: +Basically, to sum things up: -import dill as pickle - -INTO: +Make these changes per this PR in these files: +https://github.com/hyperopt/hyperopt/pull/287/files /usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py +removetrial['state'] == base.JOB_STATE_RUNNING +addtrial['state'] = base.JOB_STATE_RUNNING -At the very top. +Make these changes per this PR into the hyperopt files: +https://github.com/hyperopt/hyperopt/pull/288/files + +remove:import six.moves.cPickle as pickle +add: import dill as pickle in the following files: + +/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py +/usr/local/lib/python3.6/dist-packages/hyperopt/main.py +/usr/local/lib/python3.6/dist-packages/hyperopt/mongoexp.py +/usr/local/lib/python3.6/dist-packages/hyperopt/utils.py + +Install dill: +pip3.6 install dill + +Get this raw file and replace your hyperopy.py with this one in freqtrade/optmize/: + +https://raw.githubusercontent.com/MoonGem/freqtrade/develop/freqtrade/optimize/hyperopt.py + +then run the mongodb scripts, wait until the worker shows that it does not appear to be working then ctrl+c the worker, however, this is as far as I could get. I think this is the correct process. +However, the queue does eventually empty after awhile and you shouldn't need to ctrlc the worker. +Kinda buggy I guess still. + +You may also need to install the latest hyperopt from their github to avoid subscription error: + +pip3.6 install git+https://github.com/hyperopt/hyperopt.git + + +Lastly, copy hyperopt-mongodb.py over freqtrade/optimise/hyperopt.py then run scripts. From 3efcaaee59b2b7b786153619396129228f5dcff1 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 17:59:28 -0500 Subject: [PATCH 15/30] Update start-hyperopt-worker.py --- scripts/start-hyperopt-worker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/start-hyperopt-worker.py b/scripts/start-hyperopt-worker.py index 8b0ae6326..a24ce1780 100755 --- a/scripts/start-hyperopt-worker.py +++ b/scripts/start-hyperopt-worker.py @@ -19,6 +19,8 @@ command = [ '--mongo=127.0.0.1:1234/{}'.format(DB_NAME), '--poll-interval=0.1', '--workdir={}'.format(WORK_DIR), + '--max-jobs=100000', + '--max-consecutive-failures=100000', ] processes = [subprocess.Popen(command) for i in range(PROC_COUNT)] From a9ee8783d7e817717e84a4548e2052f8861cd717 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 18:10:40 -0500 Subject: [PATCH 16/30] Update hyperoptNote.md --- hyperoptNote.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hyperoptNote.md b/hyperoptNote.md index 038b61563..0fc997d25 100644 --- a/hyperoptNote.md +++ b/hyperoptNote.md @@ -14,8 +14,11 @@ remove:import six.moves.cPickle as pickle add: import dill as pickle in the following files: /usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py + /usr/local/lib/python3.6/dist-packages/hyperopt/main.py + /usr/local/lib/python3.6/dist-packages/hyperopt/mongoexp.py + /usr/local/lib/python3.6/dist-packages/hyperopt/utils.py Install dill: @@ -33,5 +36,8 @@ You may also need to install the latest hyperopt from their github to avoid subs pip3.6 install git+https://github.com/hyperopt/hyperopt.git +You also need to edit: --max-jobs=1 to --max-jobs=100000 in mongoexp.py: + +/usr/local/lib/python3.6/dist-packages/hyperopt/mongoexp.py Lastly, copy hyperopt-mongodb.py over freqtrade/optimise/hyperopt.py then run scripts. From c6102a55af0ac3653c3b05e0c8c9969d3c7cfda7 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 18:30:08 -0500 Subject: [PATCH 17/30] Update start-hyperopt-worker.py --- scripts/start-hyperopt-worker.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/start-hyperopt-worker.py b/scripts/start-hyperopt-worker.py index a24ce1780..db80ec8de 100755 --- a/scripts/start-hyperopt-worker.py +++ b/scripts/start-hyperopt-worker.py @@ -2,7 +2,8 @@ import multiprocessing import os import subprocess - +import sys +import shlex PROC_COUNT = multiprocessing.cpu_count() - 1 DB_NAME = 'freqtrade_hyperopt' WORK_DIR = os.path.join( @@ -22,8 +23,6 @@ command = [ '--max-jobs=100000', '--max-consecutive-failures=100000', ] -processes = [subprocess.Popen(command) for i in range(PROC_COUNT)] - -# Join all workers -for proc in processes: - proc.wait() +def run(command): + subprocess.call(command, shell=True)) +run(command) From d46eaae7a3afc424d03475354c12c3d83aa476ff Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 18:44:37 -0500 Subject: [PATCH 18/30] Update start-hyperopt-worker.py --- scripts/start-hyperopt-worker.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/start-hyperopt-worker.py b/scripts/start-hyperopt-worker.py index db80ec8de..8b0ae6326 100755 --- a/scripts/start-hyperopt-worker.py +++ b/scripts/start-hyperopt-worker.py @@ -2,8 +2,7 @@ import multiprocessing import os import subprocess -import sys -import shlex + PROC_COUNT = multiprocessing.cpu_count() - 1 DB_NAME = 'freqtrade_hyperopt' WORK_DIR = os.path.join( @@ -20,9 +19,9 @@ command = [ '--mongo=127.0.0.1:1234/{}'.format(DB_NAME), '--poll-interval=0.1', '--workdir={}'.format(WORK_DIR), - '--max-jobs=100000', - '--max-consecutive-failures=100000', ] -def run(command): - subprocess.call(command, shell=True)) -run(command) +processes = [subprocess.Popen(command) for i in range(PROC_COUNT)] + +# Join all workers +for proc in processes: + proc.wait() From 76245245ac9a89d9f6e57ebcc018e65571f1fc84 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 18:45:23 -0500 Subject: [PATCH 19/30] Update hyperoptNote.md --- hyperoptNote.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hyperoptNote.md b/hyperoptNote.md index 0fc997d25..b8ea0fdaa 100644 --- a/hyperoptNote.md +++ b/hyperoptNote.md @@ -36,7 +36,7 @@ You may also need to install the latest hyperopt from their github to avoid subs pip3.6 install git+https://github.com/hyperopt/hyperopt.git -You also need to edit: --max-jobs=1 to --max-jobs=100000 in mongoexp.py: +You also need to edit: --max-jobs=1 to --max-jobs=500 in mongoexp.py: /usr/local/lib/python3.6/dist-packages/hyperopt/mongoexp.py From d7d87317adf7449e002a79764dbe63adc69a6704 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 20:31:38 -0500 Subject: [PATCH 20/30] Update hyperoptNote.md --- hyperoptNote.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/hyperoptNote.md b/hyperoptNote.md index b8ea0fdaa..19bb32493 100644 --- a/hyperoptNote.md +++ b/hyperoptNote.md @@ -36,8 +36,5 @@ You may also need to install the latest hyperopt from their github to avoid subs pip3.6 install git+https://github.com/hyperopt/hyperopt.git -You also need to edit: --max-jobs=1 to --max-jobs=500 in mongoexp.py: - -/usr/local/lib/python3.6/dist-packages/hyperopt/mongoexp.py Lastly, copy hyperopt-mongodb.py over freqtrade/optimise/hyperopt.py then run scripts. From a397c0b868d549db0f5c06613760a59d01674f5c Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 20:31:53 -0500 Subject: [PATCH 21/30] Update hyperoptNote.md --- hyperoptNote.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hyperoptNote.md b/hyperoptNote.md index 19bb32493..76f94fa76 100644 --- a/hyperoptNote.md +++ b/hyperoptNote.md @@ -37,4 +37,4 @@ You may also need to install the latest hyperopt from their github to avoid subs pip3.6 install git+https://github.com/hyperopt/hyperopt.git -Lastly, copy hyperopt-mongodb.py over freqtrade/optimise/hyperopt.py then run scripts. +Lastly, copy hyperopt-mongodb.py over freqtrade/optimise/hyperopt.py then run scripts. Wait until the database fills and get result! From 37c517792cf5b47fd5943ded09f8d06053c3d103 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 20:32:38 -0500 Subject: [PATCH 22/30] Keep workers from falling off --- scripts/start-hyperopt-worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/start-hyperopt-worker.py b/scripts/start-hyperopt-worker.py index 8b0ae6326..e38243702 100755 --- a/scripts/start-hyperopt-worker.py +++ b/scripts/start-hyperopt-worker.py @@ -19,6 +19,7 @@ command = [ '--mongo=127.0.0.1:1234/{}'.format(DB_NAME), '--poll-interval=0.1', '--workdir={}'.format(WORK_DIR), + '--reserve-timeout=900, ] processes = [subprocess.Popen(command) for i in range(PROC_COUNT)] From f43f9246932610f1df0bd87db793a0dc59024d6e Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 20:33:03 -0500 Subject: [PATCH 23/30] Updated 3/23/18 8:32PMC --- scripts/start-hyperopt-worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start-hyperopt-worker.py b/scripts/start-hyperopt-worker.py index e38243702..b04ad1029 100755 --- a/scripts/start-hyperopt-worker.py +++ b/scripts/start-hyperopt-worker.py @@ -19,7 +19,7 @@ command = [ '--mongo=127.0.0.1:1234/{}'.format(DB_NAME), '--poll-interval=0.1', '--workdir={}'.format(WORK_DIR), - '--reserve-timeout=900, + '--reserve-timeout=900', ] processes = [subprocess.Popen(command) for i in range(PROC_COUNT)] From 194ec415eb5597d05399904145c4cd5d1ba270cd Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 20:36:40 -0500 Subject: [PATCH 24/30] Update hyperoptNote.md --- hyperoptNote.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hyperoptNote.md b/hyperoptNote.md index 76f94fa76..ca5a0771f 100644 --- a/hyperoptNote.md +++ b/hyperoptNote.md @@ -1,16 +1,21 @@ Basically, to sum things up: Make these changes per this PR in these files: + https://github.com/hyperopt/hyperopt/pull/287/files /usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py + removetrial['state'] == base.JOB_STATE_RUNNING + addtrial['state'] = base.JOB_STATE_RUNNING Make these changes per this PR into the hyperopt files: + https://github.com/hyperopt/hyperopt/pull/288/files remove:import six.moves.cPickle as pickle + add: import dill as pickle in the following files: /usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py @@ -22,6 +27,7 @@ add: import dill as pickle in the following files: /usr/local/lib/python3.6/dist-packages/hyperopt/utils.py Install dill: + pip3.6 install dill Get this raw file and replace your hyperopy.py with this one in freqtrade/optmize/: @@ -38,3 +44,7 @@ pip3.6 install git+https://github.com/hyperopt/hyperopt.git Lastly, copy hyperopt-mongodb.py over freqtrade/optimise/hyperopt.py then run scripts. Wait until the database fills and get result! + +Management tool for mongodb: + +https://robomongo.org/download From fad1015cc1a935c8d3e939e468458645ba5610f0 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 20:52:14 -0500 Subject: [PATCH 25/30] Fixed name --- hyperopy-mongodb.py => hyperopt-mongodb.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename hyperopy-mongodb.py => hyperopt-mongodb.py (100%) diff --git a/hyperopy-mongodb.py b/hyperopt-mongodb.py similarity index 100% rename from hyperopy-mongodb.py rename to hyperopt-mongodb.py From e3c5b4ecd4341ff943eed78cac4dda0845c39b9a Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 20:52:57 -0500 Subject: [PATCH 26/30] For -s all all spaces including roi timeframes --- config.json => config-hyperopt.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename config.json => config-hyperopt.json (100%) diff --git a/config.json b/config-hyperopt.json similarity index 100% rename from config.json rename to config-hyperopt.json From 07a90f51c0a17e5a4f70b82e7f9b3ba1a598eaf0 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 20:53:25 -0500 Subject: [PATCH 27/30] For organization --- config-hyperopt.json => hyperopt-config.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename config-hyperopt.json => hyperopt-config.json (100%) diff --git a/config-hyperopt.json b/hyperopt-config.json similarity index 100% rename from config-hyperopt.json rename to hyperopt-config.json From ad3541b376e8d4cd5ddd26ea899d47107a08b466 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 20:59:26 -0500 Subject: [PATCH 28/30] Dev test notes. --- hyperoptNote.md => hyperopt-note.md | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) rename hyperoptNote.md => hyperopt-note.md (51%) diff --git a/hyperoptNote.md b/hyperopt-note.md similarity index 51% rename from hyperoptNote.md rename to hyperopt-note.md index ca5a0771f..1e88bd0e0 100644 --- a/hyperoptNote.md +++ b/hyperopt-note.md @@ -1,5 +1,7 @@ Basically, to sum things up: +The locations of your hyperopt directories may vary. Some reside in .local. + Make these changes per this PR in these files: https://github.com/hyperopt/hyperopt/pull/287/files @@ -30,21 +32,18 @@ Install dill: pip3.6 install dill -Get this raw file and replace your hyperopy.py with this one in freqtrade/optmize/: - -https://raw.githubusercontent.com/MoonGem/freqtrade/develop/freqtrade/optimize/hyperopt.py - -then run the mongodb scripts, wait until the worker shows that it does not appear to be working then ctrl+c the worker, however, this is as far as I could get. I think this is the correct process. -However, the queue does eventually empty after awhile and you shouldn't need to ctrlc the worker. -Kinda buggy I guess still. - -You may also need to install the latest hyperopt from their github to avoid subscription error: - -pip3.6 install git+https://github.com/hyperopt/hyperopt.git - - -Lastly, copy hyperopt-mongodb.py over freqtrade/optimise/hyperopt.py then run scripts. Wait until the database fills and get result! +replace freqtrade/optimize/hyperopt.py with the hyperopt-mongodb.py in the freqtrade root. Management tool for mongodb: https://robomongo.org/download + + +Then, run the scipts. 5000 epochs can take a few hours even on a high-end machine. You can use the mongodb took above to check the progress on the local machine. + + +Lastly, read the test_strategy.py file located in user_data/strategy/test_strategy.py + +Once you've done that, run scripts. You can restart the worker and the freqtrader hyperopt process but I do not advise it. Monitor the output with mongodb. + + From b964efbb4241ed796444e3f09a537dfc2daa3ac1 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 21:00:00 -0500 Subject: [PATCH 29/30] Rename hyperopt-note.md to hyperopt-mongodb-note.md --- hyperopt-note.md => hyperopt-mongodb-note.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename hyperopt-note.md => hyperopt-mongodb-note.md (100%) diff --git a/hyperopt-note.md b/hyperopt-mongodb-note.md similarity index 100% rename from hyperopt-note.md rename to hyperopt-mongodb-note.md From 5b953f03e7be39dd10b037f945e8a899045a9181 Mon Sep 17 00:00:00 2001 From: MoonGem <34537029+MoonGem@users.noreply.github.com> Date: Fri, 23 Mar 2018 21:07:25 -0500 Subject: [PATCH 30/30] Hardcoded fee to remove hyperopt-mongo-worker error Hardcoded fee to remove hyperopt-mongo-worker error --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d8af47326..17ddcec09 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -113,7 +113,7 @@ class Backtesting(object): open_date=buy_row.date, stake_amount=stake_amount, amount=stake_amount / buy_row.open, - fee=exchange.get_fee() + fee=0.0025 ) # calculate win/lose forwards from buy point