From a68c90c51229d9586cd5ff38cbf565006cc01984 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 21:04:07 +0300 Subject: [PATCH 001/105] avoid calling exchange.get_fee inside loop --- freqtrade/optimize/backtesting.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9a19d1412..ffb808a24 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -62,6 +62,7 @@ class Backtesting(object): self.config['exchange']['uid'] = '' self.config['dry_run'] = True self.exchange = Exchange(self.config) + self.fee = self.exchange.get_fee() @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: @@ -130,14 +131,13 @@ class Backtesting(object): stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) - fee = self.exchange.get_fee() trade = Trade( open_rate=buy_row.close, open_date=buy_row.date, stake_amount=stake_amount, amount=stake_amount / buy_row.open, - fee_open=fee, - fee_close=fee + fee_open=self.fee, + fee_close=self.fee ) # calculate win/lose forwards from buy point From c1691f21f3e3887d5842b091656084cc07de9088 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 21:55:09 +0300 Subject: [PATCH 002/105] check that we set fee on backtesting init --- freqtrade/tests/optimize/test_backtesting.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 1702818b1..22536e42f 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -298,6 +298,7 @@ def test_backtesting_init(mocker, default_conf) -> None: Test Backtesting._init() method """ patch_exchange(mocker) + get_fee = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5)) backtesting = Backtesting(default_conf) assert backtesting.config == default_conf assert isinstance(backtesting.analyze, Analyze) @@ -305,6 +306,8 @@ def test_backtesting_init(mocker, default_conf) -> None: assert callable(backtesting.tickerdata_to_dataframe) assert callable(backtesting.populate_buy_trend) assert callable(backtesting.populate_sell_trend) + get_fee.assert_called() + assert backtesting.fee == 0.5 def test_tickerdata_to_dataframe(default_conf, mocker) -> None: From 01d45bee76addbec4b66c999ae12d17409d9016e Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 07:58:25 +0300 Subject: [PATCH 003/105] fix flake8 --- freqtrade/tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 22536e42f..65aa00a70 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -307,7 +307,7 @@ def test_backtesting_init(mocker, default_conf) -> None: assert callable(backtesting.populate_buy_trend) assert callable(backtesting.populate_sell_trend) get_fee.assert_called() - assert backtesting.fee == 0.5 + assert backtesting.fee == 0.5 def test_tickerdata_to_dataframe(default_conf, mocker) -> None: From 2738d3aed85ea5088af595751590a5000a190c68 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 13:12:36 +0200 Subject: [PATCH 004/105] update plotly --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a31976c8c..e26c9c8f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,4 @@ tabulate==0.8.2 coinmarketcap==5.0.3 # Required for plotting data -#plotly==2.3.0 +#plotly==2.7.0 From 5aae215c94cbe37b3981e51a6b460e36a7fbfa8e Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 10:25:03 +0200 Subject: [PATCH 005/105] wrap strategies with HyperoptStrategy for module lookups with pickle --- freqtrade/optimize/hyperopt.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 19e732d7b..e52581491 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,6 +11,7 @@ import pickle import signal import sys from argparse import Namespace +from copy import deepcopy from functools import reduce from math import exp from operator import itemgetter @@ -26,10 +27,25 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting +from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) +HyperoptStrategy = None + + +def wrap_strategy(strategy: IStrategy) -> Optional[HyperoptStrategy]: + """Wraps a given Strategy instance to HyperoptStrategy""" + global HyperoptStrategy + + attr = deepcopy(dict(strategy.__class__.__dict__)) + # Patch module name to make it compatible with pickle + attr['__module__'] = 'freqtrade.optimize.hyperopt' + HyperoptStrategy = type('HyperoptStrategy', (IStrategy,), attr) + return HyperoptStrategy() + + class Hyperopt(Backtesting): """ Hyperopt class, this class contains all the logic to run a hyperopt simulation @@ -39,7 +55,6 @@ class Hyperopt(Backtesting): hyperopt.start() """ def __init__(self, config: Dict[str, Any]) -> None: - super().__init__(config) # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days @@ -57,6 +72,9 @@ class Hyperopt(Backtesting): # check that the reported Σ% values do not exceed this! self.expected_max_profit = 3.0 + # Wrap strategy to make it compatible with pickle + self.analyze.strategy = wrap_strategy(self.analyze.strategy) + # Configuration and data used by hyperopt self.processed: Optional[Dict[str, Any]] = None From 78f50a14713c70c1b444e9c2ec0ec7569230f25e Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 11:13:49 +0200 Subject: [PATCH 006/105] move logic from hyperopt to freqtrade.strategy --- freqtrade/optimize/hyperopt.py | 19 ------------------- freqtrade/strategy/__init__.py | 32 ++++++++++++++++++++++++++++++++ freqtrade/strategy/resolver.py | 5 +++-- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e52581491..7a313a3ac 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,7 +11,6 @@ import pickle import signal import sys from argparse import Namespace -from copy import deepcopy from functools import reduce from math import exp from operator import itemgetter @@ -27,25 +26,10 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) -HyperoptStrategy = None - - -def wrap_strategy(strategy: IStrategy) -> Optional[HyperoptStrategy]: - """Wraps a given Strategy instance to HyperoptStrategy""" - global HyperoptStrategy - - attr = deepcopy(dict(strategy.__class__.__dict__)) - # Patch module name to make it compatible with pickle - attr['__module__'] = 'freqtrade.optimize.hyperopt' - HyperoptStrategy = type('HyperoptStrategy', (IStrategy,), attr) - return HyperoptStrategy() - - class Hyperopt(Backtesting): """ Hyperopt class, this class contains all the logic to run a hyperopt simulation @@ -72,9 +56,6 @@ class Hyperopt(Backtesting): # check that the reported Σ% values do not exceed this! self.expected_max_profit = 3.0 - # Wrap strategy to make it compatible with pickle - self.analyze.strategy = wrap_strategy(self.analyze.strategy) - # Configuration and data used by hyperopt self.processed: Optional[Dict[str, Any]] = None diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index e69de29bb..e1dc7bb3f 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -0,0 +1,32 @@ +import logging +from copy import deepcopy + +from freqtrade.strategy.interface import IStrategy + + +logger = logging.getLogger(__name__) + + +def import_strategy(strategy: IStrategy) -> IStrategy: + """ + Imports given Strategy instance to global scope + of freqtrade.strategy and returns an instance of it + """ + # Copy all attributes from base class and class + attr = deepcopy({**strategy.__class__.__dict__, **strategy.__dict__}) + # Adjust module name + attr['__module__'] = 'freqtrade.strategy' + + name = strategy.__class__.__name__ + clazz = type(name, (IStrategy,), attr) + + logger.debug( + 'Imported strategy %s.%s as %s.%s', + strategy.__module__, strategy.__class__.__name__, + clazz.__module__, strategy.__class__.__name__, + ) + + # Modify global scope to declare class + globals()[name] = clazz + + return clazz() diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 3fd39bca3..3c7836291 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -11,6 +11,7 @@ from collections import OrderedDict from typing import Optional, Dict, Type from freqtrade import constants +from freqtrade.strategy import import_strategy from freqtrade.strategy.interface import IStrategy @@ -83,7 +84,7 @@ class StrategyResolver(object): strategy = self._search_strategy(path, strategy_name) if strategy: logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) - return strategy + return import_strategy(strategy) raise ImportError( "Impossible to load Strategy '{}'. This class does not exist" @@ -100,7 +101,7 @@ class StrategyResolver(object): """ # Generate spec based on absolute path - spec = importlib.util.spec_from_file_location('user_data.strategies', module_path) + spec = importlib.util.spec_from_file_location('unknown', module_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # type: ignore # importlib does not use typehints From 398b21a11d175d02622bc3fb88ad38e52e04bbaf Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 11:14:31 +0200 Subject: [PATCH 007/105] implement test for import_strategy --- freqtrade/tests/strategy/test_strategy.py | 27 +++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 244910790..26cd798f4 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -5,10 +5,34 @@ import os import pytest +from freqtrade.strategy import import_strategy +from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.resolver import StrategyResolver +def test_import_strategy(caplog): + caplog.set_level(logging.DEBUG) + + strategy = DefaultStrategy() + strategy.some_method = lambda *args, **kwargs: 42 + + assert strategy.__module__ == 'freqtrade.strategy.default_strategy' + assert strategy.some_method() == 42 + + imported_strategy = import_strategy(strategy) + + assert imported_strategy.__module__ == 'freqtrade.strategy' + assert imported_strategy.some_method() == 42 + + assert ( + 'freqtrade.strategy', + logging.DEBUG, + 'Imported strategy freqtrade.strategy.default_strategy.DefaultStrategy ' + 'as freqtrade.strategy.DefaultStrategy', + ) in caplog.record_tuples + + def test_search_strategy(): default_location = os.path.join(os.path.dirname( os.path.realpath(__file__)), '..', '..', 'strategy' @@ -20,8 +44,7 @@ def test_search_strategy(): def test_load_strategy(result): - resolver = StrategyResolver() - resolver._load_strategy('TestStrategy') + resolver = StrategyResolver({'strategy': 'TestStrategy'}) assert hasattr(resolver.strategy, 'populate_indicators') assert 'adx' in resolver.strategy.populate_indicators(result) From 810d7de8693ddbc4e0d3acf14c7684e9f51cd9dc Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 11:57:26 +0200 Subject: [PATCH 008/105] tests: add dir() assertion --- freqtrade/tests/strategy/test_strategy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 26cd798f4..13eac3261 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,5 +1,4 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 - import logging import os @@ -22,6 +21,8 @@ def test_import_strategy(caplog): imported_strategy = import_strategy(strategy) + assert dir(strategy) == dir(imported_strategy) + assert imported_strategy.__module__ == 'freqtrade.strategy' assert imported_strategy.some_method() == 42 From b485e6e0ba151d23a41d5d0bcbf11be9e22ae827 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 18 Jun 2018 22:40:36 +0300 Subject: [PATCH 009/105] start small --- freqtrade/optimize/hyperopt.py | 185 ++------------------------------- 1 file changed, 6 insertions(+), 179 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7a313a3ac..31764fbd7 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -21,6 +21,8 @@ import talib.abstract as ta from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from pandas import DataFrame +from skopt.space import Real, Integer, Categorical + import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration @@ -65,121 +67,18 @@ class Hyperopt(Backtesting): @staticmethod def populate_indicators(dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - """ dataframe['adx'] = ta.ADX(dataframe) - dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - dataframe['cci'] = ta.CCI(dataframe) macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] - dataframe['macdhist'] = macd['macdhist'] dataframe['mfi'] = ta.MFI(dataframe) - dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - dataframe['plus_di'] = ta.PLUS_DI(dataframe) - dataframe['roc'] = ta.ROC(dataframe) dataframe['rsi'] = ta.RSI(dataframe) - # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - rsi = 0.1 * (dataframe['rsi'] - 50) - dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) - # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - # Stoch - stoch = ta.STOCH(dataframe) - dataframe['slowd'] = stoch['slowd'] - dataframe['slowk'] = stoch['slowk'] - # Stoch fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - # Stoch RSI - stoch_rsi = ta.STOCHRSI(dataframe) - dataframe['fastd_rsi'] = stoch_rsi['fastd'] - dataframe['fastk_rsi'] = stoch_rsi['fastk'] + dataframe['minus_di'] = ta.MINUS_DI(dataframe) # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_middleband'] = bollinger['mid'] - dataframe['bb_upperband'] = bollinger['upper'] - # EMA - Exponential Moving Average - dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) - dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) - dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - # SAR Parabolic - dataframe['sar'] = ta.SAR(dataframe) - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] - - # Pattern Recognition - Bullish candlestick patterns - # ------------------------------------ - """ - # Hammer: values [0, 100] - dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) - # Inverted Hammer: values [0, 100] - dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) - # Dragonfly Doji: values [0, 100] - dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) - # Piercing Line: values [0, 100] - dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] - # Morningstar: values [0, 100] - dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] - # Three White Soldiers: values [0, 100] - dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ - - # Pattern Recognition - Bearish candlestick patterns - # ------------------------------------ - """ - # Hanging Man: values [0, 100] - dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) - # Shooting Star: values [0, 100] - dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) - # Gravestone Doji: values [0, 100] - dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) - # Dark Cloud Cover: values [0, 100] - dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) - # Evening Doji Star: values [0, 100] - dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) - # Evening Star: values [0, 100] - dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - - # Pattern Recognition - Bullish/Bearish candlestick patterns - # ------------------------------------ - """ - # Three Line Strike: values [0, -100, 100] - dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) - # Spinning Top: values [0, -100, 100] - dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] - # Engulfing: values [0, -100, 100] - dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] - # Harami: values [0, -100, 100] - dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] - # Three Outside Up/Down: values [0, -100, 100] - dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] - # Three Inside Up/Down: values [0, -100, 100] - dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ - - # Chart type - # ------------------------------------ - # Heikinashi stategy - heikinashi = qtpylib.heikinashi(dataframe) - dataframe['ha_open'] = heikinashi['open'] - dataframe['ha_close'] = heikinashi['close'] - dataframe['ha_high'] = heikinashi['high'] - dataframe['ha_low'] = heikinashi['low'] return dataframe @@ -295,38 +194,6 @@ class Hyperopt(Backtesting): {'enabled': False}, {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} ]), - 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'over_sar': hp.choice('over_sar', [ - {'enabled': False}, - {'enabled': True} - ]), - 'green_candle': hp.choice('green_candle', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_sma': hp.choice('uptrend_sma', [ - {'enabled': False}, - {'enabled': True} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'lower_bb'}, - {'type': 'lower_bb_tema'}, - {'type': 'faststoch10'}, - {'type': 'ao_cross_zero'}, - {'type': 'ema3_cross_ema10'}, - {'type': 'macd_cross_signal'}, - {'type': 'sar_reversal'}, - {'type': 'ht_sine'}, - {'type': 'heiken_reversal_bull'}, - {'type': 'di_cross'}, - ]), } def has_space(self, space: str) -> bool: @@ -361,12 +228,8 @@ class Hyperopt(Backtesting): """ conditions = [] # GUARDS AND TRENDS - if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: - conditions.append(dataframe['ema50'] > dataframe['ema100']) if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: conditions.append(dataframe['macd'] < 0) - if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: - conditions.append(dataframe['ema5'] > dataframe['ema10']) if 'mfi' in params and params['mfi']['enabled']: conditions.append(dataframe['mfi'] < params['mfi']['value']) if 'fastd' in params and params['fastd']['enabled']: @@ -375,49 +238,13 @@ class Hyperopt(Backtesting): conditions.append(dataframe['adx'] > params['adx']['value']) if 'rsi' in params and params['rsi']['enabled']: conditions.append(dataframe['rsi'] < params['rsi']['value']) - if 'over_sar' in params and params['over_sar']['enabled']: - conditions.append(dataframe['close'] > dataframe['sar']) - if 'green_candle' in params and params['green_candle']['enabled']: - conditions.append(dataframe['close'] > dataframe['open']) - if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: - prevsma = dataframe['sma'].shift(1) - conditions.append(dataframe['sma'] > prevsma) # TRIGGERS triggers = { - 'lower_bb': ( - dataframe['close'] < dataframe['bb_lowerband'] - ), - 'lower_bb_tema': ( - dataframe['tema'] < dataframe['bb_lowerband'] - ), - 'faststoch10': (qtpylib.crossed_above( - dataframe['fastd'], 10.0 - )), - 'ao_cross_zero': (qtpylib.crossed_above( - dataframe['ao'], 0.0 - )), - 'ema3_cross_ema10': (qtpylib.crossed_above( - dataframe['ema3'], dataframe['ema10'] - )), - 'macd_cross_signal': (qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )), - 'sar_reversal': (qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )), - 'ht_sine': (qtpylib.crossed_above( - dataframe['htleadsine'], dataframe['htsine'] - )), - 'heiken_reversal_bull': ( - (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & - (dataframe['ha_low'] == dataframe['ha_open']) - ), - 'di_cross': (qtpylib.crossed_above( - dataframe['plus_di'], dataframe['minus_di'] - )), } - conditions.append(triggers.get(params['trigger']['type'])) + #conditions.append(triggers.get(params['trigger']['type'])) + + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) # single trigger dataframe.loc[ reduce(lambda x, y: x & y, conditions), From 0cb1aedf5bd7a6f746d68319d8f6bff327efbf2d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 19 Jun 2018 09:09:54 +0300 Subject: [PATCH 010/105] problem with pickling --- freqtrade/optimize/hyperopt.py | 157 +++++++++++++++++++-------------- 1 file changed, 91 insertions(+), 66 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 31764fbd7..07e618b15 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -10,6 +10,8 @@ import os import pickle import signal import sys +import multiprocessing + from argparse import Namespace from functools import reduce from math import exp @@ -22,6 +24,8 @@ from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from pandas import DataFrame from skopt.space import Real, Integer, Categorical +from skopt import Optimizer +from sklearn.externals.joblib import Parallel, delayed import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments @@ -65,6 +69,21 @@ class Hyperopt(Backtesting): self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') self.trials = Trials() + def get_args(self, params): + dimensions = self.hyperopt_space() + # Ensure the number of dimensions match + # the number of parameters in the list x. + if len(params) != len(dimensions): + msg = "Mismatch in number of search-space dimensions. " \ + "len(dimensions)=={} and len(x)=={}" + msg = msg.format(len(dimensions), len(params)) + raise ValueError(msg) + + # Create a dict where the keys are the names of the dimensions + # and the values are taken from the list of parameters x. + arg_dict = {dim.name: value for dim, value in zip(dimensions, params)} + return arg_dict + @staticmethod def populate_indicators(dataframe: DataFrame) -> DataFrame: dataframe['adx'] = ta.ADX(dataframe) @@ -173,28 +192,17 @@ class Hyperopt(Backtesting): """ Define your Hyperopt space for searching strategy parameters """ - return { - 'macd_below_zero': hp.choice('macd_below_zero', [ - {'enabled': False}, - {'enabled': True} - ]), - 'mfi': hp.choice('mfi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} - ]), - 'fastd': hp.choice('fastd', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} - ]), - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} - ]), - 'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} - ]), - } + return [ + Integer(10, 25, name='mfi-value'), + Integer(15, 45, name='fastd-value'), + Integer(20, 50, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='mfi-enabled'), + Categorical([True, False], name='fastd-enabled'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + ] + def has_space(self, space: str) -> bool: """ @@ -208,14 +216,15 @@ class Hyperopt(Backtesting): """ Return the space to use during Hyperopt """ - spaces: Dict = {} - if self.has_space('buy'): - spaces = {**spaces, **Hyperopt.indicator_space()} - if self.has_space('roi'): - spaces = {**spaces, **Hyperopt.roi_space()} - if self.has_space('stoploss'): - spaces = {**spaces, **Hyperopt.stoploss_space()} - return spaces + return Hyperopt.indicator_space() + # spaces: Dict = {} + # if self.has_space('buy'): + # spaces = {**spaces, **Hyperopt.indicator_space()} + # if self.has_space('roi'): + # spaces = {**spaces, **Hyperopt.roi_space()} + # if self.has_space('stoploss'): + # spaces = {**spaces, **Hyperopt.stoploss_space()} + # return spaces @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: @@ -228,16 +237,16 @@ class Hyperopt(Backtesting): """ conditions = [] # GUARDS AND TRENDS - if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: - conditions.append(dataframe['macd'] < 0) - if 'mfi' in params and params['mfi']['enabled']: - conditions.append(dataframe['mfi'] < params['mfi']['value']) - if 'fastd' in params and params['fastd']['enabled']: - conditions.append(dataframe['fastd'] < params['fastd']['value']) - if 'adx' in params and params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if 'rsi' in params and params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) +# if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: +# conditions.append(dataframe['macd'] < 0) + if 'mfi-enabled' in params and params['mfi-enabled']: + conditions.append(dataframe['mfi'] < params['mfi-value']) + if 'fastd' in params and params['fastd-enabled']: + conditions.append(dataframe['fastd'] < params['fastd-value']) + if 'adx' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS triggers = { @@ -254,7 +263,9 @@ class Hyperopt(Backtesting): return populate_buy_trend - def generate_optimizer(self, params: Dict) -> Dict: + def generate_optimizer(self, _params) -> Dict: + params = self.get_args(_params) + if self.has_space('roi'): self.analyze.strategy.minimal_roi = self.generate_roi_table(params) @@ -297,12 +308,13 @@ class Hyperopt(Backtesting): 'result': result_explanation, } ) + return loss - return { - 'loss': loss, - 'status': STATUS_OK, - 'result': result_explanation, - } +# return { +# 'loss': loss, +# 'status': STATUS_OK, +# 'result': result_explanation, +# } def format_results(self, results: DataFrame) -> str: """ @@ -347,16 +359,29 @@ class Hyperopt(Backtesting): ) try: - best_parameters = fmin( - fn=self.generate_optimizer, - space=self.hyperopt_space(), - algo=tpe.suggest, - max_evals=self.total_tries, - trials=self.trials - ) + # best_parameters = fmin( + # fn=self.generate_optimizer, + # space=self.hyperopt_space(), + # algo=tpe.suggest, + # max_evals=self.total_tries, + # trials=self.trials + # ) + + # results = sorted(self.trials.results, key=itemgetter('loss')) + # best_result = results[0]['result'] + cpus = multiprocessing.cpu_count() + print(f'Found {cpus}. Let\'s make them scream!') + + opt = Optimizer(self.hyperopt_space(), "ET", acq_optimizer="sampling") + + for i in range(self.total_tries//cpus): + asked = opt.ask(n_points=cpus) + #asked = opt.ask() + #f_val = self.generate_optimizer(asked) + f_val = Parallel(n_jobs=-1)(delayed(self.generate_optimizer)(v) for v in asked) + opt.tell(asked, f_val) + print(f'got value {f_val}') - results = sorted(self.trials.results, key=itemgetter('loss')) - best_result = results[0]['result'] except ValueError: best_parameters = {} @@ -364,20 +389,20 @@ class Hyperopt(Backtesting): 'try with more epochs (param: -e).' # Improve best parameter logging display - if best_parameters: - best_parameters = space_eval( - self.hyperopt_space(), - best_parameters - ) + # if best_parameters: + # best_parameters = space_eval( + # self.hyperopt_space(), + # best_parameters + # ) - logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) - if 'roi_t1' in best_parameters: - logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) + # logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) + # if 'roi_t1' in best_parameters: + # logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) - logger.info('Best Result:\n%s', best_result) + # logger.info('Best Result:\n%s', best_result) - # Store trials result to file to resume next time - self.save_trials() + # # Store trials result to file to resume next time + # self.save_trials() def signal_handler(self, sig, frame) -> None: """ From a46badd5c066f13bbbad1c38bf309c6346e965bf Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 19 Jun 2018 21:57:42 +0300 Subject: [PATCH 011/105] reuse pool workers --- freqtrade/optimize/hyperopt.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 07e618b15..e49e66137 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -308,13 +308,12 @@ class Hyperopt(Backtesting): 'result': result_explanation, } ) - return loss -# return { -# 'loss': loss, -# 'status': STATUS_OK, -# 'result': result_explanation, -# } + return { + 'loss': loss, + 'status': STATUS_OK, + 'result': result_explanation, + } def format_results(self, results: DataFrame) -> str: """ @@ -374,13 +373,14 @@ class Hyperopt(Backtesting): opt = Optimizer(self.hyperopt_space(), "ET", acq_optimizer="sampling") - for i in range(self.total_tries//cpus): - asked = opt.ask(n_points=cpus) - #asked = opt.ask() - #f_val = self.generate_optimizer(asked) - f_val = Parallel(n_jobs=-1)(delayed(self.generate_optimizer)(v) for v in asked) - opt.tell(asked, f_val) - print(f'got value {f_val}') + with Parallel(n_jobs=-1) as parallel: + for i in range(self.total_tries//cpus): + asked = opt.ask(n_points=cpus) + #asked = opt.ask() + #f_val = self.generate_optimizer(asked) + f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) + opt.tell(asked, [i['loss'] for i in f_val]) + print(f'got value {f_val}') except ValueError: From 964cbdc262115772b7fdb6f2322de0c93facfc0c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 20 Jun 2018 09:13:51 +0300 Subject: [PATCH 012/105] increase initial sampling points --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e49e66137..919526e4a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -371,7 +371,7 @@ class Hyperopt(Backtesting): cpus = multiprocessing.cpu_count() print(f'Found {cpus}. Let\'s make them scream!') - opt = Optimizer(self.hyperopt_space(), "ET", acq_optimizer="sampling") + opt = Optimizer(self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30) with Parallel(n_jobs=-1) as parallel: for i in range(self.total_tries//cpus): From c415014153ab2e15291264e20af5a7f38db7adc6 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Thu, 21 Jun 2018 14:05:17 +0300 Subject: [PATCH 013/105] use multiple jobs in acq --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 919526e4a..57429af7c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -371,7 +371,7 @@ class Hyperopt(Backtesting): cpus = multiprocessing.cpu_count() print(f'Found {cpus}. Let\'s make them scream!') - opt = Optimizer(self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30) + opt = Optimizer(self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30, acq_optimizer_kwargs={'n_jobs': -1}) with Parallel(n_jobs=-1) as parallel: for i in range(self.total_tries//cpus): From 8fee2e2409b87c6ee2964452439b3c431fd58019 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Thu, 21 Jun 2018 14:59:36 +0300 Subject: [PATCH 014/105] move result logging out from optimizer --- freqtrade/optimize/hyperopt.py | 72 ++++++++++++---------------------- 1 file changed, 25 insertions(+), 47 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 57429af7c..15c1a5e48 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -50,7 +50,6 @@ class Hyperopt(Backtesting): # to the number of days self.target_trades = 600 self.total_tries = config.get('epochs', 0) - self.current_tries = 0 self.current_best_loss = 100 # max average trade duration in minutes @@ -288,27 +287,8 @@ class Hyperopt(Backtesting): trade_count = len(results.index) trade_duration = results.trade_duration.mean() - if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: - print('.', end='') - sys.stdout.flush() - return { - 'status': STATUS_FAIL, - 'loss': float('inf') - } - loss = self.calculate_loss(total_profit, trade_count, trade_duration) - self.current_tries += 1 - - self.log_results( - { - 'loss': loss, - 'current_tries': self.current_tries, - 'total_tries': self.total_tries, - 'result': result_explanation, - } - ) - return { 'loss': loss, 'status': STATUS_OK, @@ -357,36 +337,34 @@ class Hyperopt(Backtesting): self.total_tries ) - try: - # best_parameters = fmin( - # fn=self.generate_optimizer, - # space=self.hyperopt_space(), - # algo=tpe.suggest, - # max_evals=self.total_tries, - # trials=self.trials - # ) + # results = sorted(self.trials.results, key=itemgetter('loss')) + # best_result = results[0]['result'] + cpus = multiprocessing.cpu_count() + print(f'Found {cpus}. Let\'s make them scream!') - # results = sorted(self.trials.results, key=itemgetter('loss')) - # best_result = results[0]['result'] - cpus = multiprocessing.cpu_count() - print(f'Found {cpus}. Let\'s make them scream!') + opt = Optimizer( + self.hyperopt_space(), + base_estimator="ET", + acq_optimizer="auto", + n_initial_points=30, + acq_optimizer_kwargs={'n_jobs': -1} + ) - opt = Optimizer(self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30, acq_optimizer_kwargs={'n_jobs': -1}) + with Parallel(n_jobs=-1) as parallel: + for i in range(self.total_tries//cpus): + asked = opt.ask(n_points=cpus) + f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) + opt.tell(asked, [i['loss'] for i in f_val]) - with Parallel(n_jobs=-1) as parallel: - for i in range(self.total_tries//cpus): - asked = opt.ask(n_points=cpus) - #asked = opt.ask() - #f_val = self.generate_optimizer(asked) - f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) - opt.tell(asked, [i['loss'] for i in f_val]) - print(f'got value {f_val}') - - - except ValueError: - best_parameters = {} - best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ - 'try with more epochs (param: -e).' + for j in range(cpus): + self.log_results( + { + 'loss': f_val[j]['loss'], + 'current_tries': i * cpus + j, + 'total_tries': self.total_tries, + 'result': f_val[j]['result'], + } + ) # Improve best parameter logging display # if best_parameters: From 8272120c3a4dae84f8935b22435c4721bdac0dd5 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 07:10:37 +0300 Subject: [PATCH 015/105] convert stoploss and ROI search spaces to skopt format --- freqtrade/optimize/hyperopt.py | 51 +++++++++++++++++----------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 15c1a5e48..35fed3b7e 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -16,14 +16,14 @@ from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, Any, Callable, Optional +from typing import Dict, Any, Callable, Optional, List import numpy import talib.abstract as ta from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from pandas import DataFrame -from skopt.space import Real, Integer, Categorical +from skopt.space import Real, Integer, Categorical, Dimension from skopt import Optimizer from sklearn.externals.joblib import Parallel, delayed @@ -164,27 +164,27 @@ class Hyperopt(Backtesting): return roi_table @staticmethod - def roi_space() -> Dict[str, Any]: + def roi_space() -> List[Dimension]: """ Values to search for each ROI steps """ - return { - 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), - 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), - 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), - 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), - 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), - 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), - } + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + Real(0.01, 0.04, name='roi_p1'), + Real(0.01, 0.07, name='roi_p2'), + Real(0.01, 0.20, name='roi_p3'), + ] @staticmethod - def stoploss_space() -> Dict[str, Any]: + def stoploss_space() -> List[Dimension]: """ - Stoploss Value to search + Stoploss search space """ - return { - 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), - } + return [ + Real(-0.5, -0.02, name='stoploss'), + ] @staticmethod def indicator_space() -> Dict[str, Any]: @@ -211,19 +211,18 @@ class Hyperopt(Backtesting): return True return False - def hyperopt_space(self) -> Dict[str, Any]: + def hyperopt_space(self) -> List[Dimension]: """ Return the space to use during Hyperopt """ - return Hyperopt.indicator_space() - # spaces: Dict = {} - # if self.has_space('buy'): - # spaces = {**spaces, **Hyperopt.indicator_space()} - # if self.has_space('roi'): - # spaces = {**spaces, **Hyperopt.roi_space()} - # if self.has_space('stoploss'): - # spaces = {**spaces, **Hyperopt.stoploss_space()} - # return spaces + spaces: List[Dimension] = [] + if self.has_space('buy'): + spaces += Hyperopt.indicator_space() + if self.has_space('roi'): + spaces += Hyperopt.roi_space() + if self.has_space('stoploss'): + spaces += Hyperopt.stoploss_space() + return spaces @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: From a525cba8e9c94771606696788c9d50b29ee2f51f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 13:02:26 +0300 Subject: [PATCH 016/105] switch signal handler to try catch. fix pickling and formatting output --- freqtrade/optimize/hyperopt.py | 102 ++++++++++++--------------------- 1 file changed, 38 insertions(+), 64 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 35fed3b7e..960812248 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -8,7 +8,6 @@ import json import logging import os import pickle -import signal import sys import multiprocessing @@ -18,9 +17,7 @@ from math import exp from operator import itemgetter from typing import Dict, Any, Callable, Optional, List -import numpy import talib.abstract as ta -from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from pandas import DataFrame from skopt.space import Real, Integer, Categorical, Dimension @@ -64,9 +61,9 @@ class Hyperopt(Backtesting): # Configuration and data used by hyperopt self.processed: Optional[Dict[str, Any]] = None - # Hyperopt Trials + # Previous evaluations self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') - self.trials = Trials() + self.trials = [] def get_args(self, params): dimensions = self.hyperopt_space() @@ -104,10 +101,11 @@ class Hyperopt(Backtesting): """ Save hyperopt trials to file """ - logger.info('Saving Trials to \'%s\'', self.trials_file) - pickle.dump(self.trials, open(self.trials_file, 'wb')) + if self.trials: + logger.info('Saving %d evaluations to \'%s\'', len(self.trials), self.trials_file) + pickle.dump(self.trials, open(self.trials_file, 'wb')) - def read_trials(self) -> Trials: + def read_trials(self) -> List: """ Read hyperopt trials file """ @@ -120,9 +118,15 @@ class Hyperopt(Backtesting): """ Display Best hyperopt result """ - vals = json.dumps(self.trials.best_trial['misc']['vals'], indent=4) - results = self.trials.best_trial['result']['result'] - logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + results = sorted(self.trials, key=itemgetter('loss')) + best_result = results[0] + logger.info( + 'Best result:\n%s\nwith values:\n%s', + best_result['result'], + best_result['params'] + ) + if 'roi_t1' in best_result['params']: + logger.info('ROI table:\n%s', self.generate_roi_table(best_result['params'])) def log_results(self, results) -> None: """ @@ -202,7 +206,6 @@ class Hyperopt(Backtesting): Categorical([True, False], name='rsi-enabled'), ] - def has_space(self, space: str) -> bool: """ Tell if a space value is contained in the configuration @@ -251,7 +254,7 @@ class Hyperopt(Backtesting): } #conditions.append(triggers.get(params['trigger']['type'])) - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) # single trigger + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) # single trigger dataframe.loc[ reduce(lambda x, y: x & y, conditions), @@ -290,7 +293,7 @@ class Hyperopt(Backtesting): return { 'loss': loss, - 'status': STATUS_OK, + 'params': params, 'result': result_explanation, } @@ -322,22 +325,15 @@ class Hyperopt(Backtesting): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) - logger.info('Preparing Trials..') - signal.signal(signal.SIGINT, self.signal_handler) + logger.info('Preparing..') # read trials file if we have one if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: self.trials = self.read_trials() - - self.current_tries = len(self.trials.results) - self.total_tries += self.current_tries logger.info( - 'Continuing with trials. Current: %d, Total: %d', - self.current_tries, - self.total_tries + 'Loaded %d previous evaluations from disk.', + len(self.trials) ) - # results = sorted(self.trials.results, key=itemgetter('loss')) - # best_result = results[0]['result'] cpus = multiprocessing.cpu_count() print(f'Found {cpus}. Let\'s make them scream!') @@ -349,50 +345,28 @@ class Hyperopt(Backtesting): acq_optimizer_kwargs={'n_jobs': -1} ) - with Parallel(n_jobs=-1) as parallel: - for i in range(self.total_tries//cpus): - asked = opt.ask(n_points=cpus) - f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) - opt.tell(asked, [i['loss'] for i in f_val]) + try: + with Parallel(n_jobs=-1) as parallel: + for i in range(self.total_tries//cpus): + asked = opt.ask(n_points=cpus) + f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) + opt.tell(asked, [i['loss'] for i in f_val]) - for j in range(cpus): - self.log_results( - { - 'loss': f_val[j]['loss'], - 'current_tries': i * cpus + j, - 'total_tries': self.total_tries, - 'result': f_val[j]['result'], - } - ) - - # Improve best parameter logging display - # if best_parameters: - # best_parameters = space_eval( - # self.hyperopt_space(), - # best_parameters - # ) - - # logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) - # if 'roi_t1' in best_parameters: - # logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) - - # logger.info('Best Result:\n%s', best_result) - - # # Store trials result to file to resume next time - # self.save_trials() - - def signal_handler(self, sig, frame) -> None: - """ - Hyperopt SIGINT handler - """ - logger.info( - 'Hyperopt received %s', - signal.Signals(sig).name - ) + self.trials += f_val + for j in range(cpus): + self.log_results( + { + 'loss': f_val[j]['loss'], + 'current_tries': i * cpus + j, + 'total_tries': self.total_tries, + 'result': f_val[j]['result'], + } + ) + except KeyboardInterrupt: + print('User interrupted..') self.save_trials() self.log_trials_result() - sys.exit(0) def start(args: Namespace) -> None: From dde7df7fd31513d8cbb6b70527910d85fb46721c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 17:08:22 +0300 Subject: [PATCH 017/105] add scikit-optimize to dependencies --- requirements.txt | 3 +++ setup.py | 1 + 2 files changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index e26c9c8f6..5727e70c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,5 +21,8 @@ networkx==1.11 # pyup: ignore tabulate==0.8.2 coinmarketcap==5.0.3 +# Required for hyperopt +scikit-optimize=0.5.2 + # Required for plotting data #plotly==2.7.0 diff --git a/setup.py b/setup.py index ee6b7ae38..cd0574fa2 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ setup(name='freqtrade', 'tabulate', 'cachetools', 'coinmarketcap', + 'scikit-optimize', ], include_package_data=True, zip_safe=False, From e8f2e6956d71026224d28ebf6ee37445764b998a Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 21:02:59 +0300 Subject: [PATCH 018/105] to avoid pickle problems, get rid of reference to exchange after initialization --- freqtrade/optimize/hyperopt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 960812248..e53a00e4f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -325,6 +325,8 @@ class Hyperopt(Backtesting): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) + self.exchange = None + logger.info('Preparing..') # read trials file if we have one if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: From 0440a1917169eb9ed2e6fde713903fe9baec311a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 14:19:50 +0200 Subject: [PATCH 019/105] export open/close rate for backtesting too preparation to allow plotting of backtest results --- freqtrade/optimize/backtesting.py | 22 +++++++++++++------- freqtrade/tests/optimize/test_backtesting.py | 18 +++++++++++----- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ffb808a24..b5ed9b167 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -37,6 +37,8 @@ class BacktestResult(NamedTuple): close_index: int trade_duration: float open_at_end: bool + open_rate: float + close_rate: float class Backtesting(object): @@ -115,11 +117,13 @@ class Backtesting(object): def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: - records = [(trade_entry.pair, trade_entry.profit_percent, - trade_entry.open_time.timestamp(), - trade_entry.close_time.timestamp(), - trade_entry.open_index - 1, trade_entry.trade_duration) - for index, trade_entry in results.iterrows()] + # columns = ["pair", "profit", "opents", "closets", "index", "duration", + # "open_rate", "close_rate", "open_at_end"] + + records = [(t.pair, t.profit_percent, t.open_time.timestamp(), + t.close_time.timestamp(), t.open_index - 1, t.trade_duration, + t.open_rate, t.close_rate, t.open_at_end) + for index, t in results.iterrows()] if records: logger.info('Dumping backtest results to %s', recordfilename) @@ -158,7 +162,9 @@ class Backtesting(object): trade_duration=(sell_row.date - buy_row.date).seconds // 60, open_index=buy_row.Index, close_index=sell_row.Index, - open_at_end=False + open_at_end=False, + open_rate=buy_row.close, + close_rate=sell_row.close ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period @@ -171,7 +177,9 @@ class Backtesting(object): trade_duration=(sell_row.date - buy_row.date).seconds // 60, open_index=buy_row.Index, close_index=sell_row.Index, - open_at_end=True + open_at_end=True, + open_rate=buy_row.close, + close_rate=sell_row.close ) logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair, btr.profit_percent, btr.profit_abs) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 65aa00a70..4dda82463 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -604,9 +604,13 @@ def test_backtest_record(default_conf, fee, mocker): Arrow(2017, 11, 14, 22, 10, 00).datetime, Arrow(2017, 11, 14, 22, 43, 00).datetime, Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], + "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], "open_index": [1, 119, 153, 185], "close_index": [118, 151, 184, 199], - "trade_duration": [123, 34, 31, 14]}) + "trade_duration": [123, 34, 31, 14], + "open_at_end": [False, False, False, True] + }) backtesting._store_backtest_result("backtest-result.json", results) assert len(results) == 4 # Assert file_dump_json was only called once @@ -617,12 +621,16 @@ def test_backtest_record(default_conf, fee, mocker): # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117) # Below follows just a typecheck of the schema/type of trade-records oix = None - for (pair, profit, date_buy, date_sell, buy_index, dur) in records: + for (pair, profit, date_buy, date_sell, buy_index, dur, + openr, closer, open_at_end) in records: assert pair == 'UNITTEST/BTC' - isinstance(profit, float) + assert isinstance(profit, float) # FIX: buy/sell should be converted to ints - isinstance(date_buy, str) - isinstance(date_sell, str) + assert isinstance(date_buy, float) + assert isinstance(date_sell, float) + assert isinstance(openr, float) + assert isinstance(closer, float) + assert isinstance(open_at_end, bool) isinstance(buy_index, pd._libs.tslib.Timestamp) if oix: assert buy_index > oix From 09261b11afa20af557a2b638bb952282a09dd5ac Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 15:22:14 +0300 Subject: [PATCH 020/105] remove hyperopt and networkx from dependencies --- requirements.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5727e70c9..5754bf925 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,9 +15,6 @@ TA-Lib==0.4.17 pytest==3.6.2 pytest-mock==1.10.0 pytest-cov==2.5.1 -hyperopt==0.1 -# do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 -networkx==1.11 # pyup: ignore tabulate==0.8.2 coinmarketcap==5.0.3 From 136456afc0d657b2a9fe6e732f999ffe97ed5622 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 15:44:51 +0300 Subject: [PATCH 021/105] add three triggers to hyperopting --- freqtrade/optimize/hyperopt.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e53a00e4f..ea92cd561 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -94,6 +94,7 @@ class Hyperopt(Backtesting): # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['sar'] = ta.SAR(dataframe) return dataframe @@ -204,6 +205,7 @@ class Hyperopt(Backtesting): Categorical([True, False], name='fastd-enabled'), Categorical([True, False], name='adx-enabled'), Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') ] def has_space(self, space: str) -> bool: @@ -238,8 +240,6 @@ class Hyperopt(Backtesting): """ conditions = [] # GUARDS AND TRENDS -# if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: -# conditions.append(dataframe['macd'] < 0) if 'mfi-enabled' in params and params['mfi-enabled']: conditions.append(dataframe['mfi'] < params['mfi-value']) if 'fastd' in params and params['fastd-enabled']: @@ -250,11 +250,16 @@ class Hyperopt(Backtesting): conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS - triggers = { - } - #conditions.append(triggers.get(params['trigger']['type'])) - - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) # single trigger + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) dataframe.loc[ reduce(lambda x, y: x & y, conditions), From ab9e2fcea08692aa30abe5f20a48082c3452d34c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 15:47:19 +0300 Subject: [PATCH 022/105] fix guard names to match search space --- freqtrade/optimize/hyperopt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index ea92cd561..f59b884b8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -242,11 +242,11 @@ class Hyperopt(Backtesting): # GUARDS AND TRENDS if 'mfi-enabled' in params and params['mfi-enabled']: conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd' in params and params['fastd-enabled']: + if 'fastd-enabled' in params and params['fastd-enabled']: conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx' in params and params['adx-enabled']: + if 'adx-enabled' in params and params['adx-enabled']: conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi' in params and params['rsi-enabled']: + if 'rsi-enabled' in params and params['rsi-enabled']: conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS From 642ad02316aaa2a934545c9fb0841aa8ee7d5e89 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 15:56:38 +0300 Subject: [PATCH 023/105] remove unused import --- freqtrade/optimize/hyperopt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f59b884b8..f91647f6e 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -4,7 +4,6 @@ This module contains the hyperopt logic """ -import json import logging import os import pickle From 3384679bad36a5b199abe39b54d3b5a080da2095 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Jun 2018 09:38:20 -0500 Subject: [PATCH 024/105] bump develop to 0.17.1 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 7cf0fa996..ac00264f0 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ FreqTrade bot """ -__version__ = '0.17.0' +__version__ = '0.17.1' class DependencyException(BaseException): From 3cedace2f6fce431cd5df784b5bcf5ce499bf02d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 19:54:27 +0200 Subject: [PATCH 025/105] add plotting for backtested trades --- scripts/plot_dataframe.py | 49 +++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 7f9641222..cd41f51b1 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -27,9 +27,12 @@ Example of usage: import logging import os import sys +import json +from pathlib import Path from argparse import Namespace from typing import Dict, List, Any +import pandas as pd import plotly.graph_objs as go from plotly import tools from plotly.offline import plot @@ -103,10 +106,42 @@ def plot_analyzed_dataframe(args: Namespace) -> None: exit() # Get trades already made from the DB - trades: List[Trade] = [] + trades: pd.DataFrame = pd.DataFrame() if args.db_url: persistence.init(_CONF) - trades = Trade.query.filter(Trade.pair.is_(pair)).all() + trades_ = Trade.query.filter(Trade.pair.is_(pair)).all() + # columns = ["pair", "profit", "opents", "closets", "index", "duration"] + columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] + + trades = pd.DataFrame([(t.pair, t.calc_profit(), + t.open_date, t.close_date, + t.open_rate, t.close_rate, + t.close_date.timestamp() - t.open_date.timestamp()) + for t in trades_], columns=columns) + + if args.exportfilename: + file = Path(args.exportfilename) + # must align with columns in backtest.py + columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] + with file.open() as f: + data = json.load(f) + trades = pd.DataFrame(data, columns=columns) + trades = trades.loc[trades["pair"] == pair] + if timerange: + if timerange.starttype == 'date': + trades = trades.loc[trades["opents"] >= timerange.startts] + if timerange.stoptype == 'date': + trades = trades.loc[trades["opents"] <= timerange.stopts] + + trades['opents'] = pd.to_datetime(trades['opents'], + unit='s', + utc=True, + infer_datetime_format=True) + trades['closets'] = pd.to_datetime(trades['closets'], + unit='s', + utc=True, + infer_datetime_format=True) dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] @@ -126,7 +161,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) -def generate_graph(pair, trades, data, args) -> tools.make_subplots: +def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: """ Generate the graph from the data generated by Backtesting or from DB :param pair: Pair to Display on the graph @@ -187,8 +222,8 @@ def generate_graph(pair, trades, data, args) -> tools.make_subplots: ) trade_buys = go.Scattergl( - x=[t.open_date.isoformat() for t in trades], - y=[t.open_rate for t in trades], + x=trades["opents"], + y=trades["open_rate"], mode='markers', name='trade_buy', marker=dict( @@ -199,8 +234,8 @@ def generate_graph(pair, trades, data, args) -> tools.make_subplots: ) ) trade_sells = go.Scattergl( - x=[t.close_date.isoformat() for t in trades], - y=[t.close_rate for t in trades], + x=trades["closets"], + y=trades["close_rate"], mode='markers', name='trade_sell', marker=dict( From f506ebcd62bf26b522afdd658dddd1fa3371deac Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 19:58:28 +0200 Subject: [PATCH 026/105] use Pathlib in the whole script --- scripts/plot_dataframe.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index cd41f51b1..763b8d9e4 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -25,7 +25,6 @@ Example of usage: --indicators2 fastk,fastd """ import logging -import os import sys import json from pathlib import Path @@ -158,7 +157,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: args=args ) - plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) + plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html'))) def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: From 5055563458597316bee1fbfafee1d42ba5eff673 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 20:14:15 +0200 Subject: [PATCH 027/105] add --plot-limit --- scripts/plot_dataframe.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 763b8d9e4..ba4da444a 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -104,19 +104,20 @@ def plot_analyzed_dataframe(args: Namespace) -> None: if tickers == {}: exit() + if args.db_url and args.exportfilename: + logger.critical("Can only specify --db-url or --export-filename") # Get trades already made from the DB trades: pd.DataFrame = pd.DataFrame() if args.db_url: persistence.init(_CONF) - trades_ = Trade.query.filter(Trade.pair.is_(pair)).all() - # columns = ["pair", "profit", "opents", "closets", "index", "duration"] columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] trades = pd.DataFrame([(t.pair, t.calc_profit(), t.open_date, t.close_date, t.open_rate, t.close_rate, t.close_date.timestamp() - t.open_date.timestamp()) - for t in trades_], columns=columns) + for t in Trade.query.filter(Trade.pair.is_(pair)).all()], + columns=columns) if args.exportfilename: file = Path(args.exportfilename) @@ -147,13 +148,15 @@ def plot_analyzed_dataframe(args: Namespace) -> None: dataframe = analyze.populate_buy_trend(dataframe) dataframe = analyze.populate_sell_trend(dataframe) - if len(dataframe.index) > 750: - logger.warning('Ticker contained more than 750 candles, clipping.') - + if len(dataframe.index) > args.plot_limit: + logger.warning('Ticker contained more than %s candles as defined ' + 'with --plot-limit, clipping.', args.plot_limit) + dataframe = dataframe.tail(args.plot_limit) + trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']] fig = generate_graph( pair=pair, trades=trades, - data=dataframe.tail(750), + data=dataframe, args=args ) @@ -333,11 +336,17 @@ def plot_parse_args(args: List[str]) -> Namespace: default='macd', dest='indicators2', ) - + arguments.parser.add_argument( + '--plot-limit', + help='Specify tick limit for plotting - too high values cause huge files - ' + 'Default: %(default)s', + dest='plot_limit', + default=750, + type=str, + ) arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) arguments.backtesting_options(arguments.parser) - return arguments.parse_args() From d8cb63efdde8d118b4939dda1d4e7a796a6402a6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 20:19:07 +0200 Subject: [PATCH 028/105] extract load_trades --- scripts/plot_dataframe.py | 77 +++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index ba4da444a..e7e38fcd3 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -39,7 +39,7 @@ from plotly.offline import plot import freqtrade.optimize as optimize from freqtrade import persistence from freqtrade.analyze import Analyze -from freqtrade.arguments import Arguments +from freqtrade.arguments import Arguments, TimeRange from freqtrade.exchange import Exchange from freqtrade.optimize.backtesting import setup_configuration from freqtrade.persistence import Trade @@ -48,6 +48,45 @@ logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} +def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame: + trades: pd.DataFrame = pd.DataFrame() + if args.db_url: + persistence.init(_CONF) + columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] + + trades = pd.DataFrame([(t.pair, t.calc_profit(), + t.open_date, t.close_date, + t.open_rate, t.close_rate, + t.close_date.timestamp() - t.open_date.timestamp()) + for t in Trade.query.filter(Trade.pair.is_(pair)).all()], + columns=columns) + + if args.exportfilename: + file = Path(args.exportfilename) + # must align with columns in backtest.py + columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] + with file.open() as f: + data = json.load(f) + trades = pd.DataFrame(data, columns=columns) + trades = trades.loc[trades["pair"] == pair] + if timerange: + if timerange.starttype == 'date': + trades = trades.loc[trades["opents"] >= timerange.startts] + if timerange.stoptype == 'date': + trades = trades.loc[trades["opents"] <= timerange.stopts] + + trades['opents'] = pd.to_datetime(trades['opents'], + unit='s', + utc=True, + infer_datetime_format=True) + trades['closets'] = pd.to_datetime(trades['closets'], + unit='s', + utc=True, + infer_datetime_format=True) + return trades + + def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe @@ -107,41 +146,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: if args.db_url and args.exportfilename: logger.critical("Can only specify --db-url or --export-filename") # Get trades already made from the DB - trades: pd.DataFrame = pd.DataFrame() - if args.db_url: - persistence.init(_CONF) - columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] - - trades = pd.DataFrame([(t.pair, t.calc_profit(), - t.open_date, t.close_date, - t.open_rate, t.close_rate, - t.close_date.timestamp() - t.open_date.timestamp()) - for t in Trade.query.filter(Trade.pair.is_(pair)).all()], - columns=columns) - - if args.exportfilename: - file = Path(args.exportfilename) - # must align with columns in backtest.py - columns = ["pair", "profit", "opents", "closets", "index", "duration", - "open_rate", "close_rate", "open_at_end"] - with file.open() as f: - data = json.load(f) - trades = pd.DataFrame(data, columns=columns) - trades = trades.loc[trades["pair"] == pair] - if timerange: - if timerange.starttype == 'date': - trades = trades.loc[trades["opents"] >= timerange.startts] - if timerange.stoptype == 'date': - trades = trades.loc[trades["opents"] <= timerange.stopts] - - trades['opents'] = pd.to_datetime(trades['opents'], - unit='s', - utc=True, - infer_datetime_format=True) - trades['closets'] = pd.to_datetime(trades['closets'], - unit='s', - utc=True, - infer_datetime_format=True) + trades = load_trades(args, pair, timerange) dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] From 660ec6f443046f6a2f02a2ec23ccd2a4ad4f9658 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 24 Jun 2018 13:43:27 +0200 Subject: [PATCH 029/105] fix parameter type --- scripts/plot_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index e7e38fcd3..1cc6b818a 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -347,7 +347,7 @@ def plot_parse_args(args: List[str]) -> Namespace: 'Default: %(default)s', dest='plot_limit', default=750, - type=str, + type=int, ) arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) From 5e7e977ffab0b4f60f31ade8087bb038ec7c5ca6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 24 Jun 2018 14:23:05 +0200 Subject: [PATCH 030/105] Update ccxt from 1.14.256 to 1.14.257 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 51312791e..437bb5533 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.256 +ccxt==1.14.257 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 118a43cbb898658e3cfed8bb415d82d644daf944 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 24 Jun 2018 15:27:53 +0300 Subject: [PATCH 031/105] fixing tests for hyperopt --- freqtrade/optimize/hyperopt.py | 38 +++-- freqtrade/tests/optimize/test_hyperopt.py | 197 +++++----------------- 2 files changed, 61 insertions(+), 174 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f91647f6e..115cce1f0 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -315,6 +315,18 @@ class Hyperopt(Backtesting): results.trade_duration.mean(), ) + def get_optimizer(self) -> Optimizer: + return Optimizer( + self.hyperopt_space(), + base_estimator="ET", + acq_optimizer="auto", + n_initial_points=30, + acq_optimizer_kwargs={'n_jobs': -1} + ) + + def run_optimizer_parallel(self, parallel, asked) -> List: + return parallel(delayed(self.generate_optimizer)(v) for v in asked) + def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) @@ -341,33 +353,25 @@ class Hyperopt(Backtesting): ) cpus = multiprocessing.cpu_count() - print(f'Found {cpus}. Let\'s make them scream!') + logger.info(f'Found {cpus}. Let\'s make them scream!') - opt = Optimizer( - self.hyperopt_space(), - base_estimator="ET", - acq_optimizer="auto", - n_initial_points=30, - acq_optimizer_kwargs={'n_jobs': -1} - ) + opt = self.get_optimizer() try: with Parallel(n_jobs=-1) as parallel: for i in range(self.total_tries//cpus): asked = opt.ask(n_points=cpus) - f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) + f_val = self.run_optimizer_parallel(parallel, asked) opt.tell(asked, [i['loss'] for i in f_val]) self.trials += f_val for j in range(cpus): - self.log_results( - { - 'loss': f_val[j]['loss'], - 'current_tries': i * cpus + j, - 'total_tries': self.total_tries, - 'result': f_val[j]['result'], - } - ) + self.log_results({ + 'loss': f_val[j]['loss'], + 'current_tries': i * cpus + j, + 'total_tries': self.total_tries, + 'result': f_val[j]['result'], + }) except KeyboardInterrupt: print('User interrupted..') diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 8ad1932af..0f3e56916 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,6 +1,5 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 import os -import signal from copy import deepcopy from unittest.mock import MagicMock @@ -42,16 +41,7 @@ def create_trials(mocker) -> None: mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True) mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) - return mocker.Mock( - results=[ - { - 'loss': 1, - 'result': 'foo', - 'status': 'ok' - } - ], - best_trial={'misc': {'vals': {'adx': 999}}} - ) + return [{'loss': 1, 'result': 'foo', 'params': {}}] # Unit tests @@ -148,74 +138,7 @@ def test_no_log_if_loss_does_not_improve(init_hyperopt, caplog) -> None: assert caplog.record_tuples == [] -def test_fmin_best_results(mocker, init_hyperopt, default_conf, caplog) -> None: - fmin_result = { - "macd_below_zero": 0, - "adx": 1, - "adx-value": 15.0, - "fastd": 1, - "fastd-value": 40.0, - "green_candle": 1, - "mfi": 0, - "over_sar": 0, - "rsi": 1, - "rsi-value": 37.0, - "trigger": 2, - "uptrend_long_ema": 1, - "uptrend_short_ema": 0, - "uptrend_sma": 0, - "stoploss": -0.1, - "roi_t1": 1, - "roi_t2": 2, - "roi_t3": 3, - "roi_p1": 1, - "roi_p2": 2, - "roi_p3": 3, - } - - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'epochs': 1}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) - - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) - patch_exchange(mocker) - - StrategyResolver({'strategy': 'DefaultStrategy'}) - hyperopt = Hyperopt(conf) - hyperopt.trials = create_trials(mocker) - hyperopt.tickerdata_to_dataframe = MagicMock() - hyperopt.start() - - exists = [ - 'Best parameters:', - '"adx": {\n "enabled": true,\n "value": 15.0\n },', - '"fastd": {\n "enabled": true,\n "value": 40.0\n },', - '"green_candle": {\n "enabled": true\n },', - '"macd_below_zero": {\n "enabled": false\n },', - '"mfi": {\n "enabled": false\n },', - '"over_sar": {\n "enabled": false\n },', - '"roi_p1": 1.0,', - '"roi_p2": 2.0,', - '"roi_p3": 3.0,', - '"roi_t1": 1.0,', - '"roi_t2": 2.0,', - '"roi_t3": 3.0,', - '"rsi": {\n "enabled": true,\n "value": 37.0\n },', - '"stoploss": -0.1,', - '"trigger": {\n "type": "faststoch10"\n },', - '"uptrend_long_ema": {\n "enabled": true\n },', - '"uptrend_short_ema": {\n "enabled": false\n },', - '"uptrend_sma": {\n "enabled": false\n }', - 'ROI table:\n{0: 6.0, 3.0: 3.0, 5.0: 1.0, 6.0: 0}', - 'Best Result:\nfoo' - ] - for line in exists: - assert line in caplog.text - - +@pytest.mark.skip(reason="Test not implemented") def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', side_effect=ValueError()) @@ -244,6 +167,7 @@ def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> assert line in caplog.text +@pytest.mark.skip(reason="Waits for fixing") def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, default_conf) -> None: trials = create_trials(mocker) @@ -286,17 +210,18 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, defa def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: - create_trials(mocker) + trials = create_trials(mocker) mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) hyperopt = _HYPEROPT mocker.patch('freqtrade.optimize.hyperopt.open', return_value=hyperopt.trials_file) + _HYPEROPT.trials = trials hyperopt.save_trials() trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') assert log_has( - 'Saving Trials to \'{}\''.format(trials_file), + 'Saving 1 evaluations to \'{}\''.format(trials_file), caplog.record_tuples ) mock_dump.assert_called_once() @@ -333,12 +258,14 @@ def test_roi_table_generation(init_hyperopt) -> None: assert hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} -def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: - trials = create_trials(mocker) - mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) +def test_start_calls_optimizer(mocker, init_hyperopt, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.multiprocessing.cpu_count', MagicMock(return_value=1)) + parallel = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + MagicMock(return_value=[{'loss': 1, 'result': 'foo result', 'params': {}}]) + ) patch_exchange(mocker) - mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) conf = deepcopy(default_conf) conf.update({'config': 'config.json.example'}) @@ -347,11 +274,12 @@ def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: conf.update({'spaces': 'all'}) hyperopt = Hyperopt(conf) - hyperopt.trials = trials hyperopt.tickerdata_to_dataframe = MagicMock() hyperopt.start() - mock_fmin.assert_called_once() + parallel.assert_called_once() + + assert 'Best result:\nfoo result\nwith values:\n{}' in caplog.text def test_format_results(init_hyperopt): @@ -384,20 +312,6 @@ def test_format_results(init_hyperopt): assert result.find('Total profit 1.00000000 EUR') -def test_signal_handler(mocker, init_hyperopt): - """ - Test Hyperopt.signal_handler() - """ - m = MagicMock() - mocker.patch('sys.exit', m) - mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.save_trials', m) - mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.log_trials_result', m) - - hyperopt = _HYPEROPT - hyperopt.signal_handler(signal.SIGTERM, None) - assert m.call_count == 3 - - def test_has_space(init_hyperopt): """ Test Hyperopt.has_space() method @@ -422,8 +336,8 @@ def test_populate_indicators(init_hyperopt) -> None: # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe - assert 'ao' in dataframe - assert 'cci' in dataframe + assert 'mfi' in dataframe + assert 'rsi' in dataframe def test_buy_strategy_generator(init_hyperopt) -> None: @@ -437,44 +351,15 @@ def test_buy_strategy_generator(init_hyperopt) -> None: populate_buy_trend = _HYPEROPT.buy_strategy_generator( { - 'uptrend_long_ema': { - 'enabled': True - }, - 'macd_below_zero': { - 'enabled': True - }, - 'uptrend_short_ema': { - 'enabled': True - }, - 'mfi': { - 'enabled': True, - 'value': 20 - }, - 'fastd': { - 'enabled': True, - 'value': 20 - }, - 'adx': { - 'enabled': True, - 'value': 20 - }, - 'rsi': { - 'enabled': True, - 'value': 20 - }, - 'over_sar': { - 'enabled': True, - }, - 'green_candle': { - 'enabled': True, - }, - 'uptrend_sma': { - 'enabled': True, - }, - - 'trigger': { - 'type': 'lower_bb' - } + 'adx-value': 20, + 'fastd-value': 20, + 'mfi-value': 20, + 'rsi-value': 20, + 'adx-enabled': True, + 'fastd-enabled': True, + 'mfi-enabled': True, + 'rsi-enabled': True, + 'trigger': 'bb_lower' } ) result = populate_buy_trend(dataframe) @@ -505,33 +390,31 @@ def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: patch_exchange(mocker) optimizer_param = { - 'adx': {'enabled': False}, - 'fastd': {'enabled': True, 'value': 35.0}, - 'green_candle': {'enabled': True}, - 'macd_below_zero': {'enabled': True}, - 'mfi': {'enabled': False}, - 'over_sar': {'enabled': False}, - 'roi_p1': 0.01, - 'roi_p2': 0.01, - 'roi_p3': 0.1, + 'adx-value': 0, + 'fastd-value': 35, + 'mfi-value': 0, + 'rsi-value': 0, + 'adx-enabled': False, + 'fastd-enabled': True, + 'mfi-enabled': False, + 'rsi-enabled': False, + 'trigger': 'macd_cross_signal', 'roi_t1': 60.0, 'roi_t2': 30.0, 'roi_t3': 20.0, - 'rsi': {'enabled': False}, + 'roi_p1': 0.01, + 'roi_p2': 0.01, + 'roi_p3': 0.1, 'stoploss': -0.4, - 'trigger': {'type': 'macd_cross_signal'}, - 'uptrend_long_ema': {'enabled': False}, - 'uptrend_short_ema': {'enabled': True}, - 'uptrend_sma': {'enabled': True} } response_expected = { 'loss': 1.9840569076926293, 'result': ' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC ' '(0.0231Σ%). Avg duration 100.0 mins.', - 'status': 'ok' + 'params': optimizer_param } hyperopt = Hyperopt(conf) - generate_optimizer_value = hyperopt.generate_optimizer(optimizer_param) + generate_optimizer_value = hyperopt.generate_optimizer(list(optimizer_param.values())) assert generate_optimizer_value == response_expected From e70cb963f7cc9f82336c2fa07875b97d31e74c94 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 24 Jun 2018 17:00:00 +0200 Subject: [PATCH 032/105] document what to do with exported backtest results --- docs/backtesting.md | 28 ++++++++++++++++++++++++++++ freqtrade/optimize/backtesting.py | 3 --- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 1efb46b43..172969ae2 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -70,6 +70,34 @@ Where `-s TestStrategy` refers to the class name within the strategy file `test_ python3 ./freqtrade/main.py backtesting --export trades ``` +The exported trades can be read using the following code for manual analysis, or can be used by the plotting script `plot_dataframe.py` in the scripts folder. + +``` python +import json +from pathlib import Path +import pandas as pd + +filename=Path('user_data/backtest_data/backtest-result.json') + +with filename.open() as file: + data = json.load(file) + +columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] +df = pd.DataFrame(data, columns=columns) + +df['opents'] = pd.to_datetime(df['opents'], + unit='s', + utc=True, + infer_datetime_format=True + ) +df['closets'] = pd.to_datetime(df['closets'], + unit='s', + utc=True, + infer_datetime_format=True + ) +``` + #### Exporting trades to file specifying a custom filename ```bash diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b5ed9b167..70f6168a4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -117,9 +117,6 @@ class Backtesting(object): def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: - # columns = ["pair", "profit", "opents", "closets", "index", "duration", - # "open_rate", "close_rate", "open_at_end"] - records = [(t.pair, t.profit_percent, t.open_time.timestamp(), t.close_time.timestamp(), t.open_index - 1, t.trade_duration, t.open_rate, t.close_rate, t.open_at_end) From 43f1a1d264765f106ad9f94b5fd1f3ffb70d20de Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 24 Jun 2018 19:52:12 +0200 Subject: [PATCH 033/105] rework download_backtest script --- freqtrade/arguments.py | 7 +++++ scripts/download_backtest_data.py | 48 ++++++++++++++++--------------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 31232f1ff..8ce7c546d 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -334,3 +334,10 @@ class Arguments(object): nargs='+', dest='timeframes', ) + + self.parser.add_argument( + '--erase', + help='Clean all existing data for the selected exchange/pairs/timeframes', + dest='erase', + action='store_true' + ) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 2f76c1232..686098f94 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -3,11 +3,14 @@ """This script generate json data from bittrex""" import json import sys -import os +from pathlib import Path import arrow -from freqtrade import (arguments, misc) +from freqtrade import arguments +from freqtrade.arguments import TimeRange from freqtrade.exchange import Exchange +from freqtrade.optimize import download_backtesting_testdata + DEFAULT_DL_PATH = 'user_data/data' @@ -17,25 +20,27 @@ args = arguments.parse_args() timeframes = args.timeframes -dl_path = os.path.join(DEFAULT_DL_PATH, args.exchange) +dl_path = Path(DEFAULT_DL_PATH).joinpath(args.exchange) if args.export: - dl_path = args.export + dl_path = Path(args.export) -if not os.path.isdir(dl_path): +if not dl_path.is_dir(): sys.exit(f'Directory {dl_path} does not exist.') -pairs_file = args.pairs_file if args.pairs_file else os.path.join(dl_path, 'pairs.json') -if not os.path.isfile(pairs_file): +pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json') +if not pairs_file.exists(): sys.exit(f'No pairs file found with path {pairs_file}.') -with open(pairs_file) as file: +with pairs_file.open() as file: PAIRS = list(set(json.load(file))) PAIRS.sort() -since_time = None + +timerange = TimeRange() if args.days: - since_time = arrow.utcnow().shift(days=-args.days).timestamp * 1000 + time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d") + timerange = arguments.parse_timerange(f'{time_since}-') print(f'About to download pairs: {PAIRS} to {dl_path}') @@ -59,21 +64,18 @@ for pair in PAIRS: print(f"skipping pair {pair}") continue for tick_interval in timeframes: - print(f'downloading pair {pair}, interval {tick_interval}') - - data = exchange.get_ticker_history(pair, tick_interval, since_ms=since_time) - if not data: - print('\tNo data was downloaded') - break - - print('\tData was downloaded for period %s - %s' % ( - arrow.get(data[0][0] / 1000).format(), - arrow.get(data[-1][0] / 1000).format())) - - # save data pair_print = pair.replace('/', '_') filename = f'{pair_print}-{tick_interval}.json' - misc.file_dump_json(os.path.join(dl_path, filename), data) + dl_file = dl_path.joinpath(filename) + if args.erase and dl_file.exists(): + print(f'Deleting existing data for pair {pair}, interval {tick_interval}') + dl_file.unlink() + + print(f'downloading pair {pair}, interval {tick_interval}') + download_backtesting_testdata(str(dl_path), exchange=exchange, + pair=pair, + tick_interval=tick_interval, + timerange=timerange) if pairs_not_available: From 17ee7f8be51ddb64b8c4ad1de88fc7edf266c647 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 25 Jun 2018 11:15:11 +0300 Subject: [PATCH 034/105] fix typo in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5754bf925..c22dcbdca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ tabulate==0.8.2 coinmarketcap==5.0.3 # Required for hyperopt -scikit-optimize=0.5.2 +scikit-optimize==0.5.2 # Required for plotting data #plotly==2.7.0 From 0bddc58ec41db007c75de5b16d1589e66cc3131d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 25 Jun 2018 11:38:14 +0300 Subject: [PATCH 035/105] extract loading previous results to a method --- freqtrade/optimize/hyperopt.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 115cce1f0..b8b53ab3c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -327,6 +327,15 @@ class Hyperopt(Backtesting): def run_optimizer_parallel(self, parallel, asked) -> List: return parallel(delayed(self.generate_optimizer)(v) for v in asked) + def load_previous_results(self): + """ read trials file if we have one """ + if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: + self.trials = self.read_trials() + logger.info( + 'Loaded %d previous evaluations from disk.', + len(self.trials) + ) + def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) @@ -340,23 +349,13 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) - self.exchange = None - - logger.info('Preparing..') - # read trials file if we have one - if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: - self.trials = self.read_trials() - logger.info( - 'Loaded %d previous evaluations from disk.', - len(self.trials) - ) + self.load_previous_results() cpus = multiprocessing.cpu_count() logger.info(f'Found {cpus}. Let\'s make them scream!') opt = self.get_optimizer() - try: with Parallel(n_jobs=-1) as parallel: for i in range(self.total_tries//cpus): From 2b6407e5982dbeb8bd1038945507c7875592bfd4 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 25 Jun 2018 11:38:42 +0300 Subject: [PATCH 036/105] remove unused tests from hyperopt --- freqtrade/tests/optimize/test_hyperopt.py | 72 ----------------------- 1 file changed, 72 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 0f3e56916..9194a66c9 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -44,7 +44,6 @@ def create_trials(mocker) -> None: return [{'loss': 1, 'result': 'foo', 'params': {}}] -# Unit tests def test_start(mocker, default_conf, caplog) -> None: """ Test start() function @@ -138,77 +137,6 @@ def test_no_log_if_loss_does_not_improve(init_hyperopt, caplog) -> None: assert caplog.record_tuples == [] -@pytest.mark.skip(reason="Test not implemented") -def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> None: - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.fmin', side_effect=ValueError()) - - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'epochs': 1}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) - patch_exchange(mocker) - - StrategyResolver({'strategy': 'DefaultStrategy'}) - hyperopt = Hyperopt(conf) - hyperopt.trials = create_trials(mocker) - hyperopt.tickerdata_to_dataframe = MagicMock() - - hyperopt.start() - - exists = [ - 'Best Result:', - 'Sorry, Hyperopt was not able to find good parameters. Please try with more epochs ' - '(param: -e).', - ] - - for line in exists: - assert line in caplog.text - - -@pytest.mark.skip(reason="Waits for fixing") -def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, default_conf) -> None: - trials = create_trials(mocker) - - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'epochs': 1}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) - - mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=True) - mocker.patch('freqtrade.optimize.hyperopt.len', return_value=len(trials.results)) - mock_read = mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.read_trials', - return_value=trials - ) - mock_save = mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.save_trials', - return_value=None - ) - mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) - patch_exchange(mocker) - - StrategyResolver({'strategy': 'DefaultStrategy'}) - hyperopt = Hyperopt(conf) - hyperopt.trials = trials - hyperopt.tickerdata_to_dataframe = MagicMock() - - hyperopt.start() - - mock_read.assert_called_once() - mock_save.assert_called_once() - - current_tries = hyperopt.current_tries - total_tries = hyperopt.total_tries - - assert current_tries == len(trials.results) - assert total_tries == (current_tries + len(trials.results)) - - def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: trials = create_trials(mocker) mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) From 4f1fa28658d8c5d1ffa556f69022d2a56f8fdda8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 25 Jun 2018 14:23:06 +0200 Subject: [PATCH 037/105] Update ccxt from 1.14.257 to 1.14.267 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 437bb5533..c9d3a6529 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.257 +ccxt==1.14.267 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 7c2a50cef9b51edb4b5ef9e4ddb442096fed3a17 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 26 Jun 2018 14:23:06 +0200 Subject: [PATCH 038/105] Update ccxt from 1.14.267 to 1.14.272 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c9d3a6529..d4f8b2f06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.267 +ccxt==1.14.272 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 54f52fb36664c375ab8b2c4ce64ea08c40cc7e48 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Fri, 1 Jun 2018 20:50:40 -0700 Subject: [PATCH 039/105] Create stoploss.md --- docs/stoploss.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/stoploss.md diff --git a/docs/stoploss.md b/docs/stoploss.md new file mode 100644 index 000000000..ff93fd1d8 --- /dev/null +++ b/docs/stoploss.md @@ -0,0 +1,50 @@ +# Stop Loss support + +at this stage the bot contains the following stoploss support modes: + +1. static stop loss, defined in either the strategy or configuration + +2. trailing stop loss, defined in the configuration + +3. trailing stop loss, custom positive loss, defined in configuration + +## Static Stop Loss + +This is very simple, basically you define a stop loss of x in your strategy file or alternative in the configuration, which +will overwrite the strategy definition. This will basically try to sell your asset, the second the loss exceeds the defined loss. + +## Trail Stop Loss + +The initial value for this stop loss, is defined in your strategy or configuration. Just as you would define your Stop Loss normally. +To enable this Feauture all you have to do, is to define the configuration element: + +``` +"trailing_stop" : True +``` +This will now actiave an algorithm, whihch automatically moves up your stop loss, every time the price of your asset increases. + +For example, simplified math, + +* you buy an asset at a price of 100$ +* your stop loss is defined at 2% +* which means your stop loss, gets triggered once your asset dropped below 98$ +* assuming your asset now increases in proce to 102$ +* your stop loss, will now be 2% of 102$ or 99.96$ +* now your asset drops in value to 101$, your stop loss, will still be 99.96$ + +basically what this means, is that your stop loss will be adjusted to be always be 2% of the highest observed price + +### Custom positive loss + +due to demand, it is possible to have a default stop loss, when you are in the red with your buy, but once your buy turns positive, +the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you are in the +black, it will be changed to be only a 1% stop loss + +this can be configured in the main confiuration file, the following way: + +``` + "trailing_stop": { + "positive" : 0.01 + }, +``` +The 0.01 would translate to a 1% stop loss, once you hit profit. From 257e1847b121dce297f5dc540ca52e3656c74517 Mon Sep 17 00:00:00 2001 From: peterkorodi Date: Sun, 17 Jun 2018 11:00:34 +0200 Subject: [PATCH 040/105] Update stoploss.md --- docs/stoploss.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index ff93fd1d8..110cfa0d1 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -40,7 +40,7 @@ due to demand, it is possible to have a default stop loss, when you are in the r the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you are in the black, it will be changed to be only a 1% stop loss -this can be configured in the main confiuration file, the following way: +this can be configured in the main configuration file, the following way: ``` "trailing_stop": { From 9ac3c559b6028bf0b382ed4a8adc8e0786f6b57c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 20:26:28 +0200 Subject: [PATCH 041/105] fix some stoploss documentation --- docs/stoploss.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index 110cfa0d1..cf6a2bcab 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -1,11 +1,9 @@ # Stop Loss support -at this stage the bot contains the following stoploss support modes: +At this stage the bot contains the following stoploss support modes: 1. static stop loss, defined in either the strategy or configuration - 2. trailing stop loss, defined in the configuration - 3. trailing stop loss, custom positive loss, defined in configuration ## Static Stop Loss @@ -16,35 +14,37 @@ will overwrite the strategy definition. This will basically try to sell your ass ## Trail Stop Loss The initial value for this stop loss, is defined in your strategy or configuration. Just as you would define your Stop Loss normally. -To enable this Feauture all you have to do, is to define the configuration element: +To enable this Feauture all you have to do is to define the configuration element: -``` +``` json "trailing_stop" : True ``` -This will now actiave an algorithm, whihch automatically moves up your stop loss, every time the price of your asset increases. + +This will now activate an algorithm, which automatically moves your stop loss up every time the price of your asset increases. For example, simplified math, * you buy an asset at a price of 100$ * your stop loss is defined at 2% * which means your stop loss, gets triggered once your asset dropped below 98$ -* assuming your asset now increases in proce to 102$ +* assuming your asset now increases to 102$ * your stop loss, will now be 2% of 102$ or 99.96$ * now your asset drops in value to 101$, your stop loss, will still be 99.96$ -basically what this means, is that your stop loss will be adjusted to be always be 2% of the highest observed price +basically what this means is that your stop loss will be adjusted to be always be 2% of the highest observed price ### Custom positive loss -due to demand, it is possible to have a default stop loss, when you are in the red with your buy, but once your buy turns positive, +Due to demand, it is possible to have a default stop loss, when you are in the red with your buy, but once your buy turns positive, the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you are in the black, it will be changed to be only a 1% stop loss this can be configured in the main configuration file, the following way: -``` +``` json "trailing_stop": { "positive" : 0.01 }, ``` + The 0.01 would translate to a 1% stop loss, once you hit profit. From 243c36b39bda5e126cd9ca6f805e70b15a57deab Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 20:49:07 +0200 Subject: [PATCH 042/105] get persistence.py for stop_loss --- freqtrade/persistence.py | 49 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 20a7db4bb..76a499eed 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -151,6 +151,12 @@ class Trade(_DECL_BASE): open_date = Column(DateTime, nullable=False, default=datetime.utcnow) close_date = Column(DateTime) open_order_id = Column(String) + # absolute value of the stop loss + stop_loss = Column(Float, nullable=True, default=0.0) + # absolute value of the initial stop loss + initial_stop_loss = Column(Float, nullable=True, default=0.0) + # absolute value of the highest reached price + max_rate = Column(Float, nullable=True, default=0.0) def __repr__(self): return 'Trade(id={}, pair={}, amount={:.8f}, open_rate={:.8f}, open_since={})'.format( @@ -161,6 +167,49 @@ class Trade(_DECL_BASE): arrow.get(self.open_date).humanize() if self.is_open else 'closed' ) + def adjust_stop_loss(self, current_price, stoploss): + """ + this adjusts the stop loss to it's most recently observed + setting + :param current_price: + :param stoploss: + :return: + """ + + new_loss = Decimal(current_price * (1 - abs(stoploss))) + + # keeping track of the highest observed rate for this trade + if self.max_rate is None: + self.max_rate = current_price + else: + if current_price > self.max_rate: + self.max_rate = current_price + + # no stop loss assigned yet + if self.stop_loss is None or self.stop_loss == 0: + logger.debug("assigning new stop loss") + self.stop_loss = new_loss + self.initial_stop_loss = new_loss + + # evaluate if the stop loss needs to be updated + else: + if new_loss > self.stop_loss: # stop losses only walk up, never down! + self.stop_loss = new_loss + logger.debug("adjusted stop loss") + else: + logger.debug("keeping current stop loss") + + logger.debug( + "{} - current price {:.8f}, bought at {:.8f} and calculated " + "stop loss is at: {:.8f} initial stop at {:.8f}. trailing stop loss saved us: {:.8f} " + "and max observed rate was {:.8f}".format( + self.pair, current_price, self.open_rate, + self.initial_stop_loss, + self.stop_loss, float(self.stop_loss) - float(self.initial_stop_loss), + self.max_rate + + )) + def update(self, order: Dict) -> None: """ Updates this entity with amount and actual open/close rates. From 5015bc9bb09d8904131e37d7d26addd47a025e3a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 22:41:28 +0200 Subject: [PATCH 043/105] slight update to persistence --- freqtrade/persistence.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 76a499eed..db97fc858 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -176,7 +176,7 @@ class Trade(_DECL_BASE): :return: """ - new_loss = Decimal(current_price * (1 - abs(stoploss))) + new_loss = float(current_price * (1 - abs(stoploss))) # keeping track of the highest observed rate for this trade if self.max_rate is None: @@ -200,15 +200,13 @@ class Trade(_DECL_BASE): logger.debug("keeping current stop loss") logger.debug( - "{} - current price {:.8f}, bought at {:.8f} and calculated " - "stop loss is at: {:.8f} initial stop at {:.8f}. trailing stop loss saved us: {:.8f} " - "and max observed rate was {:.8f}".format( - self.pair, current_price, self.open_rate, - self.initial_stop_loss, - self.stop_loss, float(self.stop_loss) - float(self.initial_stop_loss), - self.max_rate - - )) + f"{self.pair} - current price {current_price:.8f}, " + f"bought at {self.open_rate:.8f} and calculated " + f"stop loss is at: {self.initial_stop_loss:.8f} initial " + f"stop at {self.stop_loss:.8f}. " + f"trailing stop loss saved us: " + f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f} " + f"and max observed rate was {self.max_rate:.8f}") def update(self, order: Dict) -> None: """ From 3e167e11705b204bb2dcc504bf511bd7b2d9f6ca Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 22:41:38 +0200 Subject: [PATCH 044/105] update sample configs --- config.json.example | 5 ++++- config_full.json.example | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index e5bdc95b1..7b53ffa99 100644 --- a/config.json.example +++ b/config.json.example @@ -5,6 +5,9 @@ "fiat_display_currency": "USD", "ticker_interval" : "5m", "dry_run": false, + "trailing_stop": { + "positive" : 0.005 + }, "unfilledtimeout": 600, "bid_strategy": { "ask_last_balance": 0.0 @@ -32,7 +35,7 @@ "experimental": { "use_sell_signal": false, "sell_profit_only": false, - "ignore_roi_if_buy_signal": false + "sell_fullfilled_at_roi": false }, "telegram": { "enabled": true, diff --git a/config_full.json.example b/config_full.json.example index 231384acc..d7d8f7369 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -5,6 +5,7 @@ "fiat_display_currency": "USD", "dry_run": false, "ticker_interval": "5m", + "trailing_stop": true, "minimal_roi": { "40": 0.0, "30": 0.01, From da5be9fbd0ea132a9490ecca007d2e8adf568c34 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 23:06:27 +0200 Subject: [PATCH 045/105] add stop_loss based on work from @berlinguyinca --- config.json.example | 4 +--- config_full.json.example | 3 ++- docs/stoploss.md | 6 ++---- freqtrade/analyze.py | 42 ++++++++++++++++++++++++++++++++++++---- freqtrade/constants.py | 2 ++ 5 files changed, 45 insertions(+), 12 deletions(-) diff --git a/config.json.example b/config.json.example index 7b53ffa99..f92849ec9 100644 --- a/config.json.example +++ b/config.json.example @@ -5,9 +5,7 @@ "fiat_display_currency": "USD", "ticker_interval" : "5m", "dry_run": false, - "trailing_stop": { - "positive" : 0.005 - }, + "trailing_stop": false, "unfilledtimeout": 600, "bid_strategy": { "ask_last_balance": 0.0 diff --git a/config_full.json.example b/config_full.json.example index d7d8f7369..244daabac 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -5,7 +5,8 @@ "fiat_display_currency": "USD", "dry_run": false, "ticker_interval": "5m", - "trailing_stop": true, + "trailing_stop": false, + "trailing_stop_positive": 0.005, "minimal_roi": { "40": 0.0, "30": 0.01, diff --git a/docs/stoploss.md b/docs/stoploss.md index cf6a2bcab..db4433a02 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -39,12 +39,10 @@ Due to demand, it is possible to have a default stop loss, when you are in the r the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you are in the black, it will be changed to be only a 1% stop loss -this can be configured in the main configuration file, the following way: +This can be configured in the main configuration file and requires `"trailing_stop": true` to be set to true. ``` json - "trailing_stop": { - "positive" : 0.01 - }, + "trailing_stop_positive": 0.01, ``` The 0.01 would translate to a 1% stop loss, once you hit profit. diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 36e00dd0e..b283e3ae9 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -180,7 +180,7 @@ class Analyze(object): :return: True if trade should be sold, False otherwise """ current_profit = trade.calc_profit_percent(rate) - if self.stop_loss_reached(current_profit=current_profit): + if self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date): return True experimental = self.config.get('experimental', {}) @@ -204,12 +204,46 @@ class Analyze(object): return False - def stop_loss_reached(self, current_profit: float) -> bool: - """Based on current profit of the trade and configured stoploss, decides to sell or not""" + def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime) -> bool: + """ + Based on current profit of the trade and configured (trailing) stoploss, + decides to sell or not + """ + + current_profit = trade.calc_profit_percent(current_rate) + trailing_stop = self.config.get('trailing_stop', False) + if trade.stop_loss is None: + # initially adjust the stop loss to the base value + trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss) + + # evaluate if the stoploss was hit + if self.strategy.stoploss is not None and trade.stop_loss >= current_rate: + + if trailing_stop: + logger.debug( + f"HIT STOP: current price at {current_rate:.6f}, " + f"stop loss is {trade.stop_loss:.6f}, " + f"initial stop loss was at {trade.initial_stop_loss:.6f}, " + f"trade opened at {trade.open_rate:.6f}") + logger.debug(f"trailing stop saved {trade.stop_loss - trade.initial_stop_loss:.6f}") - if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: logger.debug('Stop loss hit.') return True + + # update the stop loss afterwards, after all by definition it's supposed to be hanging + if trailing_stop: + + # check if we have a special stop loss for positive condition + # and if profit is positive + stop_loss_value = self.strategy.stoploss + if 'trailing_stop_positive' in self.config and current_profit > 0: + + stop_loss_value = self.config.get('trailing_stop_positive') + logger.debug(f"using positive stop loss mode: {stop_loss_value} " + f"since we have profit {current_profit}") + + trade.adjust_stop_loss(current_rate, stop_loss_value) + return False def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 0f12905e3..4ba1a7094 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -61,6 +61,8 @@ CONF_SCHEMA = { 'minProperties': 1 }, 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, + 'trailing_stop': {'type': 'boolean'}, + 'trailing_stop_positive': {'type': 'number', 'minimum': 0}, 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, 'bid_strategy': { 'type': 'object', From 03005bc0f14f9f7747652a13980c8f8a209ded11 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 23:14:12 +0200 Subject: [PATCH 046/105] update documentation --- docs/configuration.md | 42 ++++++++++++++++++++++++++++++++++-------- docs/index.md | 2 ++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5279ed06e..f24adb61e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,12 +1,15 @@ # Configure the bot + This page explains how to configure your `config.json` file. ## Table of Contents + - [Bot commands](#bot-commands) - [Backtesting commands](#backtesting-commands) - [Hyperopt commands](#hyperopt-commands) ## Setup config.json + We recommend to copy and use the `config.json.example` as a template for your bot configuration. @@ -22,6 +25,8 @@ The table below will list all configuration parameters. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. +| `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). +| `trailing_stoploss_positve` | 0 | No | Changes stop-loss once profit has been reached. | `unfilledtimeout` | 0 | No | How long (in minutes) the bot will wait for an unfilled order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). @@ -41,10 +46,10 @@ The table below will list all configuration parameters. | `strategy_path` | null | No | Adds an additional strategy lookup path (must be a folder). | `internals.process_throttle_secs` | 5 | Yes | Set the process throttle. Value in second. -The definition of each config parameters is in -[misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc.py#L205). +The definition of each config parameters is in [misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc.py#L205). ### Understand stake_amount + `stake_amount` is an amount of crypto-currency your bot will use for each trade. The minimal value is 0.0005. If there is not enough crypto-currency in the account an exception is generated. @@ -52,10 +57,12 @@ To allow the bot to trade all the avaliable `stake_currency` in your account set In this case a trade amount is calclulated as `currency_balanse / (max_open_trades - current_open_trades)`. ### Understand minimal_roi + `minimal_roi` is a JSON object where the key is a duration in minutes and the value is the minimum ROI in percent. See the example below: -``` + +``` json "minimal_roi": { "40": 0.0, # Sell after 40 minutes if the profit is not negative "30": 0.01, # Sell after 30 minutes if there is at least 1% profit @@ -69,6 +76,7 @@ value. This parameter is optional. If you use it, it will take over the `minimal_roi` value from the strategy file. ### Understand stoploss + `stoploss` is loss in percentage that should trigger a sale. For example value `-0.10` will cause immediate sell if the profit dips below -10% for a given trade. This parameter is optional. @@ -77,82 +85,100 @@ Most of the strategy files already include the optimal `stoploss` value. This parameter is optional. If you use it, it will take over the `stoploss` value from the strategy file. +### Understand trailing stoploss + +Go to the [trailing stoploss Documentation](stoploss.md) for details on trailing stoploss. + ### Understand initial_state + `initial_state` is an optional field that defines the initial application state. Possible values are `running` or `stopped`. (default=`running`) If the value is `stopped` the bot has to be started with `/start` first. ### Understand process_throttle_secs + `process_throttle_secs` is an optional field that defines in seconds how long the bot should wait before asking the strategy if we should buy or a sell an asset. After each wait period, the strategy is asked again for every opened trade wether or not we should sell, and for all the remaining pairs (either the dynamic list of pairs or the static list of pairs) if we should buy. ### Understand ask_last_balance + `ask_last_balance` sets the bidding price. Value `0.0` will use `ask` price, `1.0` will use the `last` price and values between those interpolate between ask and last price. Using `ask` price will guarantee quick success in bid, but bot will also end up paying more then would probably have been necessary. ### What values for exchange.name? + Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports 115 cryptocurrency exchange markets and trading APIs. The complete up-to-date list can be found in the [CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). However, the bot was tested with only Bittrex and Binance. The bot was tested with the following exchanges: + - [Bittrex](https://bittrex.com/): "bittrex" - [Binance](https://www.binance.com/): "binance" Feel free to test other exchanges and submit your PR to improve the bot. ### What values for fiat_display_currency? + `fiat_display_currency` set the base currency to use for the conversion from coin to fiat in Telegram. The valid values are: "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD". In addition to central bank currencies, a range of cryto currencies are supported. The valid values are: "BTC", "ETH", "XRP", "LTC", "BCH", "USDT". ## Switch to dry-run mode + We recommend starting the bot in dry-run mode to see how your bot will behave and how is the performance of your strategy. In Dry-run mode the bot does not engage your money. It only runs a live simulation without creating trades. ### To switch your bot in Dry-run mode: + 1. Edit your `config.json` file 2. Switch dry-run to true and specify db_url for a persistent db + ```json "dry_run": true, "db_url": "sqlite///tradesv3.dryrun.sqlite", ``` 3. Remove your Exchange API key (change them by fake api credentials) + ```json "exchange": { "name": "bittrex", "key": "key", "secret": "secret", ... -} +} ``` Once you will be happy with your bot performance, you can switch it to production mode. ## Switch to production mode + In production mode, the bot will engage your money. Be careful a wrong strategy can lose all your money. Be aware of what you are doing when you run it in production mode. ### To switch your bot in production mode: + 1. Edit your `config.json` file 2. Switch dry-run to false and don't forget to adapt your database URL if set + ```json "dry_run": false, ``` 3. Insert your Exchange API key (change them by fake api keys) + ```json "exchange": { "name": "bittrex", @@ -160,10 +186,10 @@ you run it in production mode. "secret": "08a9dc6db3d7b53e1acebd9275677f4b0a04f1a5", ... } + ``` -If you have not your Bittrex API key yet, -[see our tutorial](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md). +If you have not your Bittrex API key yet, [see our tutorial](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md). ## Next step -Now you have configured your config.json, the next step is to -[start your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md). + +Now you have configured your config.json, the next step is to [start your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md). diff --git a/docs/index.md b/docs/index.md index afde2d5eb..fd6bf4378 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,5 @@ # freqtrade documentation + Welcome to freqtrade documentation. Please feel free to contribute to this documentation if you see it became outdated by sending us a Pull-request. Do not hesitate to reach us on @@ -6,6 +7,7 @@ Pull-request. Do not hesitate to reach us on if you do not find the answer to your questions. ## Table of Contents + - [Pre-requisite](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md) - [Setup your Bittrex account](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-bittrex-account) - [Setup your Telegram bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-telegram-bot) From a3708bc56e17912af630cd2258f2920ec9d4ec17 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 23:40:20 +0200 Subject: [PATCH 047/105] add missing test --- freqtrade/tests/test_analyze.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 89dac21c6..02225acca 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -42,6 +42,7 @@ def test_analyze_object() -> None: assert hasattr(Analyze, 'get_signal') assert hasattr(Analyze, 'should_sell') assert hasattr(Analyze, 'min_roi_reached') + assert hasattr(Analyze, 'stop_loss_reached') def test_dataframe_correct_length(result): From 8bec505bbe23919da0035f86627d63a150ab445d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 23:40:36 +0200 Subject: [PATCH 048/105] add test for trailing_stoploss --- freqtrade/tests/test_freqtradebot.py | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..3568dad1d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1654,6 +1654,43 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> assert freqtrade.handle_trade(trade) is True +def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) -> None: + """ + Test sell_profit_only feature when enabled and we have a loss + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.0000172, + 'ask': 0.0000173, + 'last': 0.0000172 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + + conf = deepcopy(default_conf) + conf['trailing_stop'] = True + print(limit_buy_order) + freqtrade = FreqtradeBot(conf) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.update(limit_buy_order) + trade.stop_loss = 3.2 + caplog.set_level(logging.DEBUG) + assert freqtrade.handle_trade(trade) is True + assert log_has( + f'HIT STOP: current price at 0.000017, stop loss is {trade.stop_loss:.6f}, ' + f'initial stop loss was at 0.000000, trade opened at 0.000011', caplog.record_tuples) + + + def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: """ From 88b898cce4a434cd15567581f42663056fe05da0 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 00:16:19 +0200 Subject: [PATCH 049/105] add test for moving stoploss --- freqtrade/tests/test_freqtradebot.py | 63 ++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 3568dad1d..468f5f8aa 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1661,14 +1661,14 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ - 'bid': 0.0000172, - 'ask': 0.0000173, - 'last': 0.0000172 + 'bid': 0.00000102, + 'ask': 0.00000103, + 'last': 0.00000102 }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -1682,14 +1682,61 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) trade = Trade.query.first() trade.update(limit_buy_order) - trade.stop_loss = 3.2 caplog.set_level(logging.DEBUG) + # Sell as trailing-stop is reached + assert freqtrade.handle_trade(trade) is True + assert log_has( + f'HIT STOP: current price at 0.000001, stop loss is {trade.stop_loss:.6f}, ' + f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) + + +def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, mocker) -> None: + """ + Test sell_profit_only feature when enabled and we have a loss + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.0000182, + 'ask': 0.0000183, + 'last': 0.0000182 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + + conf = deepcopy(default_conf) + conf['trailing_stop'] = True + conf['trailing_stop_positive'] = 0.01 + print(limit_buy_order) + freqtrade = FreqtradeBot(conf) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.update(limit_buy_order) + caplog.set_level(logging.DEBUG) + # stop-loss not reached + assert freqtrade.handle_trade(trade) is False + # Adjusted stoploss + assert log_has(f'using positive stop loss mode: 0.01 since we have profit 0.64779142', + caplog.record_tuples) + assert log_has(f'adjusted stop loss', caplog.record_tuples) + assert trade.stop_loss == 0.000018018 + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(return_value={ + 'bid': 0.0000172, + 'ask': 0.0000173, + 'last': 0.0000172 + })) assert freqtrade.handle_trade(trade) is True assert log_has( f'HIT STOP: current price at 0.000017, stop loss is {trade.stop_loss:.6f}, ' - f'initial stop loss was at 0.000000, trade opened at 0.000011', caplog.record_tuples) - - + f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: From e9d5bceeb98b12aa35c020fea29d84bf24088ed4 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 00:18:50 +0200 Subject: [PATCH 050/105] cleanly check if stop_loss is initialized --- freqtrade/analyze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index b283e3ae9..b8caa45b7 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -212,7 +212,7 @@ class Analyze(object): current_profit = trade.calc_profit_percent(current_rate) trailing_stop = self.config.get('trailing_stop', False) - if trade.stop_loss is None: + if trade.stop_loss is None or trade.stop_loss == 0: # initially adjust the stop loss to the base value trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss) From a91d75b3b2acdcce30a5e7bc9745430072da00d3 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:23:49 +0200 Subject: [PATCH 051/105] Add test for adjust_stop-loss --- freqtrade/tests/test_persistence.py | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 30ad239a1..6ffcbbf65 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -453,3 +453,37 @@ def test_migrate_new(mocker, default_conf, fee): assert trade.stake_amount == default_conf.get("stake_amount") assert trade.pair == "ETC/BTC" assert trade.exchange == "binance" + + +def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + fee_open=fee.return_value, + fee_close=fee.return_value, + exchange='bittrex', + open_rate=1, + ) + + # Get percent of profit with a custom rate (Higher than open rate) + trade.adjust_stop_loss(1, 0.05) + assert trade.stop_loss == 0.95 + assert trade.max_rate == 1 + assert trade.initial_stop_loss == 0.95 + + trade.adjust_stop_loss(1.3, -0.1) + assert round(trade.stop_loss, 8) == 1.17 + assert trade.max_rate == 1.3 + assert trade.initial_stop_loss == 0.95 + + # current rate lower ... should not change + trade.adjust_stop_loss(1.2, 0.1) + assert round(trade.stop_loss, 8) == 1.17 + assert trade.max_rate == 1.3 + assert trade.initial_stop_loss == 0.95 + + # current rate higher... should raise stoploss + trade.adjust_stop_loss(1.4, 0.1) + assert round(trade.stop_loss, 8) == 1.26 + assert trade.max_rate == 1.4 + assert trade.initial_stop_loss == 0.95 From c997aa9864ef0aec7b89bfb59982f7b99a609e39 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:38:49 +0200 Subject: [PATCH 052/105] move initial logic to persistence --- freqtrade/analyze.py | 5 ++--- freqtrade/persistence.py | 12 ++++++++---- freqtrade/tests/test_persistence.py | 18 +++++++++++++++--- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index b8caa45b7..d74f9566a 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -212,9 +212,8 @@ class Analyze(object): current_profit = trade.calc_profit_percent(current_rate) trailing_stop = self.config.get('trailing_stop', False) - if trade.stop_loss is None or trade.stop_loss == 0: - # initially adjust the stop loss to the base value - trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss) + + trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) # evaluate if the stoploss was hit if self.strategy.stoploss is not None and trade.stop_loss >= current_rate: diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index db97fc858..4fc83a18f 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -167,15 +167,19 @@ class Trade(_DECL_BASE): arrow.get(self.open_date).humanize() if self.is_open else 'closed' ) - def adjust_stop_loss(self, current_price, stoploss): + def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool =False): """ - this adjusts the stop loss to it's most recently observed - setting + this adjusts the stop loss to it's most recently observed setting :param current_price: - :param stoploss: + :param stoploss in percent: + :param initial: :return: """ + if initial and not (self.stop_loss is None or self.stop_loss == 0): + # Don't modify if called with initial and nothing to do + return + new_loss = float(current_price * (1 - abs(stoploss))) # keeping track of the highest observed rate for this trade diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 6ffcbbf65..45957853f 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -465,18 +465,24 @@ def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): open_rate=1, ) - # Get percent of profit with a custom rate (Higher than open rate) - trade.adjust_stop_loss(1, 0.05) + trade.adjust_stop_loss(trade.open_rate, 0.05, True) assert trade.stop_loss == 0.95 assert trade.max_rate == 1 assert trade.initial_stop_loss == 0.95 + # Get percent of profit with a lowre rate + trade.adjust_stop_loss(0.96, 0.05) + assert trade.stop_loss == 0.95 + assert trade.max_rate == 1 + assert trade.initial_stop_loss == 0.95 + + # Get percent of profit with a custom rate (Higher than open rate) trade.adjust_stop_loss(1.3, -0.1) assert round(trade.stop_loss, 8) == 1.17 assert trade.max_rate == 1.3 assert trade.initial_stop_loss == 0.95 - # current rate lower ... should not change + # current rate lower again ... should not change trade.adjust_stop_loss(1.2, 0.1) assert round(trade.stop_loss, 8) == 1.17 assert trade.max_rate == 1.3 @@ -487,3 +493,9 @@ def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): assert round(trade.stop_loss, 8) == 1.26 assert trade.max_rate == 1.4 assert trade.initial_stop_loss == 0.95 + + # Initial is true but stop_loss set - so doesn't do anything + trade.adjust_stop_loss(1.7, 0.1, True) + assert round(trade.stop_loss, 8) == 1.26 + assert trade.max_rate == 1.4 + assert trade.initial_stop_loss == 0.95 From 78e6c9fdf680307c262e1246d02e4d76a877c90a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:51:48 +0200 Subject: [PATCH 053/105] add tests for trailing stoploss --- freqtrade/tests/test_freqtradebot.py | 41 ++++++++++++++++++---------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 468f5f8aa..7b2786421 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1694,6 +1694,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, """ Test sell_profit_only feature when enabled and we have a loss """ + buy_price = limit_buy_order['price'] patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) @@ -1702,9 +1703,9 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ - 'bid': 0.0000182, - 'ask': 0.0000183, - 'last': 0.0000182 + 'bid': buy_price - 0.000001, + 'ask': buy_price - 0.000001, + 'last': buy_price - 0.000001 }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -1713,7 +1714,6 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, conf = deepcopy(default_conf) conf['trailing_stop'] = True conf['trailing_stop_positive'] = 0.01 - print(limit_buy_order) freqtrade = FreqtradeBot(conf) freqtrade.create_trade() @@ -1722,22 +1722,35 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, caplog.set_level(logging.DEBUG) # stop-loss not reached assert freqtrade.handle_trade(trade) is False - # Adjusted stoploss - assert log_has(f'using positive stop loss mode: 0.01 since we have profit 0.64779142', - caplog.record_tuples) - assert log_has(f'adjusted stop loss', caplog.record_tuples) - assert trade.stop_loss == 0.000018018 + + # Raise ticker above buy price mocker.patch('freqtrade.exchange.Exchange.get_ticker', MagicMock(return_value={ - 'bid': 0.0000172, - 'ask': 0.0000173, - 'last': 0.0000172 - })) + 'bid': buy_price + 0.000003, + 'ask': buy_price + 0.000003, + 'last': buy_price + 0.000003 + })) + # stop-loss not reached, adjusted stoploss + assert freqtrade.handle_trade(trade) is False + assert log_has(f'using positive stop loss mode: 0.01 since we have profit 0.26662643', + caplog.record_tuples) + assert log_has(f'adjusted stop loss', caplog.record_tuples) + assert trade.stop_loss == 0.0000138501 + + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(return_value={ + 'bid': buy_price + 0.000002, + 'ask': buy_price + 0.000002, + 'last': buy_price + 0.000002 + })) + # Lower price again (but still positive) assert freqtrade.handle_trade(trade) is True assert log_has( - f'HIT STOP: current price at 0.000017, stop loss is {trade.stop_loss:.6f}, ' + f'HIT STOP: current price at {buy_price + 0.000002:.6f}, ' + f'stop loss is {trade.stop_loss:.6f}, ' f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) + def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: """ From e6e868a03c3f4b6f94d89449509b8c618f53b84f Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:54:29 +0200 Subject: [PATCH 054/105] remove markdown code type as it is not valid json --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index f24adb61e..984f2529b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -62,7 +62,7 @@ In this case a trade amount is calclulated as `currency_balanse / (max_open_trad in minutes and the value is the minimum ROI in percent. See the example below: -``` json +``` "minimal_roi": { "40": 0.0, # Sell after 40 minutes if the profit is not negative "30": 0.01, # Sell after 30 minutes if there is at least 1% profit From 8ecdae67e1417de14177da25ce30df8a28c7ad66 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:57:41 +0200 Subject: [PATCH 055/105] add mypy ignore (and comment as to why) --- freqtrade/analyze.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index d74f9566a..bf6b89b17 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -237,7 +237,8 @@ class Analyze(object): stop_loss_value = self.strategy.stoploss if 'trailing_stop_positive' in self.config and current_profit > 0: - stop_loss_value = self.config.get('trailing_stop_positive') + # Ignore mypy error check in configuration that this is a float + stop_loss_value = self.config.get('trailing_stop_positive') # type: ignore logger.debug(f"using positive stop loss mode: {stop_loss_value} " f"since we have profit {current_profit}") From 19beb0941f4354e00a89238ff1630d86d5df9050 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 27 Jun 2018 14:23:07 +0200 Subject: [PATCH 056/105] Update ccxt from 1.14.272 to 1.14.288 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d4f8b2f06..62e2e3983 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.272 +ccxt==1.14.288 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 860b270e309a022e607d789754eb3ab338475d58 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 19:49:08 +0200 Subject: [PATCH 057/105] update db migrate script to work for more changes --- freqtrade/persistence.py | 33 +++++++++++++++++++---------- freqtrade/tests/test_persistence.py | 3 +++ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 4fc83a18f..c72974c61 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -66,6 +66,10 @@ def has_column(columns, searchname: str) -> bool: return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 +def get_column_def(columns, column: str, default: str) -> str: + return default if not has_column(columns, column) else column + + def check_migrate(engine) -> None: """ Checks if migration is necessary and migrates if necessary @@ -74,17 +78,26 @@ def check_migrate(engine) -> None: cols = inspector.get_columns('trades') - if not has_column(cols, 'fee_open'): + # Check for latest column + if not has_column(cols, 'max_rate'): + open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') + close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') + stop_loss = get_column_def(cols, 'stop_loss', '0.0') + initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') + max_rate = get_column_def(cols, 'max_rate', '0.0') + # Schema migration necessary engine.execute("alter table trades rename to trades_bak") # let SQLAlchemy create the schema as required _DECL_BASE.metadata.create_all(engine) # Copy data back - following the correct schema - engine.execute("""insert into trades + engine.execute(f"""insert into trades (id, exchange, pair, is_open, fee_open, fee_close, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, - stake_amount, amount, open_date, close_date, open_order_id) + stake_amount, amount, open_date, close_date, open_order_id, + stop_loss, initial_stop_loss, max_rate + ) select id, lower(exchange), case when instr(pair, '_') != 0 then @@ -94,9 +107,12 @@ def check_migrate(engine) -> None: end pair, is_open, fee fee_open, fee fee_close, - open_rate, null open_rate_requested, close_rate, - null close_rate_requested, close_profit, - stake_amount, amount, open_date, close_date, open_order_id + open_rate, {open_rate_requested} open_rate_requested, close_rate, + {close_rate_requested} close_rate_requested, close_profit, + stake_amount, amount, open_date, close_date, open_order_id, + {stop_loss} stop_loss, {initial_stop_loss} initial_stop_loss, + {max_rate} max_rate + from trades_bak """) @@ -104,11 +120,6 @@ def check_migrate(engine) -> None: inspector = inspect(engine) cols = inspector.get_columns('trades') - if not has_column(cols, 'open_rate_requested'): - engine.execute("alter table trades add open_rate_requested float") - if not has_column(cols, 'close_rate_requested'): - engine.execute("alter table trades add close_rate_requested float") - def cleanup() -> None: """ diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 45957853f..0cd2d089c 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -400,6 +400,9 @@ def test_migrate_old(mocker, default_conf, fee): assert trade.stake_amount == default_conf.get("stake_amount") assert trade.pair == "ETC/BTC" assert trade.exchange == "bittrex" + assert trade.max_rate == 0.0 + assert trade.stop_loss == 0.0 + assert trade.initial_stop_loss == 0.0 def test_migrate_new(mocker, default_conf, fee): From d5ad066f8d4af30340da2e12eaef61a88bf48224 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 20:15:25 +0200 Subject: [PATCH 058/105] support multiple db transitions by keeping the backup-table dynamic --- freqtrade/persistence.py | 12 +++++++++--- freqtrade/tests/test_persistence.py | 10 +++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index c72974c61..2b226e53a 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -77,6 +77,13 @@ def check_migrate(engine) -> None: inspector = inspect(engine) cols = inspector.get_columns('trades') + tabs = inspector.get_table_names() + table_back_name = 'trades_bak' + i = 0 + while table_back_name in tabs: + i += 1 + table_back_name = f'trades_bak{i}' + logger.info(f'trying {table_back_name}') # Check for latest column if not has_column(cols, 'max_rate'): @@ -87,7 +94,7 @@ def check_migrate(engine) -> None: max_rate = get_column_def(cols, 'max_rate', '0.0') # Schema migration necessary - engine.execute("alter table trades rename to trades_bak") + engine.execute(f"alter table trades rename to {table_back_name}") # let SQLAlchemy create the schema as required _DECL_BASE.metadata.create_all(engine) @@ -112,8 +119,7 @@ def check_migrate(engine) -> None: stake_amount, amount, open_date, close_date, open_order_id, {stop_loss} stop_loss, {initial_stop_loss} initial_stop_loss, {max_rate} max_rate - - from trades_bak + from {table_back_name} """) # Reread columns - the above recreated the table! diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 0cd2d089c..7ef4d2c25 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -7,6 +7,7 @@ from sqlalchemy import create_engine from freqtrade import constants, OperationalException from freqtrade.persistence import Trade, init, clean_dry_run_db +from freqtrade.tests.conftest import log_has @pytest.fixture(scope='function') @@ -405,7 +406,7 @@ def test_migrate_old(mocker, default_conf, fee): assert trade.initial_stop_loss == 0.0 -def test_migrate_new(mocker, default_conf, fee): +def test_migrate_new(mocker, default_conf, fee, caplog): """ Test Database migration (starting with new pairformat) """ @@ -442,6 +443,9 @@ def test_migrate_new(mocker, default_conf, fee): # Create table using the old format engine.execute(create_table_old) engine.execute(insert_table_old) + + # fake previous backup + engine.execute("create table trades_bak as select * from trades") # Run init to test migration init(default_conf) @@ -456,6 +460,10 @@ def test_migrate_new(mocker, default_conf, fee): assert trade.stake_amount == default_conf.get("stake_amount") assert trade.pair == "ETC/BTC" assert trade.exchange == "binance" + assert trade.max_rate == 0.0 + assert trade.stop_loss == 0.0 + assert trade.initial_stop_loss == 0.0 + assert log_has("trying trades_bak1", caplog.record_tuples) def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): From 7cecae52799eb480e2265802e41ef0bd863d5a1c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 28 Jun 2018 14:23:07 +0200 Subject: [PATCH 059/105] Update ccxt from 1.14.288 to 1.14.289 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 62e2e3983..76dba3c49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.288 +ccxt==1.14.289 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 2d4ce593b511e3f9cd977a307c002307b4c6a052 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 19:48:05 +0200 Subject: [PATCH 060/105] catch crash with cobinhood fixes #959 --- freqtrade/freqtradebot.py | 11 +++++++---- freqtrade/tests/test_freqtradebot.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e25ed66cf..985ecb370 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -277,11 +277,14 @@ class FreqtradeBot(object): return None min_stake_amounts = [] - if 'cost' in market['limits'] and 'min' in market['limits']['cost']: - min_stake_amounts.append(market['limits']['cost']['min']) + limits = market['limits'] + if ('cost' in limits and 'min' in limits['cost'] + and limits['cost']['min'] is not None): + min_stake_amounts.append(limits['cost']['min']) - if 'amount' in market['limits'] and 'min' in market['limits']['amount']: - min_stake_amounts.append(market['limits']['amount']['min'] * price) + if ('amount' in limits and 'min' in limits['amount'] + and limits['amount']['min'] is not None): + min_stake_amounts.append(limits['amount']['min'] * price) if not min_stake_amounts: return None diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..ff1b7161d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -348,6 +348,34 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) assert result is None + # no cost Min + mocker.patch( + 'freqtrade.exchange.Exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {"min": None}, + 'amount': {} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result is None + + # no amount Min + mocker.patch( + 'freqtrade.exchange.Exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {}, + 'amount': {"min": None} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result is None + # empty 'cost'/'amount' section mocker.patch( 'freqtrade.exchange.Exchange.get_markets', From 8ec9a0974978242fb1bd47587151eec6a0a5bda6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 21:22:43 +0200 Subject: [PATCH 061/105] Standardize retrier exception testing --- freqtrade/tests/exchange/test_exchange.py | 141 ++++++---------------- 1 file changed, 39 insertions(+), 102 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 53e59b34b..4d46d7e61 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,6 +14,22 @@ from freqtrade.exchange import Exchange, API_RETRY_COUNT from freqtrade.tests.conftest import log_has, get_patched_exchange +def validate_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): + """Function to test exception handling """ + + with pytest.raises(TemporaryError): + api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[innerfun].call_count == API_RETRY_COUNT + 1 + + with pytest.raises(OperationalException): + api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[innerfun].call_count == 1 + + def test_init(default_conf, mocker, caplog): caplog.set_level(logging.INFO) get_patched_exchange(mocker, default_conf) @@ -243,17 +259,8 @@ def test_get_balances_prod(default_conf, mocker): assert exchange.get_balances()['1ST']['total'] == 10.0 assert exchange.get_balances()['1ST']['used'] == 0.0 - with pytest.raises(TemporaryError): - api_mock.fetch_balance = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_balances() - assert api_mock.fetch_balance.call_count == API_RETRY_COUNT + 1 - - with pytest.raises(OperationalException): - api_mock.fetch_balance = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_balances() - assert api_mock.fetch_balance.call_count == 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_balances", "fetch_balance") def test_get_tickers(default_conf, mocker): @@ -282,15 +289,8 @@ def test_get_tickers(default_conf, mocker): assert tickers['BCH/BTC']['bid'] == 0.6 assert tickers['BCH/BTC']['ask'] == 0.5 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_tickers() - - with pytest.raises(OperationalException): - api_mock.fetch_tickers = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_tickers() + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_tickers", "fetch_tickers") with pytest.raises(OperationalException): api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NotSupported) @@ -345,15 +345,9 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=False) assert api_mock.fetch_ticker.call_count == 0 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_ticker = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_ticker(pair='ETH/BTC', refresh=True) - - with pytest.raises(OperationalException): - api_mock.fetch_ticker = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_ticker(pair='ETH/BTC', refresh=True) + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker", "fetch_ticker", + pair='ETH/BTC', refresh=True) api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock) @@ -416,17 +410,9 @@ def test_get_ticker_history(default_conf, mocker): assert ticks[0][4] == 9 assert ticks[0][5] == 10 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - # new symbol to get around cache - exchange.get_ticker_history('ABCD/BTC', default_conf['ticker_interval']) - - with pytest.raises(OperationalException): - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - # new symbol to get around cache - exchange.get_ticker_history('EFGH/BTC', default_conf['ticker_interval']) + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker_history", "fetch_ohlcv", + pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) def test_get_ticker_history_sort(default_conf, mocker): @@ -515,24 +501,15 @@ def test_cancel_order(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) assert exchange.cancel_order(order_id='_', pair='TKN/BTC') == 123 - with pytest.raises(TemporaryError): - api_mock.cancel_order = MagicMock(side_effect=ccxt.NetworkError) - - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.cancel_order(order_id='_', pair='TKN/BTC') - assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(DependencyException): api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.cancel_order(order_id='_', pair='TKN/BTC') assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(OperationalException): - api_mock.cancel_order = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.cancel_order(order_id='_', pair='TKN/BTC') - assert api_mock.cancel_order.call_count == 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + "cancel_order", "cancel_order", + order_id='_', pair='TKN/BTC') def test_get_order(default_conf, mocker): @@ -550,23 +527,15 @@ def test_get_order(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) assert exchange.get_order('X', 'TKN/BTC') == 456 - with pytest.raises(TemporaryError): - api_mock.fetch_order = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_order(order_id='_', pair='TKN/BTC') - assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(DependencyException): api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_order(order_id='_', pair='TKN/BTC') assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(OperationalException): - api_mock.fetch_order = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_order(order_id='_', pair='TKN/BTC') - assert api_mock.fetch_order.call_count == 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_order', 'fetch_order', + order_id='_', pair='TKN/BTC') def test_name(default_conf, mocker): @@ -651,19 +620,9 @@ def test_get_trades_for_order(default_conf, mocker): assert len(orders) == 1 assert orders[0]['price'] == 165 - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_trades_for_order(order_id, 'LTC/BTC', since) - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_trades_for_order(order_id, 'LTC/BTC', since) - assert api_mock.fetch_my_trades.call_count == API_RETRY_COUNT + 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_trades_for_order', 'fetch_my_trades', + order_id=order_id, pair='LTC/BTC', since=since) def test_get_markets(default_conf, mocker, markets): @@ -677,19 +636,8 @@ def test_get_markets(default_conf, mocker, markets): assert ret[0]["id"] == "ethbtc" assert ret[0]["symbol"] == "ETH/BTC" - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.fetch_markets = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_markets() - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.fetch_markets = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_markets() - assert api_mock.fetch_markets.call_count == API_RETRY_COUNT + 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_markets', 'fetch_markets') def test_get_fee(default_conf, mocker): @@ -704,19 +652,8 @@ def test_get_fee(default_conf, mocker): assert exchange.get_fee() == 0.025 - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.calculate_fee = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_fee() - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.calculate_fee = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_fee() - assert api_mock.calculate_fee.call_count == API_RETRY_COUNT + 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_fee', 'calculate_fee') def test_get_amount_lots(default_conf, mocker): From ebbfc720b24c6a378dcf79b5b1c5205f8dd95dea Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 21:51:59 +0200 Subject: [PATCH 062/105] increase test coverage --- freqtrade/tests/exchange/test_exchange.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 4d46d7e61..75394ca84 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -232,6 +232,12 @@ def test_get_balance_prod(default_conf, mocker): exchange.get_balance(currency='BTC') + with pytest.raises(TemporaryError): + # api_mock.fetch_balance = MagicMock(return_value={}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + mocker.patch('freqtrade.exchange.Exchange.get_balances', MagicMock(return_value={})) + exchange.get_balance(currency='BTC') + def test_get_balances_dry_run(default_conf, mocker): default_conf['dry_run'] = True @@ -624,6 +630,9 @@ def test_get_trades_for_order(default_conf, mocker): 'get_trades_for_order', 'fetch_my_trades', order_id=order_id, pair='LTC/BTC', since=since) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + assert exchange.get_trades_for_order(order_id, 'LTC/BTC', since) == [] + def test_get_markets(default_conf, mocker, markets): api_mock = MagicMock() From fe8a21681e3fd8af90093a9fd004e7acd2c51f77 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 21:56:37 +0200 Subject: [PATCH 063/105] add test for Not supported --- freqtrade/tests/exchange/test_exchange.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 75394ca84..012c55356 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -232,8 +232,7 @@ def test_get_balance_prod(default_conf, mocker): exchange.get_balance(currency='BTC') - with pytest.raises(TemporaryError): - # api_mock.fetch_balance = MagicMock(return_value={}) + with pytest.raises(TemporaryError, match=r'.*balance due to malformed exchange response:.*'): exchange = get_patched_exchange(mocker, default_conf, api_mock) mocker.patch('freqtrade.exchange.Exchange.get_balances', MagicMock(return_value={})) exchange.get_balance(currency='BTC') @@ -420,6 +419,11 @@ def test_get_ticker_history(default_conf, mocker): "get_ticker_history", "fetch_ohlcv", pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): + api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_ticker_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + def test_get_ticker_history_sort(default_conf, mocker): api_mock = MagicMock() From 15c7854e7f79c03132e86d789f7725108aec4529 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 22:11:45 +0200 Subject: [PATCH 064/105] add test for exchange_has --- freqtrade/tests/exchange/test_exchange.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 012c55356..25c6fa0c2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -113,6 +113,20 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): Exchange(conf) +def test_exchangehas(default_conf, mocker): + exchange = get_patched_exchange(mocker, default_conf) + assert not exchange.exchange_has('ASDFASDF') + api_mock = MagicMock() + + type(api_mock).has = PropertyMock(return_value={'deadbeef': True}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + assert exchange.exchange_has("deadbeef") + + type(api_mock).has = PropertyMock(return_value={'deadbeef': False}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + assert not exchange.exchange_has("deadbeef") + + def test_buy_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) From dcdc18a33868a5630b5b747eb2773819154fb88c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 22:18:38 +0200 Subject: [PATCH 065/105] rename test-function --- freqtrade/tests/exchange/test_exchange.py | 50 +++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 25c6fa0c2..712205893 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,8 +14,8 @@ from freqtrade.exchange import Exchange, API_RETRY_COUNT from freqtrade.tests.conftest import log_has, get_patched_exchange -def validate_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): - """Function to test exception handling """ +def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): + """Function to test ccxt exception handling """ with pytest.raises(TemporaryError): api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.NetworkError) @@ -278,8 +278,8 @@ def test_get_balances_prod(default_conf, mocker): assert exchange.get_balances()['1ST']['total'] == 10.0 assert exchange.get_balances()['1ST']['used'] == 0.0 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_balances", "fetch_balance") + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_balances", "fetch_balance") def test_get_tickers(default_conf, mocker): @@ -308,8 +308,8 @@ def test_get_tickers(default_conf, mocker): assert tickers['BCH/BTC']['bid'] == 0.6 assert tickers['BCH/BTC']['ask'] == 0.5 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_tickers", "fetch_tickers") + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_tickers", "fetch_tickers") with pytest.raises(OperationalException): api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NotSupported) @@ -364,9 +364,9 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=False) assert api_mock.fetch_ticker.call_count == 0 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_ticker", "fetch_ticker", - pair='ETH/BTC', refresh=True) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker", "fetch_ticker", + pair='ETH/BTC', refresh=True) api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock) @@ -429,9 +429,9 @@ def test_get_ticker_history(default_conf, mocker): assert ticks[0][4] == 9 assert ticks[0][5] == 10 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_ticker_history", "fetch_ohlcv", - pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker_history", "fetch_ohlcv", + pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) @@ -531,9 +531,9 @@ def test_cancel_order(default_conf, mocker): exchange.cancel_order(order_id='_', pair='TKN/BTC') assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "cancel_order", "cancel_order", - order_id='_', pair='TKN/BTC') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "cancel_order", "cancel_order", + order_id='_', pair='TKN/BTC') def test_get_order(default_conf, mocker): @@ -557,9 +557,9 @@ def test_get_order(default_conf, mocker): exchange.get_order(order_id='_', pair='TKN/BTC') assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_order', 'fetch_order', - order_id='_', pair='TKN/BTC') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_order', 'fetch_order', + order_id='_', pair='TKN/BTC') def test_name(default_conf, mocker): @@ -644,9 +644,9 @@ def test_get_trades_for_order(default_conf, mocker): assert len(orders) == 1 assert orders[0]['price'] == 165 - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_trades_for_order', 'fetch_my_trades', - order_id=order_id, pair='LTC/BTC', since=since) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_trades_for_order', 'fetch_my_trades', + order_id=order_id, pair='LTC/BTC', since=since) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) assert exchange.get_trades_for_order(order_id, 'LTC/BTC', since) == [] @@ -663,8 +663,8 @@ def test_get_markets(default_conf, mocker, markets): assert ret[0]["id"] == "ethbtc" assert ret[0]["symbol"] == "ETH/BTC" - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_markets', 'fetch_markets') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_markets', 'fetch_markets') def test_get_fee(default_conf, mocker): @@ -679,8 +679,8 @@ def test_get_fee(default_conf, mocker): assert exchange.get_fee() == 0.025 - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_fee', 'calculate_fee') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_fee', 'calculate_fee') def test_get_amount_lots(default_conf, mocker): From cf6b1a637ac0a2312c15d926093851a3a1adf4f1 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 22:32:28 +0200 Subject: [PATCH 066/105] increase exchange code coverage --- freqtrade/tests/exchange/test_exchange.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 712205893..99b960885 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -36,7 +36,7 @@ def test_init(default_conf, mocker, caplog): assert log_has('Instance is running with dry_run enabled', caplog.record_tuples) -def test_init_exception(default_conf): +def test_init_exception(default_conf, mocker): default_conf['exchange']['name'] = 'wrong_exchange_name' with pytest.raises( @@ -44,6 +44,13 @@ def test_init_exception(default_conf): match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): Exchange(default_conf) + default_conf['exchange']['name'] = 'binance' + with pytest.raises( + OperationalException, + match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): + mocker.patch("ccxt.binance", MagicMock(side_effect=AttributeError)) + Exchange(default_conf) + def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() From 8a941f3aa81ee6aad4778be0e1ae0952ce3108c9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 29 Jun 2018 14:23:06 +0200 Subject: [PATCH 067/105] Update ccxt from 1.14.289 to 1.14.295 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 76dba3c49..6995c2513 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.289 +ccxt==1.14.295 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 0ce08932ed31edde076b91589526ffcd06f428be Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 30 Jun 2018 09:54:31 +0300 Subject: [PATCH 068/105] mypy fixes --- freqtrade/optimize/hyperopt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b8b53ab3c..eb9015356 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -62,7 +62,7 @@ class Hyperopt(Backtesting): # Previous evaluations self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') - self.trials = [] + self.trials: List = [] def get_args(self, params): dimensions = self.hyperopt_space() @@ -191,7 +191,7 @@ class Hyperopt(Backtesting): ] @staticmethod - def indicator_space() -> Dict[str, Any]: + def indicator_space() -> List[Dimension]: """ Define your Hyperopt space for searching strategy parameters """ @@ -349,7 +349,7 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) - self.exchange = None + self.exchange = None # type: ignore self.load_previous_results() cpus = multiprocessing.cpu_count() From 98108a78f1ff5ccd0ca4390a20885137f2d65637 Mon Sep 17 00:00:00 2001 From: Nullart Date: Thu, 14 Jun 2018 09:32:52 +0800 Subject: [PATCH 069/105] separating unfulfilled timeouts for buy and sell --- config.json.example | 5 ++++- config_full.json.example | 5 ++++- freqtrade/constants.py | 9 ++++++++- freqtrade/freqtradebot.py | 19 ++++++++++++------- freqtrade/tests/conftest.py | 5 ++++- freqtrade/tests/test_freqtradebot.py | 8 ++++---- 6 files changed, 36 insertions(+), 15 deletions(-) diff --git a/config.json.example b/config.json.example index e5bdc95b1..336a0c374 100644 --- a/config.json.example +++ b/config.json.example @@ -5,7 +5,10 @@ "fiat_display_currency": "USD", "ticker_interval" : "5m", "dry_run": false, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy":10, + "sell":30 + } "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/config_full.json.example b/config_full.json.example index 231384acc..1985edc70 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -12,7 +12,10 @@ "0": 0.04 }, "stoploss": -0.10, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy":10, + "sell":30 + } "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 0f12905e3..44bd64fe9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -61,7 +61,14 @@ CONF_SCHEMA = { 'minProperties': 1 }, 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, - 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, + 'unfilledtimeout': { + 'type': 'object', + 'properties': { + 'use_book_order': {'type': 'boolean'}, + 'buy': {'type': 'number', 'minimum': 3}, + 'sell': {'type': 'number', 'minimum': 10} + } + }, 'bid_strategy': { 'type': 'object', 'properties': { diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e25ed66cf..c5f932119 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -160,7 +160,7 @@ class FreqtradeBot(object): if 'unfilledtimeout' in self.config: # Check and handle any timed out open orders - self.check_handle_timedout(self.config['unfilledtimeout']) + self.check_handle_timedout() Trade.session.flush() except TemporaryError as error: @@ -492,13 +492,16 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False - def check_handle_timedout(self, timeoutvalue: int) -> None: + def check_handle_timedout(self) -> None: """ Check if any orders are timed out and cancel if neccessary :param timeoutvalue: Number of minutes until order is considered timed out :return: None """ - timeoutthreashold = arrow.utcnow().shift(minutes=-timeoutvalue).datetime + buy_timeout = self.config['unfilledtimeout']['buy'] + sell_timeout = self.config['unfilledtimeout']['sell'] + buy_timeoutthreashold = arrow.utcnow().shift(minutes=-buy_timeout).datetime + sell_timeoutthreashold = arrow.utcnow().shift(minutes=-sell_timeout).datetime for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all(): try: @@ -521,10 +524,12 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ if int(order['remaining']) == 0: continue - if order['side'] == 'buy' and ordertime < timeoutthreashold: - self.handle_timedout_limit_buy(trade, order) - elif order['side'] == 'sell' and ordertime < timeoutthreashold: - self.handle_timedout_limit_sell(trade, order) + # Check if trade is still actually open + if (order['status'] == 'open'): + if order['side'] == 'buy' and ordertime < buy_timeoutthreashold: + self.handle_timedout_limit_buy(trade, order) + elif order['side'] == 'sell' and ordertime < sell_timeoutthreashold: + self.handle_timedout_limit_sell(trade, order) # FIX: 20180110, why is cancel.order unconditionally here, whereas # it is conditionally called in the diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index d9ccaa325..4877fe556 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -100,7 +100,10 @@ def default_conf(): "0": 0.04 }, "stoploss": -0.10, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy": 10, + "sell": 30 + }, "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..fb7579d45 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1124,7 +1124,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe Trade.session.add(trade_buy) # check it does cancel buy orders over the time limit - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() @@ -1165,7 +1165,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, Trade.session.add(trade_sell) # check it does cancel sell orders over the time limit - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 assert trade_sell.is_open is True @@ -1205,7 +1205,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old # check it does cancel buy orders over the time limit # note this is for a partially-complete buy order - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() @@ -1256,7 +1256,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - 'recent call last):\n.*' ) - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert filter(regexp.match, caplog.record_tuples) From c447644fd16386868c6a4a1aa6b8a5e139af8c22 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 30 Jun 2018 14:23:06 +0200 Subject: [PATCH 070/105] Update ccxt from 1.14.295 to 1.14.301 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6995c2513..72decf95a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.295 +ccxt==1.14.301 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 5a591e01c08fa0c60db152da2c04ec04dfd4e58a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 30 Jun 2018 14:23:07 +0200 Subject: [PATCH 071/105] Update sqlalchemy from 1.2.8 to 1.2.9 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 72decf95a..50441879a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ccxt==1.14.301 -SQLAlchemy==1.2.8 +SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 From 14e12bd3c09c42a4ed20c170765a55826e304b04 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 30 Jun 2018 17:37:34 +0200 Subject: [PATCH 072/105] Fix missing comma in example.json --- config.json.example | 2 +- config_full.json.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index 336a0c374..606167ea3 100644 --- a/config.json.example +++ b/config.json.example @@ -8,7 +8,7 @@ "unfilledtimeout": { "buy":10, "sell":30 - } + }, "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/config_full.json.example b/config_full.json.example index 1985edc70..4bb794937 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -15,7 +15,7 @@ "unfilledtimeout": { "buy":10, "sell":30 - } + }, "bid_strategy": { "ask_last_balance": 0.0 }, From 9e3e900f78d5105f6d145d80fd5d97dbea0b9cac Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 30 Jun 2018 17:49:46 +0200 Subject: [PATCH 073/105] Add get_markets mock to new tests --- freqtrade/tests/test_freqtradebot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..f66864946 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1599,6 +1599,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) @@ -1616,7 +1617,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke assert freqtrade.handle_trade(trade) is True -def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None: +def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ @@ -1634,6 +1635,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) From 8f49d5eb1002230fb424e4b505ec5f9561e778ea Mon Sep 17 00:00:00 2001 From: Nullart Date: Thu, 14 Jun 2018 18:49:43 +0800 Subject: [PATCH 074/105] documentation updates --- docs/configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5279ed06e..ea3e99f15 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -22,7 +22,8 @@ The table below will list all configuration parameters. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. -| `unfilledtimeout` | 0 | No | How long (in minutes) the bot will wait for an unfilled order to complete, after which the order will be cancelled. +| `unfilledtimeout.buy` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. +| `unfilledtimeout.sell` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. From c66f858b988bd18ee24045839211f4248640b097 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:37:55 +0200 Subject: [PATCH 075/105] rename innerfun to mock_ccxt_fun --- freqtrade/tests/exchange/test_exchange.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 99b960885..6cf7f11b0 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,20 +14,20 @@ from freqtrade.exchange import Exchange, API_RETRY_COUNT from freqtrade.tests.conftest import log_has, get_patched_exchange -def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): +def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): """Function to test ccxt exception handling """ with pytest.raises(TemporaryError): - api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.NetworkError) + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) getattr(exchange, fun)(**kwargs) - assert api_mock.__dict__[innerfun].call_count == API_RETRY_COUNT + 1 + assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1 with pytest.raises(OperationalException): - api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.BaseError) + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) getattr(exchange, fun)(**kwargs) - assert api_mock.__dict__[innerfun].call_count == 1 + assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 def test_init(default_conf, mocker, caplog): From 2dc881558d65b19dd6596d3a8847da618f7d6885 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:41:19 +0200 Subject: [PATCH 076/105] address PR comments --- config.json.example | 4 ++-- config_full.json.example | 4 ++-- freqtrade/constants.py | 1 - freqtrade/freqtradebot.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/config.json.example b/config.json.example index 606167ea3..79ba2ac64 100644 --- a/config.json.example +++ b/config.json.example @@ -6,8 +6,8 @@ "ticker_interval" : "5m", "dry_run": false, "unfilledtimeout": { - "buy":10, - "sell":30 + "buy": 10, + "sell": 30 }, "bid_strategy": { "ask_last_balance": 0.0 diff --git a/config_full.json.example b/config_full.json.example index 4bb794937..d40c11a91 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -13,8 +13,8 @@ }, "stoploss": -0.10, "unfilledtimeout": { - "buy":10, - "sell":30 + "buy": 10, + "sell": 30 }, "bid_strategy": { "ask_last_balance": 0.0 diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 44bd64fe9..04d4a44ac 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -64,7 +64,6 @@ CONF_SCHEMA = { 'unfilledtimeout': { 'type': 'object', 'properties': { - 'use_book_order': {'type': 'boolean'}, 'buy': {'type': 'number', 'minimum': 3}, 'sell': {'type': 'number', 'minimum': 10} } diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c5f932119..4d6352e55 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -525,7 +525,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ continue # Check if trade is still actually open - if (order['status'] == 'open'): + if order['status'] == 'open': if order['side'] == 'buy' and ordertime < buy_timeoutthreashold: self.handle_timedout_limit_buy(trade, order) elif order['side'] == 'sell' and ordertime < sell_timeoutthreashold: From e39d88ef65e72387f18198da269a2d7660a2d6c6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:54:26 +0200 Subject: [PATCH 077/105] Address some PR comments --- freqtrade/constants.py | 2 +- freqtrade/persistence.py | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 4ba1a7094..d80eea6f4 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -62,7 +62,7 @@ CONF_SCHEMA = { }, 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, 'trailing_stop': {'type': 'boolean'}, - 'trailing_stop_positive': {'type': 'number', 'minimum': 0}, + 'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1}, 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, 'bid_strategy': { 'type': 'object', diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 2b226e53a..ad90d3f92 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -184,14 +184,8 @@ class Trade(_DECL_BASE): arrow.get(self.open_date).humanize() if self.is_open else 'closed' ) - def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool =False): - """ - this adjusts the stop loss to it's most recently observed setting - :param current_price: - :param stoploss in percent: - :param initial: - :return: - """ + def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool = False): + """this adjusts the stop loss to it's most recently observed setting""" if initial and not (self.stop_loss is None or self.stop_loss == 0): # Don't modify if called with initial and nothing to do @@ -207,7 +201,7 @@ class Trade(_DECL_BASE): self.max_rate = current_price # no stop loss assigned yet - if self.stop_loss is None or self.stop_loss == 0: + if not self.stop_loss: logger.debug("assigning new stop loss") self.stop_loss = new_loss self.initial_stop_loss = new_loss From 937644a04b96adbdb26ed54c5969945e09f314c2 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:55:51 +0200 Subject: [PATCH 078/105] change while-loop to enumerate - add intensified test for this scenario --- freqtrade/tests/test_persistence.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 7ef4d2c25..12ae6579f 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -446,6 +446,8 @@ def test_migrate_new(mocker, default_conf, fee, caplog): # fake previous backup engine.execute("create table trades_bak as select * from trades") + + engine.execute("create table trades_bak1 as select * from trades") # Run init to test migration init(default_conf) @@ -464,6 +466,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 assert log_has("trying trades_bak1", caplog.record_tuples) + assert log_has("trying trades_bak2", caplog.record_tuples) def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): From 782570e71eb6fca9c5a70cf36896d692661a14a7 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 20:03:07 +0200 Subject: [PATCH 079/105] Address PR comment --- config.json.example | 2 +- freqtrade/persistence.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index 87c27090e..e8473e919 100644 --- a/config.json.example +++ b/config.json.example @@ -36,7 +36,7 @@ "experimental": { "use_sell_signal": false, "sell_profit_only": false, - "sell_fullfilled_at_roi": false + "ignore_roi_if_buy_signal": false }, "telegram": { "enabled": true, diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index ad90d3f92..dfeb6145c 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -80,7 +80,7 @@ def check_migrate(engine) -> None: tabs = inspector.get_table_names() table_back_name = 'trades_bak' i = 0 - while table_back_name in tabs: + for i, table_back_name in enumerate(tabs): i += 1 table_back_name = f'trades_bak{i}' logger.info(f'trying {table_back_name}') From 3c5be55eb907878c73f0cfb26a4f21b3af2a4745 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 20:17:30 +0200 Subject: [PATCH 080/105] remove unnecessary variable --- freqtrade/persistence.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index dfeb6145c..aa33f7063 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -79,9 +79,7 @@ def check_migrate(engine) -> None: cols = inspector.get_columns('trades') tabs = inspector.get_table_names() table_back_name = 'trades_bak' - i = 0 for i, table_back_name in enumerate(tabs): - i += 1 table_back_name = f'trades_bak{i}' logger.info(f'trying {table_back_name}') From a58d51ded0fcfd06cbe7344da202e353e9deff10 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 2 Jul 2018 09:56:58 +0300 Subject: [PATCH 081/105] update hyperopt documentation --- docs/hyperopt.md | 279 ++++++++++++++++------------------------------- 1 file changed, 96 insertions(+), 183 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 2ad94896a..8d3fe6704 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -1,153 +1,109 @@ # Hyperopt This page explains how to tune your strategy by finding the optimal -parameters with Hyperopt. +parameters, process called hyperparameter optimization. The bot uses several +algorithms included in `scikit-optimize` package to accomplish this. The +search will burn all your CPU cores, make your laptop sound like a fighter jet +and still take a long time. ## Table of Contents - [Prepare your Hyperopt](#prepare-hyperopt) - - [1. Configure your Guards and Triggers](#1-configure-your-guards-and-triggers) - - [2. Update the hyperopt config file](#2-update-the-hyperopt-config-file) -- [Advanced Hyperopt notions](#advanced-notions) - - [Understand the Guards and Triggers](#understand-the-guards-and-triggers) +- [Configure your Guards and Triggers](#configure-your-guards-and-triggers) +- [Solving a Mystery](#solving-a-mystery) +- [Adding New Indicators](#adding-new-indicators) - [Execute Hyperopt](#execute-hyperopt) - [Understand the hyperopts result](#understand-the-backtesting-result) -## Prepare Hyperopt -Before we start digging in Hyperopt, we recommend you to take a look at -your strategy file located into [user_data/strategies/](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) +## Prepare Hyperopting +We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) -### 1. Configure your Guards and Triggers +### Configure your Guards and Triggers There are two places you need to change in your strategy file to add a new buy strategy for testing: -- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L278-L294). -- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297) known as `SPACE`. +- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L278-L294). +- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L218-L229) +and the associated methods `indicator_space`, `roi_space`, `stoploss_space`. -There you have two different type of indicators: 1. `guards` and 2. -`triggers`. +There you have two different type of indicators: 1. `guards` and 2. `triggers`. 1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. 2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or buy when close price touches lower bollinger band. -HyperOpt will, for each eval round, pick just ONE trigger, and possibly -multiple guards. So that the constructed strategy will be something like +Hyperoptimization will, for each eval round, pick one trigger and possibly +multiple guards. The constructed strategy will be something like "*buy exactly when close price touches lower bollinger band, BUT only if ADX > 10*". - -If you have updated the buy strategy, means change the content of +If you have updated the buy strategy, ie. changed the contents of `populate_buy_trend()` method you have to update the `guards` and -`triggers` hyperopts must used. +`triggers` hyperopts must use. -As for an example if your `populate_buy_trend()` method is: -```python -def populate_buy_trend(dataframe: DataFrame) -> DataFrame: - dataframe.loc[ - (dataframe['rsi'] < 35) & - (dataframe['adx'] > 65), - 'buy'] = 1 +## Solving a Mystery - return dataframe -``` +Let's say you are curious: should you use MACD crossings or lower Bollinger +Bands to trigger your buys. And you also wonder should you use RSI or ADX to +help with those buy decisions. If you decide to use RSI or ADX, which values +should I use for them? So let's use hyperparameter optimization to solve this +mystery. -Your hyperopt file must contain `guards` to find the right value for -`(dataframe['adx'] > 65)` & and `(dataframe['plus_di'] > 0.5)`. That -means you will need to enable/disable triggers. - -In our case the `SPACE` and `populate_buy_trend` in your strategy file -will look like: -```python -space = { - 'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)} - ]), - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 15, 50, 1)} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'lower_bb'}, - {'type': 'faststoch10'}, - {'type': 'ao_cross_zero'}, - {'type': 'ema5_cross_ema10'}, - {'type': 'macd_cross_signal'}, - {'type': 'sar_reversal'}, - {'type': 'stochf_cross'}, - {'type': 'ht_sine'}, - ]), -} - -... - -def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - conditions = [] - # GUARDS AND TRENDS - if params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) - - # TRIGGERS - triggers = { - 'lower_bb': dataframe['tema'] <= dataframe['blower'], - 'faststoch10': (crossed_above(dataframe['fastd'], 10.0)), - 'ao_cross_zero': (crossed_above(dataframe['ao'], 0.0)), - 'ema5_cross_ema10': (crossed_above(dataframe['ema5'], dataframe['ema10'])), - 'macd_cross_signal': (crossed_above(dataframe['macd'], dataframe['macdsignal'])), - 'sar_reversal': (crossed_above(dataframe['close'], dataframe['sar'])), - 'stochf_cross': (crossed_above(dataframe['fastk'], dataframe['fastd'])), - 'ht_sine': (crossed_above(dataframe['htleadsine'], dataframe['htsine'])), - } - ... -``` - - -### 2. Update the hyperopt config file -Hyperopt is using a dedicated config file. Currently hyperopt -cannot use your config file. It is also made on purpose to allow you -testing your strategy with different configurations. - -The Hyperopt configuration is located in -[user_data/hyperopt_conf.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopt_conf.py). - - -## Advanced notions -### Understand the Guards and Triggers -When you need to add the new guards and triggers to be hyperopt -parameters, you do this by adding them into the [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297). - -If it's a trigger, you add one line to the 'trigger' choice group and that's it. - -If it's a guard, you will add a line like this: -``` -'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)} -]), -``` -This says, "*one of the guards is RSI, it can have two values, enabled or -disabled. If it is enabled, try different values for it between 20 and 40*". - -So, the part of the strategy builder using the above setting looks like -this: +We will start by defining a search space: ``` -if params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return [ + Integer(20, 40, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal'], name='trigger') + ] ``` -It checks if Hyperopt wants the RSI guard to be enabled for this -round `params['rsi']['enabled']` and if it is, then it will add a -condition that says RSI must be smaller than the value hyperopt picked -for this evaluation, which is given in the `params['rsi']['value']`. +Above definition says: I have five parameters I want you to randomly combine +to find the best combination. Two of them are integer values (`adx-value` +and `rsi-value`) and I want you test in the range of values 20 to 40. +Then we have three category variables. First two are either `True` or `False`. +We use these to either enable or disable the ADX and RSI guards. The last +one we call `trigger` and use it to decide which buy trigger we want to use. -That's it. Now you can add new parts of strategies to Hyperopt and it -will try all the combinations with all different values in the search -for best working algo. +So let's write the buy strategy using these values: +``` + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + conditions = [] + # GUARDS AND TRENDS + if 'adx-enabled' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi-enabled' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) -### Add a new Indicators + # TRIGGERS + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend +``` + +Hyperopting will now call this `populate_buy_trend` as many times you ask it (`epochs`) +with different value combinations. It will then use the given historical data and make +buys based on the buy signals generated with the above function and based on the results +it will end with telling you which paramter combination produced the best profits. + +### Adding New Indicators If you want to test an indicator that isn't used by the bot currently, you need to add it to the `populate_indicators()` method in `hyperopt.py`. @@ -164,12 +120,12 @@ python3 ./freqtrade/main.py -c config.json hyperopt -e 5000 The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. -### Execute hyperopt with different ticker-data source +### Execute Hyperopt with Different Ticker-Data Source If you would like to hyperopt parameters using an alternate ticker data that you have on-disk, use the `--datadir PATH` option. Default hyperopt will use data from directory `user_data/data`. -### Running hyperopt with smaller testset +### Running Hyperopt with Smaller Testset Use the `--timeperiod` argument to change how much of the testset you want to use. The last N ticks/timeframes will be used. Example: @@ -178,7 +134,7 @@ Example: python3 ./freqtrade/main.py hyperopt --timeperiod -200 ``` -### Running hyperopt with smaller search space +### Running Hyperopt with Smaller Search Space Use the `--spaces` argument to limit the search space used by hyperopt. Letting Hyperopt optimize everything is a huuuuge search space. Often it might make more sense to start by just searching for initial buy algorithm. @@ -193,87 +149,44 @@ Legal values are: - `stoploss`: search for the best stoploss value - space-separated list of any of the above values for example `--spaces roi stoploss` -## Understand the hyperopts result -Once Hyperopt is completed you can use the result to adding new buy -signal. Given following result from hyperopt: -``` -Best parameters: -{ - "adx": { - "enabled": true, - "value": 15.0 - }, - "fastd": { - "enabled": true, - "value": 40.0 - }, - "green_candle": { - "enabled": true - }, - "mfi": { - "enabled": false - }, - "over_sar": { - "enabled": false - }, - "rsi": { - "enabled": true, - "value": 37.0 - }, - "trigger": { - "type": "lower_bb" - }, - "uptrend_long_ema": { - "enabled": true - }, - "uptrend_short_ema": { - "enabled": false - }, - "uptrend_sma": { - "enabled": false - } -} +## Understand the Hyperopts Result +Once Hyperopt is completed you can use the result to creating a new strategy. +Given following result from hyperopt: -Best Result: - 2197 trades. Avg profit 1.84%. Total profit 0.79367541 BTC. Avg duration 241.0 mins. +``` +Best result: + 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. +with values: +{'adx-value': 44, 'rsi-value': 29, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'bb_lower'} ``` You should understand this result like: -- You should **consider** the guard "adx" (`"adx"` is `"enabled": true`) -and the best value is `15.0` (`"value": 15.0,`) -- You should **consider** the guard "fastd" (`"fastd"` is `"enabled": -true`) and the best value is `40.0` (`"value": 40.0,`) -- You should **consider** to enable the guard "green_candle" -(`"green_candle"` is `"enabled": true`) but this guards as no -customizable value. -- You should **ignore** the guard "mfi" (`"mfi"` is `"enabled": false`) -- and so on... +- The buy trigger that worked best was `bb_lower`. +- You should not use ADX because `adx-enabled: False`) +- You should **consider** using the RSI indicator (`rsi-enabled: True` and the best value is `29.0` (`rsi-value: 29.0`) You have to look inside your strategy file into `buy_strategy_generator()` method, what those values match to. -So for example you had `adx:` with the `value: 15.0` so we would look -at `adx`-block, that translates to the following code block: +So for example you had `rsi-value: 29.0` so we would look +at `rsi`-block, that translates to the following code block: ``` -(dataframe['adx'] > 15.0) +(dataframe['rsi'] < 29.0) ``` -Translating your whole hyperopt result to as the new buy-signal -would be the following: +Translating your whole hyperopt result as the new buy-signal +would then look like: ``` def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: dataframe.loc[ ( - (dataframe['adx'] > 15.0) & # adx-value - (dataframe['fastd'] < 40.0) & # fastd-value - (dataframe['close'] > dataframe['open']) & # green_candle - (dataframe['rsi'] < 37.0) & # rsi-value - (dataframe['ema50'] > dataframe['ema100']) # uptrend_long_ema + (dataframe['rsi'] < 29.0) & # rsi-value + dataframe['close'] < dataframe['bb_lowerband'] # trigger ), 'buy'] = 1 return dataframe ``` -## Next step +## Next Step Now you have a perfect bot and want to control it from Telegram. Your next step is to learn the [Telegram usage](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md). From fa8fc3e4ce19b566a9ea5f079cb26db38b143089 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 2 Jul 2018 11:44:33 +0300 Subject: [PATCH 082/105] handle the case where we have zero buys --- freqtrade/optimize/hyperopt.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index eb9015356..e499def70 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -31,6 +31,8 @@ from freqtrade.optimize.backtesting import Backtesting logger = logging.getLogger(__name__) +MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization + class Hyperopt(Backtesting): """ @@ -152,7 +154,8 @@ class Hyperopt(Backtesting): trade_loss = 1 - 0.25 * exp(-(trade_count - self.target_trades) ** 2 / 10 ** 5.8) profit_loss = max(0, 1 - total_profit / self.expected_max_profit) duration_loss = 0.4 * min(trade_duration / self.max_accepted_trade_duration, 1) - return trade_loss + profit_loss + duration_loss + result = trade_loss + profit_loss + duration_loss + return result @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: @@ -293,6 +296,13 @@ class Hyperopt(Backtesting): trade_count = len(results.index) trade_duration = results.trade_duration.mean() + if trade_count == 0: + return { + 'loss': MAX_LOSS, + 'params': params, + 'result': result_explanation, + } + loss = self.calculate_loss(total_profit, trade_count, trade_duration) return { From 76343ecb77e2869a80f76adabf748baeeb314e1a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 2 Jul 2018 14:23:06 +0200 Subject: [PATCH 083/105] Update ccxt from 1.14.301 to 1.15.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 50441879a..ebb5b5b49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.301 +ccxt==1.15.3 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From 79aab4cce2f516df4d9504d99c31f3ccf4b70394 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 11:17:41 +0300 Subject: [PATCH 084/105] use fstring --- freqtrade/optimize/hyperopt.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e499def70..9b26b5b09 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -71,10 +71,8 @@ class Hyperopt(Backtesting): # Ensure the number of dimensions match # the number of parameters in the list x. if len(params) != len(dimensions): - msg = "Mismatch in number of search-space dimensions. " \ - "len(dimensions)=={} and len(x)=={}" - msg = msg.format(len(dimensions), len(params)) - raise ValueError(msg) + raise ValueError('Mismatch in number of search-space dimensions. ' + f'len(dimensions)=={len(dimensions)} and len(x)=={len(params)}') # Create a dict where the keys are the names of the dimensions # and the values are taken from the list of parameters x. From 2713fdb8607b01602d34c37e7c31facc7f9cb78e Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 11:46:56 +0300 Subject: [PATCH 085/105] use cpu count explicitly in job count --- freqtrade/optimize/hyperopt.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 9b26b5b09..8290eb2d8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -323,13 +323,13 @@ class Hyperopt(Backtesting): results.trade_duration.mean(), ) - def get_optimizer(self) -> Optimizer: + def get_optimizer(self, cpu_count) -> Optimizer: return Optimizer( self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30, - acq_optimizer_kwargs={'n_jobs': -1} + acq_optimizer_kwargs={'n_jobs': cpu_count} ) def run_optimizer_parallel(self, parallel, asked) -> List: @@ -361,11 +361,11 @@ class Hyperopt(Backtesting): self.load_previous_results() cpus = multiprocessing.cpu_count() - logger.info(f'Found {cpus}. Let\'s make them scream!') + logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') - opt = self.get_optimizer() + opt = self.get_optimizer(cpus) try: - with Parallel(n_jobs=-1) as parallel: + with Parallel(n_jobs=cpus) as parallel: for i in range(self.total_tries//cpus): asked = opt.ask(n_points=cpus) f_val = self.run_optimizer_parallel(parallel, asked) From 4a26b88a17af950f2bdedcfde43e2cd684dfcd4d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 12:43:25 +0300 Subject: [PATCH 086/105] improve documentation --- docs/hyperopt.md | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 8d3fe6704..f4b69b632 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -1,7 +1,7 @@ # Hyperopt -This page explains how to tune your strategy by finding the optimal -parameters, process called hyperparameter optimization. The bot uses several -algorithms included in `scikit-optimize` package to accomplish this. The +This page explains how to tune your strategy by finding the optimal +parameters, a process called hyperparameter optimization. The bot uses several +algorithms included in the `scikit-optimize` package to accomplish this. The search will burn all your CPU cores, make your laptop sound like a fighter jet and still take a long time. @@ -17,18 +17,17 @@ and still take a long time. We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) ### Configure your Guards and Triggers -There are two places you need to change in your strategy file to add a -new buy strategy for testing: +There are two places you need to change to add a new buy strategy for testing: - Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L278-L294). - Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L218-L229) and the associated methods `indicator_space`, `roi_space`, `stoploss_space`. There you have two different type of indicators: 1. `guards` and 2. `triggers`. -1. Guards are conditions like "never buy if ADX < 10", or never buy if -current price is over EMA10. +1. Guards are conditions like "never buy if ADX < 10", or "never buy if +current price is over EMA10". 2. Triggers are ones that actually trigger buy in specific moment, like -"buy when EMA5 crosses over EMA10" or buy when close price touches lower -bollinger band. +"buy when EMA5 crosses over EMA10" or "buy when close price touches lower +bollinger band". Hyperoptimization will, for each eval round, pick one trigger and possibly multiple guards. The constructed strategy will be something like @@ -103,9 +102,13 @@ with different value combinations. It will then use the given historical data an buys based on the buy signals generated with the above function and based on the results it will end with telling you which paramter combination produced the best profits. -### Adding New Indicators -If you want to test an indicator that isn't used by the bot currently, -you need to add it to the `populate_indicators()` method in `hyperopt.py`. +The search for best parameters starts with a few random combinations and then uses a +regressor algorithm (currently ExtraTreesRegressor) to quickly find a parameter combination +that minimizes the value of the objective function `calculate_loss` in `hyperopt.py`. + +The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. +When you want to test an indicator that isn't used by the bot currently, remember to +add it to the `populate_indicators()` method in `hyperopt.py`. ## Execute Hyperopt Once you have updated your hyperopt configuration you can run it. @@ -150,8 +153,8 @@ Legal values are: - space-separated list of any of the above values for example `--spaces roi stoploss` ## Understand the Hyperopts Result -Once Hyperopt is completed you can use the result to creating a new strategy. -Given following result from hyperopt: +Once Hyperopt is completed you can use the result to create a new strategy. +Given the following result from hyperopt: ``` Best result: From ee4754cfb985fbe608ff02b21c65102223e1e005 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 14:46:16 +0300 Subject: [PATCH 087/105] avoid re-serialization of whole dataframe --- freqtrade/optimize/hyperopt.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 8290eb2d8..57bf75742 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -14,14 +14,14 @@ from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, Any, Callable, Optional, List +from typing import Dict, Any, Callable, List import talib.abstract as ta from pandas import DataFrame from skopt.space import Real, Integer, Categorical, Dimension from skopt import Optimizer -from sklearn.externals.joblib import Parallel, delayed +from sklearn.externals.joblib import Parallel, delayed, load, dump import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments @@ -32,6 +32,7 @@ from freqtrade.optimize.backtesting import Backtesting logger = logging.getLogger(__name__) MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization +TICKERDATA_PICKLE = os.path.join('user_data', 'hyperopt_tickerdata.pkl') class Hyperopt(Backtesting): @@ -60,7 +61,7 @@ class Hyperopt(Backtesting): self.expected_max_profit = 3.0 # Configuration and data used by hyperopt - self.processed: Optional[Dict[str, Any]] = None +# self.processed: Optional[Dict[str, Any]] = None # Previous evaluations self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') @@ -281,10 +282,11 @@ class Hyperopt(Backtesting): if self.has_space('stoploss'): self.analyze.strategy.stoploss = params['stoploss'] + processed = load(TICKERDATA_PICKLE) results = self.backtest( { 'stake_amount': self.config['stake_amount'], - 'processed': self.processed, + 'processed': processed, 'realistic': self.config.get('realistic_simulation', False), } ) @@ -356,7 +358,7 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore - self.processed = self.tickerdata_to_dataframe(data) + dump(self.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() From 2c0e950486ef8e7368160b35934ca0c817f3d0a9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 3 Jul 2018 14:23:05 +0200 Subject: [PATCH 088/105] Update ccxt from 1.15.3 to 1.15.7 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ebb5b5b49..98721b74c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.3 +ccxt==1.15.7 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From ef59f9ad249f8a24a0ab29b2cb4c2f28d180c206 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 21:50:24 +0300 Subject: [PATCH 089/105] sort imports in hyperopt.py --- freqtrade/optimize/hyperopt.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 57bf75742..a1d86bd18 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -5,23 +5,21 @@ This module contains the hyperopt logic """ import logging +import multiprocessing import os import pickle import sys -import multiprocessing - from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, Any, Callable, List +from typing import Any, Callable, Dict, List import talib.abstract as ta from pandas import DataFrame - -from skopt.space import Real, Integer, Categorical, Dimension +from sklearn.externals.joblib import Parallel, delayed, dump, load from skopt import Optimizer -from sklearn.externals.joblib import Parallel, delayed, load, dump +from skopt.space import Categorical, Dimension, Integer, Real import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments From 2cde540645bf321360594769768f017c2ceabe0f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 21:50:45 +0300 Subject: [PATCH 090/105] remove dead code --- freqtrade/optimize/hyperopt.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a1d86bd18..861523965 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -58,9 +58,6 @@ class Hyperopt(Backtesting): # check that the reported Σ% values do not exceed this! self.expected_max_profit = 3.0 - # Configuration and data used by hyperopt -# self.processed: Optional[Dict[str, Any]] = None - # Previous evaluations self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') self.trials: List = [] From 3a7056ea1bab0a2cea913e2eca3c6e590481ee93 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 21:54:32 +0300 Subject: [PATCH 091/105] run at least one epoch --- freqtrade/optimize/hyperopt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 861523965..71e923f00 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -361,9 +361,10 @@ class Hyperopt(Backtesting): logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') opt = self.get_optimizer(cpus) + EVALS = max(self.total_tries//cpus, 1) try: with Parallel(n_jobs=cpus) as parallel: - for i in range(self.total_tries//cpus): + for i in range(EVALS): asked = opt.ask(n_points=cpus) f_val = self.run_optimizer_parallel(parallel, asked) opt.tell(asked, [i['loss'] for i in f_val]) From 9dbe0f50a3e6a708a621bea70757bf9e0b6f7577 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 22:09:59 +0300 Subject: [PATCH 092/105] fix tests after changing the dumping and pickling dataframe in hyperopt --- freqtrade/tests/optimize/test_hyperopt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 9194a66c9..bbe16ae7a 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -187,6 +187,7 @@ def test_roi_table_generation(init_hyperopt) -> None: def test_start_calls_optimizer(mocker, init_hyperopt, default_conf, caplog) -> None: + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.multiprocessing.cpu_count', MagicMock(return_value=1)) parallel = mocker.patch( @@ -208,6 +209,7 @@ def test_start_calls_optimizer(mocker, init_hyperopt, default_conf, caplog) -> N parallel.assert_called_once() assert 'Best result:\nfoo result\nwith values:\n{}' in caplog.text + assert dumper.called def test_format_results(init_hyperopt): @@ -316,6 +318,7 @@ def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: MagicMock(return_value=backtest_result) ) patch_exchange(mocker) + mocker.patch('freqtrade.optimize.hyperopt.load', MagicMock()) optimizer_param = { 'adx-value': 0, From c4a8435e007db87e157cf65e43213c2b4da81129 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 22:17:43 +0300 Subject: [PATCH 093/105] change pickle file name to better suit it's current purpose --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 71e923f00..d90468cff 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -59,7 +59,7 @@ class Hyperopt(Backtesting): self.expected_max_profit = 3.0 # Previous evaluations - self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') + self.trials_file = os.path.join('user_data', 'hyperopt_results.pickle') self.trials: List = [] def get_args(self, params): From 96bb2efe69ded2e6cbd2998804445e9d8726ded0 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 22:51:48 +0300 Subject: [PATCH 094/105] use joblib.dump and load for trials --- freqtrade/optimize/hyperopt.py | 5 ++--- freqtrade/tests/optimize/test_hyperopt.py | 9 +++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index d90468cff..7138e2601 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -7,7 +7,6 @@ This module contains the hyperopt logic import logging import multiprocessing import os -import pickle import sys from argparse import Namespace from functools import reduce @@ -99,14 +98,14 @@ class Hyperopt(Backtesting): """ if self.trials: logger.info('Saving %d evaluations to \'%s\'', len(self.trials), self.trials_file) - pickle.dump(self.trials, open(self.trials_file, 'wb')) + dump(self.trials, self.trials_file) def read_trials(self) -> List: """ Read hyperopt trials file """ logger.info('Reading Trials from \'%s\'', self.trials_file) - trials = pickle.load(open(self.trials_file, 'rb')) + trials = load(self.trials_file) os.remove(self.trials_file) return trials diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index bbe16ae7a..72a102c22 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -39,7 +39,7 @@ def create_trials(mocker) -> None: mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=False) mocker.patch('freqtrade.optimize.hyperopt.os.path.getsize', return_value=1) mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True) - mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) return [{'loss': 1, 'result': 'foo', 'params': {}}] @@ -139,10 +139,9 @@ def test_no_log_if_loss_does_not_improve(init_hyperopt, caplog) -> None: def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: trials = create_trials(mocker) - mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) + mock_dump = mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) hyperopt = _HYPEROPT - mocker.patch('freqtrade.optimize.hyperopt.open', return_value=hyperopt.trials_file) _HYPEROPT.trials = trials hyperopt.save_trials() @@ -157,8 +156,7 @@ def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: def test_read_trials_returns_trials_file(mocker, init_hyperopt, caplog) -> None: trials = create_trials(mocker) - mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', return_value=trials) - mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', return_value=mock_load) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=trials) hyperopt = _HYPEROPT hyperopt_trial = hyperopt.read_trials() @@ -168,7 +166,6 @@ def test_read_trials_returns_trials_file(mocker, init_hyperopt, caplog) -> None: caplog.record_tuples ) assert hyperopt_trial == trials - mock_open.assert_called_once() mock_load.assert_called_once() From bf4d0a9b7088eb012c1090144d52569c791848b6 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 4 Jul 2018 10:31:35 +0300 Subject: [PATCH 095/105] sort imports --- freqtrade/__main__.py | 2 +- freqtrade/analyze.py | 3 +-- freqtrade/arguments.py | 5 +++-- freqtrade/configuration.py | 8 ++++---- freqtrade/fiat_convert.py | 4 +++- freqtrade/freqtradebot.py | 8 +++----- freqtrade/indicator_helpers.py | 2 +- freqtrade/misc.py | 2 +- freqtrade/optimize/backtesting.py | 6 +++--- freqtrade/persistence.py | 5 ++--- freqtrade/rpc/rpc.py | 4 ++-- freqtrade/strategy/interface.py | 2 +- freqtrade/strategy/resolver.py | 3 +-- freqtrade/tests/conftest.py | 4 ++-- freqtrade/tests/exchange/test_exchange.py | 8 ++++---- freqtrade/tests/optimize/test_backtesting.py | 7 ++++--- freqtrade/tests/optimize/test_optimize.py | 15 +++++++++------ freqtrade/tests/rpc/test_rpc.py | 3 ++- freqtrade/tests/rpc/test_rpc_manager.py | 2 +- freqtrade/tests/rpc/test_rpc_telegram.py | 11 ++++++----- freqtrade/tests/test_acl_pair.py | 3 ++- freqtrade/tests/test_analyze.py | 4 ++-- freqtrade/tests/test_configuration.py | 6 +++--- freqtrade/tests/test_fiat_convert.py | 1 - freqtrade/tests/test_freqtradebot.py | 6 ++++-- freqtrade/tests/test_indicator_helpers.py | 2 +- freqtrade/tests/test_main.py | 2 +- freqtrade/tests/test_misc.py | 4 ++-- freqtrade/tests/test_persistence.py | 4 ++-- 29 files changed, 71 insertions(+), 65 deletions(-) diff --git a/freqtrade/__main__.py b/freqtrade/__main__.py index fe1318a35..7d271dfd1 100644 --- a/freqtrade/__main__.py +++ b/freqtrade/__main__.py @@ -7,8 +7,8 @@ To launch Freqtrade as a module """ import sys -from freqtrade import main +from freqtrade import main if __name__ == '__main__': main.set_loggers() diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index bf6b89b17..6d6a85596 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -12,8 +12,7 @@ from pandas import DataFrame, to_datetime from freqtrade import constants from freqtrade.exchange import Exchange from freqtrade.persistence import Trade -from freqtrade.strategy.resolver import StrategyResolver, IStrategy - +from freqtrade.strategy.resolver import IStrategy, StrategyResolver logger = logging.getLogger(__name__) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 8ce7c546d..731c5d88c 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -2,12 +2,13 @@ This module contains the argument manager class """ -import os import argparse import logging +import os import re +from typing import List, NamedTuple, Optional + import arrow -from typing import List, Optional, NamedTuple from freqtrade import __version__, constants diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 7c3a5eb4b..c90eddeb8 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -1,18 +1,18 @@ """ This module contains the configuration class """ -import os import json import logging +import os from argparse import Namespace -from typing import Optional, Dict, Any +from typing import Any, Dict, Optional + +import ccxt from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match -import ccxt from freqtrade import OperationalException, constants - logger = logging.getLogger(__name__) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 44a4f3054..2e1a7cac8 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -7,10 +7,12 @@ import logging import time from typing import Dict, List -from coinmarketcap import Market from requests.exceptions import RequestException +from coinmarketcap import Market + from freqtrade.constants import SUPPORTED_FIAT + logger = logging.getLogger(__name__) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 530535710..64ff170fd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -7,16 +7,14 @@ import logging import time import traceback from datetime import datetime -from typing import Dict, List, Optional, Any, Callable +from typing import Any, Callable, Dict, List, Optional import arrow import requests from cachetools import TTLCache, cached -from freqtrade import ( - DependencyException, OperationalException, TemporaryError, persistence, __version__, -) -from freqtrade import constants +from freqtrade import (DependencyException, OperationalException, + TemporaryError, __version__, constants, persistence) from freqtrade.analyze import Analyze from freqtrade.exchange import Exchange from freqtrade.fiat_convert import CryptoToFiatConverter diff --git a/freqtrade/indicator_helpers.py b/freqtrade/indicator_helpers.py index 50586578a..f8ea0d939 100644 --- a/freqtrade/indicator_helpers.py +++ b/freqtrade/indicator_helpers.py @@ -1,4 +1,4 @@ -from math import exp, pi, sqrt, cos +from math import cos, exp, pi, sqrt import numpy as np import talib as ta diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 90a1db42b..832437951 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -2,10 +2,10 @@ Various tool function for Freqtrade and scripts """ +import gzip import json import logging import re -import gzip from datetime import datetime from typing import Dict diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 316fba508..16c21258f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -7,18 +7,18 @@ import logging import operator from argparse import Namespace from datetime import datetime -from typing import Dict, Tuple, Any, List, Optional, NamedTuple +from typing import Any, Dict, List, NamedTuple, Optional, Tuple import arrow from pandas import DataFrame from tabulate import tabulate import freqtrade.optimize as optimize -from freqtrade import constants, DependencyException -from freqtrade.exchange import Exchange +from freqtrade import DependencyException, constants from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration +from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index aa33f7063..764ba6509 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -5,12 +5,11 @@ This module contains the class to persist trades into SQLite import logging from datetime import datetime from decimal import Decimal, getcontext -from typing import Dict, Optional, Any +from typing import Any, Dict, Optional import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, - create_engine) -from sqlalchemy import inspect + create_engine, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index ee6ecb770..f8cb136e4 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -3,9 +3,9 @@ This module contains class to define a RPC communications """ import logging from abc import abstractmethod -from datetime import datetime, timedelta, date +from datetime import date, datetime, timedelta from decimal import Decimal -from typing import Dict, Tuple, Any, List +from typing import Any, Dict, List, Tuple import arrow import sqlalchemy as sql diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 4ae358c6f..f73617f46 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -2,8 +2,8 @@ IStrategy interface This module defines the interface to apply for strategies """ -from typing import Dict from abc import ABC, abstractmethod +from typing import Dict from pandas import DataFrame diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 0dcd3fc6a..10cedb073 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -8,13 +8,12 @@ import inspect import logging import os from collections import OrderedDict -from typing import Optional, Dict, Type +from typing import Dict, Optional, Type from freqtrade import constants from freqtrade.strategy import import_strategy from freqtrade.strategy.interface import IStrategy - logger = logging.getLogger(__name__) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 4877fe556..9c86d1ece 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -2,8 +2,8 @@ import json import logging from datetime import datetime -from typing import Dict, Optional from functools import reduce +from typing import Dict, Optional from unittest.mock import MagicMock import arrow @@ -11,8 +11,8 @@ import pytest from jsonschema import validate from telegram import Chat, Message, Update -from freqtrade.analyze import Analyze from freqtrade import constants +from freqtrade.analyze import Analyze from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 6cf7f11b0..3ddec0ded 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -2,16 +2,16 @@ # pragma pylint: disable=protected-access import logging from copy import deepcopy -from random import randint from datetime import datetime +from random import randint from unittest.mock import MagicMock, PropertyMock import ccxt import pytest -from freqtrade import OperationalException, DependencyException, TemporaryError -from freqtrade.exchange import Exchange, API_RETRY_COUNT -from freqtrade.tests.conftest import log_has, get_patched_exchange +from freqtrade import DependencyException, OperationalException, TemporaryError +from freqtrade.exchange import API_RETRY_COUNT, Exchange +from freqtrade.tests.conftest import get_patched_exchange, log_has def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 49a6370bb..cb225e465 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -3,19 +3,20 @@ import json import math import random -import pytest from copy import deepcopy from typing import List from unittest.mock import MagicMock import numpy as np import pandas as pd +import pytest from arrow import Arrow -from freqtrade import optimize, constants, DependencyException +from freqtrade import DependencyException, constants, optimize from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments, TimeRange -from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration +from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, + start) from freqtrade.tests.conftest import log_has, patch_exchange diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 624f8ed77..4ab32343a 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -3,16 +3,19 @@ import json import os import uuid -import arrow from shutil import copyfile +import arrow + from freqtrade import optimize -from freqtrade.misc import file_dump_json -from freqtrade.optimize.__init__ import make_testdata_path, download_pairs, \ - download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, \ - load_cached_data_for_updating from freqtrade.arguments import TimeRange -from freqtrade.tests.conftest import log_has, get_patched_exchange +from freqtrade.misc import file_dump_json +from freqtrade.optimize.__init__ import (download_backtesting_testdata, + download_pairs, + load_cached_data_for_updating, + load_tickerdata_file, + make_testdata_path, trim_tickerlist) +from freqtrade.tests.conftest import get_patched_exchange, log_has # Change this if modifying UNITTEST/BTC testdatafile _BTC_UNITTEST_LENGTH = 13681 diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 11db7ffb3..58514d1c0 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -13,7 +13,8 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc.rpc import RPC, RPCException from freqtrade.state import State -from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap +from freqtrade.tests.test_freqtradebot import (patch_coinmarketcap, + patch_get_signal) # Functions for recurrent object patching diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 805424d26..5aea98d48 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -7,7 +7,7 @@ from copy import deepcopy from unittest.mock import MagicMock from freqtrade.rpc.rpc_manager import RPCManager -from freqtrade.tests.conftest import log_has, get_patched_freqtradebot +from freqtrade.tests.conftest import get_patched_freqtradebot, log_has def test_rpc_manager_object() -> None: diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index b2cca9b9a..2710328bd 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -11,17 +11,18 @@ from datetime import datetime from random import randint from unittest.mock import MagicMock -from telegram import Update, Message, Chat +from telegram import Chat, Message, Update from telegram.error import NetworkError from freqtrade import __version__ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade -from freqtrade.rpc.telegram import Telegram -from freqtrade.rpc.telegram import authorized_only +from freqtrade.rpc.telegram import Telegram, authorized_only from freqtrade.state import State -from freqtrade.tests.conftest import get_patched_freqtradebot, patch_exchange, log_has -from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap +from freqtrade.tests.conftest import (get_patched_freqtradebot, log_has, + patch_exchange) +from freqtrade.tests.test_freqtradebot import (patch_coinmarketcap, + patch_get_signal) class DummyCls(Telegram): diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 608e6a4a1..094c166b8 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -1,8 +1,9 @@ # pragma pylint: disable=missing-docstring,C0103,protected-access -import freqtrade.tests.conftest as tt # test tools from unittest.mock import MagicMock +import freqtrade.tests.conftest as tt # test tools + # whitelist, blacklist, filtering, all of that will # eventually become some rules to run on a generic ACL engine # perhaps try to anticipate that by using some python package diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 02225acca..e6108e8f8 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -12,9 +12,9 @@ import arrow from pandas import DataFrame from freqtrade.analyze import Analyze, SignalType -from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.arguments import TimeRange -from freqtrade.tests.conftest import log_has, get_patched_exchange +from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.tests.conftest import get_patched_exchange, log_has # Avoid to reinit the same object again and again _ANALYZE = Analyze({'strategy': 'DefaultStrategy'}) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index a4096cc01..e64e1b486 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -4,18 +4,18 @@ Unit test file for configuration.py """ import json +from argparse import Namespace from copy import deepcopy from unittest.mock import MagicMock -from argparse import Namespace import pytest from jsonschema import ValidationError +from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration -from freqtrade.constants import DEFAULT_DB_PROD_URL, DEFAULT_DB_DRYRUN_URL +from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.tests.conftest import log_has -from freqtrade import OperationalException def test_configuration_object() -> None: diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index 2fb9219ca..5af85d268 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -5,7 +5,6 @@ import time from unittest.mock import MagicMock import pytest - from requests.exceptions import RequestException from freqtrade.fiat_convert import CryptoFiat, CryptoToFiatConverter diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index dde2dd6d9..17bd6aa7c 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -14,11 +14,13 @@ import arrow import pytest import requests -from freqtrade import constants, DependencyException, OperationalException, TemporaryError +from freqtrade import (DependencyException, OperationalException, + TemporaryError, constants) from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.state import State -from freqtrade.tests.conftest import log_has, patch_coinmarketcap, patch_exchange +from freqtrade.tests.conftest import (log_has, patch_coinmarketcap, + patch_exchange) # Functions for recurrent object patching diff --git a/freqtrade/tests/test_indicator_helpers.py b/freqtrade/tests/test_indicator_helpers.py index 87b085a0b..f3d34ec0b 100644 --- a/freqtrade/tests/test_indicator_helpers.py +++ b/freqtrade/tests/test_indicator_helpers.py @@ -1,6 +1,6 @@ import pandas as pd -from freqtrade.indicator_helpers import went_up, went_down +from freqtrade.indicator_helpers import went_down, went_up def test_went_up(): diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 414dcdbe9..20a02eedc 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -11,7 +11,7 @@ import pytest from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.main import main, set_loggers, reconfigure +from freqtrade.main import main, reconfigure, set_loggers from freqtrade.state import State from freqtrade.tests.conftest import log_has, patch_exchange diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index b57900428..e2ba40dee 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -8,8 +8,8 @@ import datetime from unittest.mock import MagicMock from freqtrade.analyze import Analyze -from freqtrade.misc import (shorten_date, datesarray_to_datetimearray, - common_datearray, file_dump_json, format_ms_time) +from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, + file_dump_json, format_ms_time, shorten_date) from freqtrade.optimize.__init__ import load_tickerdata_file diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 12ae6579f..b24f2dd6c 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -5,8 +5,8 @@ from unittest.mock import MagicMock import pytest from sqlalchemy import create_engine -from freqtrade import constants, OperationalException -from freqtrade.persistence import Trade, init, clean_dry_run_db +from freqtrade import OperationalException, constants +from freqtrade.persistence import Trade, clean_dry_run_db, init from freqtrade.tests.conftest import log_has From ac20bf31dfab8589f3f3ae8b09fb3fa8fa83a685 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 4 Jul 2018 14:23:06 +0200 Subject: [PATCH 096/105] Update ccxt from 1.15.7 to 1.15.8 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 98721b74c..8ab13d394 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.7 +ccxt==1.15.8 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From bfd1e90154b4db42fac2e867f381f9dfba6c825a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 5 Jul 2018 14:23:11 +0200 Subject: [PATCH 097/105] Update ccxt from 1.15.8 to 1.15.13 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ca1575d7a..c2303f11d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.8 +ccxt==1.15.13 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From 239f8606e15fe0fe02d3b02db1fd2bbf28f1171e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 5 Jul 2018 14:23:12 +0200 Subject: [PATCH 098/105] Update pytest from 3.6.2 to 3.6.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c2303f11d..f87241e32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.14.5 TA-Lib==0.4.17 -pytest==3.6.2 +pytest==3.6.3 pytest-mock==1.10.0 pytest-cov==2.5.1 tabulate==0.8.2 From 03c112a6010c07941c55d565e4be0fa47e00b2b6 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Thu, 14 Jun 2018 08:31:29 +0300 Subject: [PATCH 099/105] config, optimize: fstrings in use --- freqtrade/configuration.py | 6 +++--- freqtrade/optimize/__init__.py | 7 ++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index c90eddeb8..582b2889c 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -62,8 +62,8 @@ class Configuration(object): conf = json.load(file) except FileNotFoundError: raise OperationalException( - 'Config file "{}" not found!' - ' Please create a config file or check whether it exists.'.format(path)) + f'Config file "{path}" not found!' + ' Please create a config file or check whether it exists.') if 'internals' not in conf: conf['internals'] = {} @@ -109,7 +109,7 @@ class Configuration(object): config['db_url'] = constants.DEFAULT_DB_PROD_URL logger.info('Dry run is disabled') - logger.info('Using DB: "{}"'.format(config['db_url'])) + logger.info(f'Using DB: "{config["db_url"]}"') # Check if the exchange set by the user is supported self.check_exchange(config) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 1e76808e7..e806ff2b8 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -54,11 +54,8 @@ def load_tickerdata_file( :return dict OR empty if unsuccesful """ path = make_testdata_path(datadir) - pair_file_string = pair.replace('/', '_') - file = os.path.join(path, '{pair}-{ticker_interval}.json'.format( - pair=pair_file_string, - ticker_interval=ticker_interval, - )) + pair_s = pair.replace('/', '_') + file = os.path.join(path, f'{pair_s}-{ticker_interval}.json') gzipfile = file + '.gz' # If the file does not exist we download it when None is returned. From 7dca3c6d03e83fe512d85280ceab0a5c41673d80 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Thu, 14 Jun 2018 08:52:13 +0300 Subject: [PATCH 100/105] freqtradebot,main,hyperopt: fstrings in use --- freqtrade/freqtradebot.py | 8 ++------ freqtrade/main.py | 5 +---- freqtrade/optimize/hyperopt.py | 30 +++++++++++++++--------------- 3 files changed, 18 insertions(+), 25 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 64ff170fd..9def7078c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -626,12 +626,8 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ # Because telegram._forcesell does not have the configuration # Ignore the FIAT value and does not show the stake_currency as well else: - message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f})`'.format( - gain="profit" if fmt_exp_profit > 0 else "loss", - profit_percent=fmt_exp_profit, - profit_coin=profit_trade - ) - + gain = "profit" if fmt_exp_profit > 0 else "loss" + message += f'` ({gain}: {fmt_exp_profit:.2f}%, {profit_trade:.8f})`' # Send the message self.rpc.send_msg(message) Trade.session.flush() diff --git a/freqtrade/main.py b/freqtrade/main.py index 9d17a403a..79080ce37 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -74,10 +74,7 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot: # Create new instance freqtrade = FreqtradeBot(Configuration(args).get_config()) freqtrade.rpc.send_msg( - '*Status:* `Config reloaded ...`'.format( - freqtrade.state.name.lower() - ) - ) + '*Status:* `Config reloaded {freqtrade.state.name.lower()}...`') return freqtrade diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7138e2601..72bf34eb3 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -128,13 +128,12 @@ class Hyperopt(Backtesting): Log results if it is better than any previous evaluation """ if results['loss'] < self.current_best_loss: + current = results['current_tries'] + total = results['total_tries'] + res = results['result'] + loss = results['loss'] self.current_best_loss = results['loss'] - log_msg = '\n{:5d}/{}: {}. Loss {:.5f}'.format( - results['current_tries'], - results['total_tries'], - results['result'], - results['loss'] - ) + log_msg = f'\n{current:5d}/{total}: {res}. Loss {loss:.5f}' print(log_msg) else: print('.', end='') @@ -309,15 +308,16 @@ class Hyperopt(Backtesting): """ Return the format result in a string """ - return ('{:6d} trades. Avg profit {: 5.2f}%. ' - 'Total profit {: 11.8f} {} ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( - len(results.index), - results.profit_percent.mean() * 100.0, - results.profit_abs.sum(), - self.config['stake_currency'], - results.profit_percent.sum(), - results.trade_duration.mean(), - ) + trades = len(results.index) + avg_profit = results.profit_percent.mean() * 100.0 + total_profit = results.profit_abs.sum() + stake_cur = self.config['stake_currency'] + profit = results.profit_percent.sum() + duration = results.trade_duration.mean() + + return (f'{trades:6d} trades. Avg profit {avg_profit: 5.2f}%. ' + f'Total profit {total_profit: 11.8f} {stake_cur} ' + f'({profit:.4f}Σ%). Avg duration {duration:5.1f} mins.') def get_optimizer(self, cpu_count) -> Optimizer: return Optimizer( From a2063ede558a28e68e94fa385fa56416c20f8341 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Jun 2018 08:27:29 -0500 Subject: [PATCH 101/105] persistence: fstrings in use --- freqtrade/persistence.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 764ba6509..0e0b22e82 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -21,6 +21,7 @@ from freqtrade import OperationalException logger = logging.getLogger(__name__) _DECL_BASE: Any = declarative_base() +_SQL_DOCS_URL = 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls' def init(config: Dict) -> None: @@ -45,10 +46,8 @@ def init(config: Dict) -> None: try: engine = create_engine(db_url, **kwargs) except NoSuchModuleError: - error = 'Given value for db_url: \'{}\' is no valid database URL! (See {}).'.format( - db_url, 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls' - ) - raise OperationalException(error) + raise OperationalException(f'Given value for db_url: \'{db_url}\' ' + f'is no valid database URL! (See {_SQL_DOCS_URL})') session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.session = session() @@ -173,13 +172,10 @@ class Trade(_DECL_BASE): max_rate = Column(Float, nullable=True, default=0.0) def __repr__(self): - return 'Trade(id={}, pair={}, amount={:.8f}, open_rate={:.8f}, open_since={})'.format( - self.id, - self.pair, - self.amount, - self.open_rate, - arrow.get(self.open_date).humanize() if self.is_open else 'closed' - ) + open_since = arrow.get(self.open_date).humanize() if self.is_open else 'closed' + + return (f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, ' + f'open_rate={self.open_rate:.8f}, open_since={open_since})') def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool = False): """this adjusts the stop loss to it's most recently observed setting""" @@ -226,6 +222,7 @@ class Trade(_DECL_BASE): :param order: order retrieved by exchange.get_order() :return: None """ + order_type = order['type'] # Ignore open and cancelled orders if order['status'] == 'open' or order['price'] is None: return @@ -233,16 +230,16 @@ class Trade(_DECL_BASE): logger.info('Updating trade (id=%d) ...', self.id) getcontext().prec = 8 # Bittrex do not go above 8 decimal - if order['type'] == 'limit' and order['side'] == 'buy': + if order_type == 'limit' and order['side'] == 'buy': # Update open rate and actual amount self.open_rate = Decimal(order['price']) self.amount = Decimal(order['amount']) logger.info('LIMIT_BUY has been fulfilled for %s.', self) self.open_order_id = None - elif order['type'] == 'limit' and order['side'] == 'sell': + elif order_type == 'limit' and order['side'] == 'sell': self.close(order['price']) else: - raise ValueError('Unknown order type: {}'.format(order['type'])) + raise ValueError(f'Unknown order type: {order_type}') cleanup() def close(self, rate: float) -> None: @@ -313,7 +310,8 @@ class Trade(_DECL_BASE): rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - return float("{0:.8f}".format(close_trade_price - open_trade_price)) + profit = close_trade_price - open_trade_price + return float(f"{profit:.8f}") def calc_profit_percent( self, @@ -333,5 +331,5 @@ class Trade(_DECL_BASE): rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - - return float("{0:.8f}".format((close_trade_price / open_trade_price) - 1)) + profit_percent = (close_trade_price / open_trade_price) - 1 + return float(f"{profit_percent:.8f}") From 21fc933678ebf938e99fdcaa19f48d64844b26e9 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Jun 2018 16:35:35 -0500 Subject: [PATCH 102/105] convert_backtesting: fstrings in use --- scripts/convert_backtestdata.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py index 698c1c829..96e0cbce8 100755 --- a/scripts/convert_backtestdata.py +++ b/scripts/convert_backtestdata.py @@ -143,15 +143,14 @@ def convert_main(args: Namespace) -> None: interval = str_interval break # change order on pairs if old ticker interval found + filename_new = path.join(path.dirname(filename), - "{}_{}-{}.json".format(currencies[1], - currencies[0], interval)) + f"{currencies[1]}_{currencies[0]}-{interval}.json") elif ret_string: interval = ret_string.group(0) filename_new = path.join(path.dirname(filename), - "{}_{}-{}.json".format(currencies[0], - currencies[1], interval)) + f"{currencies[0]}_{currencies[1]}-{interval}.json") else: logger.warning("file %s could not be converted, interval not found", filename) From adbffc69e152a9a9b4413a9b9c3f9e20baa3796b Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Jun 2018 17:17:10 -0500 Subject: [PATCH 103/105] telegram: fstrings in use --- freqtrade/rpc/telegram.py | 72 +++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4dd23971b..13a2b1913 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -153,7 +153,7 @@ class Telegram(RPC): try: df_statuses = self._rpc_status_table() message = tabulate(df_statuses, headers='keys', tablefmt='simple') - self._send_msg("
{}
".format(message), parse_mode=ParseMode.HTML) + self._send_msg(f"
{message}
", parse_mode=ParseMode.HTML) except RPCException as e: self._send_msg(str(e), bot=bot) @@ -166,6 +166,8 @@ class Telegram(RPC): :param update: message update :return: None """ + stake_cur = self._config['stake_currency'] + fiat_disp_cur = self._config['fiat_display_currency'] try: timescale = int(update.message.text.replace('/daily', '').strip()) except (TypeError, ValueError): @@ -173,18 +175,17 @@ class Telegram(RPC): try: stats = self._rpc_daily_profit( timescale, - self._config['stake_currency'], - self._config['fiat_display_currency'] + stake_cur, + fiat_disp_cur ) stats = tabulate(stats, headers=[ 'Day', - 'Profit {}'.format(self._config['stake_currency']), - 'Profit {}'.format(self._config['fiat_display_currency']) + f'Profit {stake_cur}', + f'Profit {fiat_disp_cur}' ], tablefmt='simple') - message = 'Daily Profit over the last {} days:\n
{}
'\ - .format(timescale, stats) + message = f'Daily Profit over the last {timescale} days:\n
{stats}
' self._send_msg(message, bot=bot, parse_mode=ParseMode.HTML) except RPCException as e: self._send_msg(str(e), bot=bot) @@ -198,39 +199,38 @@ class Telegram(RPC): :param update: message update :return: None """ + stake_cur = self._config['stake_currency'] + fiat_disp_cur = self._config['fiat_display_currency'] + try: stats = self._rpc_trade_statistics( - self._config['stake_currency'], - self._config['fiat_display_currency']) - + stake_cur, + fiat_disp_cur) + profit_closed_coin = stats['profit_closed_coin'] + profit_closed_percent = stats['profit_closed_percent'] + profit_closed_fiat = stats['profit_closed_fiat'] + profit_all_coin = stats['profit_all_coin'] + profit_all_percent = stats['profit_all_percent'] + profit_all_fiat = stats['profit_all_fiat'] + trade_count = stats['trade_count'] + first_trade_date = stats['first_trade_date'] + latest_trade_date = stats['latest_trade_date'] + avg_duration = stats['avg_duration'] + best_pair = stats['best_pair'] + best_rate = stats['best_rate'] # Message to display markdown_msg = "*ROI:* Close trades\n" \ - "∙ `{profit_closed_coin:.8f} {coin} ({profit_closed_percent:.2f}%)`\n" \ - "∙ `{profit_closed_fiat:.3f} {fiat}`\n" \ - "*ROI:* All trades\n" \ - "∙ `{profit_all_coin:.8f} {coin} ({profit_all_percent:.2f}%)`\n" \ - "∙ `{profit_all_fiat:.3f} {fiat}`\n" \ - "*Total Trade Count:* `{trade_count}`\n" \ - "*First Trade opened:* `{first_trade_date}`\n" \ - "*Latest Trade opened:* `{latest_trade_date}`\n" \ - "*Avg. Duration:* `{avg_duration}`\n" \ - "*Best Performing:* `{best_pair}: {best_rate:.2f}%`"\ - .format( - coin=self._config['stake_currency'], - fiat=self._config['fiat_display_currency'], - profit_closed_coin=stats['profit_closed_coin'], - profit_closed_percent=stats['profit_closed_percent'], - profit_closed_fiat=stats['profit_closed_fiat'], - profit_all_coin=stats['profit_all_coin'], - profit_all_percent=stats['profit_all_percent'], - profit_all_fiat=stats['profit_all_fiat'], - trade_count=stats['trade_count'], - first_trade_date=stats['first_trade_date'], - latest_trade_date=stats['latest_trade_date'], - avg_duration=stats['avg_duration'], - best_pair=stats['best_pair'], - best_rate=stats['best_rate'] - ) + f"∙ `{profit_closed_coin:.8f} {stake_cur} "\ + f"({profit_closed_percent:.2f}%)`\n" \ + f"∙ `{profit_closed_fiat:.3f} {fiat_disp_cur}`\n" \ + f"*ROI:* All trades\n" \ + f"∙ `{profit_all_coin:.8f} {stake_cur} ({profit_all_percent:.2f}%)`\n" \ + f"∙ `{profit_all_fiat:.3f} {fiat_disp_cur}`\n" \ + f"*Total Trade Count:* `{trade_count}`\n" \ + f"*First Trade opened:* `{first_trade_date}`\n" \ + f"*Latest Trade opened:* `{latest_trade_date}`\n" \ + f"*Avg. Duration:* `{avg_duration}`\n" \ + f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`" self._send_msg(markdown_msg, bot=bot) except RPCException as e: self._send_msg(str(e), bot=bot) From df68b0990ffc991032eadaabdf04d22f8db9ca4f Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Wed, 4 Jul 2018 13:53:45 -0500 Subject: [PATCH 104/105] rpc: fstrings --- freqtrade/rpc/rpc.py | 60 ++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f8cb136e4..263f293c6 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -74,34 +74,33 @@ class RPC(object): # calculate profit and send message to user current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] current_profit = trade.calc_profit_percent(current_rate) - fmt_close_profit = '{:.2f}%'.format( - round(trade.close_profit * 100, 2) - ) if trade.close_profit else None - message = "*Trade ID:* `{trade_id}`\n" \ - "*Current Pair:* [{pair}]({market_url})\n" \ - "*Open Since:* `{date}`\n" \ - "*Amount:* `{amount}`\n" \ - "*Open Rate:* `{open_rate:.8f}`\n" \ - "*Close Rate:* `{close_rate}`\n" \ - "*Current Rate:* `{current_rate:.8f}`\n" \ - "*Close Profit:* `{close_profit}`\n" \ - "*Current Profit:* `{current_profit:.2f}%`\n" \ - "*Open Order:* `{open_order}`"\ - .format( - trade_id=trade.id, - pair=trade.pair, - market_url=self._freqtrade.exchange.get_pair_detail_url(trade.pair), - date=arrow.get(trade.open_date).humanize(), - open_rate=trade.open_rate, - close_rate=trade.close_rate, - current_rate=current_rate, - amount=round(trade.amount, 8), - close_profit=fmt_close_profit, - current_profit=round(current_profit * 100, 2), - open_order='({} {} rem={:.8f})'.format( - order['type'], order['side'], order['remaining'] - ) if order else None, - ) + fmt_close_profit = (f'{round(trade.close_profit * 100, 2):.2f}%' + if trade.close_profit else None) + market_url = self._freqtrade.exchange.get_pair_detail_url(trade.pair) + trade_date = arrow.get(trade.open_date).humanize() + open_rate = trade.open_rate + close_rate = trade.close_rate + amount = round(trade.amount, 8) + current_profit = round(current_profit * 100, 2) + if order: + order_type = order['type'] + order_side = order['side'] + order_rem = order['remaining'] + open_order = f'({order_type} {order_side} rem={order_rem:.8f})' + else: + open_order = None + + message = f"*Trade ID:* `{trade.id}`\n" \ + f"*Current Pair:* [{trade.pair}]({market_url})\n" \ + f"*Open Since:* `{trade_date}`\n" \ + f"*Amount:* `{amount}`\n" \ + f"*Open Rate:* `{open_rate:.8f}`\n" \ + f"*Close Rate:* `{close_rate}`\n" \ + f"*Current Rate:* `{current_rate:.8f}`\n" \ + f"*Close Profit:* `{fmt_close_profit}`\n" \ + f"*Current Profit:* `{current_profit:.2f}%`\n" \ + f"*Open Order:* `{open_order}`"\ + result.append(message) return result @@ -116,11 +115,12 @@ class RPC(object): for trade in trades: # calculate profit and send message to user current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] + trade_perc = (100 * trade.calc_profit_percent(current_rate)) trades_list.append([ trade.id, trade.pair, shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), - '{:.2f}%'.format(100 * trade.calc_profit_percent(current_rate)) + f'{trade_perc:.2f}%' ]) columns = ['ID', 'Pair', 'Since', 'Profit'] @@ -148,7 +148,7 @@ class RPC(object): .all() curdayprofit = sum(trade.calc_profit() for trade in trades) profit_days[profitday] = { - 'amount': format(curdayprofit, '.8f'), + 'amount': f'{curdayprofit:.8f}', 'trades': len(trades) } From e808b3a2a1c73a1f30d33d073cf2245013d46d0e Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Thu, 5 Jul 2018 10:47:08 -0500 Subject: [PATCH 105/105] rpc: get rid of extra else and fix mypy warning --- freqtrade/rpc/rpc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 263f293c6..11658c6fb 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -82,13 +82,12 @@ class RPC(object): close_rate = trade.close_rate amount = round(trade.amount, 8) current_profit = round(current_profit * 100, 2) + open_order = '' if order: order_type = order['type'] order_side = order['side'] order_rem = order['remaining'] open_order = f'({order_type} {order_side} rem={order_rem:.8f})' - else: - open_order = None message = f"*Trade ID:* `{trade.id}`\n" \ f"*Current Pair:* [{trade.pair}]({market_url})\n" \