From e759a90b2df010446f2fab3fd3c8d72c80310d28 Mon Sep 17 00:00:00 2001 From: Pan Long Date: Fri, 22 Jun 2018 19:16:48 +0530 Subject: [PATCH 001/395] Update doc for manually fix trade The profit should be close_rate/open_rate-1 not close_rate/open_rate --- docs/sql_cheatsheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index ba26f1707..ff0b92347 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -59,7 +59,7 @@ SELECT * FROM trades; ```sql UPDATE trades -SET is_open=0, close_date=, close_rate=, close_profit=close_rate/open_rate +SET is_open=0, close_date=, close_rate=, close_profit=close_rate/open_rate-1 WHERE id=; ``` From a68c90c51229d9586cd5ff38cbf565006cc01984 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 21:04:07 +0300 Subject: [PATCH 002/395] avoid calling exchange.get_fee inside loop --- freqtrade/optimize/backtesting.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9a19d1412..ffb808a24 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -62,6 +62,7 @@ class Backtesting(object): self.config['exchange']['uid'] = '' self.config['dry_run'] = True self.exchange = Exchange(self.config) + self.fee = self.exchange.get_fee() @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: @@ -130,14 +131,13 @@ class Backtesting(object): stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) - fee = self.exchange.get_fee() trade = Trade( open_rate=buy_row.close, open_date=buy_row.date, stake_amount=stake_amount, amount=stake_amount / buy_row.open, - fee_open=fee, - fee_close=fee + fee_open=self.fee, + fee_close=self.fee ) # calculate win/lose forwards from buy point From c1691f21f3e3887d5842b091656084cc07de9088 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 21:55:09 +0300 Subject: [PATCH 003/395] check that we set fee on backtesting init --- freqtrade/tests/optimize/test_backtesting.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 1702818b1..22536e42f 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -298,6 +298,7 @@ def test_backtesting_init(mocker, default_conf) -> None: Test Backtesting._init() method """ patch_exchange(mocker) + get_fee = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5)) backtesting = Backtesting(default_conf) assert backtesting.config == default_conf assert isinstance(backtesting.analyze, Analyze) @@ -305,6 +306,8 @@ def test_backtesting_init(mocker, default_conf) -> None: assert callable(backtesting.tickerdata_to_dataframe) assert callable(backtesting.populate_buy_trend) assert callable(backtesting.populate_sell_trend) + get_fee.assert_called() + assert backtesting.fee == 0.5 def test_tickerdata_to_dataframe(default_conf, mocker) -> None: From 01d45bee76addbec4b66c999ae12d17409d9016e Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 07:58:25 +0300 Subject: [PATCH 004/395] fix flake8 --- freqtrade/tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 22536e42f..65aa00a70 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -307,7 +307,7 @@ def test_backtesting_init(mocker, default_conf) -> None: assert callable(backtesting.populate_buy_trend) assert callable(backtesting.populate_sell_trend) get_fee.assert_called() - assert backtesting.fee == 0.5 + assert backtesting.fee == 0.5 def test_tickerdata_to_dataframe(default_conf, mocker) -> None: From 2738d3aed85ea5088af595751590a5000a190c68 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 13:12:36 +0200 Subject: [PATCH 005/395] update plotly --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a31976c8c..e26c9c8f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,4 @@ tabulate==0.8.2 coinmarketcap==5.0.3 # Required for plotting data -#plotly==2.3.0 +#plotly==2.7.0 From 5aae215c94cbe37b3981e51a6b460e36a7fbfa8e Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 10:25:03 +0200 Subject: [PATCH 006/395] wrap strategies with HyperoptStrategy for module lookups with pickle --- freqtrade/optimize/hyperopt.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 19e732d7b..e52581491 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,6 +11,7 @@ import pickle import signal import sys from argparse import Namespace +from copy import deepcopy from functools import reduce from math import exp from operator import itemgetter @@ -26,10 +27,25 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting +from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) +HyperoptStrategy = None + + +def wrap_strategy(strategy: IStrategy) -> Optional[HyperoptStrategy]: + """Wraps a given Strategy instance to HyperoptStrategy""" + global HyperoptStrategy + + attr = deepcopy(dict(strategy.__class__.__dict__)) + # Patch module name to make it compatible with pickle + attr['__module__'] = 'freqtrade.optimize.hyperopt' + HyperoptStrategy = type('HyperoptStrategy', (IStrategy,), attr) + return HyperoptStrategy() + + class Hyperopt(Backtesting): """ Hyperopt class, this class contains all the logic to run a hyperopt simulation @@ -39,7 +55,6 @@ class Hyperopt(Backtesting): hyperopt.start() """ def __init__(self, config: Dict[str, Any]) -> None: - super().__init__(config) # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days @@ -57,6 +72,9 @@ class Hyperopt(Backtesting): # check that the reported Σ% values do not exceed this! self.expected_max_profit = 3.0 + # Wrap strategy to make it compatible with pickle + self.analyze.strategy = wrap_strategy(self.analyze.strategy) + # Configuration and data used by hyperopt self.processed: Optional[Dict[str, Any]] = None From 78f50a14713c70c1b444e9c2ec0ec7569230f25e Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 11:13:49 +0200 Subject: [PATCH 007/395] move logic from hyperopt to freqtrade.strategy --- freqtrade/optimize/hyperopt.py | 19 ------------------- freqtrade/strategy/__init__.py | 32 ++++++++++++++++++++++++++++++++ freqtrade/strategy/resolver.py | 5 +++-- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e52581491..7a313a3ac 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,7 +11,6 @@ import pickle import signal import sys from argparse import Namespace -from copy import deepcopy from functools import reduce from math import exp from operator import itemgetter @@ -27,25 +26,10 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) -HyperoptStrategy = None - - -def wrap_strategy(strategy: IStrategy) -> Optional[HyperoptStrategy]: - """Wraps a given Strategy instance to HyperoptStrategy""" - global HyperoptStrategy - - attr = deepcopy(dict(strategy.__class__.__dict__)) - # Patch module name to make it compatible with pickle - attr['__module__'] = 'freqtrade.optimize.hyperopt' - HyperoptStrategy = type('HyperoptStrategy', (IStrategy,), attr) - return HyperoptStrategy() - - class Hyperopt(Backtesting): """ Hyperopt class, this class contains all the logic to run a hyperopt simulation @@ -72,9 +56,6 @@ class Hyperopt(Backtesting): # check that the reported Σ% values do not exceed this! self.expected_max_profit = 3.0 - # Wrap strategy to make it compatible with pickle - self.analyze.strategy = wrap_strategy(self.analyze.strategy) - # Configuration and data used by hyperopt self.processed: Optional[Dict[str, Any]] = None diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index e69de29bb..e1dc7bb3f 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -0,0 +1,32 @@ +import logging +from copy import deepcopy + +from freqtrade.strategy.interface import IStrategy + + +logger = logging.getLogger(__name__) + + +def import_strategy(strategy: IStrategy) -> IStrategy: + """ + Imports given Strategy instance to global scope + of freqtrade.strategy and returns an instance of it + """ + # Copy all attributes from base class and class + attr = deepcopy({**strategy.__class__.__dict__, **strategy.__dict__}) + # Adjust module name + attr['__module__'] = 'freqtrade.strategy' + + name = strategy.__class__.__name__ + clazz = type(name, (IStrategy,), attr) + + logger.debug( + 'Imported strategy %s.%s as %s.%s', + strategy.__module__, strategy.__class__.__name__, + clazz.__module__, strategy.__class__.__name__, + ) + + # Modify global scope to declare class + globals()[name] = clazz + + return clazz() diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 3fd39bca3..3c7836291 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -11,6 +11,7 @@ from collections import OrderedDict from typing import Optional, Dict, Type from freqtrade import constants +from freqtrade.strategy import import_strategy from freqtrade.strategy.interface import IStrategy @@ -83,7 +84,7 @@ class StrategyResolver(object): strategy = self._search_strategy(path, strategy_name) if strategy: logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) - return strategy + return import_strategy(strategy) raise ImportError( "Impossible to load Strategy '{}'. This class does not exist" @@ -100,7 +101,7 @@ class StrategyResolver(object): """ # Generate spec based on absolute path - spec = importlib.util.spec_from_file_location('user_data.strategies', module_path) + spec = importlib.util.spec_from_file_location('unknown', module_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # type: ignore # importlib does not use typehints From 398b21a11d175d02622bc3fb88ad38e52e04bbaf Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 11:14:31 +0200 Subject: [PATCH 008/395] implement test for import_strategy --- freqtrade/tests/strategy/test_strategy.py | 27 +++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 244910790..26cd798f4 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -5,10 +5,34 @@ import os import pytest +from freqtrade.strategy import import_strategy +from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.resolver import StrategyResolver +def test_import_strategy(caplog): + caplog.set_level(logging.DEBUG) + + strategy = DefaultStrategy() + strategy.some_method = lambda *args, **kwargs: 42 + + assert strategy.__module__ == 'freqtrade.strategy.default_strategy' + assert strategy.some_method() == 42 + + imported_strategy = import_strategy(strategy) + + assert imported_strategy.__module__ == 'freqtrade.strategy' + assert imported_strategy.some_method() == 42 + + assert ( + 'freqtrade.strategy', + logging.DEBUG, + 'Imported strategy freqtrade.strategy.default_strategy.DefaultStrategy ' + 'as freqtrade.strategy.DefaultStrategy', + ) in caplog.record_tuples + + def test_search_strategy(): default_location = os.path.join(os.path.dirname( os.path.realpath(__file__)), '..', '..', 'strategy' @@ -20,8 +44,7 @@ def test_search_strategy(): def test_load_strategy(result): - resolver = StrategyResolver() - resolver._load_strategy('TestStrategy') + resolver = StrategyResolver({'strategy': 'TestStrategy'}) assert hasattr(resolver.strategy, 'populate_indicators') assert 'adx' in resolver.strategy.populate_indicators(result) From 810d7de8693ddbc4e0d3acf14c7684e9f51cd9dc Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 11:57:26 +0200 Subject: [PATCH 009/395] tests: add dir() assertion --- freqtrade/tests/strategy/test_strategy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 26cd798f4..13eac3261 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,5 +1,4 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 - import logging import os @@ -22,6 +21,8 @@ def test_import_strategy(caplog): imported_strategy = import_strategy(strategy) + assert dir(strategy) == dir(imported_strategy) + assert imported_strategy.__module__ == 'freqtrade.strategy' assert imported_strategy.some_method() == 42 From b485e6e0ba151d23a41d5d0bcbf11be9e22ae827 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 18 Jun 2018 22:40:36 +0300 Subject: [PATCH 010/395] start small --- freqtrade/optimize/hyperopt.py | 185 ++------------------------------- 1 file changed, 6 insertions(+), 179 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7a313a3ac..31764fbd7 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -21,6 +21,8 @@ import talib.abstract as ta from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from pandas import DataFrame +from skopt.space import Real, Integer, Categorical + import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration @@ -65,121 +67,18 @@ class Hyperopt(Backtesting): @staticmethod def populate_indicators(dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - """ dataframe['adx'] = ta.ADX(dataframe) - dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - dataframe['cci'] = ta.CCI(dataframe) macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] - dataframe['macdhist'] = macd['macdhist'] dataframe['mfi'] = ta.MFI(dataframe) - dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - dataframe['plus_di'] = ta.PLUS_DI(dataframe) - dataframe['roc'] = ta.ROC(dataframe) dataframe['rsi'] = ta.RSI(dataframe) - # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - rsi = 0.1 * (dataframe['rsi'] - 50) - dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) - # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - # Stoch - stoch = ta.STOCH(dataframe) - dataframe['slowd'] = stoch['slowd'] - dataframe['slowk'] = stoch['slowk'] - # Stoch fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - # Stoch RSI - stoch_rsi = ta.STOCHRSI(dataframe) - dataframe['fastd_rsi'] = stoch_rsi['fastd'] - dataframe['fastk_rsi'] = stoch_rsi['fastk'] + dataframe['minus_di'] = ta.MINUS_DI(dataframe) # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_middleband'] = bollinger['mid'] - dataframe['bb_upperband'] = bollinger['upper'] - # EMA - Exponential Moving Average - dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) - dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) - dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - # SAR Parabolic - dataframe['sar'] = ta.SAR(dataframe) - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] - - # Pattern Recognition - Bullish candlestick patterns - # ------------------------------------ - """ - # Hammer: values [0, 100] - dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) - # Inverted Hammer: values [0, 100] - dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) - # Dragonfly Doji: values [0, 100] - dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) - # Piercing Line: values [0, 100] - dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] - # Morningstar: values [0, 100] - dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] - # Three White Soldiers: values [0, 100] - dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ - - # Pattern Recognition - Bearish candlestick patterns - # ------------------------------------ - """ - # Hanging Man: values [0, 100] - dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) - # Shooting Star: values [0, 100] - dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) - # Gravestone Doji: values [0, 100] - dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) - # Dark Cloud Cover: values [0, 100] - dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) - # Evening Doji Star: values [0, 100] - dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) - # Evening Star: values [0, 100] - dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - - # Pattern Recognition - Bullish/Bearish candlestick patterns - # ------------------------------------ - """ - # Three Line Strike: values [0, -100, 100] - dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) - # Spinning Top: values [0, -100, 100] - dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] - # Engulfing: values [0, -100, 100] - dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] - # Harami: values [0, -100, 100] - dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] - # Three Outside Up/Down: values [0, -100, 100] - dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] - # Three Inside Up/Down: values [0, -100, 100] - dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ - - # Chart type - # ------------------------------------ - # Heikinashi stategy - heikinashi = qtpylib.heikinashi(dataframe) - dataframe['ha_open'] = heikinashi['open'] - dataframe['ha_close'] = heikinashi['close'] - dataframe['ha_high'] = heikinashi['high'] - dataframe['ha_low'] = heikinashi['low'] return dataframe @@ -295,38 +194,6 @@ class Hyperopt(Backtesting): {'enabled': False}, {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} ]), - 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'over_sar': hp.choice('over_sar', [ - {'enabled': False}, - {'enabled': True} - ]), - 'green_candle': hp.choice('green_candle', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_sma': hp.choice('uptrend_sma', [ - {'enabled': False}, - {'enabled': True} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'lower_bb'}, - {'type': 'lower_bb_tema'}, - {'type': 'faststoch10'}, - {'type': 'ao_cross_zero'}, - {'type': 'ema3_cross_ema10'}, - {'type': 'macd_cross_signal'}, - {'type': 'sar_reversal'}, - {'type': 'ht_sine'}, - {'type': 'heiken_reversal_bull'}, - {'type': 'di_cross'}, - ]), } def has_space(self, space: str) -> bool: @@ -361,12 +228,8 @@ class Hyperopt(Backtesting): """ conditions = [] # GUARDS AND TRENDS - if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: - conditions.append(dataframe['ema50'] > dataframe['ema100']) if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: conditions.append(dataframe['macd'] < 0) - if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: - conditions.append(dataframe['ema5'] > dataframe['ema10']) if 'mfi' in params and params['mfi']['enabled']: conditions.append(dataframe['mfi'] < params['mfi']['value']) if 'fastd' in params and params['fastd']['enabled']: @@ -375,49 +238,13 @@ class Hyperopt(Backtesting): conditions.append(dataframe['adx'] > params['adx']['value']) if 'rsi' in params and params['rsi']['enabled']: conditions.append(dataframe['rsi'] < params['rsi']['value']) - if 'over_sar' in params and params['over_sar']['enabled']: - conditions.append(dataframe['close'] > dataframe['sar']) - if 'green_candle' in params and params['green_candle']['enabled']: - conditions.append(dataframe['close'] > dataframe['open']) - if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: - prevsma = dataframe['sma'].shift(1) - conditions.append(dataframe['sma'] > prevsma) # TRIGGERS triggers = { - 'lower_bb': ( - dataframe['close'] < dataframe['bb_lowerband'] - ), - 'lower_bb_tema': ( - dataframe['tema'] < dataframe['bb_lowerband'] - ), - 'faststoch10': (qtpylib.crossed_above( - dataframe['fastd'], 10.0 - )), - 'ao_cross_zero': (qtpylib.crossed_above( - dataframe['ao'], 0.0 - )), - 'ema3_cross_ema10': (qtpylib.crossed_above( - dataframe['ema3'], dataframe['ema10'] - )), - 'macd_cross_signal': (qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )), - 'sar_reversal': (qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )), - 'ht_sine': (qtpylib.crossed_above( - dataframe['htleadsine'], dataframe['htsine'] - )), - 'heiken_reversal_bull': ( - (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & - (dataframe['ha_low'] == dataframe['ha_open']) - ), - 'di_cross': (qtpylib.crossed_above( - dataframe['plus_di'], dataframe['minus_di'] - )), } - conditions.append(triggers.get(params['trigger']['type'])) + #conditions.append(triggers.get(params['trigger']['type'])) + + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) # single trigger dataframe.loc[ reduce(lambda x, y: x & y, conditions), From 0cb1aedf5bd7a6f746d68319d8f6bff327efbf2d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 19 Jun 2018 09:09:54 +0300 Subject: [PATCH 011/395] problem with pickling --- freqtrade/optimize/hyperopt.py | 157 +++++++++++++++++++-------------- 1 file changed, 91 insertions(+), 66 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 31764fbd7..07e618b15 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -10,6 +10,8 @@ import os import pickle import signal import sys +import multiprocessing + from argparse import Namespace from functools import reduce from math import exp @@ -22,6 +24,8 @@ from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from pandas import DataFrame from skopt.space import Real, Integer, Categorical +from skopt import Optimizer +from sklearn.externals.joblib import Parallel, delayed import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments @@ -65,6 +69,21 @@ class Hyperopt(Backtesting): self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') self.trials = Trials() + def get_args(self, params): + dimensions = self.hyperopt_space() + # Ensure the number of dimensions match + # the number of parameters in the list x. + if len(params) != len(dimensions): + msg = "Mismatch in number of search-space dimensions. " \ + "len(dimensions)=={} and len(x)=={}" + msg = msg.format(len(dimensions), len(params)) + raise ValueError(msg) + + # Create a dict where the keys are the names of the dimensions + # and the values are taken from the list of parameters x. + arg_dict = {dim.name: value for dim, value in zip(dimensions, params)} + return arg_dict + @staticmethod def populate_indicators(dataframe: DataFrame) -> DataFrame: dataframe['adx'] = ta.ADX(dataframe) @@ -173,28 +192,17 @@ class Hyperopt(Backtesting): """ Define your Hyperopt space for searching strategy parameters """ - return { - 'macd_below_zero': hp.choice('macd_below_zero', [ - {'enabled': False}, - {'enabled': True} - ]), - 'mfi': hp.choice('mfi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} - ]), - 'fastd': hp.choice('fastd', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} - ]), - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} - ]), - 'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} - ]), - } + return [ + Integer(10, 25, name='mfi-value'), + Integer(15, 45, name='fastd-value'), + Integer(20, 50, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='mfi-enabled'), + Categorical([True, False], name='fastd-enabled'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + ] + def has_space(self, space: str) -> bool: """ @@ -208,14 +216,15 @@ class Hyperopt(Backtesting): """ Return the space to use during Hyperopt """ - spaces: Dict = {} - if self.has_space('buy'): - spaces = {**spaces, **Hyperopt.indicator_space()} - if self.has_space('roi'): - spaces = {**spaces, **Hyperopt.roi_space()} - if self.has_space('stoploss'): - spaces = {**spaces, **Hyperopt.stoploss_space()} - return spaces + return Hyperopt.indicator_space() + # spaces: Dict = {} + # if self.has_space('buy'): + # spaces = {**spaces, **Hyperopt.indicator_space()} + # if self.has_space('roi'): + # spaces = {**spaces, **Hyperopt.roi_space()} + # if self.has_space('stoploss'): + # spaces = {**spaces, **Hyperopt.stoploss_space()} + # return spaces @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: @@ -228,16 +237,16 @@ class Hyperopt(Backtesting): """ conditions = [] # GUARDS AND TRENDS - if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: - conditions.append(dataframe['macd'] < 0) - if 'mfi' in params and params['mfi']['enabled']: - conditions.append(dataframe['mfi'] < params['mfi']['value']) - if 'fastd' in params and params['fastd']['enabled']: - conditions.append(dataframe['fastd'] < params['fastd']['value']) - if 'adx' in params and params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if 'rsi' in params and params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) +# if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: +# conditions.append(dataframe['macd'] < 0) + if 'mfi-enabled' in params and params['mfi-enabled']: + conditions.append(dataframe['mfi'] < params['mfi-value']) + if 'fastd' in params and params['fastd-enabled']: + conditions.append(dataframe['fastd'] < params['fastd-value']) + if 'adx' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS triggers = { @@ -254,7 +263,9 @@ class Hyperopt(Backtesting): return populate_buy_trend - def generate_optimizer(self, params: Dict) -> Dict: + def generate_optimizer(self, _params) -> Dict: + params = self.get_args(_params) + if self.has_space('roi'): self.analyze.strategy.minimal_roi = self.generate_roi_table(params) @@ -297,12 +308,13 @@ class Hyperopt(Backtesting): 'result': result_explanation, } ) + return loss - return { - 'loss': loss, - 'status': STATUS_OK, - 'result': result_explanation, - } +# return { +# 'loss': loss, +# 'status': STATUS_OK, +# 'result': result_explanation, +# } def format_results(self, results: DataFrame) -> str: """ @@ -347,16 +359,29 @@ class Hyperopt(Backtesting): ) try: - best_parameters = fmin( - fn=self.generate_optimizer, - space=self.hyperopt_space(), - algo=tpe.suggest, - max_evals=self.total_tries, - trials=self.trials - ) + # best_parameters = fmin( + # fn=self.generate_optimizer, + # space=self.hyperopt_space(), + # algo=tpe.suggest, + # max_evals=self.total_tries, + # trials=self.trials + # ) + + # results = sorted(self.trials.results, key=itemgetter('loss')) + # best_result = results[0]['result'] + cpus = multiprocessing.cpu_count() + print(f'Found {cpus}. Let\'s make them scream!') + + opt = Optimizer(self.hyperopt_space(), "ET", acq_optimizer="sampling") + + for i in range(self.total_tries//cpus): + asked = opt.ask(n_points=cpus) + #asked = opt.ask() + #f_val = self.generate_optimizer(asked) + f_val = Parallel(n_jobs=-1)(delayed(self.generate_optimizer)(v) for v in asked) + opt.tell(asked, f_val) + print(f'got value {f_val}') - results = sorted(self.trials.results, key=itemgetter('loss')) - best_result = results[0]['result'] except ValueError: best_parameters = {} @@ -364,20 +389,20 @@ class Hyperopt(Backtesting): 'try with more epochs (param: -e).' # Improve best parameter logging display - if best_parameters: - best_parameters = space_eval( - self.hyperopt_space(), - best_parameters - ) + # if best_parameters: + # best_parameters = space_eval( + # self.hyperopt_space(), + # best_parameters + # ) - logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) - if 'roi_t1' in best_parameters: - logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) + # logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) + # if 'roi_t1' in best_parameters: + # logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) - logger.info('Best Result:\n%s', best_result) + # logger.info('Best Result:\n%s', best_result) - # Store trials result to file to resume next time - self.save_trials() + # # Store trials result to file to resume next time + # self.save_trials() def signal_handler(self, sig, frame) -> None: """ From a46badd5c066f13bbbad1c38bf309c6346e965bf Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 19 Jun 2018 21:57:42 +0300 Subject: [PATCH 012/395] reuse pool workers --- freqtrade/optimize/hyperopt.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 07e618b15..e49e66137 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -308,13 +308,12 @@ class Hyperopt(Backtesting): 'result': result_explanation, } ) - return loss -# return { -# 'loss': loss, -# 'status': STATUS_OK, -# 'result': result_explanation, -# } + return { + 'loss': loss, + 'status': STATUS_OK, + 'result': result_explanation, + } def format_results(self, results: DataFrame) -> str: """ @@ -374,13 +373,14 @@ class Hyperopt(Backtesting): opt = Optimizer(self.hyperopt_space(), "ET", acq_optimizer="sampling") - for i in range(self.total_tries//cpus): - asked = opt.ask(n_points=cpus) - #asked = opt.ask() - #f_val = self.generate_optimizer(asked) - f_val = Parallel(n_jobs=-1)(delayed(self.generate_optimizer)(v) for v in asked) - opt.tell(asked, f_val) - print(f'got value {f_val}') + with Parallel(n_jobs=-1) as parallel: + for i in range(self.total_tries//cpus): + asked = opt.ask(n_points=cpus) + #asked = opt.ask() + #f_val = self.generate_optimizer(asked) + f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) + opt.tell(asked, [i['loss'] for i in f_val]) + print(f'got value {f_val}') except ValueError: From 964cbdc262115772b7fdb6f2322de0c93facfc0c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 20 Jun 2018 09:13:51 +0300 Subject: [PATCH 013/395] increase initial sampling points --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e49e66137..919526e4a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -371,7 +371,7 @@ class Hyperopt(Backtesting): cpus = multiprocessing.cpu_count() print(f'Found {cpus}. Let\'s make them scream!') - opt = Optimizer(self.hyperopt_space(), "ET", acq_optimizer="sampling") + opt = Optimizer(self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30) with Parallel(n_jobs=-1) as parallel: for i in range(self.total_tries//cpus): From c415014153ab2e15291264e20af5a7f38db7adc6 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Thu, 21 Jun 2018 14:05:17 +0300 Subject: [PATCH 014/395] use multiple jobs in acq --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 919526e4a..57429af7c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -371,7 +371,7 @@ class Hyperopt(Backtesting): cpus = multiprocessing.cpu_count() print(f'Found {cpus}. Let\'s make them scream!') - opt = Optimizer(self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30) + opt = Optimizer(self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30, acq_optimizer_kwargs={'n_jobs': -1}) with Parallel(n_jobs=-1) as parallel: for i in range(self.total_tries//cpus): From 8fee2e2409b87c6ee2964452439b3c431fd58019 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Thu, 21 Jun 2018 14:59:36 +0300 Subject: [PATCH 015/395] move result logging out from optimizer --- freqtrade/optimize/hyperopt.py | 72 ++++++++++++---------------------- 1 file changed, 25 insertions(+), 47 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 57429af7c..15c1a5e48 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -50,7 +50,6 @@ class Hyperopt(Backtesting): # to the number of days self.target_trades = 600 self.total_tries = config.get('epochs', 0) - self.current_tries = 0 self.current_best_loss = 100 # max average trade duration in minutes @@ -288,27 +287,8 @@ class Hyperopt(Backtesting): trade_count = len(results.index) trade_duration = results.trade_duration.mean() - if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: - print('.', end='') - sys.stdout.flush() - return { - 'status': STATUS_FAIL, - 'loss': float('inf') - } - loss = self.calculate_loss(total_profit, trade_count, trade_duration) - self.current_tries += 1 - - self.log_results( - { - 'loss': loss, - 'current_tries': self.current_tries, - 'total_tries': self.total_tries, - 'result': result_explanation, - } - ) - return { 'loss': loss, 'status': STATUS_OK, @@ -357,36 +337,34 @@ class Hyperopt(Backtesting): self.total_tries ) - try: - # best_parameters = fmin( - # fn=self.generate_optimizer, - # space=self.hyperopt_space(), - # algo=tpe.suggest, - # max_evals=self.total_tries, - # trials=self.trials - # ) + # results = sorted(self.trials.results, key=itemgetter('loss')) + # best_result = results[0]['result'] + cpus = multiprocessing.cpu_count() + print(f'Found {cpus}. Let\'s make them scream!') - # results = sorted(self.trials.results, key=itemgetter('loss')) - # best_result = results[0]['result'] - cpus = multiprocessing.cpu_count() - print(f'Found {cpus}. Let\'s make them scream!') + opt = Optimizer( + self.hyperopt_space(), + base_estimator="ET", + acq_optimizer="auto", + n_initial_points=30, + acq_optimizer_kwargs={'n_jobs': -1} + ) - opt = Optimizer(self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30, acq_optimizer_kwargs={'n_jobs': -1}) + with Parallel(n_jobs=-1) as parallel: + for i in range(self.total_tries//cpus): + asked = opt.ask(n_points=cpus) + f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) + opt.tell(asked, [i['loss'] for i in f_val]) - with Parallel(n_jobs=-1) as parallel: - for i in range(self.total_tries//cpus): - asked = opt.ask(n_points=cpus) - #asked = opt.ask() - #f_val = self.generate_optimizer(asked) - f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) - opt.tell(asked, [i['loss'] for i in f_val]) - print(f'got value {f_val}') - - - except ValueError: - best_parameters = {} - best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ - 'try with more epochs (param: -e).' + for j in range(cpus): + self.log_results( + { + 'loss': f_val[j]['loss'], + 'current_tries': i * cpus + j, + 'total_tries': self.total_tries, + 'result': f_val[j]['result'], + } + ) # Improve best parameter logging display # if best_parameters: From 8272120c3a4dae84f8935b22435c4721bdac0dd5 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 07:10:37 +0300 Subject: [PATCH 016/395] convert stoploss and ROI search spaces to skopt format --- freqtrade/optimize/hyperopt.py | 51 +++++++++++++++++----------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 15c1a5e48..35fed3b7e 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -16,14 +16,14 @@ from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, Any, Callable, Optional +from typing import Dict, Any, Callable, Optional, List import numpy import talib.abstract as ta from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from pandas import DataFrame -from skopt.space import Real, Integer, Categorical +from skopt.space import Real, Integer, Categorical, Dimension from skopt import Optimizer from sklearn.externals.joblib import Parallel, delayed @@ -164,27 +164,27 @@ class Hyperopt(Backtesting): return roi_table @staticmethod - def roi_space() -> Dict[str, Any]: + def roi_space() -> List[Dimension]: """ Values to search for each ROI steps """ - return { - 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), - 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), - 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), - 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), - 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), - 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), - } + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + Real(0.01, 0.04, name='roi_p1'), + Real(0.01, 0.07, name='roi_p2'), + Real(0.01, 0.20, name='roi_p3'), + ] @staticmethod - def stoploss_space() -> Dict[str, Any]: + def stoploss_space() -> List[Dimension]: """ - Stoploss Value to search + Stoploss search space """ - return { - 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), - } + return [ + Real(-0.5, -0.02, name='stoploss'), + ] @staticmethod def indicator_space() -> Dict[str, Any]: @@ -211,19 +211,18 @@ class Hyperopt(Backtesting): return True return False - def hyperopt_space(self) -> Dict[str, Any]: + def hyperopt_space(self) -> List[Dimension]: """ Return the space to use during Hyperopt """ - return Hyperopt.indicator_space() - # spaces: Dict = {} - # if self.has_space('buy'): - # spaces = {**spaces, **Hyperopt.indicator_space()} - # if self.has_space('roi'): - # spaces = {**spaces, **Hyperopt.roi_space()} - # if self.has_space('stoploss'): - # spaces = {**spaces, **Hyperopt.stoploss_space()} - # return spaces + spaces: List[Dimension] = [] + if self.has_space('buy'): + spaces += Hyperopt.indicator_space() + if self.has_space('roi'): + spaces += Hyperopt.roi_space() + if self.has_space('stoploss'): + spaces += Hyperopt.stoploss_space() + return spaces @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: From a525cba8e9c94771606696788c9d50b29ee2f51f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 13:02:26 +0300 Subject: [PATCH 017/395] switch signal handler to try catch. fix pickling and formatting output --- freqtrade/optimize/hyperopt.py | 102 ++++++++++++--------------------- 1 file changed, 38 insertions(+), 64 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 35fed3b7e..960812248 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -8,7 +8,6 @@ import json import logging import os import pickle -import signal import sys import multiprocessing @@ -18,9 +17,7 @@ from math import exp from operator import itemgetter from typing import Dict, Any, Callable, Optional, List -import numpy import talib.abstract as ta -from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe from pandas import DataFrame from skopt.space import Real, Integer, Categorical, Dimension @@ -64,9 +61,9 @@ class Hyperopt(Backtesting): # Configuration and data used by hyperopt self.processed: Optional[Dict[str, Any]] = None - # Hyperopt Trials + # Previous evaluations self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') - self.trials = Trials() + self.trials = [] def get_args(self, params): dimensions = self.hyperopt_space() @@ -104,10 +101,11 @@ class Hyperopt(Backtesting): """ Save hyperopt trials to file """ - logger.info('Saving Trials to \'%s\'', self.trials_file) - pickle.dump(self.trials, open(self.trials_file, 'wb')) + if self.trials: + logger.info('Saving %d evaluations to \'%s\'', len(self.trials), self.trials_file) + pickle.dump(self.trials, open(self.trials_file, 'wb')) - def read_trials(self) -> Trials: + def read_trials(self) -> List: """ Read hyperopt trials file """ @@ -120,9 +118,15 @@ class Hyperopt(Backtesting): """ Display Best hyperopt result """ - vals = json.dumps(self.trials.best_trial['misc']['vals'], indent=4) - results = self.trials.best_trial['result']['result'] - logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + results = sorted(self.trials, key=itemgetter('loss')) + best_result = results[0] + logger.info( + 'Best result:\n%s\nwith values:\n%s', + best_result['result'], + best_result['params'] + ) + if 'roi_t1' in best_result['params']: + logger.info('ROI table:\n%s', self.generate_roi_table(best_result['params'])) def log_results(self, results) -> None: """ @@ -202,7 +206,6 @@ class Hyperopt(Backtesting): Categorical([True, False], name='rsi-enabled'), ] - def has_space(self, space: str) -> bool: """ Tell if a space value is contained in the configuration @@ -251,7 +254,7 @@ class Hyperopt(Backtesting): } #conditions.append(triggers.get(params['trigger']['type'])) - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) # single trigger + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) # single trigger dataframe.loc[ reduce(lambda x, y: x & y, conditions), @@ -290,7 +293,7 @@ class Hyperopt(Backtesting): return { 'loss': loss, - 'status': STATUS_OK, + 'params': params, 'result': result_explanation, } @@ -322,22 +325,15 @@ class Hyperopt(Backtesting): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) - logger.info('Preparing Trials..') - signal.signal(signal.SIGINT, self.signal_handler) + logger.info('Preparing..') # read trials file if we have one if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: self.trials = self.read_trials() - - self.current_tries = len(self.trials.results) - self.total_tries += self.current_tries logger.info( - 'Continuing with trials. Current: %d, Total: %d', - self.current_tries, - self.total_tries + 'Loaded %d previous evaluations from disk.', + len(self.trials) ) - # results = sorted(self.trials.results, key=itemgetter('loss')) - # best_result = results[0]['result'] cpus = multiprocessing.cpu_count() print(f'Found {cpus}. Let\'s make them scream!') @@ -349,50 +345,28 @@ class Hyperopt(Backtesting): acq_optimizer_kwargs={'n_jobs': -1} ) - with Parallel(n_jobs=-1) as parallel: - for i in range(self.total_tries//cpus): - asked = opt.ask(n_points=cpus) - f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) - opt.tell(asked, [i['loss'] for i in f_val]) + try: + with Parallel(n_jobs=-1) as parallel: + for i in range(self.total_tries//cpus): + asked = opt.ask(n_points=cpus) + f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) + opt.tell(asked, [i['loss'] for i in f_val]) - for j in range(cpus): - self.log_results( - { - 'loss': f_val[j]['loss'], - 'current_tries': i * cpus + j, - 'total_tries': self.total_tries, - 'result': f_val[j]['result'], - } - ) - - # Improve best parameter logging display - # if best_parameters: - # best_parameters = space_eval( - # self.hyperopt_space(), - # best_parameters - # ) - - # logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) - # if 'roi_t1' in best_parameters: - # logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) - - # logger.info('Best Result:\n%s', best_result) - - # # Store trials result to file to resume next time - # self.save_trials() - - def signal_handler(self, sig, frame) -> None: - """ - Hyperopt SIGINT handler - """ - logger.info( - 'Hyperopt received %s', - signal.Signals(sig).name - ) + self.trials += f_val + for j in range(cpus): + self.log_results( + { + 'loss': f_val[j]['loss'], + 'current_tries': i * cpus + j, + 'total_tries': self.total_tries, + 'result': f_val[j]['result'], + } + ) + except KeyboardInterrupt: + print('User interrupted..') self.save_trials() self.log_trials_result() - sys.exit(0) def start(args: Namespace) -> None: From dde7df7fd31513d8cbb6b70527910d85fb46721c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 17:08:22 +0300 Subject: [PATCH 018/395] add scikit-optimize to dependencies --- requirements.txt | 3 +++ setup.py | 1 + 2 files changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index e26c9c8f6..5727e70c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,5 +21,8 @@ networkx==1.11 # pyup: ignore tabulate==0.8.2 coinmarketcap==5.0.3 +# Required for hyperopt +scikit-optimize=0.5.2 + # Required for plotting data #plotly==2.7.0 diff --git a/setup.py b/setup.py index ee6b7ae38..cd0574fa2 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ setup(name='freqtrade', 'tabulate', 'cachetools', 'coinmarketcap', + 'scikit-optimize', ], include_package_data=True, zip_safe=False, From e8f2e6956d71026224d28ebf6ee37445764b998a Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 22 Jun 2018 21:02:59 +0300 Subject: [PATCH 019/395] to avoid pickle problems, get rid of reference to exchange after initialization --- freqtrade/optimize/hyperopt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 960812248..e53a00e4f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -325,6 +325,8 @@ class Hyperopt(Backtesting): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) + self.exchange = None + logger.info('Preparing..') # read trials file if we have one if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: From 0440a1917169eb9ed2e6fde713903fe9baec311a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 14:19:50 +0200 Subject: [PATCH 020/395] export open/close rate for backtesting too preparation to allow plotting of backtest results --- freqtrade/optimize/backtesting.py | 22 +++++++++++++------- freqtrade/tests/optimize/test_backtesting.py | 18 +++++++++++----- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ffb808a24..b5ed9b167 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -37,6 +37,8 @@ class BacktestResult(NamedTuple): close_index: int trade_duration: float open_at_end: bool + open_rate: float + close_rate: float class Backtesting(object): @@ -115,11 +117,13 @@ class Backtesting(object): def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: - records = [(trade_entry.pair, trade_entry.profit_percent, - trade_entry.open_time.timestamp(), - trade_entry.close_time.timestamp(), - trade_entry.open_index - 1, trade_entry.trade_duration) - for index, trade_entry in results.iterrows()] + # columns = ["pair", "profit", "opents", "closets", "index", "duration", + # "open_rate", "close_rate", "open_at_end"] + + records = [(t.pair, t.profit_percent, t.open_time.timestamp(), + t.close_time.timestamp(), t.open_index - 1, t.trade_duration, + t.open_rate, t.close_rate, t.open_at_end) + for index, t in results.iterrows()] if records: logger.info('Dumping backtest results to %s', recordfilename) @@ -158,7 +162,9 @@ class Backtesting(object): trade_duration=(sell_row.date - buy_row.date).seconds // 60, open_index=buy_row.Index, close_index=sell_row.Index, - open_at_end=False + open_at_end=False, + open_rate=buy_row.close, + close_rate=sell_row.close ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period @@ -171,7 +177,9 @@ class Backtesting(object): trade_duration=(sell_row.date - buy_row.date).seconds // 60, open_index=buy_row.Index, close_index=sell_row.Index, - open_at_end=True + open_at_end=True, + open_rate=buy_row.close, + close_rate=sell_row.close ) logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair, btr.profit_percent, btr.profit_abs) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 65aa00a70..4dda82463 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -604,9 +604,13 @@ def test_backtest_record(default_conf, fee, mocker): Arrow(2017, 11, 14, 22, 10, 00).datetime, Arrow(2017, 11, 14, 22, 43, 00).datetime, Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], + "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], "open_index": [1, 119, 153, 185], "close_index": [118, 151, 184, 199], - "trade_duration": [123, 34, 31, 14]}) + "trade_duration": [123, 34, 31, 14], + "open_at_end": [False, False, False, True] + }) backtesting._store_backtest_result("backtest-result.json", results) assert len(results) == 4 # Assert file_dump_json was only called once @@ -617,12 +621,16 @@ def test_backtest_record(default_conf, fee, mocker): # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117) # Below follows just a typecheck of the schema/type of trade-records oix = None - for (pair, profit, date_buy, date_sell, buy_index, dur) in records: + for (pair, profit, date_buy, date_sell, buy_index, dur, + openr, closer, open_at_end) in records: assert pair == 'UNITTEST/BTC' - isinstance(profit, float) + assert isinstance(profit, float) # FIX: buy/sell should be converted to ints - isinstance(date_buy, str) - isinstance(date_sell, str) + assert isinstance(date_buy, float) + assert isinstance(date_sell, float) + assert isinstance(openr, float) + assert isinstance(closer, float) + assert isinstance(open_at_end, bool) isinstance(buy_index, pd._libs.tslib.Timestamp) if oix: assert buy_index > oix From 09261b11afa20af557a2b638bb952282a09dd5ac Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 15:22:14 +0300 Subject: [PATCH 021/395] remove hyperopt and networkx from dependencies --- requirements.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5727e70c9..5754bf925 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,9 +15,6 @@ TA-Lib==0.4.17 pytest==3.6.2 pytest-mock==1.10.0 pytest-cov==2.5.1 -hyperopt==0.1 -# do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 -networkx==1.11 # pyup: ignore tabulate==0.8.2 coinmarketcap==5.0.3 From 136456afc0d657b2a9fe6e732f999ffe97ed5622 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 15:44:51 +0300 Subject: [PATCH 022/395] add three triggers to hyperopting --- freqtrade/optimize/hyperopt.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e53a00e4f..ea92cd561 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -94,6 +94,7 @@ class Hyperopt(Backtesting): # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['sar'] = ta.SAR(dataframe) return dataframe @@ -204,6 +205,7 @@ class Hyperopt(Backtesting): Categorical([True, False], name='fastd-enabled'), Categorical([True, False], name='adx-enabled'), Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') ] def has_space(self, space: str) -> bool: @@ -238,8 +240,6 @@ class Hyperopt(Backtesting): """ conditions = [] # GUARDS AND TRENDS -# if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: -# conditions.append(dataframe['macd'] < 0) if 'mfi-enabled' in params and params['mfi-enabled']: conditions.append(dataframe['mfi'] < params['mfi-value']) if 'fastd' in params and params['fastd-enabled']: @@ -250,11 +250,16 @@ class Hyperopt(Backtesting): conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS - triggers = { - } - #conditions.append(triggers.get(params['trigger']['type'])) - - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) # single trigger + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) dataframe.loc[ reduce(lambda x, y: x & y, conditions), From ab9e2fcea08692aa30abe5f20a48082c3452d34c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 15:47:19 +0300 Subject: [PATCH 023/395] fix guard names to match search space --- freqtrade/optimize/hyperopt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index ea92cd561..f59b884b8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -242,11 +242,11 @@ class Hyperopt(Backtesting): # GUARDS AND TRENDS if 'mfi-enabled' in params and params['mfi-enabled']: conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd' in params and params['fastd-enabled']: + if 'fastd-enabled' in params and params['fastd-enabled']: conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx' in params and params['adx-enabled']: + if 'adx-enabled' in params and params['adx-enabled']: conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi' in params and params['rsi-enabled']: + if 'rsi-enabled' in params and params['rsi-enabled']: conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS From 642ad02316aaa2a934545c9fb0841aa8ee7d5e89 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 23 Jun 2018 15:56:38 +0300 Subject: [PATCH 024/395] remove unused import --- freqtrade/optimize/hyperopt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f59b884b8..f91647f6e 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -4,7 +4,6 @@ This module contains the hyperopt logic """ -import json import logging import os import pickle From 3384679bad36a5b199abe39b54d3b5a080da2095 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Jun 2018 09:38:20 -0500 Subject: [PATCH 025/395] bump develop to 0.17.1 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 7cf0fa996..ac00264f0 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ FreqTrade bot """ -__version__ = '0.17.0' +__version__ = '0.17.1' class DependencyException(BaseException): From 3cedace2f6fce431cd5df784b5bcf5ce499bf02d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 19:54:27 +0200 Subject: [PATCH 026/395] add plotting for backtested trades --- scripts/plot_dataframe.py | 49 +++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 7f9641222..cd41f51b1 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -27,9 +27,12 @@ Example of usage: import logging import os import sys +import json +from pathlib import Path from argparse import Namespace from typing import Dict, List, Any +import pandas as pd import plotly.graph_objs as go from plotly import tools from plotly.offline import plot @@ -103,10 +106,42 @@ def plot_analyzed_dataframe(args: Namespace) -> None: exit() # Get trades already made from the DB - trades: List[Trade] = [] + trades: pd.DataFrame = pd.DataFrame() if args.db_url: persistence.init(_CONF) - trades = Trade.query.filter(Trade.pair.is_(pair)).all() + trades_ = Trade.query.filter(Trade.pair.is_(pair)).all() + # columns = ["pair", "profit", "opents", "closets", "index", "duration"] + columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] + + trades = pd.DataFrame([(t.pair, t.calc_profit(), + t.open_date, t.close_date, + t.open_rate, t.close_rate, + t.close_date.timestamp() - t.open_date.timestamp()) + for t in trades_], columns=columns) + + if args.exportfilename: + file = Path(args.exportfilename) + # must align with columns in backtest.py + columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] + with file.open() as f: + data = json.load(f) + trades = pd.DataFrame(data, columns=columns) + trades = trades.loc[trades["pair"] == pair] + if timerange: + if timerange.starttype == 'date': + trades = trades.loc[trades["opents"] >= timerange.startts] + if timerange.stoptype == 'date': + trades = trades.loc[trades["opents"] <= timerange.stopts] + + trades['opents'] = pd.to_datetime(trades['opents'], + unit='s', + utc=True, + infer_datetime_format=True) + trades['closets'] = pd.to_datetime(trades['closets'], + unit='s', + utc=True, + infer_datetime_format=True) dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] @@ -126,7 +161,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) -def generate_graph(pair, trades, data, args) -> tools.make_subplots: +def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: """ Generate the graph from the data generated by Backtesting or from DB :param pair: Pair to Display on the graph @@ -187,8 +222,8 @@ def generate_graph(pair, trades, data, args) -> tools.make_subplots: ) trade_buys = go.Scattergl( - x=[t.open_date.isoformat() for t in trades], - y=[t.open_rate for t in trades], + x=trades["opents"], + y=trades["open_rate"], mode='markers', name='trade_buy', marker=dict( @@ -199,8 +234,8 @@ def generate_graph(pair, trades, data, args) -> tools.make_subplots: ) ) trade_sells = go.Scattergl( - x=[t.close_date.isoformat() for t in trades], - y=[t.close_rate for t in trades], + x=trades["closets"], + y=trades["close_rate"], mode='markers', name='trade_sell', marker=dict( From f506ebcd62bf26b522afdd658dddd1fa3371deac Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 19:58:28 +0200 Subject: [PATCH 027/395] use Pathlib in the whole script --- scripts/plot_dataframe.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index cd41f51b1..763b8d9e4 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -25,7 +25,6 @@ Example of usage: --indicators2 fastk,fastd """ import logging -import os import sys import json from pathlib import Path @@ -158,7 +157,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: args=args ) - plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html')) + plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html'))) def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: From 5055563458597316bee1fbfafee1d42ba5eff673 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 20:14:15 +0200 Subject: [PATCH 028/395] add --plot-limit --- scripts/plot_dataframe.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 763b8d9e4..ba4da444a 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -104,19 +104,20 @@ def plot_analyzed_dataframe(args: Namespace) -> None: if tickers == {}: exit() + if args.db_url and args.exportfilename: + logger.critical("Can only specify --db-url or --export-filename") # Get trades already made from the DB trades: pd.DataFrame = pd.DataFrame() if args.db_url: persistence.init(_CONF) - trades_ = Trade.query.filter(Trade.pair.is_(pair)).all() - # columns = ["pair", "profit", "opents", "closets", "index", "duration"] columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] trades = pd.DataFrame([(t.pair, t.calc_profit(), t.open_date, t.close_date, t.open_rate, t.close_rate, t.close_date.timestamp() - t.open_date.timestamp()) - for t in trades_], columns=columns) + for t in Trade.query.filter(Trade.pair.is_(pair)).all()], + columns=columns) if args.exportfilename: file = Path(args.exportfilename) @@ -147,13 +148,15 @@ def plot_analyzed_dataframe(args: Namespace) -> None: dataframe = analyze.populate_buy_trend(dataframe) dataframe = analyze.populate_sell_trend(dataframe) - if len(dataframe.index) > 750: - logger.warning('Ticker contained more than 750 candles, clipping.') - + if len(dataframe.index) > args.plot_limit: + logger.warning('Ticker contained more than %s candles as defined ' + 'with --plot-limit, clipping.', args.plot_limit) + dataframe = dataframe.tail(args.plot_limit) + trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']] fig = generate_graph( pair=pair, trades=trades, - data=dataframe.tail(750), + data=dataframe, args=args ) @@ -333,11 +336,17 @@ def plot_parse_args(args: List[str]) -> Namespace: default='macd', dest='indicators2', ) - + arguments.parser.add_argument( + '--plot-limit', + help='Specify tick limit for plotting - too high values cause huge files - ' + 'Default: %(default)s', + dest='plot_limit', + default=750, + type=str, + ) arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) arguments.backtesting_options(arguments.parser) - return arguments.parse_args() From d8cb63efdde8d118b4939dda1d4e7a796a6402a6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 23 Jun 2018 20:19:07 +0200 Subject: [PATCH 029/395] extract load_trades --- scripts/plot_dataframe.py | 77 +++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index ba4da444a..e7e38fcd3 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -39,7 +39,7 @@ from plotly.offline import plot import freqtrade.optimize as optimize from freqtrade import persistence from freqtrade.analyze import Analyze -from freqtrade.arguments import Arguments +from freqtrade.arguments import Arguments, TimeRange from freqtrade.exchange import Exchange from freqtrade.optimize.backtesting import setup_configuration from freqtrade.persistence import Trade @@ -48,6 +48,45 @@ logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} +def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame: + trades: pd.DataFrame = pd.DataFrame() + if args.db_url: + persistence.init(_CONF) + columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] + + trades = pd.DataFrame([(t.pair, t.calc_profit(), + t.open_date, t.close_date, + t.open_rate, t.close_rate, + t.close_date.timestamp() - t.open_date.timestamp()) + for t in Trade.query.filter(Trade.pair.is_(pair)).all()], + columns=columns) + + if args.exportfilename: + file = Path(args.exportfilename) + # must align with columns in backtest.py + columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] + with file.open() as f: + data = json.load(f) + trades = pd.DataFrame(data, columns=columns) + trades = trades.loc[trades["pair"] == pair] + if timerange: + if timerange.starttype == 'date': + trades = trades.loc[trades["opents"] >= timerange.startts] + if timerange.stoptype == 'date': + trades = trades.loc[trades["opents"] <= timerange.stopts] + + trades['opents'] = pd.to_datetime(trades['opents'], + unit='s', + utc=True, + infer_datetime_format=True) + trades['closets'] = pd.to_datetime(trades['closets'], + unit='s', + utc=True, + infer_datetime_format=True) + return trades + + def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe @@ -107,41 +146,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: if args.db_url and args.exportfilename: logger.critical("Can only specify --db-url or --export-filename") # Get trades already made from the DB - trades: pd.DataFrame = pd.DataFrame() - if args.db_url: - persistence.init(_CONF) - columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] - - trades = pd.DataFrame([(t.pair, t.calc_profit(), - t.open_date, t.close_date, - t.open_rate, t.close_rate, - t.close_date.timestamp() - t.open_date.timestamp()) - for t in Trade.query.filter(Trade.pair.is_(pair)).all()], - columns=columns) - - if args.exportfilename: - file = Path(args.exportfilename) - # must align with columns in backtest.py - columns = ["pair", "profit", "opents", "closets", "index", "duration", - "open_rate", "close_rate", "open_at_end"] - with file.open() as f: - data = json.load(f) - trades = pd.DataFrame(data, columns=columns) - trades = trades.loc[trades["pair"] == pair] - if timerange: - if timerange.starttype == 'date': - trades = trades.loc[trades["opents"] >= timerange.startts] - if timerange.stoptype == 'date': - trades = trades.loc[trades["opents"] <= timerange.stopts] - - trades['opents'] = pd.to_datetime(trades['opents'], - unit='s', - utc=True, - infer_datetime_format=True) - trades['closets'] = pd.to_datetime(trades['closets'], - unit='s', - utc=True, - infer_datetime_format=True) + trades = load_trades(args, pair, timerange) dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] From 660ec6f443046f6a2f02a2ec23ccd2a4ad4f9658 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 24 Jun 2018 13:43:27 +0200 Subject: [PATCH 030/395] fix parameter type --- scripts/plot_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index e7e38fcd3..1cc6b818a 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -347,7 +347,7 @@ def plot_parse_args(args: List[str]) -> Namespace: 'Default: %(default)s', dest='plot_limit', default=750, - type=str, + type=int, ) arguments.common_args_parser() arguments.optimizer_shared_options(arguments.parser) From 5e7e977ffab0b4f60f31ade8087bb038ec7c5ca6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 24 Jun 2018 14:23:05 +0200 Subject: [PATCH 031/395] Update ccxt from 1.14.256 to 1.14.257 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 51312791e..437bb5533 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.256 +ccxt==1.14.257 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 118a43cbb898658e3cfed8bb415d82d644daf944 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 24 Jun 2018 15:27:53 +0300 Subject: [PATCH 032/395] fixing tests for hyperopt --- freqtrade/optimize/hyperopt.py | 38 +++-- freqtrade/tests/optimize/test_hyperopt.py | 197 +++++----------------- 2 files changed, 61 insertions(+), 174 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f91647f6e..115cce1f0 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -315,6 +315,18 @@ class Hyperopt(Backtesting): results.trade_duration.mean(), ) + def get_optimizer(self) -> Optimizer: + return Optimizer( + self.hyperopt_space(), + base_estimator="ET", + acq_optimizer="auto", + n_initial_points=30, + acq_optimizer_kwargs={'n_jobs': -1} + ) + + def run_optimizer_parallel(self, parallel, asked) -> List: + return parallel(delayed(self.generate_optimizer)(v) for v in asked) + def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) @@ -341,33 +353,25 @@ class Hyperopt(Backtesting): ) cpus = multiprocessing.cpu_count() - print(f'Found {cpus}. Let\'s make them scream!') + logger.info(f'Found {cpus}. Let\'s make them scream!') - opt = Optimizer( - self.hyperopt_space(), - base_estimator="ET", - acq_optimizer="auto", - n_initial_points=30, - acq_optimizer_kwargs={'n_jobs': -1} - ) + opt = self.get_optimizer() try: with Parallel(n_jobs=-1) as parallel: for i in range(self.total_tries//cpus): asked = opt.ask(n_points=cpus) - f_val = parallel(delayed(self.generate_optimizer)(v) for v in asked) + f_val = self.run_optimizer_parallel(parallel, asked) opt.tell(asked, [i['loss'] for i in f_val]) self.trials += f_val for j in range(cpus): - self.log_results( - { - 'loss': f_val[j]['loss'], - 'current_tries': i * cpus + j, - 'total_tries': self.total_tries, - 'result': f_val[j]['result'], - } - ) + self.log_results({ + 'loss': f_val[j]['loss'], + 'current_tries': i * cpus + j, + 'total_tries': self.total_tries, + 'result': f_val[j]['result'], + }) except KeyboardInterrupt: print('User interrupted..') diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 8ad1932af..0f3e56916 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,6 +1,5 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 import os -import signal from copy import deepcopy from unittest.mock import MagicMock @@ -42,16 +41,7 @@ def create_trials(mocker) -> None: mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True) mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) - return mocker.Mock( - results=[ - { - 'loss': 1, - 'result': 'foo', - 'status': 'ok' - } - ], - best_trial={'misc': {'vals': {'adx': 999}}} - ) + return [{'loss': 1, 'result': 'foo', 'params': {}}] # Unit tests @@ -148,74 +138,7 @@ def test_no_log_if_loss_does_not_improve(init_hyperopt, caplog) -> None: assert caplog.record_tuples == [] -def test_fmin_best_results(mocker, init_hyperopt, default_conf, caplog) -> None: - fmin_result = { - "macd_below_zero": 0, - "adx": 1, - "adx-value": 15.0, - "fastd": 1, - "fastd-value": 40.0, - "green_candle": 1, - "mfi": 0, - "over_sar": 0, - "rsi": 1, - "rsi-value": 37.0, - "trigger": 2, - "uptrend_long_ema": 1, - "uptrend_short_ema": 0, - "uptrend_sma": 0, - "stoploss": -0.1, - "roi_t1": 1, - "roi_t2": 2, - "roi_t3": 3, - "roi_p1": 1, - "roi_p2": 2, - "roi_p3": 3, - } - - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'epochs': 1}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) - - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) - patch_exchange(mocker) - - StrategyResolver({'strategy': 'DefaultStrategy'}) - hyperopt = Hyperopt(conf) - hyperopt.trials = create_trials(mocker) - hyperopt.tickerdata_to_dataframe = MagicMock() - hyperopt.start() - - exists = [ - 'Best parameters:', - '"adx": {\n "enabled": true,\n "value": 15.0\n },', - '"fastd": {\n "enabled": true,\n "value": 40.0\n },', - '"green_candle": {\n "enabled": true\n },', - '"macd_below_zero": {\n "enabled": false\n },', - '"mfi": {\n "enabled": false\n },', - '"over_sar": {\n "enabled": false\n },', - '"roi_p1": 1.0,', - '"roi_p2": 2.0,', - '"roi_p3": 3.0,', - '"roi_t1": 1.0,', - '"roi_t2": 2.0,', - '"roi_t3": 3.0,', - '"rsi": {\n "enabled": true,\n "value": 37.0\n },', - '"stoploss": -0.1,', - '"trigger": {\n "type": "faststoch10"\n },', - '"uptrend_long_ema": {\n "enabled": true\n },', - '"uptrend_short_ema": {\n "enabled": false\n },', - '"uptrend_sma": {\n "enabled": false\n }', - 'ROI table:\n{0: 6.0, 3.0: 3.0, 5.0: 1.0, 6.0: 0}', - 'Best Result:\nfoo' - ] - for line in exists: - assert line in caplog.text - - +@pytest.mark.skip(reason="Test not implemented") def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', side_effect=ValueError()) @@ -244,6 +167,7 @@ def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> assert line in caplog.text +@pytest.mark.skip(reason="Waits for fixing") def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, default_conf) -> None: trials = create_trials(mocker) @@ -286,17 +210,18 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, defa def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: - create_trials(mocker) + trials = create_trials(mocker) mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) hyperopt = _HYPEROPT mocker.patch('freqtrade.optimize.hyperopt.open', return_value=hyperopt.trials_file) + _HYPEROPT.trials = trials hyperopt.save_trials() trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') assert log_has( - 'Saving Trials to \'{}\''.format(trials_file), + 'Saving 1 evaluations to \'{}\''.format(trials_file), caplog.record_tuples ) mock_dump.assert_called_once() @@ -333,12 +258,14 @@ def test_roi_table_generation(init_hyperopt) -> None: assert hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} -def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: - trials = create_trials(mocker) - mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) +def test_start_calls_optimizer(mocker, init_hyperopt, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.multiprocessing.cpu_count', MagicMock(return_value=1)) + parallel = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + MagicMock(return_value=[{'loss': 1, 'result': 'foo result', 'params': {}}]) + ) patch_exchange(mocker) - mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) conf = deepcopy(default_conf) conf.update({'config': 'config.json.example'}) @@ -347,11 +274,12 @@ def test_start_calls_fmin(mocker, init_hyperopt, default_conf) -> None: conf.update({'spaces': 'all'}) hyperopt = Hyperopt(conf) - hyperopt.trials = trials hyperopt.tickerdata_to_dataframe = MagicMock() hyperopt.start() - mock_fmin.assert_called_once() + parallel.assert_called_once() + + assert 'Best result:\nfoo result\nwith values:\n{}' in caplog.text def test_format_results(init_hyperopt): @@ -384,20 +312,6 @@ def test_format_results(init_hyperopt): assert result.find('Total profit 1.00000000 EUR') -def test_signal_handler(mocker, init_hyperopt): - """ - Test Hyperopt.signal_handler() - """ - m = MagicMock() - mocker.patch('sys.exit', m) - mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.save_trials', m) - mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.log_trials_result', m) - - hyperopt = _HYPEROPT - hyperopt.signal_handler(signal.SIGTERM, None) - assert m.call_count == 3 - - def test_has_space(init_hyperopt): """ Test Hyperopt.has_space() method @@ -422,8 +336,8 @@ def test_populate_indicators(init_hyperopt) -> None: # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe - assert 'ao' in dataframe - assert 'cci' in dataframe + assert 'mfi' in dataframe + assert 'rsi' in dataframe def test_buy_strategy_generator(init_hyperopt) -> None: @@ -437,44 +351,15 @@ def test_buy_strategy_generator(init_hyperopt) -> None: populate_buy_trend = _HYPEROPT.buy_strategy_generator( { - 'uptrend_long_ema': { - 'enabled': True - }, - 'macd_below_zero': { - 'enabled': True - }, - 'uptrend_short_ema': { - 'enabled': True - }, - 'mfi': { - 'enabled': True, - 'value': 20 - }, - 'fastd': { - 'enabled': True, - 'value': 20 - }, - 'adx': { - 'enabled': True, - 'value': 20 - }, - 'rsi': { - 'enabled': True, - 'value': 20 - }, - 'over_sar': { - 'enabled': True, - }, - 'green_candle': { - 'enabled': True, - }, - 'uptrend_sma': { - 'enabled': True, - }, - - 'trigger': { - 'type': 'lower_bb' - } + 'adx-value': 20, + 'fastd-value': 20, + 'mfi-value': 20, + 'rsi-value': 20, + 'adx-enabled': True, + 'fastd-enabled': True, + 'mfi-enabled': True, + 'rsi-enabled': True, + 'trigger': 'bb_lower' } ) result = populate_buy_trend(dataframe) @@ -505,33 +390,31 @@ def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: patch_exchange(mocker) optimizer_param = { - 'adx': {'enabled': False}, - 'fastd': {'enabled': True, 'value': 35.0}, - 'green_candle': {'enabled': True}, - 'macd_below_zero': {'enabled': True}, - 'mfi': {'enabled': False}, - 'over_sar': {'enabled': False}, - 'roi_p1': 0.01, - 'roi_p2': 0.01, - 'roi_p3': 0.1, + 'adx-value': 0, + 'fastd-value': 35, + 'mfi-value': 0, + 'rsi-value': 0, + 'adx-enabled': False, + 'fastd-enabled': True, + 'mfi-enabled': False, + 'rsi-enabled': False, + 'trigger': 'macd_cross_signal', 'roi_t1': 60.0, 'roi_t2': 30.0, 'roi_t3': 20.0, - 'rsi': {'enabled': False}, + 'roi_p1': 0.01, + 'roi_p2': 0.01, + 'roi_p3': 0.1, 'stoploss': -0.4, - 'trigger': {'type': 'macd_cross_signal'}, - 'uptrend_long_ema': {'enabled': False}, - 'uptrend_short_ema': {'enabled': True}, - 'uptrend_sma': {'enabled': True} } response_expected = { 'loss': 1.9840569076926293, 'result': ' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC ' '(0.0231Σ%). Avg duration 100.0 mins.', - 'status': 'ok' + 'params': optimizer_param } hyperopt = Hyperopt(conf) - generate_optimizer_value = hyperopt.generate_optimizer(optimizer_param) + generate_optimizer_value = hyperopt.generate_optimizer(list(optimizer_param.values())) assert generate_optimizer_value == response_expected From e70cb963f7cc9f82336c2fa07875b97d31e74c94 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 24 Jun 2018 17:00:00 +0200 Subject: [PATCH 033/395] document what to do with exported backtest results --- docs/backtesting.md | 28 ++++++++++++++++++++++++++++ freqtrade/optimize/backtesting.py | 3 --- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 1efb46b43..172969ae2 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -70,6 +70,34 @@ Where `-s TestStrategy` refers to the class name within the strategy file `test_ python3 ./freqtrade/main.py backtesting --export trades ``` +The exported trades can be read using the following code for manual analysis, or can be used by the plotting script `plot_dataframe.py` in the scripts folder. + +``` python +import json +from pathlib import Path +import pandas as pd + +filename=Path('user_data/backtest_data/backtest-result.json') + +with filename.open() as file: + data = json.load(file) + +columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end"] +df = pd.DataFrame(data, columns=columns) + +df['opents'] = pd.to_datetime(df['opents'], + unit='s', + utc=True, + infer_datetime_format=True + ) +df['closets'] = pd.to_datetime(df['closets'], + unit='s', + utc=True, + infer_datetime_format=True + ) +``` + #### Exporting trades to file specifying a custom filename ```bash diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b5ed9b167..70f6168a4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -117,9 +117,6 @@ class Backtesting(object): def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: - # columns = ["pair", "profit", "opents", "closets", "index", "duration", - # "open_rate", "close_rate", "open_at_end"] - records = [(t.pair, t.profit_percent, t.open_time.timestamp(), t.close_time.timestamp(), t.open_index - 1, t.trade_duration, t.open_rate, t.close_rate, t.open_at_end) From 43f1a1d264765f106ad9f94b5fd1f3ffb70d20de Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 24 Jun 2018 19:52:12 +0200 Subject: [PATCH 034/395] rework download_backtest script --- freqtrade/arguments.py | 7 +++++ scripts/download_backtest_data.py | 48 ++++++++++++++++--------------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 31232f1ff..8ce7c546d 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -334,3 +334,10 @@ class Arguments(object): nargs='+', dest='timeframes', ) + + self.parser.add_argument( + '--erase', + help='Clean all existing data for the selected exchange/pairs/timeframes', + dest='erase', + action='store_true' + ) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 2f76c1232..686098f94 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -3,11 +3,14 @@ """This script generate json data from bittrex""" import json import sys -import os +from pathlib import Path import arrow -from freqtrade import (arguments, misc) +from freqtrade import arguments +from freqtrade.arguments import TimeRange from freqtrade.exchange import Exchange +from freqtrade.optimize import download_backtesting_testdata + DEFAULT_DL_PATH = 'user_data/data' @@ -17,25 +20,27 @@ args = arguments.parse_args() timeframes = args.timeframes -dl_path = os.path.join(DEFAULT_DL_PATH, args.exchange) +dl_path = Path(DEFAULT_DL_PATH).joinpath(args.exchange) if args.export: - dl_path = args.export + dl_path = Path(args.export) -if not os.path.isdir(dl_path): +if not dl_path.is_dir(): sys.exit(f'Directory {dl_path} does not exist.') -pairs_file = args.pairs_file if args.pairs_file else os.path.join(dl_path, 'pairs.json') -if not os.path.isfile(pairs_file): +pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json') +if not pairs_file.exists(): sys.exit(f'No pairs file found with path {pairs_file}.') -with open(pairs_file) as file: +with pairs_file.open() as file: PAIRS = list(set(json.load(file))) PAIRS.sort() -since_time = None + +timerange = TimeRange() if args.days: - since_time = arrow.utcnow().shift(days=-args.days).timestamp * 1000 + time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d") + timerange = arguments.parse_timerange(f'{time_since}-') print(f'About to download pairs: {PAIRS} to {dl_path}') @@ -59,21 +64,18 @@ for pair in PAIRS: print(f"skipping pair {pair}") continue for tick_interval in timeframes: - print(f'downloading pair {pair}, interval {tick_interval}') - - data = exchange.get_ticker_history(pair, tick_interval, since_ms=since_time) - if not data: - print('\tNo data was downloaded') - break - - print('\tData was downloaded for period %s - %s' % ( - arrow.get(data[0][0] / 1000).format(), - arrow.get(data[-1][0] / 1000).format())) - - # save data pair_print = pair.replace('/', '_') filename = f'{pair_print}-{tick_interval}.json' - misc.file_dump_json(os.path.join(dl_path, filename), data) + dl_file = dl_path.joinpath(filename) + if args.erase and dl_file.exists(): + print(f'Deleting existing data for pair {pair}, interval {tick_interval}') + dl_file.unlink() + + print(f'downloading pair {pair}, interval {tick_interval}') + download_backtesting_testdata(str(dl_path), exchange=exchange, + pair=pair, + tick_interval=tick_interval, + timerange=timerange) if pairs_not_available: From 17ee7f8be51ddb64b8c4ad1de88fc7edf266c647 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 25 Jun 2018 11:15:11 +0300 Subject: [PATCH 035/395] fix typo in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5754bf925..c22dcbdca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ tabulate==0.8.2 coinmarketcap==5.0.3 # Required for hyperopt -scikit-optimize=0.5.2 +scikit-optimize==0.5.2 # Required for plotting data #plotly==2.7.0 From 0bddc58ec41db007c75de5b16d1589e66cc3131d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 25 Jun 2018 11:38:14 +0300 Subject: [PATCH 036/395] extract loading previous results to a method --- freqtrade/optimize/hyperopt.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 115cce1f0..b8b53ab3c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -327,6 +327,15 @@ class Hyperopt(Backtesting): def run_optimizer_parallel(self, parallel, asked) -> List: return parallel(delayed(self.generate_optimizer)(v) for v in asked) + def load_previous_results(self): + """ read trials file if we have one """ + if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: + self.trials = self.read_trials() + logger.info( + 'Loaded %d previous evaluations from disk.', + len(self.trials) + ) + def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) @@ -340,23 +349,13 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) - self.exchange = None - - logger.info('Preparing..') - # read trials file if we have one - if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: - self.trials = self.read_trials() - logger.info( - 'Loaded %d previous evaluations from disk.', - len(self.trials) - ) + self.load_previous_results() cpus = multiprocessing.cpu_count() logger.info(f'Found {cpus}. Let\'s make them scream!') opt = self.get_optimizer() - try: with Parallel(n_jobs=-1) as parallel: for i in range(self.total_tries//cpus): From 2b6407e5982dbeb8bd1038945507c7875592bfd4 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 25 Jun 2018 11:38:42 +0300 Subject: [PATCH 037/395] remove unused tests from hyperopt --- freqtrade/tests/optimize/test_hyperopt.py | 72 ----------------------- 1 file changed, 72 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 0f3e56916..9194a66c9 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -44,7 +44,6 @@ def create_trials(mocker) -> None: return [{'loss': 1, 'result': 'foo', 'params': {}}] -# Unit tests def test_start(mocker, default_conf, caplog) -> None: """ Test start() function @@ -138,77 +137,6 @@ def test_no_log_if_loss_does_not_improve(init_hyperopt, caplog) -> None: assert caplog.record_tuples == [] -@pytest.mark.skip(reason="Test not implemented") -def test_fmin_throw_value_error(mocker, init_hyperopt, default_conf, caplog) -> None: - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.fmin', side_effect=ValueError()) - - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'epochs': 1}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) - patch_exchange(mocker) - - StrategyResolver({'strategy': 'DefaultStrategy'}) - hyperopt = Hyperopt(conf) - hyperopt.trials = create_trials(mocker) - hyperopt.tickerdata_to_dataframe = MagicMock() - - hyperopt.start() - - exists = [ - 'Best Result:', - 'Sorry, Hyperopt was not able to find good parameters. Please try with more epochs ' - '(param: -e).', - ] - - for line in exists: - assert line in caplog.text - - -@pytest.mark.skip(reason="Waits for fixing") -def test_resuming_previous_hyperopt_results_succeeds(mocker, init_hyperopt, default_conf) -> None: - trials = create_trials(mocker) - - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'epochs': 1}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) - - mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=True) - mocker.patch('freqtrade.optimize.hyperopt.len', return_value=len(trials.results)) - mock_read = mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.read_trials', - return_value=trials - ) - mock_save = mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.save_trials', - return_value=None - ) - mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) - patch_exchange(mocker) - - StrategyResolver({'strategy': 'DefaultStrategy'}) - hyperopt = Hyperopt(conf) - hyperopt.trials = trials - hyperopt.tickerdata_to_dataframe = MagicMock() - - hyperopt.start() - - mock_read.assert_called_once() - mock_save.assert_called_once() - - current_tries = hyperopt.current_tries - total_tries = hyperopt.total_tries - - assert current_tries == len(trials.results) - assert total_tries == (current_tries + len(trials.results)) - - def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: trials = create_trials(mocker) mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) From 4f1fa28658d8c5d1ffa556f69022d2a56f8fdda8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 25 Jun 2018 14:23:06 +0200 Subject: [PATCH 038/395] Update ccxt from 1.14.257 to 1.14.267 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 437bb5533..c9d3a6529 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.257 +ccxt==1.14.267 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 7c2a50cef9b51edb4b5ef9e4ddb442096fed3a17 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 26 Jun 2018 14:23:06 +0200 Subject: [PATCH 039/395] Update ccxt from 1.14.267 to 1.14.272 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c9d3a6529..d4f8b2f06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.267 +ccxt==1.14.272 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 54f52fb36664c375ab8b2c4ce64ea08c40cc7e48 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Fri, 1 Jun 2018 20:50:40 -0700 Subject: [PATCH 040/395] Create stoploss.md --- docs/stoploss.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/stoploss.md diff --git a/docs/stoploss.md b/docs/stoploss.md new file mode 100644 index 000000000..ff93fd1d8 --- /dev/null +++ b/docs/stoploss.md @@ -0,0 +1,50 @@ +# Stop Loss support + +at this stage the bot contains the following stoploss support modes: + +1. static stop loss, defined in either the strategy or configuration + +2. trailing stop loss, defined in the configuration + +3. trailing stop loss, custom positive loss, defined in configuration + +## Static Stop Loss + +This is very simple, basically you define a stop loss of x in your strategy file or alternative in the configuration, which +will overwrite the strategy definition. This will basically try to sell your asset, the second the loss exceeds the defined loss. + +## Trail Stop Loss + +The initial value for this stop loss, is defined in your strategy or configuration. Just as you would define your Stop Loss normally. +To enable this Feauture all you have to do, is to define the configuration element: + +``` +"trailing_stop" : True +``` +This will now actiave an algorithm, whihch automatically moves up your stop loss, every time the price of your asset increases. + +For example, simplified math, + +* you buy an asset at a price of 100$ +* your stop loss is defined at 2% +* which means your stop loss, gets triggered once your asset dropped below 98$ +* assuming your asset now increases in proce to 102$ +* your stop loss, will now be 2% of 102$ or 99.96$ +* now your asset drops in value to 101$, your stop loss, will still be 99.96$ + +basically what this means, is that your stop loss will be adjusted to be always be 2% of the highest observed price + +### Custom positive loss + +due to demand, it is possible to have a default stop loss, when you are in the red with your buy, but once your buy turns positive, +the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you are in the +black, it will be changed to be only a 1% stop loss + +this can be configured in the main confiuration file, the following way: + +``` + "trailing_stop": { + "positive" : 0.01 + }, +``` +The 0.01 would translate to a 1% stop loss, once you hit profit. From 257e1847b121dce297f5dc540ca52e3656c74517 Mon Sep 17 00:00:00 2001 From: peterkorodi Date: Sun, 17 Jun 2018 11:00:34 +0200 Subject: [PATCH 041/395] Update stoploss.md --- docs/stoploss.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index ff93fd1d8..110cfa0d1 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -40,7 +40,7 @@ due to demand, it is possible to have a default stop loss, when you are in the r the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you are in the black, it will be changed to be only a 1% stop loss -this can be configured in the main confiuration file, the following way: +this can be configured in the main configuration file, the following way: ``` "trailing_stop": { From 9ac3c559b6028bf0b382ed4a8adc8e0786f6b57c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 20:26:28 +0200 Subject: [PATCH 042/395] fix some stoploss documentation --- docs/stoploss.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index 110cfa0d1..cf6a2bcab 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -1,11 +1,9 @@ # Stop Loss support -at this stage the bot contains the following stoploss support modes: +At this stage the bot contains the following stoploss support modes: 1. static stop loss, defined in either the strategy or configuration - 2. trailing stop loss, defined in the configuration - 3. trailing stop loss, custom positive loss, defined in configuration ## Static Stop Loss @@ -16,35 +14,37 @@ will overwrite the strategy definition. This will basically try to sell your ass ## Trail Stop Loss The initial value for this stop loss, is defined in your strategy or configuration. Just as you would define your Stop Loss normally. -To enable this Feauture all you have to do, is to define the configuration element: +To enable this Feauture all you have to do is to define the configuration element: -``` +``` json "trailing_stop" : True ``` -This will now actiave an algorithm, whihch automatically moves up your stop loss, every time the price of your asset increases. + +This will now activate an algorithm, which automatically moves your stop loss up every time the price of your asset increases. For example, simplified math, * you buy an asset at a price of 100$ * your stop loss is defined at 2% * which means your stop loss, gets triggered once your asset dropped below 98$ -* assuming your asset now increases in proce to 102$ +* assuming your asset now increases to 102$ * your stop loss, will now be 2% of 102$ or 99.96$ * now your asset drops in value to 101$, your stop loss, will still be 99.96$ -basically what this means, is that your stop loss will be adjusted to be always be 2% of the highest observed price +basically what this means is that your stop loss will be adjusted to be always be 2% of the highest observed price ### Custom positive loss -due to demand, it is possible to have a default stop loss, when you are in the red with your buy, but once your buy turns positive, +Due to demand, it is possible to have a default stop loss, when you are in the red with your buy, but once your buy turns positive, the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you are in the black, it will be changed to be only a 1% stop loss this can be configured in the main configuration file, the following way: -``` +``` json "trailing_stop": { "positive" : 0.01 }, ``` + The 0.01 would translate to a 1% stop loss, once you hit profit. From 243c36b39bda5e126cd9ca6f805e70b15a57deab Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 20:49:07 +0200 Subject: [PATCH 043/395] get persistence.py for stop_loss --- freqtrade/persistence.py | 49 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 20a7db4bb..76a499eed 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -151,6 +151,12 @@ class Trade(_DECL_BASE): open_date = Column(DateTime, nullable=False, default=datetime.utcnow) close_date = Column(DateTime) open_order_id = Column(String) + # absolute value of the stop loss + stop_loss = Column(Float, nullable=True, default=0.0) + # absolute value of the initial stop loss + initial_stop_loss = Column(Float, nullable=True, default=0.0) + # absolute value of the highest reached price + max_rate = Column(Float, nullable=True, default=0.0) def __repr__(self): return 'Trade(id={}, pair={}, amount={:.8f}, open_rate={:.8f}, open_since={})'.format( @@ -161,6 +167,49 @@ class Trade(_DECL_BASE): arrow.get(self.open_date).humanize() if self.is_open else 'closed' ) + def adjust_stop_loss(self, current_price, stoploss): + """ + this adjusts the stop loss to it's most recently observed + setting + :param current_price: + :param stoploss: + :return: + """ + + new_loss = Decimal(current_price * (1 - abs(stoploss))) + + # keeping track of the highest observed rate for this trade + if self.max_rate is None: + self.max_rate = current_price + else: + if current_price > self.max_rate: + self.max_rate = current_price + + # no stop loss assigned yet + if self.stop_loss is None or self.stop_loss == 0: + logger.debug("assigning new stop loss") + self.stop_loss = new_loss + self.initial_stop_loss = new_loss + + # evaluate if the stop loss needs to be updated + else: + if new_loss > self.stop_loss: # stop losses only walk up, never down! + self.stop_loss = new_loss + logger.debug("adjusted stop loss") + else: + logger.debug("keeping current stop loss") + + logger.debug( + "{} - current price {:.8f}, bought at {:.8f} and calculated " + "stop loss is at: {:.8f} initial stop at {:.8f}. trailing stop loss saved us: {:.8f} " + "and max observed rate was {:.8f}".format( + self.pair, current_price, self.open_rate, + self.initial_stop_loss, + self.stop_loss, float(self.stop_loss) - float(self.initial_stop_loss), + self.max_rate + + )) + def update(self, order: Dict) -> None: """ Updates this entity with amount and actual open/close rates. From 5015bc9bb09d8904131e37d7d26addd47a025e3a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 22:41:28 +0200 Subject: [PATCH 044/395] slight update to persistence --- freqtrade/persistence.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 76a499eed..db97fc858 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -176,7 +176,7 @@ class Trade(_DECL_BASE): :return: """ - new_loss = Decimal(current_price * (1 - abs(stoploss))) + new_loss = float(current_price * (1 - abs(stoploss))) # keeping track of the highest observed rate for this trade if self.max_rate is None: @@ -200,15 +200,13 @@ class Trade(_DECL_BASE): logger.debug("keeping current stop loss") logger.debug( - "{} - current price {:.8f}, bought at {:.8f} and calculated " - "stop loss is at: {:.8f} initial stop at {:.8f}. trailing stop loss saved us: {:.8f} " - "and max observed rate was {:.8f}".format( - self.pair, current_price, self.open_rate, - self.initial_stop_loss, - self.stop_loss, float(self.stop_loss) - float(self.initial_stop_loss), - self.max_rate - - )) + f"{self.pair} - current price {current_price:.8f}, " + f"bought at {self.open_rate:.8f} and calculated " + f"stop loss is at: {self.initial_stop_loss:.8f} initial " + f"stop at {self.stop_loss:.8f}. " + f"trailing stop loss saved us: " + f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f} " + f"and max observed rate was {self.max_rate:.8f}") def update(self, order: Dict) -> None: """ From 3e167e11705b204bb2dcc504bf511bd7b2d9f6ca Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 22:41:38 +0200 Subject: [PATCH 045/395] update sample configs --- config.json.example | 5 ++++- config_full.json.example | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index e5bdc95b1..7b53ffa99 100644 --- a/config.json.example +++ b/config.json.example @@ -5,6 +5,9 @@ "fiat_display_currency": "USD", "ticker_interval" : "5m", "dry_run": false, + "trailing_stop": { + "positive" : 0.005 + }, "unfilledtimeout": 600, "bid_strategy": { "ask_last_balance": 0.0 @@ -32,7 +35,7 @@ "experimental": { "use_sell_signal": false, "sell_profit_only": false, - "ignore_roi_if_buy_signal": false + "sell_fullfilled_at_roi": false }, "telegram": { "enabled": true, diff --git a/config_full.json.example b/config_full.json.example index 231384acc..d7d8f7369 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -5,6 +5,7 @@ "fiat_display_currency": "USD", "dry_run": false, "ticker_interval": "5m", + "trailing_stop": true, "minimal_roi": { "40": 0.0, "30": 0.01, From da5be9fbd0ea132a9490ecca007d2e8adf568c34 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 23:06:27 +0200 Subject: [PATCH 046/395] add stop_loss based on work from @berlinguyinca --- config.json.example | 4 +--- config_full.json.example | 3 ++- docs/stoploss.md | 6 ++---- freqtrade/analyze.py | 42 ++++++++++++++++++++++++++++++++++++---- freqtrade/constants.py | 2 ++ 5 files changed, 45 insertions(+), 12 deletions(-) diff --git a/config.json.example b/config.json.example index 7b53ffa99..f92849ec9 100644 --- a/config.json.example +++ b/config.json.example @@ -5,9 +5,7 @@ "fiat_display_currency": "USD", "ticker_interval" : "5m", "dry_run": false, - "trailing_stop": { - "positive" : 0.005 - }, + "trailing_stop": false, "unfilledtimeout": 600, "bid_strategy": { "ask_last_balance": 0.0 diff --git a/config_full.json.example b/config_full.json.example index d7d8f7369..244daabac 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -5,7 +5,8 @@ "fiat_display_currency": "USD", "dry_run": false, "ticker_interval": "5m", - "trailing_stop": true, + "trailing_stop": false, + "trailing_stop_positive": 0.005, "minimal_roi": { "40": 0.0, "30": 0.01, diff --git a/docs/stoploss.md b/docs/stoploss.md index cf6a2bcab..db4433a02 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -39,12 +39,10 @@ Due to demand, it is possible to have a default stop loss, when you are in the r the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you are in the black, it will be changed to be only a 1% stop loss -this can be configured in the main configuration file, the following way: +This can be configured in the main configuration file and requires `"trailing_stop": true` to be set to true. ``` json - "trailing_stop": { - "positive" : 0.01 - }, + "trailing_stop_positive": 0.01, ``` The 0.01 would translate to a 1% stop loss, once you hit profit. diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 36e00dd0e..b283e3ae9 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -180,7 +180,7 @@ class Analyze(object): :return: True if trade should be sold, False otherwise """ current_profit = trade.calc_profit_percent(rate) - if self.stop_loss_reached(current_profit=current_profit): + if self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date): return True experimental = self.config.get('experimental', {}) @@ -204,12 +204,46 @@ class Analyze(object): return False - def stop_loss_reached(self, current_profit: float) -> bool: - """Based on current profit of the trade and configured stoploss, decides to sell or not""" + def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime) -> bool: + """ + Based on current profit of the trade and configured (trailing) stoploss, + decides to sell or not + """ + + current_profit = trade.calc_profit_percent(current_rate) + trailing_stop = self.config.get('trailing_stop', False) + if trade.stop_loss is None: + # initially adjust the stop loss to the base value + trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss) + + # evaluate if the stoploss was hit + if self.strategy.stoploss is not None and trade.stop_loss >= current_rate: + + if trailing_stop: + logger.debug( + f"HIT STOP: current price at {current_rate:.6f}, " + f"stop loss is {trade.stop_loss:.6f}, " + f"initial stop loss was at {trade.initial_stop_loss:.6f}, " + f"trade opened at {trade.open_rate:.6f}") + logger.debug(f"trailing stop saved {trade.stop_loss - trade.initial_stop_loss:.6f}") - if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: logger.debug('Stop loss hit.') return True + + # update the stop loss afterwards, after all by definition it's supposed to be hanging + if trailing_stop: + + # check if we have a special stop loss for positive condition + # and if profit is positive + stop_loss_value = self.strategy.stoploss + if 'trailing_stop_positive' in self.config and current_profit > 0: + + stop_loss_value = self.config.get('trailing_stop_positive') + logger.debug(f"using positive stop loss mode: {stop_loss_value} " + f"since we have profit {current_profit}") + + trade.adjust_stop_loss(current_rate, stop_loss_value) + return False def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 0f12905e3..4ba1a7094 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -61,6 +61,8 @@ CONF_SCHEMA = { 'minProperties': 1 }, 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, + 'trailing_stop': {'type': 'boolean'}, + 'trailing_stop_positive': {'type': 'number', 'minimum': 0}, 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, 'bid_strategy': { 'type': 'object', From 03005bc0f14f9f7747652a13980c8f8a209ded11 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 23:14:12 +0200 Subject: [PATCH 047/395] update documentation --- docs/configuration.md | 42 ++++++++++++++++++++++++++++++++++-------- docs/index.md | 2 ++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5279ed06e..f24adb61e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,12 +1,15 @@ # Configure the bot + This page explains how to configure your `config.json` file. ## Table of Contents + - [Bot commands](#bot-commands) - [Backtesting commands](#backtesting-commands) - [Hyperopt commands](#hyperopt-commands) ## Setup config.json + We recommend to copy and use the `config.json.example` as a template for your bot configuration. @@ -22,6 +25,8 @@ The table below will list all configuration parameters. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. +| `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). +| `trailing_stoploss_positve` | 0 | No | Changes stop-loss once profit has been reached. | `unfilledtimeout` | 0 | No | How long (in minutes) the bot will wait for an unfilled order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). @@ -41,10 +46,10 @@ The table below will list all configuration parameters. | `strategy_path` | null | No | Adds an additional strategy lookup path (must be a folder). | `internals.process_throttle_secs` | 5 | Yes | Set the process throttle. Value in second. -The definition of each config parameters is in -[misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc.py#L205). +The definition of each config parameters is in [misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc.py#L205). ### Understand stake_amount + `stake_amount` is an amount of crypto-currency your bot will use for each trade. The minimal value is 0.0005. If there is not enough crypto-currency in the account an exception is generated. @@ -52,10 +57,12 @@ To allow the bot to trade all the avaliable `stake_currency` in your account set In this case a trade amount is calclulated as `currency_balanse / (max_open_trades - current_open_trades)`. ### Understand minimal_roi + `minimal_roi` is a JSON object where the key is a duration in minutes and the value is the minimum ROI in percent. See the example below: -``` + +``` json "minimal_roi": { "40": 0.0, # Sell after 40 minutes if the profit is not negative "30": 0.01, # Sell after 30 minutes if there is at least 1% profit @@ -69,6 +76,7 @@ value. This parameter is optional. If you use it, it will take over the `minimal_roi` value from the strategy file. ### Understand stoploss + `stoploss` is loss in percentage that should trigger a sale. For example value `-0.10` will cause immediate sell if the profit dips below -10% for a given trade. This parameter is optional. @@ -77,82 +85,100 @@ Most of the strategy files already include the optimal `stoploss` value. This parameter is optional. If you use it, it will take over the `stoploss` value from the strategy file. +### Understand trailing stoploss + +Go to the [trailing stoploss Documentation](stoploss.md) for details on trailing stoploss. + ### Understand initial_state + `initial_state` is an optional field that defines the initial application state. Possible values are `running` or `stopped`. (default=`running`) If the value is `stopped` the bot has to be started with `/start` first. ### Understand process_throttle_secs + `process_throttle_secs` is an optional field that defines in seconds how long the bot should wait before asking the strategy if we should buy or a sell an asset. After each wait period, the strategy is asked again for every opened trade wether or not we should sell, and for all the remaining pairs (either the dynamic list of pairs or the static list of pairs) if we should buy. ### Understand ask_last_balance + `ask_last_balance` sets the bidding price. Value `0.0` will use `ask` price, `1.0` will use the `last` price and values between those interpolate between ask and last price. Using `ask` price will guarantee quick success in bid, but bot will also end up paying more then would probably have been necessary. ### What values for exchange.name? + Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports 115 cryptocurrency exchange markets and trading APIs. The complete up-to-date list can be found in the [CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). However, the bot was tested with only Bittrex and Binance. The bot was tested with the following exchanges: + - [Bittrex](https://bittrex.com/): "bittrex" - [Binance](https://www.binance.com/): "binance" Feel free to test other exchanges and submit your PR to improve the bot. ### What values for fiat_display_currency? + `fiat_display_currency` set the base currency to use for the conversion from coin to fiat in Telegram. The valid values are: "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD". In addition to central bank currencies, a range of cryto currencies are supported. The valid values are: "BTC", "ETH", "XRP", "LTC", "BCH", "USDT". ## Switch to dry-run mode + We recommend starting the bot in dry-run mode to see how your bot will behave and how is the performance of your strategy. In Dry-run mode the bot does not engage your money. It only runs a live simulation without creating trades. ### To switch your bot in Dry-run mode: + 1. Edit your `config.json` file 2. Switch dry-run to true and specify db_url for a persistent db + ```json "dry_run": true, "db_url": "sqlite///tradesv3.dryrun.sqlite", ``` 3. Remove your Exchange API key (change them by fake api credentials) + ```json "exchange": { "name": "bittrex", "key": "key", "secret": "secret", ... -} +} ``` Once you will be happy with your bot performance, you can switch it to production mode. ## Switch to production mode + In production mode, the bot will engage your money. Be careful a wrong strategy can lose all your money. Be aware of what you are doing when you run it in production mode. ### To switch your bot in production mode: + 1. Edit your `config.json` file 2. Switch dry-run to false and don't forget to adapt your database URL if set + ```json "dry_run": false, ``` 3. Insert your Exchange API key (change them by fake api keys) + ```json "exchange": { "name": "bittrex", @@ -160,10 +186,10 @@ you run it in production mode. "secret": "08a9dc6db3d7b53e1acebd9275677f4b0a04f1a5", ... } + ``` -If you have not your Bittrex API key yet, -[see our tutorial](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md). +If you have not your Bittrex API key yet, [see our tutorial](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md). ## Next step -Now you have configured your config.json, the next step is to -[start your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md). + +Now you have configured your config.json, the next step is to [start your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md). diff --git a/docs/index.md b/docs/index.md index afde2d5eb..fd6bf4378 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,5 @@ # freqtrade documentation + Welcome to freqtrade documentation. Please feel free to contribute to this documentation if you see it became outdated by sending us a Pull-request. Do not hesitate to reach us on @@ -6,6 +7,7 @@ Pull-request. Do not hesitate to reach us on if you do not find the answer to your questions. ## Table of Contents + - [Pre-requisite](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md) - [Setup your Bittrex account](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-bittrex-account) - [Setup your Telegram bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md#setup-your-telegram-bot) From a3708bc56e17912af630cd2258f2920ec9d4ec17 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 23:40:20 +0200 Subject: [PATCH 048/395] add missing test --- freqtrade/tests/test_analyze.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 89dac21c6..02225acca 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -42,6 +42,7 @@ def test_analyze_object() -> None: assert hasattr(Analyze, 'get_signal') assert hasattr(Analyze, 'should_sell') assert hasattr(Analyze, 'min_roi_reached') + assert hasattr(Analyze, 'stop_loss_reached') def test_dataframe_correct_length(result): From 8bec505bbe23919da0035f86627d63a150ab445d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 23:40:36 +0200 Subject: [PATCH 049/395] add test for trailing_stoploss --- freqtrade/tests/test_freqtradebot.py | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..3568dad1d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1654,6 +1654,43 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> assert freqtrade.handle_trade(trade) is True +def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) -> None: + """ + Test sell_profit_only feature when enabled and we have a loss + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.0000172, + 'ask': 0.0000173, + 'last': 0.0000172 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + + conf = deepcopy(default_conf) + conf['trailing_stop'] = True + print(limit_buy_order) + freqtrade = FreqtradeBot(conf) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.update(limit_buy_order) + trade.stop_loss = 3.2 + caplog.set_level(logging.DEBUG) + assert freqtrade.handle_trade(trade) is True + assert log_has( + f'HIT STOP: current price at 0.000017, stop loss is {trade.stop_loss:.6f}, ' + f'initial stop loss was at 0.000000, trade opened at 0.000011', caplog.record_tuples) + + + def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: """ From 88b898cce4a434cd15567581f42663056fe05da0 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 00:16:19 +0200 Subject: [PATCH 050/395] add test for moving stoploss --- freqtrade/tests/test_freqtradebot.py | 63 ++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 3568dad1d..468f5f8aa 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1661,14 +1661,14 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ - 'bid': 0.0000172, - 'ask': 0.0000173, - 'last': 0.0000172 + 'bid': 0.00000102, + 'ask': 0.00000103, + 'last': 0.00000102 }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -1682,14 +1682,61 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) trade = Trade.query.first() trade.update(limit_buy_order) - trade.stop_loss = 3.2 caplog.set_level(logging.DEBUG) + # Sell as trailing-stop is reached + assert freqtrade.handle_trade(trade) is True + assert log_has( + f'HIT STOP: current price at 0.000001, stop loss is {trade.stop_loss:.6f}, ' + f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) + + +def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, mocker) -> None: + """ + Test sell_profit_only feature when enabled and we have a loss + """ + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.0000182, + 'ask': 0.0000183, + 'last': 0.0000182 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + + conf = deepcopy(default_conf) + conf['trailing_stop'] = True + conf['trailing_stop_positive'] = 0.01 + print(limit_buy_order) + freqtrade = FreqtradeBot(conf) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.update(limit_buy_order) + caplog.set_level(logging.DEBUG) + # stop-loss not reached + assert freqtrade.handle_trade(trade) is False + # Adjusted stoploss + assert log_has(f'using positive stop loss mode: 0.01 since we have profit 0.64779142', + caplog.record_tuples) + assert log_has(f'adjusted stop loss', caplog.record_tuples) + assert trade.stop_loss == 0.000018018 + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(return_value={ + 'bid': 0.0000172, + 'ask': 0.0000173, + 'last': 0.0000172 + })) assert freqtrade.handle_trade(trade) is True assert log_has( f'HIT STOP: current price at 0.000017, stop loss is {trade.stop_loss:.6f}, ' - f'initial stop loss was at 0.000000, trade opened at 0.000011', caplog.record_tuples) - - + f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: From e9d5bceeb98b12aa35c020fea29d84bf24088ed4 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 00:18:50 +0200 Subject: [PATCH 051/395] cleanly check if stop_loss is initialized --- freqtrade/analyze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index b283e3ae9..b8caa45b7 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -212,7 +212,7 @@ class Analyze(object): current_profit = trade.calc_profit_percent(current_rate) trailing_stop = self.config.get('trailing_stop', False) - if trade.stop_loss is None: + if trade.stop_loss is None or trade.stop_loss == 0: # initially adjust the stop loss to the base value trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss) From a91d75b3b2acdcce30a5e7bc9745430072da00d3 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:23:49 +0200 Subject: [PATCH 052/395] Add test for adjust_stop-loss --- freqtrade/tests/test_persistence.py | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 30ad239a1..6ffcbbf65 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -453,3 +453,37 @@ def test_migrate_new(mocker, default_conf, fee): assert trade.stake_amount == default_conf.get("stake_amount") assert trade.pair == "ETC/BTC" assert trade.exchange == "binance" + + +def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + fee_open=fee.return_value, + fee_close=fee.return_value, + exchange='bittrex', + open_rate=1, + ) + + # Get percent of profit with a custom rate (Higher than open rate) + trade.adjust_stop_loss(1, 0.05) + assert trade.stop_loss == 0.95 + assert trade.max_rate == 1 + assert trade.initial_stop_loss == 0.95 + + trade.adjust_stop_loss(1.3, -0.1) + assert round(trade.stop_loss, 8) == 1.17 + assert trade.max_rate == 1.3 + assert trade.initial_stop_loss == 0.95 + + # current rate lower ... should not change + trade.adjust_stop_loss(1.2, 0.1) + assert round(trade.stop_loss, 8) == 1.17 + assert trade.max_rate == 1.3 + assert trade.initial_stop_loss == 0.95 + + # current rate higher... should raise stoploss + trade.adjust_stop_loss(1.4, 0.1) + assert round(trade.stop_loss, 8) == 1.26 + assert trade.max_rate == 1.4 + assert trade.initial_stop_loss == 0.95 From c997aa9864ef0aec7b89bfb59982f7b99a609e39 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:38:49 +0200 Subject: [PATCH 053/395] move initial logic to persistence --- freqtrade/analyze.py | 5 ++--- freqtrade/persistence.py | 12 ++++++++---- freqtrade/tests/test_persistence.py | 18 +++++++++++++++--- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index b8caa45b7..d74f9566a 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -212,9 +212,8 @@ class Analyze(object): current_profit = trade.calc_profit_percent(current_rate) trailing_stop = self.config.get('trailing_stop', False) - if trade.stop_loss is None or trade.stop_loss == 0: - # initially adjust the stop loss to the base value - trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss) + + trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) # evaluate if the stoploss was hit if self.strategy.stoploss is not None and trade.stop_loss >= current_rate: diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index db97fc858..4fc83a18f 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -167,15 +167,19 @@ class Trade(_DECL_BASE): arrow.get(self.open_date).humanize() if self.is_open else 'closed' ) - def adjust_stop_loss(self, current_price, stoploss): + def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool =False): """ - this adjusts the stop loss to it's most recently observed - setting + this adjusts the stop loss to it's most recently observed setting :param current_price: - :param stoploss: + :param stoploss in percent: + :param initial: :return: """ + if initial and not (self.stop_loss is None or self.stop_loss == 0): + # Don't modify if called with initial and nothing to do + return + new_loss = float(current_price * (1 - abs(stoploss))) # keeping track of the highest observed rate for this trade diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 6ffcbbf65..45957853f 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -465,18 +465,24 @@ def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): open_rate=1, ) - # Get percent of profit with a custom rate (Higher than open rate) - trade.adjust_stop_loss(1, 0.05) + trade.adjust_stop_loss(trade.open_rate, 0.05, True) assert trade.stop_loss == 0.95 assert trade.max_rate == 1 assert trade.initial_stop_loss == 0.95 + # Get percent of profit with a lowre rate + trade.adjust_stop_loss(0.96, 0.05) + assert trade.stop_loss == 0.95 + assert trade.max_rate == 1 + assert trade.initial_stop_loss == 0.95 + + # Get percent of profit with a custom rate (Higher than open rate) trade.adjust_stop_loss(1.3, -0.1) assert round(trade.stop_loss, 8) == 1.17 assert trade.max_rate == 1.3 assert trade.initial_stop_loss == 0.95 - # current rate lower ... should not change + # current rate lower again ... should not change trade.adjust_stop_loss(1.2, 0.1) assert round(trade.stop_loss, 8) == 1.17 assert trade.max_rate == 1.3 @@ -487,3 +493,9 @@ def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): assert round(trade.stop_loss, 8) == 1.26 assert trade.max_rate == 1.4 assert trade.initial_stop_loss == 0.95 + + # Initial is true but stop_loss set - so doesn't do anything + trade.adjust_stop_loss(1.7, 0.1, True) + assert round(trade.stop_loss, 8) == 1.26 + assert trade.max_rate == 1.4 + assert trade.initial_stop_loss == 0.95 From 78e6c9fdf680307c262e1246d02e4d76a877c90a Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:51:48 +0200 Subject: [PATCH 054/395] add tests for trailing stoploss --- freqtrade/tests/test_freqtradebot.py | 41 ++++++++++++++++++---------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 468f5f8aa..7b2786421 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1694,6 +1694,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, """ Test sell_profit_only feature when enabled and we have a loss """ + buy_price = limit_buy_order['price'] patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) @@ -1702,9 +1703,9 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=MagicMock(return_value={ - 'bid': 0.0000182, - 'ask': 0.0000183, - 'last': 0.0000182 + 'bid': buy_price - 0.000001, + 'ask': buy_price - 0.000001, + 'last': buy_price - 0.000001 }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -1713,7 +1714,6 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, conf = deepcopy(default_conf) conf['trailing_stop'] = True conf['trailing_stop_positive'] = 0.01 - print(limit_buy_order) freqtrade = FreqtradeBot(conf) freqtrade.create_trade() @@ -1722,22 +1722,35 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, caplog.set_level(logging.DEBUG) # stop-loss not reached assert freqtrade.handle_trade(trade) is False - # Adjusted stoploss - assert log_has(f'using positive stop loss mode: 0.01 since we have profit 0.64779142', - caplog.record_tuples) - assert log_has(f'adjusted stop loss', caplog.record_tuples) - assert trade.stop_loss == 0.000018018 + + # Raise ticker above buy price mocker.patch('freqtrade.exchange.Exchange.get_ticker', MagicMock(return_value={ - 'bid': 0.0000172, - 'ask': 0.0000173, - 'last': 0.0000172 - })) + 'bid': buy_price + 0.000003, + 'ask': buy_price + 0.000003, + 'last': buy_price + 0.000003 + })) + # stop-loss not reached, adjusted stoploss + assert freqtrade.handle_trade(trade) is False + assert log_has(f'using positive stop loss mode: 0.01 since we have profit 0.26662643', + caplog.record_tuples) + assert log_has(f'adjusted stop loss', caplog.record_tuples) + assert trade.stop_loss == 0.0000138501 + + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(return_value={ + 'bid': buy_price + 0.000002, + 'ask': buy_price + 0.000002, + 'last': buy_price + 0.000002 + })) + # Lower price again (but still positive) assert freqtrade.handle_trade(trade) is True assert log_has( - f'HIT STOP: current price at 0.000017, stop loss is {trade.stop_loss:.6f}, ' + f'HIT STOP: current price at {buy_price + 0.000002:.6f}, ' + f'stop loss is {trade.stop_loss:.6f}, ' f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) + def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: """ From e6e868a03c3f4b6f94d89449509b8c618f53b84f Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:54:29 +0200 Subject: [PATCH 055/395] remove markdown code type as it is not valid json --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index f24adb61e..984f2529b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -62,7 +62,7 @@ In this case a trade amount is calclulated as `currency_balanse / (max_open_trad in minutes and the value is the minimum ROI in percent. See the example below: -``` json +``` "minimal_roi": { "40": 0.0, # Sell after 40 minutes if the profit is not negative "30": 0.01, # Sell after 30 minutes if there is at least 1% profit From 8ecdae67e1417de14177da25ce30df8a28c7ad66 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 06:57:41 +0200 Subject: [PATCH 056/395] add mypy ignore (and comment as to why) --- freqtrade/analyze.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index d74f9566a..bf6b89b17 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -237,7 +237,8 @@ class Analyze(object): stop_loss_value = self.strategy.stoploss if 'trailing_stop_positive' in self.config and current_profit > 0: - stop_loss_value = self.config.get('trailing_stop_positive') + # Ignore mypy error check in configuration that this is a float + stop_loss_value = self.config.get('trailing_stop_positive') # type: ignore logger.debug(f"using positive stop loss mode: {stop_loss_value} " f"since we have profit {current_profit}") From 19beb0941f4354e00a89238ff1630d86d5df9050 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 27 Jun 2018 14:23:07 +0200 Subject: [PATCH 057/395] Update ccxt from 1.14.272 to 1.14.288 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d4f8b2f06..62e2e3983 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.272 +ccxt==1.14.288 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 860b270e309a022e607d789754eb3ab338475d58 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 19:49:08 +0200 Subject: [PATCH 058/395] update db migrate script to work for more changes --- freqtrade/persistence.py | 33 +++++++++++++++++++---------- freqtrade/tests/test_persistence.py | 3 +++ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 4fc83a18f..c72974c61 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -66,6 +66,10 @@ def has_column(columns, searchname: str) -> bool: return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 +def get_column_def(columns, column: str, default: str) -> str: + return default if not has_column(columns, column) else column + + def check_migrate(engine) -> None: """ Checks if migration is necessary and migrates if necessary @@ -74,17 +78,26 @@ def check_migrate(engine) -> None: cols = inspector.get_columns('trades') - if not has_column(cols, 'fee_open'): + # Check for latest column + if not has_column(cols, 'max_rate'): + open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') + close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') + stop_loss = get_column_def(cols, 'stop_loss', '0.0') + initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') + max_rate = get_column_def(cols, 'max_rate', '0.0') + # Schema migration necessary engine.execute("alter table trades rename to trades_bak") # let SQLAlchemy create the schema as required _DECL_BASE.metadata.create_all(engine) # Copy data back - following the correct schema - engine.execute("""insert into trades + engine.execute(f"""insert into trades (id, exchange, pair, is_open, fee_open, fee_close, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, - stake_amount, amount, open_date, close_date, open_order_id) + stake_amount, amount, open_date, close_date, open_order_id, + stop_loss, initial_stop_loss, max_rate + ) select id, lower(exchange), case when instr(pair, '_') != 0 then @@ -94,9 +107,12 @@ def check_migrate(engine) -> None: end pair, is_open, fee fee_open, fee fee_close, - open_rate, null open_rate_requested, close_rate, - null close_rate_requested, close_profit, - stake_amount, amount, open_date, close_date, open_order_id + open_rate, {open_rate_requested} open_rate_requested, close_rate, + {close_rate_requested} close_rate_requested, close_profit, + stake_amount, amount, open_date, close_date, open_order_id, + {stop_loss} stop_loss, {initial_stop_loss} initial_stop_loss, + {max_rate} max_rate + from trades_bak """) @@ -104,11 +120,6 @@ def check_migrate(engine) -> None: inspector = inspect(engine) cols = inspector.get_columns('trades') - if not has_column(cols, 'open_rate_requested'): - engine.execute("alter table trades add open_rate_requested float") - if not has_column(cols, 'close_rate_requested'): - engine.execute("alter table trades add close_rate_requested float") - def cleanup() -> None: """ diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 45957853f..0cd2d089c 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -400,6 +400,9 @@ def test_migrate_old(mocker, default_conf, fee): assert trade.stake_amount == default_conf.get("stake_amount") assert trade.pair == "ETC/BTC" assert trade.exchange == "bittrex" + assert trade.max_rate == 0.0 + assert trade.stop_loss == 0.0 + assert trade.initial_stop_loss == 0.0 def test_migrate_new(mocker, default_conf, fee): From d5ad066f8d4af30340da2e12eaef61a88bf48224 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Wed, 27 Jun 2018 20:15:25 +0200 Subject: [PATCH 059/395] support multiple db transitions by keeping the backup-table dynamic --- freqtrade/persistence.py | 12 +++++++++--- freqtrade/tests/test_persistence.py | 10 +++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index c72974c61..2b226e53a 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -77,6 +77,13 @@ def check_migrate(engine) -> None: inspector = inspect(engine) cols = inspector.get_columns('trades') + tabs = inspector.get_table_names() + table_back_name = 'trades_bak' + i = 0 + while table_back_name in tabs: + i += 1 + table_back_name = f'trades_bak{i}' + logger.info(f'trying {table_back_name}') # Check for latest column if not has_column(cols, 'max_rate'): @@ -87,7 +94,7 @@ def check_migrate(engine) -> None: max_rate = get_column_def(cols, 'max_rate', '0.0') # Schema migration necessary - engine.execute("alter table trades rename to trades_bak") + engine.execute(f"alter table trades rename to {table_back_name}") # let SQLAlchemy create the schema as required _DECL_BASE.metadata.create_all(engine) @@ -112,8 +119,7 @@ def check_migrate(engine) -> None: stake_amount, amount, open_date, close_date, open_order_id, {stop_loss} stop_loss, {initial_stop_loss} initial_stop_loss, {max_rate} max_rate - - from trades_bak + from {table_back_name} """) # Reread columns - the above recreated the table! diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 0cd2d089c..7ef4d2c25 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -7,6 +7,7 @@ from sqlalchemy import create_engine from freqtrade import constants, OperationalException from freqtrade.persistence import Trade, init, clean_dry_run_db +from freqtrade.tests.conftest import log_has @pytest.fixture(scope='function') @@ -405,7 +406,7 @@ def test_migrate_old(mocker, default_conf, fee): assert trade.initial_stop_loss == 0.0 -def test_migrate_new(mocker, default_conf, fee): +def test_migrate_new(mocker, default_conf, fee, caplog): """ Test Database migration (starting with new pairformat) """ @@ -442,6 +443,9 @@ def test_migrate_new(mocker, default_conf, fee): # Create table using the old format engine.execute(create_table_old) engine.execute(insert_table_old) + + # fake previous backup + engine.execute("create table trades_bak as select * from trades") # Run init to test migration init(default_conf) @@ -456,6 +460,10 @@ def test_migrate_new(mocker, default_conf, fee): assert trade.stake_amount == default_conf.get("stake_amount") assert trade.pair == "ETC/BTC" assert trade.exchange == "binance" + assert trade.max_rate == 0.0 + assert trade.stop_loss == 0.0 + assert trade.initial_stop_loss == 0.0 + assert log_has("trying trades_bak1", caplog.record_tuples) def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): From 7cecae52799eb480e2265802e41ef0bd863d5a1c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 28 Jun 2018 14:23:07 +0200 Subject: [PATCH 060/395] Update ccxt from 1.14.288 to 1.14.289 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 62e2e3983..76dba3c49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.288 +ccxt==1.14.289 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 2d4ce593b511e3f9cd977a307c002307b4c6a052 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 19:48:05 +0200 Subject: [PATCH 061/395] catch crash with cobinhood fixes #959 --- freqtrade/freqtradebot.py | 11 +++++++---- freqtrade/tests/test_freqtradebot.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e25ed66cf..985ecb370 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -277,11 +277,14 @@ class FreqtradeBot(object): return None min_stake_amounts = [] - if 'cost' in market['limits'] and 'min' in market['limits']['cost']: - min_stake_amounts.append(market['limits']['cost']['min']) + limits = market['limits'] + if ('cost' in limits and 'min' in limits['cost'] + and limits['cost']['min'] is not None): + min_stake_amounts.append(limits['cost']['min']) - if 'amount' in market['limits'] and 'min' in market['limits']['amount']: - min_stake_amounts.append(market['limits']['amount']['min'] * price) + if ('amount' in limits and 'min' in limits['amount'] + and limits['amount']['min'] is not None): + min_stake_amounts.append(limits['amount']['min'] * price) if not min_stake_amounts: return None diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..ff1b7161d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -348,6 +348,34 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) assert result is None + # no cost Min + mocker.patch( + 'freqtrade.exchange.Exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {"min": None}, + 'amount': {} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result is None + + # no amount Min + mocker.patch( + 'freqtrade.exchange.Exchange.get_markets', + MagicMock(return_value=[{ + 'symbol': 'ETH/BTC', + 'limits': { + 'cost': {}, + 'amount': {"min": None} + } + }]) + ) + result = freqtrade._get_min_pair_stake_amount('ETH/BTC', 1) + assert result is None + # empty 'cost'/'amount' section mocker.patch( 'freqtrade.exchange.Exchange.get_markets', From 8ec9a0974978242fb1bd47587151eec6a0a5bda6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 21:22:43 +0200 Subject: [PATCH 062/395] Standardize retrier exception testing --- freqtrade/tests/exchange/test_exchange.py | 141 ++++++---------------- 1 file changed, 39 insertions(+), 102 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 53e59b34b..4d46d7e61 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,6 +14,22 @@ from freqtrade.exchange import Exchange, API_RETRY_COUNT from freqtrade.tests.conftest import log_has, get_patched_exchange +def validate_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): + """Function to test exception handling """ + + with pytest.raises(TemporaryError): + api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[innerfun].call_count == API_RETRY_COUNT + 1 + + with pytest.raises(OperationalException): + api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[innerfun].call_count == 1 + + def test_init(default_conf, mocker, caplog): caplog.set_level(logging.INFO) get_patched_exchange(mocker, default_conf) @@ -243,17 +259,8 @@ def test_get_balances_prod(default_conf, mocker): assert exchange.get_balances()['1ST']['total'] == 10.0 assert exchange.get_balances()['1ST']['used'] == 0.0 - with pytest.raises(TemporaryError): - api_mock.fetch_balance = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_balances() - assert api_mock.fetch_balance.call_count == API_RETRY_COUNT + 1 - - with pytest.raises(OperationalException): - api_mock.fetch_balance = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_balances() - assert api_mock.fetch_balance.call_count == 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_balances", "fetch_balance") def test_get_tickers(default_conf, mocker): @@ -282,15 +289,8 @@ def test_get_tickers(default_conf, mocker): assert tickers['BCH/BTC']['bid'] == 0.6 assert tickers['BCH/BTC']['ask'] == 0.5 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_tickers() - - with pytest.raises(OperationalException): - api_mock.fetch_tickers = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_tickers() + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_tickers", "fetch_tickers") with pytest.raises(OperationalException): api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NotSupported) @@ -345,15 +345,9 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=False) assert api_mock.fetch_ticker.call_count == 0 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_ticker = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_ticker(pair='ETH/BTC', refresh=True) - - with pytest.raises(OperationalException): - api_mock.fetch_ticker = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_ticker(pair='ETH/BTC', refresh=True) + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker", "fetch_ticker", + pair='ETH/BTC', refresh=True) api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock) @@ -416,17 +410,9 @@ def test_get_ticker_history(default_conf, mocker): assert ticks[0][4] == 9 assert ticks[0][5] == 10 - with pytest.raises(TemporaryError): # test retrier - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - # new symbol to get around cache - exchange.get_ticker_history('ABCD/BTC', default_conf['ticker_interval']) - - with pytest.raises(OperationalException): - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - # new symbol to get around cache - exchange.get_ticker_history('EFGH/BTC', default_conf['ticker_interval']) + validate_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker_history", "fetch_ohlcv", + pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) def test_get_ticker_history_sort(default_conf, mocker): @@ -515,24 +501,15 @@ def test_cancel_order(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) assert exchange.cancel_order(order_id='_', pair='TKN/BTC') == 123 - with pytest.raises(TemporaryError): - api_mock.cancel_order = MagicMock(side_effect=ccxt.NetworkError) - - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.cancel_order(order_id='_', pair='TKN/BTC') - assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(DependencyException): api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.cancel_order(order_id='_', pair='TKN/BTC') assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(OperationalException): - api_mock.cancel_order = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.cancel_order(order_id='_', pair='TKN/BTC') - assert api_mock.cancel_order.call_count == 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + "cancel_order", "cancel_order", + order_id='_', pair='TKN/BTC') def test_get_order(default_conf, mocker): @@ -550,23 +527,15 @@ def test_get_order(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) assert exchange.get_order('X', 'TKN/BTC') == 456 - with pytest.raises(TemporaryError): - api_mock.fetch_order = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_order(order_id='_', pair='TKN/BTC') - assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(DependencyException): api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_order(order_id='_', pair='TKN/BTC') assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - with pytest.raises(OperationalException): - api_mock.fetch_order = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_order(order_id='_', pair='TKN/BTC') - assert api_mock.fetch_order.call_count == 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_order', 'fetch_order', + order_id='_', pair='TKN/BTC') def test_name(default_conf, mocker): @@ -651,19 +620,9 @@ def test_get_trades_for_order(default_conf, mocker): assert len(orders) == 1 assert orders[0]['price'] == 165 - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_trades_for_order(order_id, 'LTC/BTC', since) - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.fetch_my_trades = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_trades_for_order(order_id, 'LTC/BTC', since) - assert api_mock.fetch_my_trades.call_count == API_RETRY_COUNT + 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_trades_for_order', 'fetch_my_trades', + order_id=order_id, pair='LTC/BTC', since=since) def test_get_markets(default_conf, mocker, markets): @@ -677,19 +636,8 @@ def test_get_markets(default_conf, mocker, markets): assert ret[0]["id"] == "ethbtc" assert ret[0]["symbol"] == "ETH/BTC" - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.fetch_markets = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_markets() - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.fetch_markets = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_markets() - assert api_mock.fetch_markets.call_count == API_RETRY_COUNT + 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_markets', 'fetch_markets') def test_get_fee(default_conf, mocker): @@ -704,19 +652,8 @@ def test_get_fee(default_conf, mocker): assert exchange.get_fee() == 0.025 - # test Exceptions - with pytest.raises(OperationalException): - api_mock = MagicMock() - api_mock.calculate_fee = MagicMock(side_effect=ccxt.BaseError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_fee() - - with pytest.raises(TemporaryError): - api_mock = MagicMock() - api_mock.calculate_fee = MagicMock(side_effect=ccxt.NetworkError) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_fee() - assert api_mock.calculate_fee.call_count == API_RETRY_COUNT + 1 + validate_exceptionhandlers(mocker, default_conf, api_mock, + 'get_fee', 'calculate_fee') def test_get_amount_lots(default_conf, mocker): From ebbfc720b24c6a378dcf79b5b1c5205f8dd95dea Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 21:51:59 +0200 Subject: [PATCH 063/395] increase test coverage --- freqtrade/tests/exchange/test_exchange.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 4d46d7e61..75394ca84 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -232,6 +232,12 @@ def test_get_balance_prod(default_conf, mocker): exchange.get_balance(currency='BTC') + with pytest.raises(TemporaryError): + # api_mock.fetch_balance = MagicMock(return_value={}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + mocker.patch('freqtrade.exchange.Exchange.get_balances', MagicMock(return_value={})) + exchange.get_balance(currency='BTC') + def test_get_balances_dry_run(default_conf, mocker): default_conf['dry_run'] = True @@ -624,6 +630,9 @@ def test_get_trades_for_order(default_conf, mocker): 'get_trades_for_order', 'fetch_my_trades', order_id=order_id, pair='LTC/BTC', since=since) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + assert exchange.get_trades_for_order(order_id, 'LTC/BTC', since) == [] + def test_get_markets(default_conf, mocker, markets): api_mock = MagicMock() From fe8a21681e3fd8af90093a9fd004e7acd2c51f77 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 21:56:37 +0200 Subject: [PATCH 064/395] add test for Not supported --- freqtrade/tests/exchange/test_exchange.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 75394ca84..012c55356 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -232,8 +232,7 @@ def test_get_balance_prod(default_conf, mocker): exchange.get_balance(currency='BTC') - with pytest.raises(TemporaryError): - # api_mock.fetch_balance = MagicMock(return_value={}) + with pytest.raises(TemporaryError, match=r'.*balance due to malformed exchange response:.*'): exchange = get_patched_exchange(mocker, default_conf, api_mock) mocker.patch('freqtrade.exchange.Exchange.get_balances', MagicMock(return_value={})) exchange.get_balance(currency='BTC') @@ -420,6 +419,11 @@ def test_get_ticker_history(default_conf, mocker): "get_ticker_history", "fetch_ohlcv", pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): + api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_ticker_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + def test_get_ticker_history_sort(default_conf, mocker): api_mock = MagicMock() From 15c7854e7f79c03132e86d789f7725108aec4529 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 22:11:45 +0200 Subject: [PATCH 065/395] add test for exchange_has --- freqtrade/tests/exchange/test_exchange.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 012c55356..25c6fa0c2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -113,6 +113,20 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): Exchange(conf) +def test_exchangehas(default_conf, mocker): + exchange = get_patched_exchange(mocker, default_conf) + assert not exchange.exchange_has('ASDFASDF') + api_mock = MagicMock() + + type(api_mock).has = PropertyMock(return_value={'deadbeef': True}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + assert exchange.exchange_has("deadbeef") + + type(api_mock).has = PropertyMock(return_value={'deadbeef': False}) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + assert not exchange.exchange_has("deadbeef") + + def test_buy_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) From dcdc18a33868a5630b5b747eb2773819154fb88c Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 22:18:38 +0200 Subject: [PATCH 066/395] rename test-function --- freqtrade/tests/exchange/test_exchange.py | 50 +++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 25c6fa0c2..712205893 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,8 +14,8 @@ from freqtrade.exchange import Exchange, API_RETRY_COUNT from freqtrade.tests.conftest import log_has, get_patched_exchange -def validate_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): - """Function to test exception handling """ +def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): + """Function to test ccxt exception handling """ with pytest.raises(TemporaryError): api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.NetworkError) @@ -278,8 +278,8 @@ def test_get_balances_prod(default_conf, mocker): assert exchange.get_balances()['1ST']['total'] == 10.0 assert exchange.get_balances()['1ST']['used'] == 0.0 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_balances", "fetch_balance") + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_balances", "fetch_balance") def test_get_tickers(default_conf, mocker): @@ -308,8 +308,8 @@ def test_get_tickers(default_conf, mocker): assert tickers['BCH/BTC']['bid'] == 0.6 assert tickers['BCH/BTC']['ask'] == 0.5 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_tickers", "fetch_tickers") + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_tickers", "fetch_tickers") with pytest.raises(OperationalException): api_mock.fetch_tickers = MagicMock(side_effect=ccxt.NotSupported) @@ -364,9 +364,9 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=False) assert api_mock.fetch_ticker.call_count == 0 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_ticker", "fetch_ticker", - pair='ETH/BTC', refresh=True) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker", "fetch_ticker", + pair='ETH/BTC', refresh=True) api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock) @@ -429,9 +429,9 @@ def test_get_ticker_history(default_conf, mocker): assert ticks[0][4] == 9 assert ticks[0][5] == 10 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "get_ticker_history", "fetch_ohlcv", - pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "get_ticker_history", "fetch_ohlcv", + pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) @@ -531,9 +531,9 @@ def test_cancel_order(default_conf, mocker): exchange.cancel_order(order_id='_', pair='TKN/BTC') assert api_mock.cancel_order.call_count == API_RETRY_COUNT + 1 - validate_exceptionhandlers(mocker, default_conf, api_mock, - "cancel_order", "cancel_order", - order_id='_', pair='TKN/BTC') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + "cancel_order", "cancel_order", + order_id='_', pair='TKN/BTC') def test_get_order(default_conf, mocker): @@ -557,9 +557,9 @@ def test_get_order(default_conf, mocker): exchange.get_order(order_id='_', pair='TKN/BTC') assert api_mock.fetch_order.call_count == API_RETRY_COUNT + 1 - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_order', 'fetch_order', - order_id='_', pair='TKN/BTC') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_order', 'fetch_order', + order_id='_', pair='TKN/BTC') def test_name(default_conf, mocker): @@ -644,9 +644,9 @@ def test_get_trades_for_order(default_conf, mocker): assert len(orders) == 1 assert orders[0]['price'] == 165 - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_trades_for_order', 'fetch_my_trades', - order_id=order_id, pair='LTC/BTC', since=since) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_trades_for_order', 'fetch_my_trades', + order_id=order_id, pair='LTC/BTC', since=since) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) assert exchange.get_trades_for_order(order_id, 'LTC/BTC', since) == [] @@ -663,8 +663,8 @@ def test_get_markets(default_conf, mocker, markets): assert ret[0]["id"] == "ethbtc" assert ret[0]["symbol"] == "ETH/BTC" - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_markets', 'fetch_markets') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_markets', 'fetch_markets') def test_get_fee(default_conf, mocker): @@ -679,8 +679,8 @@ def test_get_fee(default_conf, mocker): assert exchange.get_fee() == 0.025 - validate_exceptionhandlers(mocker, default_conf, api_mock, - 'get_fee', 'calculate_fee') + ccxt_exceptionhandlers(mocker, default_conf, api_mock, + 'get_fee', 'calculate_fee') def test_get_amount_lots(default_conf, mocker): From cf6b1a637ac0a2312c15d926093851a3a1adf4f1 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Thu, 28 Jun 2018 22:32:28 +0200 Subject: [PATCH 067/395] increase exchange code coverage --- freqtrade/tests/exchange/test_exchange.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 712205893..99b960885 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -36,7 +36,7 @@ def test_init(default_conf, mocker, caplog): assert log_has('Instance is running with dry_run enabled', caplog.record_tuples) -def test_init_exception(default_conf): +def test_init_exception(default_conf, mocker): default_conf['exchange']['name'] = 'wrong_exchange_name' with pytest.raises( @@ -44,6 +44,13 @@ def test_init_exception(default_conf): match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): Exchange(default_conf) + default_conf['exchange']['name'] = 'binance' + with pytest.raises( + OperationalException, + match='Exchange {} is not supported'.format(default_conf['exchange']['name'])): + mocker.patch("ccxt.binance", MagicMock(side_effect=AttributeError)) + Exchange(default_conf) + def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() From 8a941f3aa81ee6aad4778be0e1ae0952ce3108c9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 29 Jun 2018 14:23:06 +0200 Subject: [PATCH 068/395] Update ccxt from 1.14.289 to 1.14.295 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 76dba3c49..6995c2513 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.289 +ccxt==1.14.295 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 0ce08932ed31edde076b91589526ffcd06f428be Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sat, 30 Jun 2018 09:54:31 +0300 Subject: [PATCH 069/395] mypy fixes --- freqtrade/optimize/hyperopt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b8b53ab3c..eb9015356 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -62,7 +62,7 @@ class Hyperopt(Backtesting): # Previous evaluations self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') - self.trials = [] + self.trials: List = [] def get_args(self, params): dimensions = self.hyperopt_space() @@ -191,7 +191,7 @@ class Hyperopt(Backtesting): ] @staticmethod - def indicator_space() -> Dict[str, Any]: + def indicator_space() -> List[Dimension]: """ Define your Hyperopt space for searching strategy parameters """ @@ -349,7 +349,7 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) - self.exchange = None + self.exchange = None # type: ignore self.load_previous_results() cpus = multiprocessing.cpu_count() From 98108a78f1ff5ccd0ca4390a20885137f2d65637 Mon Sep 17 00:00:00 2001 From: Nullart Date: Thu, 14 Jun 2018 09:32:52 +0800 Subject: [PATCH 070/395] separating unfulfilled timeouts for buy and sell --- config.json.example | 5 ++++- config_full.json.example | 5 ++++- freqtrade/constants.py | 9 ++++++++- freqtrade/freqtradebot.py | 19 ++++++++++++------- freqtrade/tests/conftest.py | 5 ++++- freqtrade/tests/test_freqtradebot.py | 8 ++++---- 6 files changed, 36 insertions(+), 15 deletions(-) diff --git a/config.json.example b/config.json.example index e5bdc95b1..336a0c374 100644 --- a/config.json.example +++ b/config.json.example @@ -5,7 +5,10 @@ "fiat_display_currency": "USD", "ticker_interval" : "5m", "dry_run": false, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy":10, + "sell":30 + } "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/config_full.json.example b/config_full.json.example index 231384acc..1985edc70 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -12,7 +12,10 @@ "0": 0.04 }, "stoploss": -0.10, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy":10, + "sell":30 + } "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 0f12905e3..44bd64fe9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -61,7 +61,14 @@ CONF_SCHEMA = { 'minProperties': 1 }, 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, - 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, + 'unfilledtimeout': { + 'type': 'object', + 'properties': { + 'use_book_order': {'type': 'boolean'}, + 'buy': {'type': 'number', 'minimum': 3}, + 'sell': {'type': 'number', 'minimum': 10} + } + }, 'bid_strategy': { 'type': 'object', 'properties': { diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e25ed66cf..c5f932119 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -160,7 +160,7 @@ class FreqtradeBot(object): if 'unfilledtimeout' in self.config: # Check and handle any timed out open orders - self.check_handle_timedout(self.config['unfilledtimeout']) + self.check_handle_timedout() Trade.session.flush() except TemporaryError as error: @@ -492,13 +492,16 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False - def check_handle_timedout(self, timeoutvalue: int) -> None: + def check_handle_timedout(self) -> None: """ Check if any orders are timed out and cancel if neccessary :param timeoutvalue: Number of minutes until order is considered timed out :return: None """ - timeoutthreashold = arrow.utcnow().shift(minutes=-timeoutvalue).datetime + buy_timeout = self.config['unfilledtimeout']['buy'] + sell_timeout = self.config['unfilledtimeout']['sell'] + buy_timeoutthreashold = arrow.utcnow().shift(minutes=-buy_timeout).datetime + sell_timeoutthreashold = arrow.utcnow().shift(minutes=-sell_timeout).datetime for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all(): try: @@ -521,10 +524,12 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ if int(order['remaining']) == 0: continue - if order['side'] == 'buy' and ordertime < timeoutthreashold: - self.handle_timedout_limit_buy(trade, order) - elif order['side'] == 'sell' and ordertime < timeoutthreashold: - self.handle_timedout_limit_sell(trade, order) + # Check if trade is still actually open + if (order['status'] == 'open'): + if order['side'] == 'buy' and ordertime < buy_timeoutthreashold: + self.handle_timedout_limit_buy(trade, order) + elif order['side'] == 'sell' and ordertime < sell_timeoutthreashold: + self.handle_timedout_limit_sell(trade, order) # FIX: 20180110, why is cancel.order unconditionally here, whereas # it is conditionally called in the diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index d9ccaa325..4877fe556 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -100,7 +100,10 @@ def default_conf(): "0": 0.04 }, "stoploss": -0.10, - "unfilledtimeout": 600, + "unfilledtimeout": { + "buy": 10, + "sell": 30 + }, "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..fb7579d45 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1124,7 +1124,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe Trade.session.add(trade_buy) # check it does cancel buy orders over the time limit - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() @@ -1165,7 +1165,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, Trade.session.add(trade_sell) # check it does cancel sell orders over the time limit - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 assert trade_sell.is_open is True @@ -1205,7 +1205,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old # check it does cancel buy orders over the time limit # note this is for a partially-complete buy order - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all() @@ -1256,7 +1256,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - 'recent call last):\n.*' ) - freqtrade.check_handle_timedout(600) + freqtrade.check_handle_timedout() assert filter(regexp.match, caplog.record_tuples) From c447644fd16386868c6a4a1aa6b8a5e139af8c22 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 30 Jun 2018 14:23:06 +0200 Subject: [PATCH 071/395] Update ccxt from 1.14.295 to 1.14.301 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6995c2513..72decf95a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.295 +ccxt==1.14.301 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 From 5a591e01c08fa0c60db152da2c04ec04dfd4e58a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 30 Jun 2018 14:23:07 +0200 Subject: [PATCH 072/395] Update sqlalchemy from 1.2.8 to 1.2.9 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 72decf95a..50441879a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ccxt==1.14.301 -SQLAlchemy==1.2.8 +SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 From 14e12bd3c09c42a4ed20c170765a55826e304b04 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 30 Jun 2018 17:37:34 +0200 Subject: [PATCH 073/395] Fix missing comma in example.json --- config.json.example | 2 +- config_full.json.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index 336a0c374..606167ea3 100644 --- a/config.json.example +++ b/config.json.example @@ -8,7 +8,7 @@ "unfilledtimeout": { "buy":10, "sell":30 - } + }, "bid_strategy": { "ask_last_balance": 0.0 }, diff --git a/config_full.json.example b/config_full.json.example index 1985edc70..4bb794937 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -15,7 +15,7 @@ "unfilledtimeout": { "buy":10, "sell":30 - } + }, "bid_strategy": { "ask_last_balance": 0.0 }, From 9e3e900f78d5105f6d145d80fd5d97dbea0b9cac Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sat, 30 Jun 2018 17:49:46 +0200 Subject: [PATCH 074/395] Add get_markets mock to new tests --- freqtrade/tests/test_freqtradebot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1cb2bfca2..f66864946 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1599,6 +1599,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) @@ -1616,7 +1617,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke assert freqtrade.handle_trade(trade) is True -def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None: +def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ @@ -1634,6 +1635,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets ) conf = deepcopy(default_conf) From 8f49d5eb1002230fb424e4b505ec5f9561e778ea Mon Sep 17 00:00:00 2001 From: Nullart Date: Thu, 14 Jun 2018 18:49:43 +0800 Subject: [PATCH 075/395] documentation updates --- docs/configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5279ed06e..ea3e99f15 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -22,7 +22,8 @@ The table below will list all configuration parameters. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. -| `unfilledtimeout` | 0 | No | How long (in minutes) the bot will wait for an unfilled order to complete, after which the order will be cancelled. +| `unfilledtimeout.buy` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. +| `unfilledtimeout.sell` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. From c66f858b988bd18ee24045839211f4248640b097 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:37:55 +0200 Subject: [PATCH 076/395] rename innerfun to mock_ccxt_fun --- freqtrade/tests/exchange/test_exchange.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 99b960885..6cf7f11b0 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,20 +14,20 @@ from freqtrade.exchange import Exchange, API_RETRY_COUNT from freqtrade.tests.conftest import log_has, get_patched_exchange -def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, innerfun, **kwargs): +def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): """Function to test ccxt exception handling """ with pytest.raises(TemporaryError): - api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.NetworkError) + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) getattr(exchange, fun)(**kwargs) - assert api_mock.__dict__[innerfun].call_count == API_RETRY_COUNT + 1 + assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1 with pytest.raises(OperationalException): - api_mock.__dict__[innerfun] = MagicMock(side_effect=ccxt.BaseError) + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) getattr(exchange, fun)(**kwargs) - assert api_mock.__dict__[innerfun].call_count == 1 + assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 def test_init(default_conf, mocker, caplog): From 2dc881558d65b19dd6596d3a8847da618f7d6885 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:41:19 +0200 Subject: [PATCH 077/395] address PR comments --- config.json.example | 4 ++-- config_full.json.example | 4 ++-- freqtrade/constants.py | 1 - freqtrade/freqtradebot.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/config.json.example b/config.json.example index 606167ea3..79ba2ac64 100644 --- a/config.json.example +++ b/config.json.example @@ -6,8 +6,8 @@ "ticker_interval" : "5m", "dry_run": false, "unfilledtimeout": { - "buy":10, - "sell":30 + "buy": 10, + "sell": 30 }, "bid_strategy": { "ask_last_balance": 0.0 diff --git a/config_full.json.example b/config_full.json.example index 4bb794937..d40c11a91 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -13,8 +13,8 @@ }, "stoploss": -0.10, "unfilledtimeout": { - "buy":10, - "sell":30 + "buy": 10, + "sell": 30 }, "bid_strategy": { "ask_last_balance": 0.0 diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 44bd64fe9..04d4a44ac 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -64,7 +64,6 @@ CONF_SCHEMA = { 'unfilledtimeout': { 'type': 'object', 'properties': { - 'use_book_order': {'type': 'boolean'}, 'buy': {'type': 'number', 'minimum': 3}, 'sell': {'type': 'number', 'minimum': 10} } diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c5f932119..4d6352e55 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -525,7 +525,7 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ continue # Check if trade is still actually open - if (order['status'] == 'open'): + if order['status'] == 'open': if order['side'] == 'buy' and ordertime < buy_timeoutthreashold: self.handle_timedout_limit_buy(trade, order) elif order['side'] == 'sell' and ordertime < sell_timeoutthreashold: From e39d88ef65e72387f18198da269a2d7660a2d6c6 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:54:26 +0200 Subject: [PATCH 078/395] Address some PR comments --- freqtrade/constants.py | 2 +- freqtrade/persistence.py | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 4ba1a7094..d80eea6f4 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -62,7 +62,7 @@ CONF_SCHEMA = { }, 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, 'trailing_stop': {'type': 'boolean'}, - 'trailing_stop_positive': {'type': 'number', 'minimum': 0}, + 'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1}, 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, 'bid_strategy': { 'type': 'object', diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 2b226e53a..ad90d3f92 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -184,14 +184,8 @@ class Trade(_DECL_BASE): arrow.get(self.open_date).humanize() if self.is_open else 'closed' ) - def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool =False): - """ - this adjusts the stop loss to it's most recently observed setting - :param current_price: - :param stoploss in percent: - :param initial: - :return: - """ + def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool = False): + """this adjusts the stop loss to it's most recently observed setting""" if initial and not (self.stop_loss is None or self.stop_loss == 0): # Don't modify if called with initial and nothing to do @@ -207,7 +201,7 @@ class Trade(_DECL_BASE): self.max_rate = current_price # no stop loss assigned yet - if self.stop_loss is None or self.stop_loss == 0: + if not self.stop_loss: logger.debug("assigning new stop loss") self.stop_loss = new_loss self.initial_stop_loss = new_loss From 937644a04b96adbdb26ed54c5969945e09f314c2 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 19:55:51 +0200 Subject: [PATCH 079/395] change while-loop to enumerate - add intensified test for this scenario --- freqtrade/tests/test_persistence.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 7ef4d2c25..12ae6579f 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -446,6 +446,8 @@ def test_migrate_new(mocker, default_conf, fee, caplog): # fake previous backup engine.execute("create table trades_bak as select * from trades") + + engine.execute("create table trades_bak1 as select * from trades") # Run init to test migration init(default_conf) @@ -464,6 +466,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 assert log_has("trying trades_bak1", caplog.record_tuples) + assert log_has("trying trades_bak2", caplog.record_tuples) def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): From 782570e71eb6fca9c5a70cf36896d692661a14a7 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 20:03:07 +0200 Subject: [PATCH 080/395] Address PR comment --- config.json.example | 2 +- freqtrade/persistence.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index 87c27090e..e8473e919 100644 --- a/config.json.example +++ b/config.json.example @@ -36,7 +36,7 @@ "experimental": { "use_sell_signal": false, "sell_profit_only": false, - "sell_fullfilled_at_roi": false + "ignore_roi_if_buy_signal": false }, "telegram": { "enabled": true, diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index ad90d3f92..dfeb6145c 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -80,7 +80,7 @@ def check_migrate(engine) -> None: tabs = inspector.get_table_names() table_back_name = 'trades_bak' i = 0 - while table_back_name in tabs: + for i, table_back_name in enumerate(tabs): i += 1 table_back_name = f'trades_bak{i}' logger.info(f'trying {table_back_name}') From 3c5be55eb907878c73f0cfb26a4f21b3af2a4745 Mon Sep 17 00:00:00 2001 From: xmatthias Date: Sun, 1 Jul 2018 20:17:30 +0200 Subject: [PATCH 081/395] remove unnecessary variable --- freqtrade/persistence.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index dfeb6145c..aa33f7063 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -79,9 +79,7 @@ def check_migrate(engine) -> None: cols = inspector.get_columns('trades') tabs = inspector.get_table_names() table_back_name = 'trades_bak' - i = 0 for i, table_back_name in enumerate(tabs): - i += 1 table_back_name = f'trades_bak{i}' logger.info(f'trying {table_back_name}') From a58d51ded0fcfd06cbe7344da202e353e9deff10 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 2 Jul 2018 09:56:58 +0300 Subject: [PATCH 082/395] update hyperopt documentation --- docs/hyperopt.md | 279 ++++++++++++++++------------------------------- 1 file changed, 96 insertions(+), 183 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 2ad94896a..8d3fe6704 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -1,153 +1,109 @@ # Hyperopt This page explains how to tune your strategy by finding the optimal -parameters with Hyperopt. +parameters, process called hyperparameter optimization. The bot uses several +algorithms included in `scikit-optimize` package to accomplish this. The +search will burn all your CPU cores, make your laptop sound like a fighter jet +and still take a long time. ## Table of Contents - [Prepare your Hyperopt](#prepare-hyperopt) - - [1. Configure your Guards and Triggers](#1-configure-your-guards-and-triggers) - - [2. Update the hyperopt config file](#2-update-the-hyperopt-config-file) -- [Advanced Hyperopt notions](#advanced-notions) - - [Understand the Guards and Triggers](#understand-the-guards-and-triggers) +- [Configure your Guards and Triggers](#configure-your-guards-and-triggers) +- [Solving a Mystery](#solving-a-mystery) +- [Adding New Indicators](#adding-new-indicators) - [Execute Hyperopt](#execute-hyperopt) - [Understand the hyperopts result](#understand-the-backtesting-result) -## Prepare Hyperopt -Before we start digging in Hyperopt, we recommend you to take a look at -your strategy file located into [user_data/strategies/](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) +## Prepare Hyperopting +We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) -### 1. Configure your Guards and Triggers +### Configure your Guards and Triggers There are two places you need to change in your strategy file to add a new buy strategy for testing: -- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L278-L294). -- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297) known as `SPACE`. +- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L278-L294). +- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L218-L229) +and the associated methods `indicator_space`, `roi_space`, `stoploss_space`. -There you have two different type of indicators: 1. `guards` and 2. -`triggers`. +There you have two different type of indicators: 1. `guards` and 2. `triggers`. 1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. 2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or buy when close price touches lower bollinger band. -HyperOpt will, for each eval round, pick just ONE trigger, and possibly -multiple guards. So that the constructed strategy will be something like +Hyperoptimization will, for each eval round, pick one trigger and possibly +multiple guards. The constructed strategy will be something like "*buy exactly when close price touches lower bollinger band, BUT only if ADX > 10*". - -If you have updated the buy strategy, means change the content of +If you have updated the buy strategy, ie. changed the contents of `populate_buy_trend()` method you have to update the `guards` and -`triggers` hyperopts must used. +`triggers` hyperopts must use. -As for an example if your `populate_buy_trend()` method is: -```python -def populate_buy_trend(dataframe: DataFrame) -> DataFrame: - dataframe.loc[ - (dataframe['rsi'] < 35) & - (dataframe['adx'] > 65), - 'buy'] = 1 +## Solving a Mystery - return dataframe -``` +Let's say you are curious: should you use MACD crossings or lower Bollinger +Bands to trigger your buys. And you also wonder should you use RSI or ADX to +help with those buy decisions. If you decide to use RSI or ADX, which values +should I use for them? So let's use hyperparameter optimization to solve this +mystery. -Your hyperopt file must contain `guards` to find the right value for -`(dataframe['adx'] > 65)` & and `(dataframe['plus_di'] > 0.5)`. That -means you will need to enable/disable triggers. - -In our case the `SPACE` and `populate_buy_trend` in your strategy file -will look like: -```python -space = { - 'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)} - ]), - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 15, 50, 1)} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'lower_bb'}, - {'type': 'faststoch10'}, - {'type': 'ao_cross_zero'}, - {'type': 'ema5_cross_ema10'}, - {'type': 'macd_cross_signal'}, - {'type': 'sar_reversal'}, - {'type': 'stochf_cross'}, - {'type': 'ht_sine'}, - ]), -} - -... - -def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - conditions = [] - # GUARDS AND TRENDS - if params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) - - # TRIGGERS - triggers = { - 'lower_bb': dataframe['tema'] <= dataframe['blower'], - 'faststoch10': (crossed_above(dataframe['fastd'], 10.0)), - 'ao_cross_zero': (crossed_above(dataframe['ao'], 0.0)), - 'ema5_cross_ema10': (crossed_above(dataframe['ema5'], dataframe['ema10'])), - 'macd_cross_signal': (crossed_above(dataframe['macd'], dataframe['macdsignal'])), - 'sar_reversal': (crossed_above(dataframe['close'], dataframe['sar'])), - 'stochf_cross': (crossed_above(dataframe['fastk'], dataframe['fastd'])), - 'ht_sine': (crossed_above(dataframe['htleadsine'], dataframe['htsine'])), - } - ... -``` - - -### 2. Update the hyperopt config file -Hyperopt is using a dedicated config file. Currently hyperopt -cannot use your config file. It is also made on purpose to allow you -testing your strategy with different configurations. - -The Hyperopt configuration is located in -[user_data/hyperopt_conf.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopt_conf.py). - - -## Advanced notions -### Understand the Guards and Triggers -When you need to add the new guards and triggers to be hyperopt -parameters, you do this by adding them into the [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py#L244-L297). - -If it's a trigger, you add one line to the 'trigger' choice group and that's it. - -If it's a guard, you will add a line like this: -``` -'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)} -]), -``` -This says, "*one of the guards is RSI, it can have two values, enabled or -disabled. If it is enabled, try different values for it between 20 and 40*". - -So, the part of the strategy builder using the above setting looks like -this: +We will start by defining a search space: ``` -if params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return [ + Integer(20, 40, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal'], name='trigger') + ] ``` -It checks if Hyperopt wants the RSI guard to be enabled for this -round `params['rsi']['enabled']` and if it is, then it will add a -condition that says RSI must be smaller than the value hyperopt picked -for this evaluation, which is given in the `params['rsi']['value']`. +Above definition says: I have five parameters I want you to randomly combine +to find the best combination. Two of them are integer values (`adx-value` +and `rsi-value`) and I want you test in the range of values 20 to 40. +Then we have three category variables. First two are either `True` or `False`. +We use these to either enable or disable the ADX and RSI guards. The last +one we call `trigger` and use it to decide which buy trigger we want to use. -That's it. Now you can add new parts of strategies to Hyperopt and it -will try all the combinations with all different values in the search -for best working algo. +So let's write the buy strategy using these values: +``` + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + conditions = [] + # GUARDS AND TRENDS + if 'adx-enabled' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi-enabled' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) -### Add a new Indicators + # TRIGGERS + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend +``` + +Hyperopting will now call this `populate_buy_trend` as many times you ask it (`epochs`) +with different value combinations. It will then use the given historical data and make +buys based on the buy signals generated with the above function and based on the results +it will end with telling you which paramter combination produced the best profits. + +### Adding New Indicators If you want to test an indicator that isn't used by the bot currently, you need to add it to the `populate_indicators()` method in `hyperopt.py`. @@ -164,12 +120,12 @@ python3 ./freqtrade/main.py -c config.json hyperopt -e 5000 The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. -### Execute hyperopt with different ticker-data source +### Execute Hyperopt with Different Ticker-Data Source If you would like to hyperopt parameters using an alternate ticker data that you have on-disk, use the `--datadir PATH` option. Default hyperopt will use data from directory `user_data/data`. -### Running hyperopt with smaller testset +### Running Hyperopt with Smaller Testset Use the `--timeperiod` argument to change how much of the testset you want to use. The last N ticks/timeframes will be used. Example: @@ -178,7 +134,7 @@ Example: python3 ./freqtrade/main.py hyperopt --timeperiod -200 ``` -### Running hyperopt with smaller search space +### Running Hyperopt with Smaller Search Space Use the `--spaces` argument to limit the search space used by hyperopt. Letting Hyperopt optimize everything is a huuuuge search space. Often it might make more sense to start by just searching for initial buy algorithm. @@ -193,87 +149,44 @@ Legal values are: - `stoploss`: search for the best stoploss value - space-separated list of any of the above values for example `--spaces roi stoploss` -## Understand the hyperopts result -Once Hyperopt is completed you can use the result to adding new buy -signal. Given following result from hyperopt: -``` -Best parameters: -{ - "adx": { - "enabled": true, - "value": 15.0 - }, - "fastd": { - "enabled": true, - "value": 40.0 - }, - "green_candle": { - "enabled": true - }, - "mfi": { - "enabled": false - }, - "over_sar": { - "enabled": false - }, - "rsi": { - "enabled": true, - "value": 37.0 - }, - "trigger": { - "type": "lower_bb" - }, - "uptrend_long_ema": { - "enabled": true - }, - "uptrend_short_ema": { - "enabled": false - }, - "uptrend_sma": { - "enabled": false - } -} +## Understand the Hyperopts Result +Once Hyperopt is completed you can use the result to creating a new strategy. +Given following result from hyperopt: -Best Result: - 2197 trades. Avg profit 1.84%. Total profit 0.79367541 BTC. Avg duration 241.0 mins. +``` +Best result: + 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. +with values: +{'adx-value': 44, 'rsi-value': 29, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'bb_lower'} ``` You should understand this result like: -- You should **consider** the guard "adx" (`"adx"` is `"enabled": true`) -and the best value is `15.0` (`"value": 15.0,`) -- You should **consider** the guard "fastd" (`"fastd"` is `"enabled": -true`) and the best value is `40.0` (`"value": 40.0,`) -- You should **consider** to enable the guard "green_candle" -(`"green_candle"` is `"enabled": true`) but this guards as no -customizable value. -- You should **ignore** the guard "mfi" (`"mfi"` is `"enabled": false`) -- and so on... +- The buy trigger that worked best was `bb_lower`. +- You should not use ADX because `adx-enabled: False`) +- You should **consider** using the RSI indicator (`rsi-enabled: True` and the best value is `29.0` (`rsi-value: 29.0`) You have to look inside your strategy file into `buy_strategy_generator()` method, what those values match to. -So for example you had `adx:` with the `value: 15.0` so we would look -at `adx`-block, that translates to the following code block: +So for example you had `rsi-value: 29.0` so we would look +at `rsi`-block, that translates to the following code block: ``` -(dataframe['adx'] > 15.0) +(dataframe['rsi'] < 29.0) ``` -Translating your whole hyperopt result to as the new buy-signal -would be the following: +Translating your whole hyperopt result as the new buy-signal +would then look like: ``` def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: dataframe.loc[ ( - (dataframe['adx'] > 15.0) & # adx-value - (dataframe['fastd'] < 40.0) & # fastd-value - (dataframe['close'] > dataframe['open']) & # green_candle - (dataframe['rsi'] < 37.0) & # rsi-value - (dataframe['ema50'] > dataframe['ema100']) # uptrend_long_ema + (dataframe['rsi'] < 29.0) & # rsi-value + dataframe['close'] < dataframe['bb_lowerband'] # trigger ), 'buy'] = 1 return dataframe ``` -## Next step +## Next Step Now you have a perfect bot and want to control it from Telegram. Your next step is to learn the [Telegram usage](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md). From fa8fc3e4ce19b566a9ea5f079cb26db38b143089 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 2 Jul 2018 11:44:33 +0300 Subject: [PATCH 083/395] handle the case where we have zero buys --- freqtrade/optimize/hyperopt.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index eb9015356..e499def70 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -31,6 +31,8 @@ from freqtrade.optimize.backtesting import Backtesting logger = logging.getLogger(__name__) +MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization + class Hyperopt(Backtesting): """ @@ -152,7 +154,8 @@ class Hyperopt(Backtesting): trade_loss = 1 - 0.25 * exp(-(trade_count - self.target_trades) ** 2 / 10 ** 5.8) profit_loss = max(0, 1 - total_profit / self.expected_max_profit) duration_loss = 0.4 * min(trade_duration / self.max_accepted_trade_duration, 1) - return trade_loss + profit_loss + duration_loss + result = trade_loss + profit_loss + duration_loss + return result @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: @@ -293,6 +296,13 @@ class Hyperopt(Backtesting): trade_count = len(results.index) trade_duration = results.trade_duration.mean() + if trade_count == 0: + return { + 'loss': MAX_LOSS, + 'params': params, + 'result': result_explanation, + } + loss = self.calculate_loss(total_profit, trade_count, trade_duration) return { From 76343ecb77e2869a80f76adabf748baeeb314e1a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 2 Jul 2018 14:23:06 +0200 Subject: [PATCH 084/395] Update ccxt from 1.14.301 to 1.15.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 50441879a..ebb5b5b49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.301 +ccxt==1.15.3 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From 79aab4cce2f516df4d9504d99c31f3ccf4b70394 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 11:17:41 +0300 Subject: [PATCH 085/395] use fstring --- freqtrade/optimize/hyperopt.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e499def70..9b26b5b09 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -71,10 +71,8 @@ class Hyperopt(Backtesting): # Ensure the number of dimensions match # the number of parameters in the list x. if len(params) != len(dimensions): - msg = "Mismatch in number of search-space dimensions. " \ - "len(dimensions)=={} and len(x)=={}" - msg = msg.format(len(dimensions), len(params)) - raise ValueError(msg) + raise ValueError('Mismatch in number of search-space dimensions. ' + f'len(dimensions)=={len(dimensions)} and len(x)=={len(params)}') # Create a dict where the keys are the names of the dimensions # and the values are taken from the list of parameters x. From 2713fdb8607b01602d34c37e7c31facc7f9cb78e Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 11:46:56 +0300 Subject: [PATCH 086/395] use cpu count explicitly in job count --- freqtrade/optimize/hyperopt.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 9b26b5b09..8290eb2d8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -323,13 +323,13 @@ class Hyperopt(Backtesting): results.trade_duration.mean(), ) - def get_optimizer(self) -> Optimizer: + def get_optimizer(self, cpu_count) -> Optimizer: return Optimizer( self.hyperopt_space(), base_estimator="ET", acq_optimizer="auto", n_initial_points=30, - acq_optimizer_kwargs={'n_jobs': -1} + acq_optimizer_kwargs={'n_jobs': cpu_count} ) def run_optimizer_parallel(self, parallel, asked) -> List: @@ -361,11 +361,11 @@ class Hyperopt(Backtesting): self.load_previous_results() cpus = multiprocessing.cpu_count() - logger.info(f'Found {cpus}. Let\'s make them scream!') + logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') - opt = self.get_optimizer() + opt = self.get_optimizer(cpus) try: - with Parallel(n_jobs=-1) as parallel: + with Parallel(n_jobs=cpus) as parallel: for i in range(self.total_tries//cpus): asked = opt.ask(n_points=cpus) f_val = self.run_optimizer_parallel(parallel, asked) From 4a26b88a17af950f2bdedcfde43e2cd684dfcd4d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 12:43:25 +0300 Subject: [PATCH 087/395] improve documentation --- docs/hyperopt.md | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 8d3fe6704..f4b69b632 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -1,7 +1,7 @@ # Hyperopt -This page explains how to tune your strategy by finding the optimal -parameters, process called hyperparameter optimization. The bot uses several -algorithms included in `scikit-optimize` package to accomplish this. The +This page explains how to tune your strategy by finding the optimal +parameters, a process called hyperparameter optimization. The bot uses several +algorithms included in the `scikit-optimize` package to accomplish this. The search will burn all your CPU cores, make your laptop sound like a fighter jet and still take a long time. @@ -17,18 +17,17 @@ and still take a long time. We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) ### Configure your Guards and Triggers -There are two places you need to change in your strategy file to add a -new buy strategy for testing: +There are two places you need to change to add a new buy strategy for testing: - Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L278-L294). - Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L218-L229) and the associated methods `indicator_space`, `roi_space`, `stoploss_space`. There you have two different type of indicators: 1. `guards` and 2. `triggers`. -1. Guards are conditions like "never buy if ADX < 10", or never buy if -current price is over EMA10. +1. Guards are conditions like "never buy if ADX < 10", or "never buy if +current price is over EMA10". 2. Triggers are ones that actually trigger buy in specific moment, like -"buy when EMA5 crosses over EMA10" or buy when close price touches lower -bollinger band. +"buy when EMA5 crosses over EMA10" or "buy when close price touches lower +bollinger band". Hyperoptimization will, for each eval round, pick one trigger and possibly multiple guards. The constructed strategy will be something like @@ -103,9 +102,13 @@ with different value combinations. It will then use the given historical data an buys based on the buy signals generated with the above function and based on the results it will end with telling you which paramter combination produced the best profits. -### Adding New Indicators -If you want to test an indicator that isn't used by the bot currently, -you need to add it to the `populate_indicators()` method in `hyperopt.py`. +The search for best parameters starts with a few random combinations and then uses a +regressor algorithm (currently ExtraTreesRegressor) to quickly find a parameter combination +that minimizes the value of the objective function `calculate_loss` in `hyperopt.py`. + +The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. +When you want to test an indicator that isn't used by the bot currently, remember to +add it to the `populate_indicators()` method in `hyperopt.py`. ## Execute Hyperopt Once you have updated your hyperopt configuration you can run it. @@ -150,8 +153,8 @@ Legal values are: - space-separated list of any of the above values for example `--spaces roi stoploss` ## Understand the Hyperopts Result -Once Hyperopt is completed you can use the result to creating a new strategy. -Given following result from hyperopt: +Once Hyperopt is completed you can use the result to create a new strategy. +Given the following result from hyperopt: ``` Best result: From ee4754cfb985fbe608ff02b21c65102223e1e005 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 14:46:16 +0300 Subject: [PATCH 088/395] avoid re-serialization of whole dataframe --- freqtrade/optimize/hyperopt.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 8290eb2d8..57bf75742 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -14,14 +14,14 @@ from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, Any, Callable, Optional, List +from typing import Dict, Any, Callable, List import talib.abstract as ta from pandas import DataFrame from skopt.space import Real, Integer, Categorical, Dimension from skopt import Optimizer -from sklearn.externals.joblib import Parallel, delayed +from sklearn.externals.joblib import Parallel, delayed, load, dump import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments @@ -32,6 +32,7 @@ from freqtrade.optimize.backtesting import Backtesting logger = logging.getLogger(__name__) MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization +TICKERDATA_PICKLE = os.path.join('user_data', 'hyperopt_tickerdata.pkl') class Hyperopt(Backtesting): @@ -60,7 +61,7 @@ class Hyperopt(Backtesting): self.expected_max_profit = 3.0 # Configuration and data used by hyperopt - self.processed: Optional[Dict[str, Any]] = None +# self.processed: Optional[Dict[str, Any]] = None # Previous evaluations self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') @@ -281,10 +282,11 @@ class Hyperopt(Backtesting): if self.has_space('stoploss'): self.analyze.strategy.stoploss = params['stoploss'] + processed = load(TICKERDATA_PICKLE) results = self.backtest( { 'stake_amount': self.config['stake_amount'], - 'processed': self.processed, + 'processed': processed, 'realistic': self.config.get('realistic_simulation', False), } ) @@ -356,7 +358,7 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore - self.processed = self.tickerdata_to_dataframe(data) + dump(self.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() From 2c0e950486ef8e7368160b35934ca0c817f3d0a9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 3 Jul 2018 14:23:05 +0200 Subject: [PATCH 089/395] Update ccxt from 1.15.3 to 1.15.7 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ebb5b5b49..98721b74c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.3 +ccxt==1.15.7 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From ef59f9ad249f8a24a0ab29b2cb4c2f28d180c206 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 21:50:24 +0300 Subject: [PATCH 090/395] sort imports in hyperopt.py --- freqtrade/optimize/hyperopt.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 57bf75742..a1d86bd18 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -5,23 +5,21 @@ This module contains the hyperopt logic """ import logging +import multiprocessing import os import pickle import sys -import multiprocessing - from argparse import Namespace from functools import reduce from math import exp from operator import itemgetter -from typing import Dict, Any, Callable, List +from typing import Any, Callable, Dict, List import talib.abstract as ta from pandas import DataFrame - -from skopt.space import Real, Integer, Categorical, Dimension +from sklearn.externals.joblib import Parallel, delayed, dump, load from skopt import Optimizer -from sklearn.externals.joblib import Parallel, delayed, load, dump +from skopt.space import Categorical, Dimension, Integer, Real import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments From 2cde540645bf321360594769768f017c2ceabe0f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 21:50:45 +0300 Subject: [PATCH 091/395] remove dead code --- freqtrade/optimize/hyperopt.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a1d86bd18..861523965 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -58,9 +58,6 @@ class Hyperopt(Backtesting): # check that the reported Σ% values do not exceed this! self.expected_max_profit = 3.0 - # Configuration and data used by hyperopt -# self.processed: Optional[Dict[str, Any]] = None - # Previous evaluations self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') self.trials: List = [] From 3a7056ea1bab0a2cea913e2eca3c6e590481ee93 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 21:54:32 +0300 Subject: [PATCH 092/395] run at least one epoch --- freqtrade/optimize/hyperopt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 861523965..71e923f00 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -361,9 +361,10 @@ class Hyperopt(Backtesting): logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') opt = self.get_optimizer(cpus) + EVALS = max(self.total_tries//cpus, 1) try: with Parallel(n_jobs=cpus) as parallel: - for i in range(self.total_tries//cpus): + for i in range(EVALS): asked = opt.ask(n_points=cpus) f_val = self.run_optimizer_parallel(parallel, asked) opt.tell(asked, [i['loss'] for i in f_val]) From 9dbe0f50a3e6a708a621bea70757bf9e0b6f7577 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 22:09:59 +0300 Subject: [PATCH 093/395] fix tests after changing the dumping and pickling dataframe in hyperopt --- freqtrade/tests/optimize/test_hyperopt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 9194a66c9..bbe16ae7a 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -187,6 +187,7 @@ def test_roi_table_generation(init_hyperopt) -> None: def test_start_calls_optimizer(mocker, init_hyperopt, default_conf, caplog) -> None: + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.multiprocessing.cpu_count', MagicMock(return_value=1)) parallel = mocker.patch( @@ -208,6 +209,7 @@ def test_start_calls_optimizer(mocker, init_hyperopt, default_conf, caplog) -> N parallel.assert_called_once() assert 'Best result:\nfoo result\nwith values:\n{}' in caplog.text + assert dumper.called def test_format_results(init_hyperopt): @@ -316,6 +318,7 @@ def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: MagicMock(return_value=backtest_result) ) patch_exchange(mocker) + mocker.patch('freqtrade.optimize.hyperopt.load', MagicMock()) optimizer_param = { 'adx-value': 0, From c4a8435e007db87e157cf65e43213c2b4da81129 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 22:17:43 +0300 Subject: [PATCH 094/395] change pickle file name to better suit it's current purpose --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 71e923f00..d90468cff 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -59,7 +59,7 @@ class Hyperopt(Backtesting): self.expected_max_profit = 3.0 # Previous evaluations - self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') + self.trials_file = os.path.join('user_data', 'hyperopt_results.pickle') self.trials: List = [] def get_args(self, params): From 96bb2efe69ded2e6cbd2998804445e9d8726ded0 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 3 Jul 2018 22:51:48 +0300 Subject: [PATCH 095/395] use joblib.dump and load for trials --- freqtrade/optimize/hyperopt.py | 5 ++--- freqtrade/tests/optimize/test_hyperopt.py | 9 +++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index d90468cff..7138e2601 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -7,7 +7,6 @@ This module contains the hyperopt logic import logging import multiprocessing import os -import pickle import sys from argparse import Namespace from functools import reduce @@ -99,14 +98,14 @@ class Hyperopt(Backtesting): """ if self.trials: logger.info('Saving %d evaluations to \'%s\'', len(self.trials), self.trials_file) - pickle.dump(self.trials, open(self.trials_file, 'wb')) + dump(self.trials, self.trials_file) def read_trials(self) -> List: """ Read hyperopt trials file """ logger.info('Reading Trials from \'%s\'', self.trials_file) - trials = pickle.load(open(self.trials_file, 'rb')) + trials = load(self.trials_file) os.remove(self.trials_file) return trials diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index bbe16ae7a..72a102c22 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -39,7 +39,7 @@ def create_trials(mocker) -> None: mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=False) mocker.patch('freqtrade.optimize.hyperopt.os.path.getsize', return_value=1) mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True) - mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) return [{'loss': 1, 'result': 'foo', 'params': {}}] @@ -139,10 +139,9 @@ def test_no_log_if_loss_does_not_improve(init_hyperopt, caplog) -> None: def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: trials = create_trials(mocker) - mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None) + mock_dump = mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) hyperopt = _HYPEROPT - mocker.patch('freqtrade.optimize.hyperopt.open', return_value=hyperopt.trials_file) _HYPEROPT.trials = trials hyperopt.save_trials() @@ -157,8 +156,7 @@ def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: def test_read_trials_returns_trials_file(mocker, init_hyperopt, caplog) -> None: trials = create_trials(mocker) - mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', return_value=trials) - mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', return_value=mock_load) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=trials) hyperopt = _HYPEROPT hyperopt_trial = hyperopt.read_trials() @@ -168,7 +166,6 @@ def test_read_trials_returns_trials_file(mocker, init_hyperopt, caplog) -> None: caplog.record_tuples ) assert hyperopt_trial == trials - mock_open.assert_called_once() mock_load.assert_called_once() From bf4d0a9b7088eb012c1090144d52569c791848b6 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 4 Jul 2018 10:31:35 +0300 Subject: [PATCH 096/395] sort imports --- freqtrade/__main__.py | 2 +- freqtrade/analyze.py | 3 +-- freqtrade/arguments.py | 5 +++-- freqtrade/configuration.py | 8 ++++---- freqtrade/fiat_convert.py | 4 +++- freqtrade/freqtradebot.py | 8 +++----- freqtrade/indicator_helpers.py | 2 +- freqtrade/misc.py | 2 +- freqtrade/optimize/backtesting.py | 6 +++--- freqtrade/persistence.py | 5 ++--- freqtrade/rpc/rpc.py | 4 ++-- freqtrade/strategy/interface.py | 2 +- freqtrade/strategy/resolver.py | 3 +-- freqtrade/tests/conftest.py | 4 ++-- freqtrade/tests/exchange/test_exchange.py | 8 ++++---- freqtrade/tests/optimize/test_backtesting.py | 7 ++++--- freqtrade/tests/optimize/test_optimize.py | 15 +++++++++------ freqtrade/tests/rpc/test_rpc.py | 3 ++- freqtrade/tests/rpc/test_rpc_manager.py | 2 +- freqtrade/tests/rpc/test_rpc_telegram.py | 11 ++++++----- freqtrade/tests/test_acl_pair.py | 3 ++- freqtrade/tests/test_analyze.py | 4 ++-- freqtrade/tests/test_configuration.py | 6 +++--- freqtrade/tests/test_fiat_convert.py | 1 - freqtrade/tests/test_freqtradebot.py | 6 ++++-- freqtrade/tests/test_indicator_helpers.py | 2 +- freqtrade/tests/test_main.py | 2 +- freqtrade/tests/test_misc.py | 4 ++-- freqtrade/tests/test_persistence.py | 4 ++-- 29 files changed, 71 insertions(+), 65 deletions(-) diff --git a/freqtrade/__main__.py b/freqtrade/__main__.py index fe1318a35..7d271dfd1 100644 --- a/freqtrade/__main__.py +++ b/freqtrade/__main__.py @@ -7,8 +7,8 @@ To launch Freqtrade as a module """ import sys -from freqtrade import main +from freqtrade import main if __name__ == '__main__': main.set_loggers() diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index bf6b89b17..6d6a85596 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -12,8 +12,7 @@ from pandas import DataFrame, to_datetime from freqtrade import constants from freqtrade.exchange import Exchange from freqtrade.persistence import Trade -from freqtrade.strategy.resolver import StrategyResolver, IStrategy - +from freqtrade.strategy.resolver import IStrategy, StrategyResolver logger = logging.getLogger(__name__) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 8ce7c546d..731c5d88c 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -2,12 +2,13 @@ This module contains the argument manager class """ -import os import argparse import logging +import os import re +from typing import List, NamedTuple, Optional + import arrow -from typing import List, Optional, NamedTuple from freqtrade import __version__, constants diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 7c3a5eb4b..c90eddeb8 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -1,18 +1,18 @@ """ This module contains the configuration class """ -import os import json import logging +import os from argparse import Namespace -from typing import Optional, Dict, Any +from typing import Any, Dict, Optional + +import ccxt from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match -import ccxt from freqtrade import OperationalException, constants - logger = logging.getLogger(__name__) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 44a4f3054..2e1a7cac8 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -7,10 +7,12 @@ import logging import time from typing import Dict, List -from coinmarketcap import Market from requests.exceptions import RequestException +from coinmarketcap import Market + from freqtrade.constants import SUPPORTED_FIAT + logger = logging.getLogger(__name__) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 530535710..64ff170fd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -7,16 +7,14 @@ import logging import time import traceback from datetime import datetime -from typing import Dict, List, Optional, Any, Callable +from typing import Any, Callable, Dict, List, Optional import arrow import requests from cachetools import TTLCache, cached -from freqtrade import ( - DependencyException, OperationalException, TemporaryError, persistence, __version__, -) -from freqtrade import constants +from freqtrade import (DependencyException, OperationalException, + TemporaryError, __version__, constants, persistence) from freqtrade.analyze import Analyze from freqtrade.exchange import Exchange from freqtrade.fiat_convert import CryptoToFiatConverter diff --git a/freqtrade/indicator_helpers.py b/freqtrade/indicator_helpers.py index 50586578a..f8ea0d939 100644 --- a/freqtrade/indicator_helpers.py +++ b/freqtrade/indicator_helpers.py @@ -1,4 +1,4 @@ -from math import exp, pi, sqrt, cos +from math import cos, exp, pi, sqrt import numpy as np import talib as ta diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 90a1db42b..832437951 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -2,10 +2,10 @@ Various tool function for Freqtrade and scripts """ +import gzip import json import logging import re -import gzip from datetime import datetime from typing import Dict diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 316fba508..16c21258f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -7,18 +7,18 @@ import logging import operator from argparse import Namespace from datetime import datetime -from typing import Dict, Tuple, Any, List, Optional, NamedTuple +from typing import Any, Dict, List, NamedTuple, Optional, Tuple import arrow from pandas import DataFrame from tabulate import tabulate import freqtrade.optimize as optimize -from freqtrade import constants, DependencyException -from freqtrade.exchange import Exchange +from freqtrade import DependencyException, constants from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration +from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index aa33f7063..764ba6509 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -5,12 +5,11 @@ This module contains the class to persist trades into SQLite import logging from datetime import datetime from decimal import Decimal, getcontext -from typing import Dict, Optional, Any +from typing import Any, Dict, Optional import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, - create_engine) -from sqlalchemy import inspect + create_engine, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index ee6ecb770..f8cb136e4 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -3,9 +3,9 @@ This module contains class to define a RPC communications """ import logging from abc import abstractmethod -from datetime import datetime, timedelta, date +from datetime import date, datetime, timedelta from decimal import Decimal -from typing import Dict, Tuple, Any, List +from typing import Any, Dict, List, Tuple import arrow import sqlalchemy as sql diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 4ae358c6f..f73617f46 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -2,8 +2,8 @@ IStrategy interface This module defines the interface to apply for strategies """ -from typing import Dict from abc import ABC, abstractmethod +from typing import Dict from pandas import DataFrame diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 0dcd3fc6a..10cedb073 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -8,13 +8,12 @@ import inspect import logging import os from collections import OrderedDict -from typing import Optional, Dict, Type +from typing import Dict, Optional, Type from freqtrade import constants from freqtrade.strategy import import_strategy from freqtrade.strategy.interface import IStrategy - logger = logging.getLogger(__name__) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 4877fe556..9c86d1ece 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -2,8 +2,8 @@ import json import logging from datetime import datetime -from typing import Dict, Optional from functools import reduce +from typing import Dict, Optional from unittest.mock import MagicMock import arrow @@ -11,8 +11,8 @@ import pytest from jsonschema import validate from telegram import Chat, Message, Update -from freqtrade.analyze import Analyze from freqtrade import constants +from freqtrade.analyze import Analyze from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 6cf7f11b0..3ddec0ded 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -2,16 +2,16 @@ # pragma pylint: disable=protected-access import logging from copy import deepcopy -from random import randint from datetime import datetime +from random import randint from unittest.mock import MagicMock, PropertyMock import ccxt import pytest -from freqtrade import OperationalException, DependencyException, TemporaryError -from freqtrade.exchange import Exchange, API_RETRY_COUNT -from freqtrade.tests.conftest import log_has, get_patched_exchange +from freqtrade import DependencyException, OperationalException, TemporaryError +from freqtrade.exchange import API_RETRY_COUNT, Exchange +from freqtrade.tests.conftest import get_patched_exchange, log_has def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 49a6370bb..cb225e465 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -3,19 +3,20 @@ import json import math import random -import pytest from copy import deepcopy from typing import List from unittest.mock import MagicMock import numpy as np import pandas as pd +import pytest from arrow import Arrow -from freqtrade import optimize, constants, DependencyException +from freqtrade import DependencyException, constants, optimize from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments, TimeRange -from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration +from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, + start) from freqtrade.tests.conftest import log_has, patch_exchange diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 624f8ed77..4ab32343a 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -3,16 +3,19 @@ import json import os import uuid -import arrow from shutil import copyfile +import arrow + from freqtrade import optimize -from freqtrade.misc import file_dump_json -from freqtrade.optimize.__init__ import make_testdata_path, download_pairs, \ - download_backtesting_testdata, load_tickerdata_file, trim_tickerlist, \ - load_cached_data_for_updating from freqtrade.arguments import TimeRange -from freqtrade.tests.conftest import log_has, get_patched_exchange +from freqtrade.misc import file_dump_json +from freqtrade.optimize.__init__ import (download_backtesting_testdata, + download_pairs, + load_cached_data_for_updating, + load_tickerdata_file, + make_testdata_path, trim_tickerlist) +from freqtrade.tests.conftest import get_patched_exchange, log_has # Change this if modifying UNITTEST/BTC testdatafile _BTC_UNITTEST_LENGTH = 13681 diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 11db7ffb3..58514d1c0 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -13,7 +13,8 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc.rpc import RPC, RPCException from freqtrade.state import State -from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap +from freqtrade.tests.test_freqtradebot import (patch_coinmarketcap, + patch_get_signal) # Functions for recurrent object patching diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 805424d26..5aea98d48 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -7,7 +7,7 @@ from copy import deepcopy from unittest.mock import MagicMock from freqtrade.rpc.rpc_manager import RPCManager -from freqtrade.tests.conftest import log_has, get_patched_freqtradebot +from freqtrade.tests.conftest import get_patched_freqtradebot, log_has def test_rpc_manager_object() -> None: diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index b2cca9b9a..2710328bd 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -11,17 +11,18 @@ from datetime import datetime from random import randint from unittest.mock import MagicMock -from telegram import Update, Message, Chat +from telegram import Chat, Message, Update from telegram.error import NetworkError from freqtrade import __version__ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade -from freqtrade.rpc.telegram import Telegram -from freqtrade.rpc.telegram import authorized_only +from freqtrade.rpc.telegram import Telegram, authorized_only from freqtrade.state import State -from freqtrade.tests.conftest import get_patched_freqtradebot, patch_exchange, log_has -from freqtrade.tests.test_freqtradebot import patch_get_signal, patch_coinmarketcap +from freqtrade.tests.conftest import (get_patched_freqtradebot, log_has, + patch_exchange) +from freqtrade.tests.test_freqtradebot import (patch_coinmarketcap, + patch_get_signal) class DummyCls(Telegram): diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 608e6a4a1..094c166b8 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -1,8 +1,9 @@ # pragma pylint: disable=missing-docstring,C0103,protected-access -import freqtrade.tests.conftest as tt # test tools from unittest.mock import MagicMock +import freqtrade.tests.conftest as tt # test tools + # whitelist, blacklist, filtering, all of that will # eventually become some rules to run on a generic ACL engine # perhaps try to anticipate that by using some python package diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 02225acca..e6108e8f8 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -12,9 +12,9 @@ import arrow from pandas import DataFrame from freqtrade.analyze import Analyze, SignalType -from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.arguments import TimeRange -from freqtrade.tests.conftest import log_has, get_patched_exchange +from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.tests.conftest import get_patched_exchange, log_has # Avoid to reinit the same object again and again _ANALYZE = Analyze({'strategy': 'DefaultStrategy'}) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index a4096cc01..e64e1b486 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -4,18 +4,18 @@ Unit test file for configuration.py """ import json +from argparse import Namespace from copy import deepcopy from unittest.mock import MagicMock -from argparse import Namespace import pytest from jsonschema import ValidationError +from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration -from freqtrade.constants import DEFAULT_DB_PROD_URL, DEFAULT_DB_DRYRUN_URL +from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.tests.conftest import log_has -from freqtrade import OperationalException def test_configuration_object() -> None: diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index 2fb9219ca..5af85d268 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -5,7 +5,6 @@ import time from unittest.mock import MagicMock import pytest - from requests.exceptions import RequestException from freqtrade.fiat_convert import CryptoFiat, CryptoToFiatConverter diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index dde2dd6d9..17bd6aa7c 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -14,11 +14,13 @@ import arrow import pytest import requests -from freqtrade import constants, DependencyException, OperationalException, TemporaryError +from freqtrade import (DependencyException, OperationalException, + TemporaryError, constants) from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.state import State -from freqtrade.tests.conftest import log_has, patch_coinmarketcap, patch_exchange +from freqtrade.tests.conftest import (log_has, patch_coinmarketcap, + patch_exchange) # Functions for recurrent object patching diff --git a/freqtrade/tests/test_indicator_helpers.py b/freqtrade/tests/test_indicator_helpers.py index 87b085a0b..f3d34ec0b 100644 --- a/freqtrade/tests/test_indicator_helpers.py +++ b/freqtrade/tests/test_indicator_helpers.py @@ -1,6 +1,6 @@ import pandas as pd -from freqtrade.indicator_helpers import went_up, went_down +from freqtrade.indicator_helpers import went_down, went_up def test_went_up(): diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 414dcdbe9..20a02eedc 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -11,7 +11,7 @@ import pytest from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.main import main, set_loggers, reconfigure +from freqtrade.main import main, reconfigure, set_loggers from freqtrade.state import State from freqtrade.tests.conftest import log_has, patch_exchange diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index b57900428..e2ba40dee 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -8,8 +8,8 @@ import datetime from unittest.mock import MagicMock from freqtrade.analyze import Analyze -from freqtrade.misc import (shorten_date, datesarray_to_datetimearray, - common_datearray, file_dump_json, format_ms_time) +from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, + file_dump_json, format_ms_time, shorten_date) from freqtrade.optimize.__init__ import load_tickerdata_file diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 12ae6579f..b24f2dd6c 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -5,8 +5,8 @@ from unittest.mock import MagicMock import pytest from sqlalchemy import create_engine -from freqtrade import constants, OperationalException -from freqtrade.persistence import Trade, init, clean_dry_run_db +from freqtrade import OperationalException, constants +from freqtrade.persistence import Trade, clean_dry_run_db, init from freqtrade.tests.conftest import log_has From ac20bf31dfab8589f3f3ae8b09fb3fa8fa83a685 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 4 Jul 2018 14:23:06 +0200 Subject: [PATCH 097/395] Update ccxt from 1.15.7 to 1.15.8 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 98721b74c..8ab13d394 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.7 +ccxt==1.15.8 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From 966668f48a5f02cf6abd71358cadebca8fed09f5 Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 5 Jul 2018 11:57:59 +0000 Subject: [PATCH 098/395] Handle if ticker_interval in config.json is not supported on exchange. Returns. Tested positive and negative data. The ticker list in constants.py may be obsolete now, im not sure. raise OperationalException(f'Invalid ticker {timeframe}, this Exchange supports {timeframes}') freqtrade.OperationalException: Invalid ticker 14m, this Exchange supports {'1m': '1m', '3m': '3m', '5m': '5m', '15m': '15m', '30m': '30m', '1h': '1h', '2h': '2h', '4h': '4h', '6h': '6h', '8h': '8h', '12h': '12h', '1d': '1d', '3d': '3d', '1w': '1w', '1M': '1M'} --- freqtrade/exchange/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index acfefdad4..23305b59f 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -70,6 +70,9 @@ class Exchange(object): # Check if all pairs are available self.validate_pairs(config['exchange']['pair_whitelist']) + # Check if timeframe is available + self.validate_timeframes(config['ticker_interval']) + def _init_ccxt(self, exchange_config: dict) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid @@ -128,6 +131,14 @@ class Exchange(object): raise OperationalException( f'Pair {pair} is not available at {self.name}') + def validate_timeframes(self, timeframe: List[str]) -> None: + """ + Checks if ticker interval from config is a supported timeframe on the exchange + """ + timeframes=self._api.timeframes + if timeframe not in timeframes: + raise OperationalException(f'Invalid ticker {timeframe}, this Exchange supports {timeframes}') + def exchange_has(self, endpoint: str) -> bool: """ Checks if exchange implements a specific API endpoint. From 5ab644dea6fe64e2e11478cf560adb8b295e6444 Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 5 Jul 2018 12:05:31 +0000 Subject: [PATCH 099/395] flake 8 fix --- freqtrade/exchange/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 23305b59f..d87d3fc85 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -135,9 +135,10 @@ class Exchange(object): """ Checks if ticker interval from config is a supported timeframe on the exchange """ - timeframes=self._api.timeframes + timeframes = self._api.timeframes if timeframe not in timeframes: - raise OperationalException(f'Invalid ticker {timeframe}, this Exchange supports {timeframes}') + raise OperationalException( + f'Invalid ticker {timeframe}, this Exchange supports {timeframes}') def exchange_has(self, endpoint: str) -> bool: """ From bfd1e90154b4db42fac2e867f381f9dfba6c825a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 5 Jul 2018 14:23:11 +0200 Subject: [PATCH 100/395] Update ccxt from 1.15.8 to 1.15.13 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ca1575d7a..c2303f11d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.8 +ccxt==1.15.13 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From 239f8606e15fe0fe02d3b02db1fd2bbf28f1171e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 5 Jul 2018 14:23:12 +0200 Subject: [PATCH 101/395] Update pytest from 3.6.2 to 3.6.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c2303f11d..f87241e32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.14.5 TA-Lib==0.4.17 -pytest==3.6.2 +pytest==3.6.3 pytest-mock==1.10.0 pytest-cov==2.5.1 tabulate==0.8.2 From 03c112a6010c07941c55d565e4be0fa47e00b2b6 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Thu, 14 Jun 2018 08:31:29 +0300 Subject: [PATCH 102/395] config, optimize: fstrings in use --- freqtrade/configuration.py | 6 +++--- freqtrade/optimize/__init__.py | 7 ++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index c90eddeb8..582b2889c 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -62,8 +62,8 @@ class Configuration(object): conf = json.load(file) except FileNotFoundError: raise OperationalException( - 'Config file "{}" not found!' - ' Please create a config file or check whether it exists.'.format(path)) + f'Config file "{path}" not found!' + ' Please create a config file or check whether it exists.') if 'internals' not in conf: conf['internals'] = {} @@ -109,7 +109,7 @@ class Configuration(object): config['db_url'] = constants.DEFAULT_DB_PROD_URL logger.info('Dry run is disabled') - logger.info('Using DB: "{}"'.format(config['db_url'])) + logger.info(f'Using DB: "{config["db_url"]}"') # Check if the exchange set by the user is supported self.check_exchange(config) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 1e76808e7..e806ff2b8 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -54,11 +54,8 @@ def load_tickerdata_file( :return dict OR empty if unsuccesful """ path = make_testdata_path(datadir) - pair_file_string = pair.replace('/', '_') - file = os.path.join(path, '{pair}-{ticker_interval}.json'.format( - pair=pair_file_string, - ticker_interval=ticker_interval, - )) + pair_s = pair.replace('/', '_') + file = os.path.join(path, f'{pair_s}-{ticker_interval}.json') gzipfile = file + '.gz' # If the file does not exist we download it when None is returned. From 7dca3c6d03e83fe512d85280ceab0a5c41673d80 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Thu, 14 Jun 2018 08:52:13 +0300 Subject: [PATCH 103/395] freqtradebot,main,hyperopt: fstrings in use --- freqtrade/freqtradebot.py | 8 ++------ freqtrade/main.py | 5 +---- freqtrade/optimize/hyperopt.py | 30 +++++++++++++++--------------- 3 files changed, 18 insertions(+), 25 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 64ff170fd..9def7078c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -626,12 +626,8 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ # Because telegram._forcesell does not have the configuration # Ignore the FIAT value and does not show the stake_currency as well else: - message += '` ({gain}: {profit_percent:.2f}%, {profit_coin:.8f})`'.format( - gain="profit" if fmt_exp_profit > 0 else "loss", - profit_percent=fmt_exp_profit, - profit_coin=profit_trade - ) - + gain = "profit" if fmt_exp_profit > 0 else "loss" + message += f'` ({gain}: {fmt_exp_profit:.2f}%, {profit_trade:.8f})`' # Send the message self.rpc.send_msg(message) Trade.session.flush() diff --git a/freqtrade/main.py b/freqtrade/main.py index 9d17a403a..79080ce37 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -74,10 +74,7 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot: # Create new instance freqtrade = FreqtradeBot(Configuration(args).get_config()) freqtrade.rpc.send_msg( - '*Status:* `Config reloaded ...`'.format( - freqtrade.state.name.lower() - ) - ) + '*Status:* `Config reloaded {freqtrade.state.name.lower()}...`') return freqtrade diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7138e2601..72bf34eb3 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -128,13 +128,12 @@ class Hyperopt(Backtesting): Log results if it is better than any previous evaluation """ if results['loss'] < self.current_best_loss: + current = results['current_tries'] + total = results['total_tries'] + res = results['result'] + loss = results['loss'] self.current_best_loss = results['loss'] - log_msg = '\n{:5d}/{}: {}. Loss {:.5f}'.format( - results['current_tries'], - results['total_tries'], - results['result'], - results['loss'] - ) + log_msg = f'\n{current:5d}/{total}: {res}. Loss {loss:.5f}' print(log_msg) else: print('.', end='') @@ -309,15 +308,16 @@ class Hyperopt(Backtesting): """ Return the format result in a string """ - return ('{:6d} trades. Avg profit {: 5.2f}%. ' - 'Total profit {: 11.8f} {} ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( - len(results.index), - results.profit_percent.mean() * 100.0, - results.profit_abs.sum(), - self.config['stake_currency'], - results.profit_percent.sum(), - results.trade_duration.mean(), - ) + trades = len(results.index) + avg_profit = results.profit_percent.mean() * 100.0 + total_profit = results.profit_abs.sum() + stake_cur = self.config['stake_currency'] + profit = results.profit_percent.sum() + duration = results.trade_duration.mean() + + return (f'{trades:6d} trades. Avg profit {avg_profit: 5.2f}%. ' + f'Total profit {total_profit: 11.8f} {stake_cur} ' + f'({profit:.4f}Σ%). Avg duration {duration:5.1f} mins.') def get_optimizer(self, cpu_count) -> Optimizer: return Optimizer( From a2063ede558a28e68e94fa385fa56416c20f8341 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Jun 2018 08:27:29 -0500 Subject: [PATCH 104/395] persistence: fstrings in use --- freqtrade/persistence.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 764ba6509..0e0b22e82 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -21,6 +21,7 @@ from freqtrade import OperationalException logger = logging.getLogger(__name__) _DECL_BASE: Any = declarative_base() +_SQL_DOCS_URL = 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls' def init(config: Dict) -> None: @@ -45,10 +46,8 @@ def init(config: Dict) -> None: try: engine = create_engine(db_url, **kwargs) except NoSuchModuleError: - error = 'Given value for db_url: \'{}\' is no valid database URL! (See {}).'.format( - db_url, 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls' - ) - raise OperationalException(error) + raise OperationalException(f'Given value for db_url: \'{db_url}\' ' + f'is no valid database URL! (See {_SQL_DOCS_URL})') session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.session = session() @@ -173,13 +172,10 @@ class Trade(_DECL_BASE): max_rate = Column(Float, nullable=True, default=0.0) def __repr__(self): - return 'Trade(id={}, pair={}, amount={:.8f}, open_rate={:.8f}, open_since={})'.format( - self.id, - self.pair, - self.amount, - self.open_rate, - arrow.get(self.open_date).humanize() if self.is_open else 'closed' - ) + open_since = arrow.get(self.open_date).humanize() if self.is_open else 'closed' + + return (f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, ' + f'open_rate={self.open_rate:.8f}, open_since={open_since})') def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool = False): """this adjusts the stop loss to it's most recently observed setting""" @@ -226,6 +222,7 @@ class Trade(_DECL_BASE): :param order: order retrieved by exchange.get_order() :return: None """ + order_type = order['type'] # Ignore open and cancelled orders if order['status'] == 'open' or order['price'] is None: return @@ -233,16 +230,16 @@ class Trade(_DECL_BASE): logger.info('Updating trade (id=%d) ...', self.id) getcontext().prec = 8 # Bittrex do not go above 8 decimal - if order['type'] == 'limit' and order['side'] == 'buy': + if order_type == 'limit' and order['side'] == 'buy': # Update open rate and actual amount self.open_rate = Decimal(order['price']) self.amount = Decimal(order['amount']) logger.info('LIMIT_BUY has been fulfilled for %s.', self) self.open_order_id = None - elif order['type'] == 'limit' and order['side'] == 'sell': + elif order_type == 'limit' and order['side'] == 'sell': self.close(order['price']) else: - raise ValueError('Unknown order type: {}'.format(order['type'])) + raise ValueError(f'Unknown order type: {order_type}') cleanup() def close(self, rate: float) -> None: @@ -313,7 +310,8 @@ class Trade(_DECL_BASE): rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - return float("{0:.8f}".format(close_trade_price - open_trade_price)) + profit = close_trade_price - open_trade_price + return float(f"{profit:.8f}") def calc_profit_percent( self, @@ -333,5 +331,5 @@ class Trade(_DECL_BASE): rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - - return float("{0:.8f}".format((close_trade_price / open_trade_price) - 1)) + profit_percent = (close_trade_price / open_trade_price) - 1 + return float(f"{profit_percent:.8f}") From 21fc933678ebf938e99fdcaa19f48d64844b26e9 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Jun 2018 16:35:35 -0500 Subject: [PATCH 105/395] convert_backtesting: fstrings in use --- scripts/convert_backtestdata.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py index 698c1c829..96e0cbce8 100755 --- a/scripts/convert_backtestdata.py +++ b/scripts/convert_backtestdata.py @@ -143,15 +143,14 @@ def convert_main(args: Namespace) -> None: interval = str_interval break # change order on pairs if old ticker interval found + filename_new = path.join(path.dirname(filename), - "{}_{}-{}.json".format(currencies[1], - currencies[0], interval)) + f"{currencies[1]}_{currencies[0]}-{interval}.json") elif ret_string: interval = ret_string.group(0) filename_new = path.join(path.dirname(filename), - "{}_{}-{}.json".format(currencies[0], - currencies[1], interval)) + f"{currencies[0]}_{currencies[1]}-{interval}.json") else: logger.warning("file %s could not be converted, interval not found", filename) From adbffc69e152a9a9b4413a9b9c3f9e20baa3796b Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 23 Jun 2018 17:17:10 -0500 Subject: [PATCH 106/395] telegram: fstrings in use --- freqtrade/rpc/telegram.py | 72 +++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4dd23971b..13a2b1913 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -153,7 +153,7 @@ class Telegram(RPC): try: df_statuses = self._rpc_status_table() message = tabulate(df_statuses, headers='keys', tablefmt='simple') - self._send_msg("
{}
".format(message), parse_mode=ParseMode.HTML) + self._send_msg(f"
{message}
", parse_mode=ParseMode.HTML) except RPCException as e: self._send_msg(str(e), bot=bot) @@ -166,6 +166,8 @@ class Telegram(RPC): :param update: message update :return: None """ + stake_cur = self._config['stake_currency'] + fiat_disp_cur = self._config['fiat_display_currency'] try: timescale = int(update.message.text.replace('/daily', '').strip()) except (TypeError, ValueError): @@ -173,18 +175,17 @@ class Telegram(RPC): try: stats = self._rpc_daily_profit( timescale, - self._config['stake_currency'], - self._config['fiat_display_currency'] + stake_cur, + fiat_disp_cur ) stats = tabulate(stats, headers=[ 'Day', - 'Profit {}'.format(self._config['stake_currency']), - 'Profit {}'.format(self._config['fiat_display_currency']) + f'Profit {stake_cur}', + f'Profit {fiat_disp_cur}' ], tablefmt='simple') - message = 'Daily Profit over the last {} days:\n
{}
'\ - .format(timescale, stats) + message = f'Daily Profit over the last {timescale} days:\n
{stats}
' self._send_msg(message, bot=bot, parse_mode=ParseMode.HTML) except RPCException as e: self._send_msg(str(e), bot=bot) @@ -198,39 +199,38 @@ class Telegram(RPC): :param update: message update :return: None """ + stake_cur = self._config['stake_currency'] + fiat_disp_cur = self._config['fiat_display_currency'] + try: stats = self._rpc_trade_statistics( - self._config['stake_currency'], - self._config['fiat_display_currency']) - + stake_cur, + fiat_disp_cur) + profit_closed_coin = stats['profit_closed_coin'] + profit_closed_percent = stats['profit_closed_percent'] + profit_closed_fiat = stats['profit_closed_fiat'] + profit_all_coin = stats['profit_all_coin'] + profit_all_percent = stats['profit_all_percent'] + profit_all_fiat = stats['profit_all_fiat'] + trade_count = stats['trade_count'] + first_trade_date = stats['first_trade_date'] + latest_trade_date = stats['latest_trade_date'] + avg_duration = stats['avg_duration'] + best_pair = stats['best_pair'] + best_rate = stats['best_rate'] # Message to display markdown_msg = "*ROI:* Close trades\n" \ - "∙ `{profit_closed_coin:.8f} {coin} ({profit_closed_percent:.2f}%)`\n" \ - "∙ `{profit_closed_fiat:.3f} {fiat}`\n" \ - "*ROI:* All trades\n" \ - "∙ `{profit_all_coin:.8f} {coin} ({profit_all_percent:.2f}%)`\n" \ - "∙ `{profit_all_fiat:.3f} {fiat}`\n" \ - "*Total Trade Count:* `{trade_count}`\n" \ - "*First Trade opened:* `{first_trade_date}`\n" \ - "*Latest Trade opened:* `{latest_trade_date}`\n" \ - "*Avg. Duration:* `{avg_duration}`\n" \ - "*Best Performing:* `{best_pair}: {best_rate:.2f}%`"\ - .format( - coin=self._config['stake_currency'], - fiat=self._config['fiat_display_currency'], - profit_closed_coin=stats['profit_closed_coin'], - profit_closed_percent=stats['profit_closed_percent'], - profit_closed_fiat=stats['profit_closed_fiat'], - profit_all_coin=stats['profit_all_coin'], - profit_all_percent=stats['profit_all_percent'], - profit_all_fiat=stats['profit_all_fiat'], - trade_count=stats['trade_count'], - first_trade_date=stats['first_trade_date'], - latest_trade_date=stats['latest_trade_date'], - avg_duration=stats['avg_duration'], - best_pair=stats['best_pair'], - best_rate=stats['best_rate'] - ) + f"∙ `{profit_closed_coin:.8f} {stake_cur} "\ + f"({profit_closed_percent:.2f}%)`\n" \ + f"∙ `{profit_closed_fiat:.3f} {fiat_disp_cur}`\n" \ + f"*ROI:* All trades\n" \ + f"∙ `{profit_all_coin:.8f} {stake_cur} ({profit_all_percent:.2f}%)`\n" \ + f"∙ `{profit_all_fiat:.3f} {fiat_disp_cur}`\n" \ + f"*Total Trade Count:* `{trade_count}`\n" \ + f"*First Trade opened:* `{first_trade_date}`\n" \ + f"*Latest Trade opened:* `{latest_trade_date}`\n" \ + f"*Avg. Duration:* `{avg_duration}`\n" \ + f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`" self._send_msg(markdown_msg, bot=bot) except RPCException as e: self._send_msg(str(e), bot=bot) From df68b0990ffc991032eadaabdf04d22f8db9ca4f Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Wed, 4 Jul 2018 13:53:45 -0500 Subject: [PATCH 107/395] rpc: fstrings --- freqtrade/rpc/rpc.py | 60 ++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f8cb136e4..263f293c6 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -74,34 +74,33 @@ class RPC(object): # calculate profit and send message to user current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] current_profit = trade.calc_profit_percent(current_rate) - fmt_close_profit = '{:.2f}%'.format( - round(trade.close_profit * 100, 2) - ) if trade.close_profit else None - message = "*Trade ID:* `{trade_id}`\n" \ - "*Current Pair:* [{pair}]({market_url})\n" \ - "*Open Since:* `{date}`\n" \ - "*Amount:* `{amount}`\n" \ - "*Open Rate:* `{open_rate:.8f}`\n" \ - "*Close Rate:* `{close_rate}`\n" \ - "*Current Rate:* `{current_rate:.8f}`\n" \ - "*Close Profit:* `{close_profit}`\n" \ - "*Current Profit:* `{current_profit:.2f}%`\n" \ - "*Open Order:* `{open_order}`"\ - .format( - trade_id=trade.id, - pair=trade.pair, - market_url=self._freqtrade.exchange.get_pair_detail_url(trade.pair), - date=arrow.get(trade.open_date).humanize(), - open_rate=trade.open_rate, - close_rate=trade.close_rate, - current_rate=current_rate, - amount=round(trade.amount, 8), - close_profit=fmt_close_profit, - current_profit=round(current_profit * 100, 2), - open_order='({} {} rem={:.8f})'.format( - order['type'], order['side'], order['remaining'] - ) if order else None, - ) + fmt_close_profit = (f'{round(trade.close_profit * 100, 2):.2f}%' + if trade.close_profit else None) + market_url = self._freqtrade.exchange.get_pair_detail_url(trade.pair) + trade_date = arrow.get(trade.open_date).humanize() + open_rate = trade.open_rate + close_rate = trade.close_rate + amount = round(trade.amount, 8) + current_profit = round(current_profit * 100, 2) + if order: + order_type = order['type'] + order_side = order['side'] + order_rem = order['remaining'] + open_order = f'({order_type} {order_side} rem={order_rem:.8f})' + else: + open_order = None + + message = f"*Trade ID:* `{trade.id}`\n" \ + f"*Current Pair:* [{trade.pair}]({market_url})\n" \ + f"*Open Since:* `{trade_date}`\n" \ + f"*Amount:* `{amount}`\n" \ + f"*Open Rate:* `{open_rate:.8f}`\n" \ + f"*Close Rate:* `{close_rate}`\n" \ + f"*Current Rate:* `{current_rate:.8f}`\n" \ + f"*Close Profit:* `{fmt_close_profit}`\n" \ + f"*Current Profit:* `{current_profit:.2f}%`\n" \ + f"*Open Order:* `{open_order}`"\ + result.append(message) return result @@ -116,11 +115,12 @@ class RPC(object): for trade in trades: # calculate profit and send message to user current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] + trade_perc = (100 * trade.calc_profit_percent(current_rate)) trades_list.append([ trade.id, trade.pair, shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), - '{:.2f}%'.format(100 * trade.calc_profit_percent(current_rate)) + f'{trade_perc:.2f}%' ]) columns = ['ID', 'Pair', 'Since', 'Profit'] @@ -148,7 +148,7 @@ class RPC(object): .all() curdayprofit = sum(trade.calc_profit() for trade in trades) profit_days[profitday] = { - 'amount': format(curdayprofit, '.8f'), + 'amount': f'{curdayprofit:.8f}', 'trades': len(trades) } From e808b3a2a1c73a1f30d33d073cf2245013d46d0e Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Thu, 5 Jul 2018 10:47:08 -0500 Subject: [PATCH 108/395] rpc: get rid of extra else and fix mypy warning --- freqtrade/rpc/rpc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 263f293c6..11658c6fb 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -82,13 +82,12 @@ class RPC(object): close_rate = trade.close_rate amount = round(trade.amount, 8) current_profit = round(current_profit * 100, 2) + open_order = '' if order: order_type = order['type'] order_side = order['side'] order_rem = order['remaining'] open_order = f'({order_type} {order_side} rem={order_rem:.8f})' - else: - open_order = None message = f"*Trade ID:* `{trade.id}`\n" \ f"*Current Pair:* [{trade.pair}]({market_url})\n" \ From c35d1b9c9d3b7a94b9cfc3b00e89976fd4b2a5fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 5 Jul 2018 23:22:35 +0200 Subject: [PATCH 109/395] Add test which checks the backtest result --- freqtrade/tests/optimize/test_backtesting.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index cb225e465..10fd1351b 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -499,6 +499,22 @@ def test_backtest(default_conf, fee, mocker) -> None: assert not results.empty assert len(results) == 2 + expected = pd.DataFrame( + {'pair': ['UNITTEST/BTC', 'UNITTEST/BTC'], + 'profit_percent': [0.00148826, 0.00075313], + 'profit_abs': [1.49e-06, 7.6e-07], + 'open_time': [Arrow(2018, 1, 29, 18, 40, 0).datetime, + Arrow(2018, 1, 30, 3, 30, 0).datetime], + 'close_time': [Arrow(2018, 1, 29, 23, 15, 0).datetime, + Arrow(2018, 1, 30, 4, 20, 0).datetime], + 'open_index': [77, 183], + 'close_index': [132, 193], + 'trade_duration': [275, 50], + 'open_at_end': [False, False], + 'open_rate': [0.10432, 0.103364], + 'close_rate': [0.104999, 0.10396]}) + pd.testing.assert_frame_equal(results, expected) + def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: """ From 8bbee4038b874c766a3739e234f098db3f5464be Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Thu, 5 Jul 2018 14:30:24 -0700 Subject: [PATCH 110/395] integrated BASE64 encoded strategy loading --- freqtrade/strategy/resolver.py | 22 ++++++++++++++++++++-- freqtrade/tests/strategy/test_strategy.py | 9 +++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 0dcd3fc6a..46f70ccc9 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -6,14 +6,16 @@ This module load custom strategies import importlib.util import inspect import logging -import os +from base64 import urlsafe_b64decode from collections import OrderedDict from typing import Optional, Dict, Type from freqtrade import constants from freqtrade.strategy import import_strategy from freqtrade.strategy.interface import IStrategy - +import tempfile +import os +from pathlib import Path logger = logging.getLogger(__name__) @@ -80,6 +82,22 @@ class StrategyResolver(object): # Add extra strategy directory on top of search paths abs_paths.insert(0, extra_dir) + if ":" in strategy_name and "http" not in strategy_name: + print("loading none http based strategy: {}".format(strategy_name)) + strat = strategy_name.split(":") + + if len(strat) == 2: + temp = Path(tempfile.mkdtemp("freq", "strategy")) + name = strat[0] + ".py" + + temp.joinpath(name).write_text(urlsafe_b64decode(strat[1]).decode('utf-8')) + temp.joinpath("__init__.py").touch() + + strategy_name = os.path.splitext(name)[0] + + # register temp path with the bot + abs_paths.insert(0, temp.absolute()) + for path in abs_paths: try: strategy = self._search_strategy(path, strategy_name) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 1e082c380..72125a244 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import logging import os +from base64 import urlsafe_b64encode import pytest @@ -50,6 +51,14 @@ def test_load_strategy(result): assert 'adx' in resolver.strategy.populate_indicators(result) +def test_load_strategy_byte64(result): + with open("freqtrade/tests/strategy/test_strategy.py", "r") as file: + encoded_string = urlsafe_b64encode(file.read().encode("utf-8")).decode("utf-8") + resolver = StrategyResolver({'strategy': 'TestStrategy:{}'.format(encoded_string)}) + assert hasattr(resolver.strategy, 'populate_indicators') + assert 'adx' in resolver.strategy.populate_indicators(result) + + def test_load_strategy_invalid_directory(result, caplog): resolver = StrategyResolver() extra_dir = os.path.join('some', 'path') From e1f5745f59400f9be4e68b026166ad9c3dbb4ea6 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Thu, 5 Jul 2018 14:50:23 -0700 Subject: [PATCH 111/395] Update resolver.py --- freqtrade/strategy/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 25eb81888..d9531ed90 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -84,7 +84,7 @@ class StrategyResolver(object): abs_paths.insert(0, extra_dir) if ":" in strategy_name: - logger.debug(("loading base64 endocded strategy".) + logger.debug(("loading base64 endocded strategy") strat = strategy_name.split(":") if len(strat) == 2: From 58879ff0125077bfaa6a3ba45cd7e3c3b2278eb4 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Thu, 5 Jul 2018 15:01:53 -0700 Subject: [PATCH 112/395] fixed braket --- freqtrade/strategy/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index d9531ed90..3ea2bf412 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -84,7 +84,7 @@ class StrategyResolver(object): abs_paths.insert(0, extra_dir) if ":" in strategy_name: - logger.debug(("loading base64 endocded strategy") + logger.debug("loading base64 endocded strategy") strat = strategy_name.split(":") if len(strat) == 2: From 1897a1cb6a97e529bf325238b8a2a5fe4ee1245c Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Thu, 5 Jul 2018 16:10:38 -0700 Subject: [PATCH 113/395] fixed mypy issues, seriosuly... --- freqtrade/strategy/resolver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 3ea2bf412..6ae779669 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -17,7 +17,6 @@ import tempfile import os from pathlib import Path - logger = logging.getLogger(__name__) @@ -97,7 +96,7 @@ class StrategyResolver(object): strategy_name = os.path.splitext(name)[0] # register temp path with the bot - abs_paths.insert(0, temp.absolute()) + abs_paths.insert(0, str(temp.resolve())) for path in abs_paths: try: From af03c172098c2996a788c91acdb1f0a4b764486f Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 6 Jul 2018 14:23:06 +0200 Subject: [PATCH 114/395] Update ccxt from 1.15.13 to 1.15.21 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f87241e32..fb2ae09cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.13 +ccxt==1.15.21 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From 54976fa103ba9ae2a9cfa2cd5b612cd4bd79531a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 6 Jul 2018 19:45:58 +0200 Subject: [PATCH 115/395] Add more tests to validate buy/sell rows --- freqtrade/tests/optimize/test_backtesting.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 10fd1351b..4af9c444e 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -485,13 +485,14 @@ def test_backtest(default_conf, fee, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) patch_exchange(mocker) backtesting = Backtesting(default_conf) - + pair = 'UNITTEST/BTC' data = optimize.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC']) data = trim_dictlist(data, -200) + data_processed = backtesting.tickerdata_to_dataframe(data) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], - 'processed': backtesting.tickerdata_to_dataframe(data), + 'processed': data_processed, 'max_open_trades': 10, 'realistic': True } @@ -500,7 +501,7 @@ def test_backtest(default_conf, fee, mocker) -> None: assert len(results) == 2 expected = pd.DataFrame( - {'pair': ['UNITTEST/BTC', 'UNITTEST/BTC'], + {'pair': [pair, pair], 'profit_percent': [0.00148826, 0.00075313], 'profit_abs': [1.49e-06, 7.6e-07], 'open_time': [Arrow(2018, 1, 29, 18, 40, 0).datetime, @@ -514,6 +515,15 @@ def test_backtest(default_conf, fee, mocker) -> None: 'open_rate': [0.10432, 0.103364], 'close_rate': [0.104999, 0.10396]}) pd.testing.assert_frame_equal(results, expected) + data_pair = data_processed[pair] + # Check open trade + for _, t in results.iterrows(): + ln = data_pair.loc[data_pair["date"] == t["open_time"]] + assert ln is not None + assert round(ln.iloc[0]["close"], 6) == round(t["open_rate"], 6) + # check close trade + ln = data_pair.loc[data_pair["date"] == t["close_time"]] + assert round(ln.iloc[0]["close"], 6) == round(t["close_rate"], 6) def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: From 9906da46f63d2031c6523bc571ef3ce71824e18d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 6 Jul 2018 20:00:39 +0200 Subject: [PATCH 116/395] move comment to correct place --- freqtrade/tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 4af9c444e..7b71c0da6 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -516,9 +516,9 @@ def test_backtest(default_conf, fee, mocker) -> None: 'close_rate': [0.104999, 0.10396]}) pd.testing.assert_frame_equal(results, expected) data_pair = data_processed[pair] - # Check open trade for _, t in results.iterrows(): ln = data_pair.loc[data_pair["date"] == t["open_time"]] + # Check open trade assert ln is not None assert round(ln.iloc[0]["close"], 6) == round(t["open_rate"], 6) # check close trade From 08fe10e302bb59f951fca17eec4f926491af3838 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 7 Jul 2018 14:23:06 +0200 Subject: [PATCH 117/395] Update ccxt from 1.15.21 to 1.15.25 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fb2ae09cc..f10ec2fed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.21 +ccxt==1.15.25 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From 742fefa786738a7eb7ea809e03d95b07570165f9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 7 Jul 2018 14:23:08 +0200 Subject: [PATCH 118/395] Update pandas from 0.23.1 to 0.23.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f10ec2fed..e18711fe9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ cachetools==2.1.0 requests==2.19.1 urllib3==1.22 wrapt==1.10.11 -pandas==0.23.1 +pandas==0.23.2 scikit-learn==0.19.1 scipy==1.1.0 jsonschema==2.6.0 From af17cef0027018de47fbee25aec9114cebc52ef7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 14:41:36 +0200 Subject: [PATCH 119/395] fix existing tests to work with validate_timeframes --- freqtrade/tests/conftest.py | 1 + freqtrade/tests/exchange/test_exchange.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 9c86d1ece..ec435ab09 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -29,6 +29,7 @@ def log_has(line, logs): def patch_exchange(mocker, api_mock=None) -> None: mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) if api_mock: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) else: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 3ddec0ded..246c9c54f 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -61,6 +61,7 @@ def test_validate_pairs(default_conf, mocker): type(api_mock).id = id_mock mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) Exchange(default_conf) @@ -68,6 +69,7 @@ def test_validate_pairs_not_available(default_conf, mocker): api_mock = MagicMock() api_mock.load_markets = MagicMock(return_value={}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) with pytest.raises(OperationalException, match=r'not available'): Exchange(default_conf) @@ -81,7 +83,7 @@ def test_validate_pairs_not_compatible(default_conf, mocker): conf = deepcopy(default_conf) conf['stake_currency'] = 'ETH' mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) with pytest.raises(OperationalException, match=r'not compatible'): Exchange(conf) @@ -93,6 +95,7 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): api_mock.load_markets = MagicMock(return_value={}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available at Binance'): Exchange(default_conf) @@ -112,6 +115,7 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): api_mock = MagicMock() api_mock.name = MagicMock(return_value='binance') mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) with pytest.raises( OperationalException, From 3f6e9cd28f3e6209f398429f1565148390c9eb15 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 14:42:53 +0200 Subject: [PATCH 120/395] Add tests for validate_timeframes --- freqtrade/tests/exchange/test_exchange.py | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 246c9c54f..89d3beb42 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -124,6 +124,39 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): Exchange(conf) +def test_validate_timeframes(default_conf, mocker): + default_conf["ticker_interval"] = "5m" + api_mock = MagicMock() + id_mock = PropertyMock(return_value='test_exchange') + type(api_mock).id = id_mock + timeframes = PropertyMock(return_value={'1m': '1m', + '5m': '5m', + '15m': '15m', + '1h': '1h'}) + type(api_mock).timeframes = timeframes + + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + Exchange(default_conf) + + +def test_validate_timeframes_failed(default_conf, mocker): + default_conf["ticker_interval"] = "3m" + api_mock = MagicMock() + id_mock = PropertyMock(return_value='test_exchange') + type(api_mock).id = id_mock + timeframes = PropertyMock(return_value={'1m': '1m', + '5m': '5m', + '15m': '15m', + '1h': '1h'}) + type(api_mock).timeframes = timeframes + + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + with pytest.raises(OperationalException, match=r'Invalid ticker 3m, this Exchange supports.*'): + Exchange(default_conf) + + def test_exchangehas(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) assert not exchange.exchange_has('ASDFASDF') From 2b488d1da2b290894b5c17a046e2c0ef4ca1b05e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 14:52:39 +0200 Subject: [PATCH 121/395] Update Dockerfile to 3.6.6 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index afafd93c1..d2c2b1b22 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6.5-slim-stretch +FROM python:3.6.6-slim-stretch # Install TA-lib RUN apt-get update && apt-get -y install curl build-essential && apt-get clean From 570d27a0c48f7b4e52ec5c9fcbc09171f113d63b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 15:30:29 +0200 Subject: [PATCH 122/395] Add testcase where ticker_interval is not in the configuration --- freqtrade/tests/exchange/test_exchange.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 89d3beb42..282d8ef01 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -157,6 +157,22 @@ def test_validate_timeframes_failed(default_conf, mocker): Exchange(default_conf) +def test_validate_timeframes_not_in_config(default_conf, mocker): + del default_conf["ticker_interval"] + api_mock = MagicMock() + id_mock = PropertyMock(return_value='test_exchange') + type(api_mock).id = id_mock + timeframes = PropertyMock(return_value={'1m': '1m', + '5m': '5m', + '15m': '15m', + '1h': '1h'}) + type(api_mock).timeframes = timeframes + + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + Exchange(default_conf) + + def test_exchangehas(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) assert not exchange.exchange_has('ASDFASDF') From 3e03a208f1e62a2f18ab633a242f2508a5da8a33 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 20:17:53 +0200 Subject: [PATCH 123/395] reduce calculation effort (slightly!) --- freqtrade/analyze.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 6d6a85596..e512947fb 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -179,7 +179,8 @@ class Analyze(object): :return: True if trade should be sold, False otherwise """ current_profit = trade.calc_profit_percent(rate) - if self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date): + if self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, + current_profit=current_profit): return True experimental = self.config.get('experimental', {}) @@ -203,13 +204,13 @@ class Analyze(object): return False - def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime) -> bool: + def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, + current_profit: float) -> bool: """ Based on current profit of the trade and configured (trailing) stoploss, decides to sell or not """ - current_profit = trade.calc_profit_percent(current_rate) trailing_stop = self.config.get('trailing_stop', False) trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) From 8dd6e29426f7546d4b5d9a0284b744318bb4ab72 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 Jul 2018 13:34:47 +0200 Subject: [PATCH 124/395] don't flag data as outdated which isn't --- freqtrade/analyze.py | 4 ++-- freqtrade/tests/test_analyze.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 6d6a85596..dc32675b6 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -2,7 +2,7 @@ Functions to analyze ticker data with indicators and produce buy and sell signals """ import logging -from datetime import datetime, timedelta +from datetime import datetime from enum import Enum from typing import Dict, List, Tuple @@ -154,7 +154,7 @@ class Analyze(object): # Check if dataframe is out of date signal_date = arrow.get(latest['date']) interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] - if signal_date < (arrow.utcnow() - timedelta(minutes=(interval_minutes + 5))): + if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))): logger.warning( 'Outdated history for pair %s. Last tick is %s minutes old', pair, diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index e6108e8f8..6e035d842 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -4,7 +4,6 @@ Unit test file for analyse.py """ -import datetime import logging from unittest.mock import MagicMock @@ -148,8 +147,9 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog): caplog.set_level(logging.INFO) mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) exchange = get_patched_exchange(mocker, default_conf) - # FIX: The get_signal function has hardcoded 10, which we must inturn hardcode - oldtime = arrow.utcnow() - datetime.timedelta(minutes=11) + # default_conf defines a 5m interval. we check interval * 2 + 5m + # this is necessary as the last candle is removed (partial candles) by default + oldtime = arrow.utcnow().shift(minutes=-16) ticks = DataFrame([{'buy': 1, 'date': oldtime}]) mocker.patch.multiple( 'freqtrade.analyze.Analyze', @@ -159,7 +159,7 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog): ) assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) assert log_has( - 'Outdated history for pair xyz. Last tick is 11 minutes old', + 'Outdated history for pair xyz. Last tick is 16 minutes old', caplog.record_tuples ) From cc107bb3cc6205a8c12773a52967f8ca3c60a462 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 8 Jul 2018 14:23:05 +0200 Subject: [PATCH 125/395] Update ccxt from 1.15.25 to 1.15.27 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e18711fe9..a9cc50a5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.25 +ccxt==1.15.27 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From 17c9c183f51f4f316ff823f0900f77830a7d6a71 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 8 Jul 2018 14:23:07 +0200 Subject: [PATCH 126/395] Update pandas from 0.23.2 to 0.23.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a9cc50a5d..abc76182c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ cachetools==2.1.0 requests==2.19.1 urllib3==1.22 wrapt==1.10.11 -pandas==0.23.2 +pandas==0.23.3 scikit-learn==0.19.1 scipy==1.1.0 jsonschema==2.6.0 From 1a24afef7775bc238cd0159c035e9db695c7a10a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 Jul 2018 19:55:04 +0200 Subject: [PATCH 127/395] add cumsum to backtest-results --- freqtrade/optimize/backtesting.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 16c21258f..56d9e3c63 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -88,9 +88,9 @@ class Backtesting(object): """ stake_currency = str(self.config.get('stake_currency')) - floatfmt = ('s', 'd', '.2f', '.8f', '.1f') + floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.1f') tabular_data = [] - headers = ['pair', 'buy count', 'avg profit %', + headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] for pair in data: result = results[results.pair == pair] @@ -98,6 +98,7 @@ class Backtesting(object): pair, len(result.index), result.profit_percent.mean() * 100.0, + result.profit_percent.sum() * 100.0, result.profit_abs.sum(), result.trade_duration.mean(), len(result[result.profit_abs > 0]), @@ -109,6 +110,7 @@ class Backtesting(object): 'TOTAL', len(results.index), results.profit_percent.mean() * 100.0, + result.profit_percent.sum() * 100.0, results.profit_abs.sum(), results.trade_duration.mean(), len(results[results.profit_abs > 0]), From 38487644f01f35d66e2914b9b0a14e28617efda9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 Jul 2018 19:55:16 +0200 Subject: [PATCH 128/395] fix tests for backtest-result output table --- freqtrade/tests/optimize/test_backtesting.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index cb225e465..0407c0708 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -391,15 +391,16 @@ def test_generate_text_table(default_conf, mocker): ) result_str = ( - '| pair | buy count | avg profit % | ' + '| pair | buy count | avg profit % | cum profit % | ' 'total profit BTC | avg duration | profit | loss |\n' - '|:--------|------------:|---------------:|' + '|:--------|------------:|---------------:|---------------:|' '-------------------:|---------------:|---------:|-------:|\n' - '| ETH/BTC | 2 | 15.00 | ' + '| ETH/BTC | 2 | 15.00 | 30.00 | ' '0.60000000 | 20.0 | 2 | 0 |\n' - '| TOTAL | 2 | 15.00 | ' + '| TOTAL | 2 | 15.00 | 30.00 | ' '0.60000000 | 20.0 | 2 | 0 |' ) + print(result_str) assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str From efaa8f16e7f485a9c206a073262df59630088ad2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 Jul 2018 20:01:33 +0200 Subject: [PATCH 129/395] Improve formattiong of table --- freqtrade/optimize/backtesting.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 56d9e3c63..80116d144 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -314,9 +314,9 @@ class Backtesting(object): self._store_backtest_result(self.config.get('exportfilename'), results) logger.info( - '\n======================================== ' + '\n================================================= ' 'BACKTESTING REPORT' - ' =========================================\n' + ' ==================================================\n' '%s', self._generate_text_table( data, @@ -325,9 +325,9 @@ class Backtesting(object): ) logger.info( - '\n====================================== ' + '\n=============================================== ' 'LEFT OPEN TRADES REPORT' - ' ======================================\n' + ' ===============================================\n' '%s', self._generate_text_table( data, From 8b06000f0fd6ed65d4b7c675a20c6ccf0ee156de Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 5 Jul 2018 20:20:52 +0200 Subject: [PATCH 130/395] Use open-rates for backtesting --- freqtrade/optimize/backtesting.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 16c21258f..16c46480b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -134,7 +134,7 @@ class Backtesting(object): stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) trade = Trade( - open_rate=buy_row.close, + open_rate=buy_row.open, open_date=buy_row.date, stake_amount=stake_amount, amount=stake_amount / buy_row.open, @@ -149,35 +149,35 @@ class Backtesting(object): trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1 buy_signal = sell_row.buy - if self.analyze.should_sell(trade, sell_row.close, sell_row.date, buy_signal, + if self.analyze.should_sell(trade, sell_row.open, sell_row.date, buy_signal, sell_row.sell): return BacktestResult(pair=pair, - profit_percent=trade.calc_profit_percent(rate=sell_row.close), - profit_abs=trade.calc_profit(rate=sell_row.close), + profit_percent=trade.calc_profit_percent(rate=sell_row.open), + profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, trade_duration=(sell_row.date - buy_row.date).seconds // 60, open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=False, - open_rate=buy_row.close, - close_rate=sell_row.close + open_rate=buy_row.open, + close_rate=sell_row.open ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period sell_row = partial_ticker[-1] btr = BacktestResult(pair=pair, - profit_percent=trade.calc_profit_percent(rate=sell_row.close), - profit_abs=trade.calc_profit(rate=sell_row.close), + profit_percent=trade.calc_profit_percent(rate=sell_row.open), + profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, trade_duration=(sell_row.date - buy_row.date).seconds // 60, open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=True, - open_rate=buy_row.close, - close_rate=sell_row.close + open_rate=buy_row.open, + close_rate=sell_row.open ) logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair, btr.profit_percent, btr.profit_abs) From 750d737b7d6d010901cec0477819148185cca28a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 Jul 2018 20:18:34 +0200 Subject: [PATCH 131/395] Add tests for change to open_rate --- freqtrade/tests/optimize/test_backtesting.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 7b71c0da6..467c50f81 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -502,28 +502,28 @@ def test_backtest(default_conf, fee, mocker) -> None: expected = pd.DataFrame( {'pair': [pair, pair], - 'profit_percent': [0.00148826, 0.00075313], + 'profit_percent': [0.00029975, 0.00056708], 'profit_abs': [1.49e-06, 7.6e-07], 'open_time': [Arrow(2018, 1, 29, 18, 40, 0).datetime, Arrow(2018, 1, 30, 3, 30, 0).datetime], - 'close_time': [Arrow(2018, 1, 29, 23, 15, 0).datetime, + 'close_time': [Arrow(2018, 1, 29, 22, 40, 0).datetime, Arrow(2018, 1, 30, 4, 20, 0).datetime], 'open_index': [77, 183], - 'close_index': [132, 193], - 'trade_duration': [275, 50], + 'close_index': [125, 193], + 'trade_duration': [240, 50], 'open_at_end': [False, False], - 'open_rate': [0.10432, 0.103364], - 'close_rate': [0.104999, 0.10396]}) + 'open_rate': [0.104445, 0.10302485], + 'close_rate': [0.105, 0.10359999]}) pd.testing.assert_frame_equal(results, expected) data_pair = data_processed[pair] for _, t in results.iterrows(): ln = data_pair.loc[data_pair["date"] == t["open_time"]] - # Check open trade + # Check open trade rate alignes to open rate assert ln is not None - assert round(ln.iloc[0]["close"], 6) == round(t["open_rate"], 6) - # check close trade + assert round(ln.iloc[0]["open"], 6) == round(t["open_rate"], 6) + # check close trade rate alignes to close rate ln = data_pair.loc[data_pair["date"] == t["close_time"]] - assert round(ln.iloc[0]["close"], 6) == round(t["close_rate"], 6) + assert round(ln.iloc[0]["open"], 6) == round(t["close_rate"], 6) def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: From 465479278471459a20a6f0809261749ca681ffd7 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Sun, 8 Jul 2018 22:43:34 -0700 Subject: [PATCH 132/395] Fixing database issues 1. if database is defined in config file, it currently tosses an exception that only export file or db is defined 2. if trades are loaded from databases, plot crashes with an exception 'cannot compare tz-naive and tz-aware datetime-like objects' 3. if Trade is not closed, crashes with exception that NoneType has no field timestamp all should be fixed --- scripts/plot_dataframe.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 1cc6b818a..9724ff389 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -24,15 +24,17 @@ Example of usage: > python3 scripts/plot_dataframe.py --pair BTC/EUR -d user_data/data/ --indicators1 sma,ema3 --indicators2 fastk,fastd """ +import json import logging import sys -import json -from pathlib import Path from argparse import Namespace +from pathlib import Path from typing import Dict, List, Any import pandas as pd import plotly.graph_objs as go +import pytz + from plotly import tools from plotly.offline import plot @@ -47,6 +49,8 @@ from freqtrade.persistence import Trade logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} +timeZone = pytz.UTC + def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame: trades: pd.DataFrame = pd.DataFrame() @@ -54,14 +58,18 @@ def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFram persistence.init(_CONF) columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] + for x in Trade.query.all(): + print("date: {}".format(x.open_date)) + trades = pd.DataFrame([(t.pair, t.calc_profit(), - t.open_date, t.close_date, + t.open_date.replace(tzinfo=timeZone), + t.close_date.replace(tzinfo=timeZone) if t.close_date else None, t.open_rate, t.close_rate, - t.close_date.timestamp() - t.open_date.timestamp()) + t.close_date.timestamp() - t.open_date.timestamp() if t.close_date else None) for t in Trade.query.filter(Trade.pair.is_(pair)).all()], columns=columns) - if args.exportfilename: + elif args.exportfilename: file = Path(args.exportfilename) # must align with columns in backtest.py columns = ["pair", "profit", "opents", "closets", "index", "duration", @@ -97,6 +105,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: # Load the configuration _CONF.update(setup_configuration(args)) + print(_CONF) # Set the pair to audit pair = args.pair @@ -136,19 +145,19 @@ def plot_analyzed_dataframe(args: Namespace) -> None: pairs=[pair], ticker_interval=tick_interval, refresh_pairs=_CONF.get('refresh_pairs', False), - timerange=timerange + timerange=timerange, + exchange=Exchange(_CONF) ) # No ticker found, or impossible to download if tickers == {}: exit() - if args.db_url and args.exportfilename: - logger.critical("Can only specify --db-url or --export-filename") # Get trades already made from the DB trades = load_trades(args, pair, timerange) dataframes = analyze.tickerdata_to_dataframe(tickers) + dataframe = dataframes[pair] dataframe = analyze.populate_buy_trend(dataframe) dataframe = analyze.populate_sell_trend(dataframe) @@ -157,6 +166,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: logger.warning('Ticker contained more than %s candles as defined ' 'with --plot-limit, clipping.', args.plot_limit) dataframe = dataframe.tail(args.plot_limit) + trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']] fig = generate_graph( pair=pair, From b773e3472a75bf9fe079762eca034bbf5572b5fa Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 9 Jul 2018 14:23:06 +0200 Subject: [PATCH 133/395] Update ccxt from 1.15.27 to 1.15.28 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index abc76182c..8e6757edc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.27 +ccxt==1.15.28 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From f5bc65b877b1653c1bb497b59369b17a80626f0c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 Jul 2018 21:56:24 +0200 Subject: [PATCH 134/395] update plotly --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8e6757edc..b04db6c56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,4 @@ coinmarketcap==5.0.3 scikit-optimize==0.5.2 # Required for plotting data -#plotly==2.7.0 +#plotly==3.0.0 From 6be6448334b5e5f4a30bee19b7854fecb0b4ce96 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 Jul 2018 21:56:29 +0200 Subject: [PATCH 135/395] replace "transparent" with rgb to fix exception in plotly 3.0.0 --- scripts/plot_dataframe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 9724ff389..3b86afc9e 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -271,7 +271,7 @@ def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tool x=data.date, y=data.bb_lowerband, name='BB lower', - line={'color': "transparent"}, + line={'color': 'rgba(255,255,255,0)'}, ) bb_upper = go.Scatter( x=data.date, @@ -279,7 +279,7 @@ def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tool name='BB upper', fill="tonexty", fillcolor="rgba(0,176,246,0.2)", - line={'color': "transparent"}, + line={'color': 'rgba(255,255,255,0)'}, ) fig.append_trace(bb_lower, 1, 1) fig.append_trace(bb_upper, 1, 1) From 85c60519b0d9c3642fd6f20e8d948b74783a6b71 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 Jul 2018 22:11:12 +0200 Subject: [PATCH 136/395] Fix test crash --- freqtrade/exchange/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index d87d3fc85..972ff49ca 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -70,8 +70,9 @@ class Exchange(object): # Check if all pairs are available self.validate_pairs(config['exchange']['pair_whitelist']) - # Check if timeframe is available - self.validate_timeframes(config['ticker_interval']) + if config.get('ticker_interval'): + # Check if timeframe is available + self.validate_timeframes(config['ticker_interval']) def _init_ccxt(self, exchange_config: dict) -> ccxt.Exchange: """ From d546a4b29f170e9e0fbeca3f1e8068428bc8e676 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 10 Jul 2018 14:23:08 +0200 Subject: [PATCH 137/395] Update ccxt from 1.15.28 to 1.15.35 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8e6757edc..5c7039286 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.28 +ccxt==1.15.35 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From 773fb5953bb586b0eaf24663fe3569ca70ada1c6 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Tue, 10 Jul 2018 15:10:56 +0200 Subject: [PATCH 138/395] Reafcotring Create Trade --- freqtrade/freqtradebot.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9def7078c..83c6a969c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -305,9 +305,6 @@ class FreqtradeBot(object): if not stake_amount: return False - stake_currency = self.config['stake_currency'] - fiat_currency = self.config['fiat_display_currency'] - exc_name = self.exchange.name logger.info( 'Checking buy signals to create a new trade with stake_amount: %f ...', @@ -328,12 +325,20 @@ class FreqtradeBot(object): for _pair in whitelist: (buy, sell) = self.analyze.get_signal(self.exchange, _pair, interval) if buy and not sell: - pair = _pair - break - else: - return False + return self.execute_buy(_pair, stake_amount) + return False + + def execute_buy(self, pair: str, stake_amount: float) -> bool: + """ + Executes a limit buy for the given pair + :param pair: pair for which we want to create a LIMIT_BUY + :return: None + """ pair_s = pair.replace('_', '/') pair_url = self.exchange.get_pair_detail_url(pair) + stake_currency = self.config['stake_currency'] + fiat_currency = self.config['fiat_display_currency'] + exc_name = self.exchange.name # Calculate amount buy_limit = self.get_target_bid(self.exchange.get_ticker(pair)) From 8f6252b312897c1a1533f98d85cac5a5bb8a639e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 11 Jul 2018 14:23:06 +0200 Subject: [PATCH 139/395] Update ccxt from 1.15.35 to 1.15.42 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fd24baa66..3fb91888c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.35 +ccxt==1.15.42 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From 06c9494a4696751c32c05b3e475e0fef642b7c50 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Jul 2018 14:50:04 +0200 Subject: [PATCH 140/395] add missing s to Backtest cum results --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ba13da298..05bcdf4b7 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -110,7 +110,7 @@ class Backtesting(object): 'TOTAL', len(results.index), results.profit_percent.mean() * 100.0, - result.profit_percent.sum() * 100.0, + results.profit_percent.sum() * 100.0, results.profit_abs.sum(), results.trade_duration.mean(), len(results[results.profit_abs > 0]), From ddfc4722b946f40fc4ecb1cab9d0919f4b68e9b2 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 12 Jul 2018 14:23:06 +0200 Subject: [PATCH 141/395] Update ccxt from 1.15.42 to 1.16.6 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3fb91888c..25c39757f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.15.42 +ccxt==1.16.6 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From df8ba28ce55103b769740da9ac50e6821bd87645 Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 22 Jun 2018 03:32:45 +0200 Subject: [PATCH 142/395] convert start, stop and reload_conf to return a dict --- freqtrade/rpc/rpc.py | 29 ++++++++++++++++-------- freqtrade/rpc/telegram.py | 6 ++--- freqtrade/tests/rpc/test_rpc.py | 9 ++++---- freqtrade/tests/rpc/test_rpc_telegram.py | 4 ++-- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 11658c6fb..a7256ca74 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -26,7 +26,17 @@ class RPCException(Exception): raise RPCException('*Status:* `no active trade`') """ - pass + def __init__(self, message: str) -> None: + super().__init__(self) + self.message = message + + def __str__(self): + return self.message + + def __json__(self): + return { + 'msg': self.message + } class RPC(object): @@ -286,28 +296,27 @@ class RPC(object): value = fiat.convert_amount(total, 'BTC', symbol) return output, total, symbol, value - def _rpc_start(self) -> str: + def _rpc_start(self) -> Dict[str, str]: """ Handler for start """ if self._freqtrade.state == State.RUNNING: - return '*Status:* `already running`' + return {'status': 'already running'} self._freqtrade.state = State.RUNNING - return '`Starting trader ...`' + return {'status': 'starting trader ...'} - def _rpc_stop(self) -> str: + def _rpc_stop(self) -> Dict[str, str]: """ Handler for stop """ if self._freqtrade.state == State.RUNNING: self._freqtrade.state = State.STOPPED - return '`Stopping trader ...`' + return {'status': 'stopping trader ...'} - return '*Status:* `already stopped`' + return {'status': 'already stopped'} - def _rpc_reload_conf(self) -> str: + def _rpc_reload_conf(self) -> Dict[str, str]: """ Handler for reload_conf. """ self._freqtrade.state = State.RELOAD_CONF - return '*Status:* `Reloading config ...`' + return {'status': 'reloading config ...'} - # FIX: no test for this!!!! def _rpc_forcesell(self, trade_id) -> None: """ Handler for forcesell . diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 13a2b1913..621ee0d16 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -266,7 +266,7 @@ class Telegram(RPC): :return: None """ msg = self._rpc_start() - self._send_msg(msg, bot=bot) + self._send_msg('Status: `{status}`'.format(**msg), bot=bot) @authorized_only def _stop(self, bot: Bot, update: Update) -> None: @@ -278,7 +278,7 @@ class Telegram(RPC): :return: None """ msg = self._rpc_stop() - self._send_msg(msg, bot=bot) + self._send_msg('Status: `{status}`'.format(**msg), bot=bot) @authorized_only def _reload_conf(self, bot: Bot, update: Update) -> None: @@ -290,7 +290,7 @@ class Telegram(RPC): :return: None """ msg = self._rpc_reload_conf() - self._send_msg(msg, bot=bot) + self._send_msg('Status: `{status}`'.format(**msg), bot=bot) @authorized_only def _forcesell(self, bot: Bot, update: Update) -> None: diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 58514d1c0..e2104836a 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -358,11 +358,11 @@ def test_rpc_start(mocker, default_conf) -> None: freqtradebot.state = State.STOPPED result = rpc._rpc_start() - assert '`Starting trader ...`' in result + assert {'status': 'starting trader ...'} == result assert freqtradebot.state == State.RUNNING result = rpc._rpc_start() - assert '*Status:* `already running`' in result + assert {'status': 'already running'} == result assert freqtradebot.state == State.RUNNING @@ -384,11 +384,12 @@ def test_rpc_stop(mocker, default_conf) -> None: freqtradebot.state = State.RUNNING result = rpc._rpc_stop() - assert '`Stopping trader ...`' in result + assert {'status': 'stopping trader ...'} == result assert freqtradebot.state == State.STOPPED result = rpc._rpc_stop() - assert '*Status:* `already stopped`' in result + + assert {'status': 'already stopped'} == result assert freqtradebot.state == State.STOPPED diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 2710328bd..386ab9dcc 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -664,7 +664,7 @@ def test_stop_handle(default_conf, update, mocker) -> None: telegram._stop(bot=MagicMock(), update=update) assert freqtradebot.state == State.STOPPED assert msg_mock.call_count == 1 - assert 'Stopping trader' in msg_mock.call_args_list[0][0][0] + assert 'stopping trader' in msg_mock.call_args_list[0][0][0] def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: @@ -708,7 +708,7 @@ def test_reload_conf_handle(default_conf, update, mocker) -> None: telegram._reload_conf(bot=MagicMock(), update=update) assert freqtradebot.state == State.RELOAD_CONF assert msg_mock.call_count == 1 - assert 'Reloading config' in msg_mock.call_args_list[0][0][0] + assert 'reloading config' in msg_mock.call_args_list[0][0][0] def test_forcesell_handle(default_conf, update, ticker, fee, From 29670b9814fe93ac3e729d02af96bbbbec35567f Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 22 Jun 2018 03:37:19 +0200 Subject: [PATCH 143/395] remove markdown formatting from exception string --- freqtrade/rpc/rpc.py | 22 +++++++++++----------- freqtrade/tests/rpc/test_rpc.py | 12 ++++++------ freqtrade/tests/rpc/test_rpc_telegram.py | 6 +++--- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index a7256ca74..36b5e9804 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -72,9 +72,9 @@ class RPC(object): # Fetch open trade trades = Trade.query.filter(Trade.is_open.is_(True)).all() if self._freqtrade.state != State.RUNNING: - raise RPCException('*Status:* `trader is not running`') + raise RPCException('trader is not running') elif not trades: - raise RPCException('*Status:* `no active trade`') + raise RPCException('no active trade') else: result = [] for trade in trades: @@ -116,9 +116,9 @@ class RPC(object): def _rpc_status_table(self) -> DataFrame: trades = Trade.query.filter(Trade.is_open.is_(True)).all() if self._freqtrade.state != State.RUNNING: - raise RPCException('*Status:* `trader is not running`') + raise RPCException('trader is not running') elif not trades: - raise RPCException('*Status:* `no active order`') + raise RPCException('no active order') else: trades_list = [] for trade in trades: @@ -144,7 +144,7 @@ class RPC(object): profit_days: Dict[date, Dict] = {} if not (isinstance(timescale, int) and timescale > 0): - raise RPCException('*Daily [n]:* `must be an integer greater than 0`') + raise RPCException('timescale must be an integer greater than 0') fiat = self._freqtrade.fiat_converter for day in range(0, timescale): @@ -224,7 +224,7 @@ class RPC(object): .order_by(sql.text('profit_sum DESC')).first() if not best_pair: - raise RPCException('*Status:* `no closed trade`') + raise RPCException('no closed trade') bp_pair, bp_rate = best_pair @@ -289,7 +289,7 @@ class RPC(object): } ) if total == 0.0: - raise RPCException('`All balances are zero.`') + raise RPCException('all balances are zero') fiat = self._freqtrade.fiat_converter symbol = fiat_display_currency @@ -350,7 +350,7 @@ class RPC(object): # ---- EOF def _exec_forcesell ---- if self._freqtrade.state != State.RUNNING: - raise RPCException('`trader is not running`') + raise RPCException('trader is not running') if trade_id == 'all': # Execute sell for all open orders @@ -367,7 +367,7 @@ class RPC(object): ).first() if not trade: logger.warning('forcesell: Invalid argument received') - raise RPCException('Invalid argument.') + raise RPCException('invalid argument') _exec_forcesell(trade) Trade.session.flush() @@ -378,7 +378,7 @@ class RPC(object): Shows a performance statistic from finished trades """ if self._freqtrade.state != State.RUNNING: - raise RPCException('`trader is not running`') + raise RPCException('trader is not running') pair_rates = Trade.session.query(Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum'), @@ -395,6 +395,6 @@ class RPC(object): def _rpc_count(self) -> List[Trade]: """ Returns the number of trades running """ if self._freqtrade.state != State.RUNNING: - raise RPCException('`trader is not running`') + raise RPCException('trader is not running') return Trade.query.filter(Trade.is_open.is_(True)).all() diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index e2104836a..37cff270c 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -92,11 +92,11 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED - with pytest.raises(RPCException, match=r'.*\*Status:\* `trader is not running``*'): + with pytest.raises(RPCException, match=r'.*trader is not running*'): rpc._rpc_status_table() freqtradebot.state = State.RUNNING - with pytest.raises(RPCException, match=r'.*\*Status:\* `no active order`*'): + with pytest.raises(RPCException, match=r'.*no active order*'): rpc._rpc_status_table() freqtradebot.create_trade() @@ -422,11 +422,11 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED - with pytest.raises(RPCException, match=r'.*`trader is not running`*'): + with pytest.raises(RPCException, match=r'.*trader is not running*'): rpc._rpc_forcesell(None) freqtradebot.state = State.RUNNING - with pytest.raises(RPCException, match=r'.*Invalid argument.*'): + with pytest.raises(RPCException, match=r'.*invalid argument*'): rpc._rpc_forcesell(None) rpc._rpc_forcesell('all') @@ -437,10 +437,10 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: rpc._rpc_forcesell('1') freqtradebot.state = State.STOPPED - with pytest.raises(RPCException, match=r'.*`trader is not running`*'): + with pytest.raises(RPCException, match=r'.*trader is not running*'): rpc._rpc_forcesell(None) - with pytest.raises(RPCException, match=r'.*`trader is not running`*'): + with pytest.raises(RPCException, match=r'.*trader is not running*'): rpc._rpc_forcesell('all') freqtradebot.state = State.RUNNING diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 386ab9dcc..740c7d0b6 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -598,7 +598,7 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: telegram._balance(bot=MagicMock(), update=update) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 - assert '`All balances are zero.`' in result + assert 'all balances are zero' in result def test_start_handle(default_conf, update, mocker) -> None: @@ -866,7 +866,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: update.message.text = '/forcesell' telegram._forcesell(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert 'Invalid argument' in msg_mock.call_args_list[0][0][0] + assert 'invalid argument' in msg_mock.call_args_list[0][0][0] # Invalid argument msg_mock.reset_mock() @@ -874,7 +874,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: update.message.text = '/forcesell 123456' telegram._forcesell(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert 'Invalid argument.' in msg_mock.call_args_list[0][0][0] + assert 'invalid argument' in msg_mock.call_args_list[0][0][0] def test_performance_handle(default_conf, update, ticker, fee, From f1a370b3b9db3158e60f5cb8cb77dddaa680dcae Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 22 Jun 2018 03:54:10 +0200 Subject: [PATCH 144/395] return dict from _rpc_status and handle rendering in module impl --- freqtrade/rpc/rpc.py | 46 ++++++++++-------------- freqtrade/rpc/telegram.py | 18 ++++++++-- freqtrade/tests/rpc/test_rpc.py | 31 ++++++++-------- freqtrade/tests/rpc/test_rpc_telegram.py | 16 +++++++-- 4 files changed, 62 insertions(+), 49 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 36b5e9804..3dc07d304 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -64,7 +64,7 @@ class RPC(object): def send_msg(self, msg: str) -> None: """ Sends a message to all registered rpc modules """ - def _rpc_trade_status(self) -> List[str]: + def _rpc_trade_status(self) -> List[Dict]: """ Below follows the RPC backend it is prefixed with rpc_ to raise awareness that it is a remotely exposed function @@ -76,7 +76,7 @@ class RPC(object): elif not trades: raise RPCException('no active trade') else: - result = [] + results = [] for trade in trades: order = None if trade.open_order_id: @@ -86,32 +86,22 @@ class RPC(object): current_profit = trade.calc_profit_percent(current_rate) fmt_close_profit = (f'{round(trade.close_profit * 100, 2):.2f}%' if trade.close_profit else None) - market_url = self._freqtrade.exchange.get_pair_detail_url(trade.pair) - trade_date = arrow.get(trade.open_date).humanize() - open_rate = trade.open_rate - close_rate = trade.close_rate - amount = round(trade.amount, 8) - current_profit = round(current_profit * 100, 2) - open_order = '' - if order: - order_type = order['type'] - order_side = order['side'] - order_rem = order['remaining'] - open_order = f'({order_type} {order_side} rem={order_rem:.8f})' - - message = f"*Trade ID:* `{trade.id}`\n" \ - f"*Current Pair:* [{trade.pair}]({market_url})\n" \ - f"*Open Since:* `{trade_date}`\n" \ - f"*Amount:* `{amount}`\n" \ - f"*Open Rate:* `{open_rate:.8f}`\n" \ - f"*Close Rate:* `{close_rate}`\n" \ - f"*Current Rate:* `{current_rate:.8f}`\n" \ - f"*Close Profit:* `{fmt_close_profit}`\n" \ - f"*Current Profit:* `{current_profit:.2f}%`\n" \ - f"*Open Order:* `{open_order}`"\ - - result.append(message) - return result + results.append(dict( + trade_id=trade.id, + pair=trade.pair, + market_url=self._freqtrade.exchange.get_pair_detail_url(trade.pair), + date=arrow.get(trade.open_date).humanize(), + open_rate=trade.open_rate, + close_rate=trade.close_rate, + current_rate=current_rate, + amount=round(trade.amount, 8), + close_profit=fmt_close_profit, + current_profit=round(current_profit * 100, 2), + open_order='({} {} rem={:.8f})'.format( + order['type'], order['side'], order['remaining'] + ) if order else None, + )) + return results def _rpc_status_table(self) -> DataFrame: trades = Trade.query.filter(Trade.is_open.is_(True)).all() diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 621ee0d16..0bce435c4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -136,8 +136,22 @@ class Telegram(RPC): return try: - for trade_msg in self._rpc_trade_status(): - self._send_msg(trade_msg, bot=bot) + results = self._rpc_trade_status() + messages = [ + "*Trade ID:* `{trade_id}`\n" + "*Current Pair:* [{pair}]({market_url})\n" + "*Open Since:* `{date}`\n" + "*Amount:* `{amount}`\n" + "*Open Rate:* `{open_rate:.8f}`\n" + "*Close Rate:* `{close_rate}`\n" + "*Current Rate:* `{current_rate:.8f}`\n" + "*Close Profit:* `{close_profit}`\n" + "*Current Profit:* `{current_profit:.2f}%`\n" + "*Open Order:* `{open_order}`".format(**result) + for result in results + ] + for msg in messages: + self._send_msg(msg, bot=bot) except RPCException as e: self._send_msg(str(e), bot=bot) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 37cff270c..d8b9043ec 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -53,24 +53,21 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: rpc._rpc_trade_status() freqtradebot.create_trade() - trades = rpc._rpc_trade_status() - trade = trades[0] + results = rpc._rpc_trade_status() - result_message = [ - '*Trade ID:* `1`\n' - '*Current Pair:* ' - '[ETH/BTC](https://bittrex.com/Market/Index?MarketName=BTC-ETH)\n' - '*Open Since:* `just now`\n' - '*Amount:* `90.99181074`\n' - '*Open Rate:* `0.00001099`\n' - '*Close Rate:* `None`\n' - '*Current Rate:* `0.00001098`\n' - '*Close Profit:* `None`\n' - '*Current Profit:* `-0.59%`\n' - '*Open Order:* `(limit buy rem=0.00000000)`' - ] - assert trades == result_message - assert trade.find('[ETH/BTC]') >= 0 + assert { + 'trade_id': 1, + 'pair': 'ETH/BTC', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'date': 'just now', + 'open_rate': 1.099e-05, + 'close_rate': None, + 'current_rate': 1.098e-05, + 'amount': 90.99181074, + 'close_profit': None, + 'current_profit': -0.59, + 'open_order': '(limit buy rem=0.00000000)' + } == results[0] def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 740c7d0b6..dc3e3aa3c 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -210,7 +210,19 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - _rpc_trade_status=MagicMock(return_value=[1, 2, 3]), + _rpc_trade_status=MagicMock(return_value=[{ + 'trade_id': 1, + 'pair': 'ETH/BTC', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'date': 'just now', + 'open_rate': 1.099e-05, + 'close_rate': None, + 'current_rate': 1.098e-05, + 'amount': 90.99181074, + 'close_profit': None, + 'current_profit': -0.59, + 'open_order': '(limit buy rem=0.00000000)' + }]), _status_table=status_table, _send_msg=msg_mock ) @@ -224,7 +236,7 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: freqtradebot.create_trade() telegram._status(bot=MagicMock(), update=update) - assert msg_mock.call_count == 3 + assert msg_mock.call_count == 1 update.message.text = MagicMock() update.message.text.replace = MagicMock(return_value='table 2 3') From 112998c2058e2b237bec3c0ffa1972a72b4c065e Mon Sep 17 00:00:00 2001 From: gcarq Date: Fri, 22 Jun 2018 04:08:51 +0200 Subject: [PATCH 145/395] refactor _rpc_balance --- freqtrade/rpc/rpc.py | 29 ++++++++++++++++------------- freqtrade/rpc/telegram.py | 9 ++++----- freqtrade/tests/rpc/test_rpc.py | 21 +++++++++++---------- 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 3dc07d304..6e9cd437f 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -3,9 +3,9 @@ This module contains class to define a RPC communications """ import logging from abc import abstractmethod -from datetime import date, datetime, timedelta +from datetime import timedelta, datetime, date from decimal import Decimal -from typing import Any, Dict, List, Tuple +from typing import Dict, Any, List import arrow import sqlalchemy as sql @@ -252,7 +252,7 @@ class RPC(object): 'best_rate': round(bp_rate * 100, 2), } - def _rpc_balance(self, fiat_display_currency: str) -> Tuple[List[Dict], float, str, float]: + def _rpc_balance(self, fiat_display_currency: str) -> Dict: """ Returns current account balance per crypto """ output = [] total = 0.0 @@ -269,22 +269,25 @@ class RPC(object): rate = self._freqtrade.exchange.get_ticker(coin + '/BTC', False)['bid'] est_btc: float = rate * balance['total'] total = total + est_btc - output.append( - { - 'currency': coin, - 'available': balance['free'], - 'balance': balance['total'], - 'pending': balance['used'], - 'est_btc': est_btc - } - ) + output.append({ + 'currency': coin, + 'available': balance['free'], + 'balance': balance['total'], + 'pending': balance['used'], + 'est_btc': est_btc, + }) if total == 0.0: raise RPCException('all balances are zero') fiat = self._freqtrade.fiat_converter symbol = fiat_display_currency value = fiat.convert_amount(total, 'BTC', symbol) - return output, total, symbol, value + return { + 'currencies': output, + 'total': total, + 'symbol': symbol, + 'value': value, + } def _rpc_start(self) -> Dict[str, str]: """ Handler for start """ diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 0bce435c4..e7ba74bee 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -253,10 +253,9 @@ class Telegram(RPC): def _balance(self, bot: Bot, update: Update) -> None: """ Handler for /balance """ try: - currencys, total, symbol, value = \ - self._rpc_balance(self._config['fiat_display_currency']) + result = self._rpc_balance(self._config['fiat_display_currency']) output = '' - for currency in currencys: + for currency in result['currencies']: output += "*{currency}:*\n" \ "\t`Available: {available: .8f}`\n" \ "\t`Balance: {balance: .8f}`\n" \ @@ -264,8 +263,8 @@ class Telegram(RPC): "\t`Est. BTC: {est_btc: .8f}`\n".format(**currency) output += "\n*Estimated Value*:\n" \ - "\t`BTC: {0: .8f}`\n" \ - "\t`{1}: {2: .2f}`\n".format(total, symbol, value) + "\t`BTC: {total: .8f}`\n" \ + "\t`{symbol}: {value: .2f}`\n".format(**result) self._send_msg(output, bot=bot) except RPCException as e: self._send_msg(str(e), bot=bot) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index d8b9043ec..654547078 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -325,16 +325,17 @@ def test_rpc_balance_handle(default_conf, mocker): freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) - output, total, symbol, value = rpc._rpc_balance(default_conf['fiat_display_currency']) - assert prec_satoshi(total, 12) - assert prec_satoshi(value, 180000) - assert 'USD' in symbol - assert len(output) == 1 - assert 'BTC' in output[0]['currency'] - assert prec_satoshi(output[0]['available'], 10) - assert prec_satoshi(output[0]['balance'], 12) - assert prec_satoshi(output[0]['pending'], 2) - assert prec_satoshi(output[0]['est_btc'], 12) + result = rpc._rpc_balance(default_conf['fiat_display_currency']) + assert prec_satoshi(result['total'], 12) + assert prec_satoshi(result['value'], 180000) + assert 'USD' == result['symbol'] + assert result['currencies'] == [{ + 'currency': 'BTC', + 'available': 10.0, + 'balance': 12.0, + 'pending': 2.0, + 'est_btc': 12.0, + }] def test_rpc_start(mocker, default_conf) -> None: From 96a405feb7bed66bd0a5e211df8c0c5604a558d5 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 23 Jun 2018 12:28:32 +0200 Subject: [PATCH 146/395] implement name property in abstract class --- freqtrade/rpc/rpc.py | 10 +++++----- freqtrade/rpc/telegram.py | 4 ---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 6e9cd437f..0ab3a14bc 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -51,15 +51,15 @@ class RPC(object): """ self._freqtrade = freqtrade + @property + def name(self) -> str: + """ Returns the lowercase name of the implementation """ + return self.__class__.__name__.lower() + @abstractmethod def cleanup(self) -> None: """ Cleanup pending module resources """ - @property - @abstractmethod - def name(self) -> str: - """ Returns the lowercase name of this module """ - @abstractmethod def send_msg(self, msg: str) -> None: """ Sends a message to all registered rpc modules """ diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e7ba74bee..bea680fba 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -55,10 +55,6 @@ def authorized_only(command_handler: Callable[[Any, Bot, Update], None]) -> Call class Telegram(RPC): """ This class handles all telegram communication """ - @property - def name(self) -> str: - return "telegram" - def __init__(self, freqtrade) -> None: """ Init the Telegram call, and init the super class RPC From 4cb1aa1d97695363c2292ce53d64410e7d1f754a Mon Sep 17 00:00:00 2001 From: gcarq Date: Mon, 25 Jun 2018 00:04:27 +0200 Subject: [PATCH 147/395] use dict as argument for rpc.send_msg --- freqtrade/freqtradebot.py | 35 ++++++++++------ freqtrade/main.py | 9 ++-- freqtrade/rpc/rpc.py | 2 +- freqtrade/rpc/rpc_manager.py | 13 +++--- freqtrade/rpc/telegram.py | 6 +-- freqtrade/tests/rpc/test_rpc_manager.py | 8 ++-- freqtrade/tests/rpc/test_rpc_telegram.py | 32 ++++++++------- freqtrade/tests/test_freqtradebot.py | 52 +++++++++++++----------- 8 files changed, 89 insertions(+), 68 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 83c6a969c..bbece2f03 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -91,7 +91,9 @@ class FreqtradeBot(object): # Log state transition state = self.state if state != old_state: - self.rpc.send_msg(f'*Status:* `{state.name.lower()}`') + self.rpc.send_msg({ + 'status': f'{state.name.lower()}' + }) logger.info('Changing state to: %s', state.name) if state == State.STOPPED: @@ -167,9 +169,9 @@ class FreqtradeBot(object): except OperationalException: tb = traceback.format_exc() hint = 'Issue `/start` if you think it is safe to restart.' - self.rpc.send_msg( - f'*Status:* OperationalException:\n```\n{tb}```{hint}' - ) + self.rpc.send_msg({ + 'status': f'OperationalException:\n```\n{tb}```{hint}' + }) logger.exception('OperationalException. Stopping trader ...') self.state = State.STOPPED return state_changed @@ -362,11 +364,12 @@ class FreqtradeBot(object): ) # Create trade entity and return - self.rpc.send_msg( - f"""*{exc_name}:* Buying [{pair_s}]({pair_url}) \ -with limit `{buy_limit:.8f} ({stake_amount:.6f} \ -{stake_currency}, {stake_amount_fiat:.3f} {fiat_currency})`""" - ) + self.rpc.send_msg({ + 'status': + f"""*{exc_name}:* Buying [{pair_s}]({pair_url}) \ + with limit `{buy_limit:.8f} ({stake_amount:.6f} \ + {stake_currency}, {stake_amount_fiat:.3f} {fiat_currency})`""" + }) # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') trade = Trade( @@ -551,7 +554,9 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ Trade.session.delete(trade) Trade.session.flush() logger.info('Buy order timeout for %s.', trade) - self.rpc.send_msg(f'*Timeout:* Unfilled buy order for {pair_s} cancelled') + self.rpc.send_msg({ + 'status': f'Unfilled buy order for {pair_s} cancelled due to timeout' + }) return True # if trade is partially complete, edit the stake details for the trade @@ -560,7 +565,9 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ trade.stake_amount = trade.amount * trade.open_rate trade.open_order_id = None logger.info('Partial buy order timeout for %s.', trade) - self.rpc.send_msg(f'*Timeout:* Remaining buy order for {pair_s} cancelled') + self.rpc.send_msg({ + 'status': f'Remaining buy order for {pair_s} cancelled due to timeout' + }) return False # FIX: 20180110, should cancel_order() be cond. or unconditionally called? @@ -578,7 +585,9 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ trade.close_date = None trade.is_open = True trade.open_order_id = None - self.rpc.send_msg(f'*Timeout:* Unfilled sell order for {pair_s} cancelled') + self.rpc.send_msg({ + 'status': f'Unfilled sell order for {pair_s} cancelled due to timeout' + }) logger.info('Sell order timeout for %s.', trade) return True @@ -634,5 +643,5 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \ gain = "profit" if fmt_exp_profit > 0 else "loss" message += f'` ({gain}: {fmt_exp_profit:.2f}%, {profit_trade:.8f})`' # Send the message - self.rpc.send_msg(message) + self.rpc.send_msg({'status': message}) Trade.session.flush() diff --git a/freqtrade/main.py b/freqtrade/main.py index 79080ce37..74d8da031 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -59,7 +59,9 @@ def main(sysargv: List[str]) -> None: logger.exception('Fatal exception!') finally: if freqtrade: - freqtrade.rpc.send_msg('*Status:* `Process died ...`') + freqtrade.rpc.send_msg({ + 'status': 'process died' + }) freqtrade.cleanup() sys.exit(return_code) @@ -73,8 +75,9 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot: # Create new instance freqtrade = FreqtradeBot(Configuration(args).get_config()) - freqtrade.rpc.send_msg( - '*Status:* `Config reloaded {freqtrade.state.name.lower()}...`') + freqtrade.rpc.send_msg({ + 'status': 'config reloaded' + }) return freqtrade diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 0ab3a14bc..9ad506b82 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -61,7 +61,7 @@ class RPC(object): """ Cleanup pending module resources """ @abstractmethod - def send_msg(self, msg: str) -> None: + def send_msg(self, msg: Dict[str, str]) -> None: """ Sends a message to all registered rpc modules """ def _rpc_trade_status(self) -> List[Dict]: diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 252bbcdd8..ec193d23d 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -2,7 +2,7 @@ This module contains class to manage RPC communications (Telegram, Slack, ...) """ import logging -from typing import List +from typing import List, Dict from freqtrade.rpc.rpc import RPC @@ -32,11 +32,14 @@ class RPCManager(object): mod.cleanup() del mod - def send_msg(self, msg: str) -> None: + def send_msg(self, msg: Dict[str, str]) -> None: """ - Send given markdown message to all registered rpc modules - :param msg: message - :return: None + Send given message to all registered rpc modules. + A message consists of one or more key value pairs of strings. + e.g.: + { + 'status': 'stopping bot' + } """ logger.info('Sending rpc message: %s', msg) for mod in self.registered_modules: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index bea680fba..cab5d1d74 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -4,7 +4,7 @@ This module manage Telegram communication """ import logging -from typing import Any, Callable +from typing import Any, Callable, Dict from tabulate import tabulate from telegram import Bot, ParseMode, ReplyKeyboardMarkup, Update @@ -110,9 +110,9 @@ class Telegram(RPC): """ self._updater.stop() - def send_msg(self, msg: str) -> None: + def send_msg(self, msg: Dict[str, str]) -> None: """ Send a message to telegram channel """ - self._send_msg(msg) + self._send_msg('*Status:* `{status}`'.format(**msg)) @authorized_only def _status(self, bot: Bot, update: Update) -> None: diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 5aea98d48..3c17a4270 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -102,9 +102,9 @@ def test_send_msg_telegram_disabled(mocker, default_conf, caplog) -> None: freqtradebot = get_patched_freqtradebot(mocker, conf) rpc_manager = RPCManager(freqtradebot) - rpc_manager.send_msg('test') + rpc_manager.send_msg({'status': 'test'}) - assert log_has('Sending rpc message: test', caplog.record_tuples) + assert log_has("Sending rpc message: {'status': 'test'}", caplog.record_tuples) assert telegram_mock.call_count == 0 @@ -117,7 +117,7 @@ def test_send_msg_telegram_enabled(mocker, default_conf, caplog) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc_manager = RPCManager(freqtradebot) - rpc_manager.send_msg('test') + rpc_manager.send_msg({'status': 'test'}) - assert log_has('Sending rpc message: test', caplog.record_tuples) + assert log_has("Sending rpc message: {'status': 'test'}", caplog.record_tuples) assert telegram_mock.call_count == 1 diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index dc3e3aa3c..30782d598 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -757,12 +757,13 @@ def test_forcesell_handle(default_conf, update, ticker, fee, telegram._forcesell(bot=MagicMock(), update=update) assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[ETH/BTC]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] - assert 'profit: 6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] - assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] + last_call = rpc_mock.call_args_list[-1][0][0]['status'] + assert 'Selling' in last_call + assert '[ETH/BTC]' in last_call + assert 'Amount' in last_call + assert '0.00001172' in last_call + assert 'profit: 6.11%, 0.00006126' in last_call + assert '0.919 USD' in last_call def test_forcesell_down_handle(default_conf, update, ticker, fee, @@ -802,13 +803,14 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, update.message.text = '/forcesell 1' telegram._forcesell(bot=MagicMock(), update=update) + last_call = rpc_mock.call_args_list[-1][0][0]['status'] assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[ETH/BTC]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] - assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] - assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] + assert 'Selling' in last_call + assert '[ETH/BTC]' in last_call + assert 'Amount' in last_call + assert '0.00001044' in last_call + assert 'loss: -5.48%, -0.00005492' in last_call + assert '-0.824 USD' in last_call def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker) -> None: @@ -842,9 +844,9 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker assert rpc_mock.call_count == 4 for args in rpc_mock.call_args_list: - assert '0.00001098' in args[0][0] - assert 'loss: -0.59%, -0.00000591 BTC' in args[0][0] - assert '-0.089 USD' in args[0][0] + assert '0.00001098' in args[0][0]['status'] + assert 'loss: -0.59%, -0.00000591 BTC' in args[0][0]['status'] + assert '-0.089 USD' in args[0][0]['status'] def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 17bd6aa7c..678e54fe6 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -755,7 +755,7 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) -> result = freqtrade._process() assert result is False assert freqtrade.state == State.STOPPED - assert 'OperationalException' in msg_mock.call_args_list[-1][0][0] + assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status'] def test_process_trade_handling( @@ -1375,13 +1375,14 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid']) assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[ETH/BTC]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert 'Profit' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] - assert 'profit: 6.11%, 0.00006126' in rpc_mock.call_args_list[-1][0][0] - assert '0.919 USD' in rpc_mock.call_args_list[-1][0][0] + last_call = rpc_mock.call_args_list[-1][0][0]['status'] + assert 'Selling' in last_call + assert '[ETH/BTC]' in last_call + assert 'Amount' in last_call + assert 'Profit' in last_call + assert '0.00001172' in last_call + assert 'profit: 6.11%, 0.00006126' in last_call + assert '0.919 USD' in last_call def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: @@ -1417,12 +1418,13 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid']) assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[ETH/BTC]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] - assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] - assert '-0.824 USD' in rpc_mock.call_args_list[-1][0][0] + last_call = rpc_mock.call_args_list[-1][0][0]['status'] + assert 'Selling' in last_call + assert '[ETH/BTC]' in last_call + assert 'Amount' in last_call + assert '0.00001044' in last_call + assert 'loss: -5.48%, -0.00005492' in last_call + assert '-0.824 USD' in last_call def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, @@ -1459,12 +1461,13 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid']) assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[ETH/BTC]' in rpc_mock.call_args_list[-1][0][0] - assert 'Amount' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001172' in rpc_mock.call_args_list[-1][0][0] - assert '(profit: 6.11%, 0.00006126)' in rpc_mock.call_args_list[-1][0][0] - assert 'USD' not in rpc_mock.call_args_list[-1][0][0] + last_call = rpc_mock.call_args_list[-1][0][0]['status'] + assert 'Selling' in last_call + assert '[ETH/BTC]' in last_call + assert 'Amount' in last_call + assert '0.00001172' in last_call + assert '(profit: 6.11%, 0.00006126)' in last_call + assert 'USD' not in last_call def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, @@ -1501,10 +1504,11 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid']) assert rpc_mock.call_count == 2 - assert 'Selling' in rpc_mock.call_args_list[-1][0][0] - assert '[ETH/BTC]' in rpc_mock.call_args_list[-1][0][0] - assert '0.00001044' in rpc_mock.call_args_list[-1][0][0] - assert 'loss: -5.48%, -0.00005492' in rpc_mock.call_args_list[-1][0][0] + last_call = rpc_mock.call_args_list[-1][0][0]['status'] + assert 'Selling' in last_call + assert '[ETH/BTC]' in last_call + assert '0.00001044' in last_call + assert 'loss: -5.48%, -0.00005492' in last_call def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, From 0920fb61200ca206be153c399b0d1422f321dead Mon Sep 17 00:00:00 2001 From: gcarq Date: Tue, 3 Jul 2018 20:26:48 +0200 Subject: [PATCH 148/395] use more granular msg dict for buy/sell notifications --- freqtrade/freqtradebot.py | 75 ++++++++++--------- freqtrade/main.py | 3 + freqtrade/rpc/__init__.py | 2 + freqtrade/rpc/rpc.py | 15 ++-- freqtrade/rpc/rpc_manager.py | 6 +- freqtrade/rpc/telegram.py | 39 +++++++++- freqtrade/tests/rpc/test_rpc.py | 2 +- freqtrade/tests/rpc/test_rpc_manager.py | 18 +++-- freqtrade/tests/rpc/test_rpc_telegram.py | 92 +++++++++++++++++------ freqtrade/tests/test_freqtradebot.py | 93 ++++++++++++++++-------- 10 files changed, 244 insertions(+), 101 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index bbece2f03..72b5190b9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -19,7 +19,8 @@ from freqtrade.analyze import Analyze from freqtrade.exchange import Exchange from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.persistence import Trade -from freqtrade.rpc.rpc_manager import RPCManager +from freqtrade.rpc import RPCMessageType +from freqtrade.rpc import RPCManager from freqtrade.state import State logger = logging.getLogger(__name__) @@ -92,6 +93,7 @@ class FreqtradeBot(object): state = self.state if state != old_state: self.rpc.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, 'status': f'{state.name.lower()}' }) logger.info('Changing state to: %s', state.name) @@ -170,6 +172,7 @@ class FreqtradeBot(object): tb = traceback.format_exc() hint = 'Issue `/start` if you think it is safe to restart.' self.rpc.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, 'status': f'OperationalException:\n```\n{tb}```{hint}' }) logger.exception('OperationalException. Stopping trader ...') @@ -340,7 +343,6 @@ class FreqtradeBot(object): pair_url = self.exchange.get_pair_detail_url(pair) stake_currency = self.config['stake_currency'] fiat_currency = self.config['fiat_display_currency'] - exc_name = self.exchange.name # Calculate amount buy_limit = self.get_target_bid(self.exchange.get_ticker(pair)) @@ -363,12 +365,16 @@ class FreqtradeBot(object): fiat_currency ) - # Create trade entity and return self.rpc.send_msg({ - 'status': - f"""*{exc_name}:* Buying [{pair_s}]({pair_url}) \ - with limit `{buy_limit:.8f} ({stake_amount:.6f} \ - {stake_currency}, {stake_amount_fiat:.3f} {fiat_currency})`""" + 'type': RPCMessageType.BUY_NOTIFICATION, + 'exchange': self.exchange.name.capitalize(), + 'pair': pair_s, + 'market_url': pair_url, + 'limit': buy_limit, + 'stake_amount': stake_amount, + 'stake_amount_fiat': stake_amount_fiat, + 'stake_currency': stake_currency, + 'fiat_currency': fiat_currency }) # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') @@ -555,6 +561,7 @@ class FreqtradeBot(object): Trade.session.flush() logger.info('Buy order timeout for %s.', trade) self.rpc.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, 'status': f'Unfilled buy order for {pair_s} cancelled due to timeout' }) return True @@ -566,6 +573,7 @@ class FreqtradeBot(object): trade.open_order_id = None logger.info('Partial buy order timeout for %s.', trade) self.rpc.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, 'status': f'Remaining buy order for {pair_s} cancelled due to timeout' }) return False @@ -586,6 +594,7 @@ class FreqtradeBot(object): trade.is_open = True trade.open_order_id = None self.rpc.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, 'status': f'Unfilled sell order for {pair_s} cancelled due to timeout' }) logger.info('Sell order timeout for %s.', trade) @@ -601,47 +610,47 @@ class FreqtradeBot(object): :param limit: limit rate for the sell order :return: None """ - exc = trade.exchange - pair = trade.pair # Execute sell and update trade record order_id = self.exchange.sell(str(trade.pair), limit, trade.amount)['id'] trade.open_order_id = order_id trade.close_rate_requested = limit - fmt_exp_profit = round(trade.calc_profit_percent(rate=limit) * 100, 2) profit_trade = trade.calc_profit(rate=limit) current_rate = self.exchange.get_ticker(trade.pair)['bid'] - profit = trade.calc_profit_percent(limit) + profit_percent = trade.calc_profit_percent(limit) pair_url = self.exchange.get_pair_detail_url(trade.pair) - gain = "profit" if fmt_exp_profit > 0 else "loss" + gain = "profit" if profit_percent > 0 else "loss" - message = f"*{exc}:* Selling\n" \ - f"*Current Pair:* [{pair}]({pair_url})\n" \ - f"*Limit:* `{limit}`\n" \ - f"*Amount:* `{round(trade.amount, 8)}`\n" \ - f"*Open Rate:* `{trade.open_rate:.8f}`\n" \ - f"*Current Rate:* `{current_rate:.8f}`\n" \ - f"*Profit:* `{round(profit * 100, 2):.2f}%`" \ - "" + msg = { + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': trade.exchange.capitalize(), + 'pair': trade.pair, + 'gain': gain, + 'market_url': pair_url, + 'limit': limit, + 'amount': trade.amount, + 'open_rate': trade.open_rate, + 'current_rate': current_rate, + 'profit_amount': profit_trade, + 'profit_percent': profit_percent, + } # For regular case, when the configuration exists if 'stake_currency' in self.config and 'fiat_display_currency' in self.config: - stake = self.config['stake_currency'] - fiat = self.config['fiat_display_currency'] + stake_currency = self.config['stake_currency'] + fiat_currency = self.config['fiat_display_currency'] fiat_converter = CryptoToFiatConverter() profit_fiat = fiat_converter.convert_amount( profit_trade, - stake, - fiat + stake_currency, + fiat_currency, ) - message += f'` ({gain}: {fmt_exp_profit:.2f}%, {profit_trade:.8f} {stake}`' \ - f'` / {profit_fiat:.3f} {fiat})`'\ - '' - # Because telegram._forcesell does not have the configuration - # Ignore the FIAT value and does not show the stake_currency as well - else: - gain = "profit" if fmt_exp_profit > 0 else "loss" - message += f'` ({gain}: {fmt_exp_profit:.2f}%, {profit_trade:.8f})`' + msg.update({ + 'profit_fiat': profit_fiat, + 'stake_currency': stake_currency, + 'fiat_currency': fiat_currency, + }) + # Send the message - self.rpc.send_msg({'status': message}) + self.rpc.send_msg(msg) Trade.session.flush() diff --git a/freqtrade/main.py b/freqtrade/main.py index 74d8da031..977212faf 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -13,6 +13,7 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.freqtradebot import FreqtradeBot from freqtrade.state import State +from freqtrade.rpc import RPCMessageType logger = logging.getLogger('freqtrade') @@ -60,6 +61,7 @@ def main(sysargv: List[str]) -> None: finally: if freqtrade: freqtrade.rpc.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, 'status': 'process died' }) freqtrade.cleanup() @@ -76,6 +78,7 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot: # Create new instance freqtrade = FreqtradeBot(Configuration(args).get_config()) freqtrade.rpc.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, 'status': 'config reloaded' }) return freqtrade diff --git a/freqtrade/rpc/__init__.py b/freqtrade/rpc/__init__.py index e69de29bb..31c854f82 100644 --- a/freqtrade/rpc/__init__.py +++ b/freqtrade/rpc/__init__.py @@ -0,0 +1,2 @@ +from .rpc import RPC, RPCMessageType, RPCException # noqa +from .rpc_manager import RPCManager # noqa diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 9ad506b82..faf0653b1 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -5,6 +5,7 @@ import logging from abc import abstractmethod from datetime import timedelta, datetime, date from decimal import Decimal +from enum import Enum from typing import Dict, Any, List import arrow @@ -19,6 +20,15 @@ from freqtrade.state import State logger = logging.getLogger(__name__) +class RPCMessageType(Enum): + STATUS_NOTIFICATION = 'status' + BUY_NOTIFICATION = 'buy' + SELL_NOTIFICATION = 'sell' + + def __repr__(self): + return self.value + + class RPCException(Exception): """ Should be raised with a rpc-formatted message in an _rpc_* method @@ -33,11 +43,6 @@ class RPCException(Exception): def __str__(self): return self.message - def __json__(self): - return { - 'msg': self.message - } - class RPC(object): """ diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index ec193d23d..34094ee20 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -2,9 +2,9 @@ This module contains class to manage RPC communications (Telegram, Slack, ...) """ import logging -from typing import List, Dict +from typing import List, Dict, Any -from freqtrade.rpc.rpc import RPC +from freqtrade.rpc import RPC logger = logging.getLogger(__name__) @@ -32,7 +32,7 @@ class RPCManager(object): mod.cleanup() del mod - def send_msg(self, msg: Dict[str, str]) -> None: + def send_msg(self, msg: Dict[str, Any]) -> None: """ Send given message to all registered rpc modules. A message consists of one or more key value pairs of strings. diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index cab5d1d74..39724a2f9 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -12,7 +12,7 @@ from telegram.error import NetworkError, TelegramError from telegram.ext import CommandHandler, Updater from freqtrade.__init__ import __version__ -from freqtrade.rpc.rpc import RPC, RPCException +from freqtrade.rpc import RPC, RPCException, RPCMessageType logger = logging.getLogger(__name__) @@ -110,9 +110,42 @@ class Telegram(RPC): """ self._updater.stop() - def send_msg(self, msg: Dict[str, str]) -> None: + def send_msg(self, msg: Dict[str, Any]) -> None: """ Send a message to telegram channel """ - self._send_msg('*Status:* `{status}`'.format(**msg)) + + if msg['type'] == RPCMessageType.BUY_NOTIFICATION: + message = "*{exchange}:* Buying [{pair}]({market_url})\n" \ + "with limit `{limit:.8f}\n" \ + "({stake_amount:.6f} {stake_currency}," \ + "{stake_amount_fiat:.3f} {fiat_currency})`" \ + .format(**msg) + + elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: + msg['amount'] = round(msg['amount'], 8) + msg['profit_percent'] = round(msg['profit_percent'] * 100, 2) + + message = "*{exchange}:* Selling [{pair}]({market_url})\n" \ + "*Limit:* `{limit:.8f}`\n" \ + "*Amount:* `{amount:.8f}`\n" \ + "*Open Rate:* `{open_rate:.8f}`\n" \ + "*Current Rate:* `{current_rate:.8f}`\n" \ + "*Profit:* `{profit_percent:.2f}%`".format(**msg) + + # Check if all sell properties are available. + # This might not be the case if the message origin is triggered by /forcesell + if all(prop in msg for prop in ['gain', 'profit_fiat', + 'fiat_currency', 'stake_currency']): + message += '` ({gain}: {profit_amount:.8f} {stake_currency}`' \ + '` / {profit_fiat:.3f} {fiat_currency})`'.format(**msg) + + self._send_msg(message) + elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION: + message = '*Status:* `{status}`'.format(**msg) + + else: + raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) + + self._send_msg(message) @authorized_only def _status(self, bot: Bot, update: Update) -> None: diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 654547078..23e14cfbb 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -11,7 +11,7 @@ import pytest from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade -from freqtrade.rpc.rpc import RPC, RPCException +from freqtrade.rpc import RPC, RPCException from freqtrade.state import State from freqtrade.tests.test_freqtradebot import (patch_coinmarketcap, patch_get_signal) diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 3c17a4270..1f9b034b9 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -6,8 +6,8 @@ import logging from copy import deepcopy from unittest.mock import MagicMock -from freqtrade.rpc.rpc_manager import RPCManager -from freqtrade.tests.conftest import get_patched_freqtradebot, log_has +from freqtrade.rpc import RPCMessageType, RPCManager +from freqtrade.tests.conftest import log_has, get_patched_freqtradebot def test_rpc_manager_object() -> None: @@ -102,9 +102,12 @@ def test_send_msg_telegram_disabled(mocker, default_conf, caplog) -> None: freqtradebot = get_patched_freqtradebot(mocker, conf) rpc_manager = RPCManager(freqtradebot) - rpc_manager.send_msg({'status': 'test'}) + rpc_manager.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, + 'status': 'test' + }) - assert log_has("Sending rpc message: {'status': 'test'}", caplog.record_tuples) + assert log_has("Sending rpc message: {'type': status, 'status': 'test'}", caplog.record_tuples) assert telegram_mock.call_count == 0 @@ -117,7 +120,10 @@ def test_send_msg_telegram_enabled(mocker, default_conf, caplog) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc_manager = RPCManager(freqtradebot) - rpc_manager.send_msg({'status': 'test'}) + rpc_manager.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, + 'status': 'test' + }) - assert log_has("Sending rpc message: {'status': 'test'}", caplog.record_tuples) + assert log_has("Sending rpc message: {'type': status, 'status': 'test'}", caplog.record_tuples) assert telegram_mock.call_count == 1 diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 30782d598..baf9f124d 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -9,7 +9,7 @@ import re from copy import deepcopy from datetime import datetime from random import randint -from unittest.mock import MagicMock +from unittest.mock import MagicMock, ANY from telegram import Chat, Message, Update from telegram.error import NetworkError @@ -17,6 +17,7 @@ from telegram.error import NetworkError from freqtrade import __version__ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade +from freqtrade.rpc import RPCMessageType from freqtrade.rpc.telegram import Telegram, authorized_only from freqtrade.state import State from freqtrade.tests.conftest import (get_patched_freqtradebot, log_has, @@ -757,13 +758,23 @@ def test_forcesell_handle(default_conf, update, ticker, fee, telegram._forcesell(bot=MagicMock(), update=update) assert rpc_mock.call_count == 2 - last_call = rpc_mock.call_args_list[-1][0][0]['status'] - assert 'Selling' in last_call - assert '[ETH/BTC]' in last_call - assert 'Amount' in last_call - assert '0.00001172' in last_call - assert 'profit: 6.11%, 0.00006126' in last_call - assert '0.919 USD' in last_call + last_msg = rpc_mock.call_args_list[-1][0][0] + assert { + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'gain': 'profit', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'limit': 1.172e-05, + 'amount': 90.99181073703367, + 'open_rate': 1.099e-05, + 'current_rate': 1.172e-05, + 'profit_amount': 6.126e-05, + 'profit_percent': 0.06110514, + 'profit_fiat': 0.9189, + 'stake_currency': 'BTC', + 'fiat_currency': 'USD', + } == last_msg def test_forcesell_down_handle(default_conf, update, ticker, fee, @@ -803,14 +814,25 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, update.message.text = '/forcesell 1' telegram._forcesell(bot=MagicMock(), update=update) - last_call = rpc_mock.call_args_list[-1][0][0]['status'] assert rpc_mock.call_count == 2 - assert 'Selling' in last_call - assert '[ETH/BTC]' in last_call - assert 'Amount' in last_call - assert '0.00001044' in last_call - assert 'loss: -5.48%, -0.00005492' in last_call - assert '-0.824 USD' in last_call + + last_msg = rpc_mock.call_args_list[-1][0][0] + assert { + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'gain': 'loss', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'limit': 1.044e-05, + 'amount': 90.99181073703367, + 'open_rate': 1.099e-05, + 'current_rate': 1.044e-05, + 'profit_amount': -5.492e-05, + 'profit_percent': -0.05478343, + 'profit_fiat': -0.8238000000000001, + 'stake_currency': 'BTC', + 'fiat_currency': 'USD', + } == last_msg def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker) -> None: @@ -843,10 +865,23 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker telegram._forcesell(bot=MagicMock(), update=update) assert rpc_mock.call_count == 4 - for args in rpc_mock.call_args_list: - assert '0.00001098' in args[0][0]['status'] - assert 'loss: -0.59%, -0.00000591 BTC' in args[0][0]['status'] - assert '-0.089 USD' in args[0][0]['status'] + msg = rpc_mock.call_args_list[0][0][0] + assert { + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'gain': 'loss', + 'market_url': ANY, + 'limit': 1.098e-05, + 'amount': 90.99181073703367, + 'open_rate': 1.099e-05, + 'current_rate': 1.098e-05, + 'profit_amount': -5.91e-06, + 'profit_percent': -0.00589292, + 'profit_fiat': -0.08865, + 'stake_currency': 'BTC', + 'fiat_currency': 'USD', + } == msg def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: @@ -1040,7 +1075,22 @@ def test_version_handle(default_conf, update, mocker) -> None: assert '*Version:* `{}`'.format(__version__) in msg_mock.call_args_list[0][0][0] -def test_send_msg(default_conf, mocker) -> None: +def test_send_msg_buy_notification() -> None: + # TODO: implement me + pass + + +def test_send_msg_sell_notification() -> None: + # TODO: implement me + pass + + +def test_send_msg_status_notification() -> None: + # TODO: implement me + pass + + +def test__send_msg(default_conf, mocker) -> None: """ Test send_msg() method """ @@ -1056,7 +1106,7 @@ def test_send_msg(default_conf, mocker) -> None: assert len(bot.method_calls) == 1 -def test_send_msg_network_error(default_conf, mocker, caplog) -> None: +def test__send_msg_network_error(default_conf, mocker, caplog) -> None: """ Test send_msg() method """ diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 678e54fe6..450504f57 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -18,9 +18,9 @@ from freqtrade import (DependencyException, OperationalException, TemporaryError, constants) from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade +from freqtrade.rpc import RPCMessageType from freqtrade.state import State -from freqtrade.tests.conftest import (log_has, patch_coinmarketcap, - patch_exchange) +from freqtrade.tests.conftest import log_has, patch_coinmarketcap, patch_exchange # Functions for recurrent object patching @@ -1375,14 +1375,23 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid']) assert rpc_mock.call_count == 2 - last_call = rpc_mock.call_args_list[-1][0][0]['status'] - assert 'Selling' in last_call - assert '[ETH/BTC]' in last_call - assert 'Amount' in last_call - assert 'Profit' in last_call - assert '0.00001172' in last_call - assert 'profit: 6.11%, 0.00006126' in last_call - assert '0.919 USD' in last_call + last_msg = rpc_mock.call_args_list[-1][0][0] + assert { + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'gain': 'profit', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'limit': 1.172e-05, + 'amount': 90.99181073703367, + 'open_rate': 1.099e-05, + 'current_rate': 1.172e-05, + 'profit_amount': 6.126e-05, + 'profit_percent': 0.06110514, + 'profit_fiat': 0.9189, + 'stake_currency': 'BTC', + 'fiat_currency': 'USD', + } == last_msg def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: @@ -1418,13 +1427,23 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid']) assert rpc_mock.call_count == 2 - last_call = rpc_mock.call_args_list[-1][0][0]['status'] - assert 'Selling' in last_call - assert '[ETH/BTC]' in last_call - assert 'Amount' in last_call - assert '0.00001044' in last_call - assert 'loss: -5.48%, -0.00005492' in last_call - assert '-0.824 USD' in last_call + last_msg = rpc_mock.call_args_list[-1][0][0] + assert { + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'gain': 'loss', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'limit': 1.044e-05, + 'amount': 90.99181073703367, + 'open_rate': 1.099e-05, + 'current_rate': 1.044e-05, + 'profit_amount': -5.492e-05, + 'profit_percent': -0.05478343, + 'profit_fiat': -0.8238000000000001, + 'stake_currency': 'BTC', + 'fiat_currency': 'USD', + } == last_msg def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, @@ -1461,13 +1480,20 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid']) assert rpc_mock.call_count == 2 - last_call = rpc_mock.call_args_list[-1][0][0]['status'] - assert 'Selling' in last_call - assert '[ETH/BTC]' in last_call - assert 'Amount' in last_call - assert '0.00001172' in last_call - assert '(profit: 6.11%, 0.00006126)' in last_call - assert 'USD' not in last_call + last_msg = rpc_mock.call_args_list[-1][0][0] + assert { + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'gain': 'profit', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'limit': 1.172e-05, + 'amount': 90.99181073703367, + 'open_rate': 1.099e-05, + 'current_rate': 1.172e-05, + 'profit_amount': 6.126e-05, + 'profit_percent': 0.06110514, + } == last_msg def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, @@ -1504,11 +1530,20 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid']) assert rpc_mock.call_count == 2 - last_call = rpc_mock.call_args_list[-1][0][0]['status'] - assert 'Selling' in last_call - assert '[ETH/BTC]' in last_call - assert '0.00001044' in last_call - assert 'loss: -5.48%, -0.00005492' in last_call + last_msg = rpc_mock.call_args_list[-1][0][0] + assert { + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'gain': 'loss', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'limit': 1.044e-05, + 'amount': 90.99181073703367, + 'open_rate': 1.099e-05, + 'current_rate': 1.044e-05, + 'profit_amount': -5.492e-05, + 'profit_percent': -0.05478343, + } == last_msg def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, From 7eaeb8d1469ef762216837bfb1a257bab8307562 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 12 Jul 2018 17:27:40 +0200 Subject: [PATCH 149/395] status: return arrow object instead humanized str --- freqtrade/rpc/rpc.py | 4 ++-- freqtrade/rpc/telegram.py | 4 ++++ freqtrade/tests/rpc/test_rpc.py | 4 ++-- freqtrade/tests/rpc/test_rpc_telegram.py | 4 +++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index faf0653b1..621c19fd2 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -69,7 +69,7 @@ class RPC(object): def send_msg(self, msg: Dict[str, str]) -> None: """ Sends a message to all registered rpc modules """ - def _rpc_trade_status(self) -> List[Dict]: + def _rpc_trade_status(self) -> List[Dict[str, Any]]: """ Below follows the RPC backend it is prefixed with rpc_ to raise awareness that it is a remotely exposed function @@ -95,7 +95,7 @@ class RPC(object): trade_id=trade.id, pair=trade.pair, market_url=self._freqtrade.exchange.get_pair_detail_url(trade.pair), - date=arrow.get(trade.open_date).humanize(), + date=arrow.get(trade.open_date), open_rate=trade.open_rate, close_rate=trade.close_rate, current_rate=current_rate, diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 39724a2f9..37dbb34ff 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -166,6 +166,10 @@ class Telegram(RPC): try: results = self._rpc_trade_status() + # pre format data + for result in results: + result['date'] = result['date'].humanize() + messages = [ "*Trade ID:* `{trade_id}`\n" "*Current Pair:* [{pair}]({market_url})\n" diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 23e14cfbb..6e59b4116 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -5,7 +5,7 @@ Unit test file for rpc/rpc.py """ from datetime import datetime -from unittest.mock import MagicMock +from unittest.mock import MagicMock, ANY import pytest @@ -59,7 +59,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: 'trade_id': 1, 'pair': 'ETH/BTC', 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', - 'date': 'just now', + 'date': ANY, 'open_rate': 1.099e-05, 'close_rate': None, 'current_rate': 1.098e-05, diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index baf9f124d..b13097eed 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -11,6 +11,7 @@ from datetime import datetime from random import randint from unittest.mock import MagicMock, ANY +import arrow from telegram import Chat, Message, Update from telegram.error import NetworkError @@ -198,6 +199,7 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) + mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -215,7 +217,7 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: 'trade_id': 1, 'pair': 'ETH/BTC', 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', - 'date': 'just now', + 'date': arrow.utcnow(), 'open_rate': 1.099e-05, 'close_rate': None, 'current_rate': 1.098e-05, From a559e22f167b2bb5000efc5e723212fff7659d6f Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 12 Jul 2018 17:29:02 +0200 Subject: [PATCH 150/395] remove duplicate send_msg invocation --- freqtrade/rpc/telegram.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 37dbb34ff..02b74358e 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -138,7 +138,6 @@ class Telegram(RPC): message += '` ({gain}: {profit_amount:.8f} {stake_currency}`' \ '` / {profit_fiat:.3f} {fiat_currency})`'.format(**msg) - self._send_msg(message) elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION: message = '*Status:* `{status}`'.format(**msg) From cb8cd21e22cd5cab5b32a68c8a26195371f058cd Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 12 Jul 2018 17:50:11 +0200 Subject: [PATCH 151/395] add tests for telegram.send_msg --- freqtrade/tests/rpc/test_rpc_telegram.py | 120 +++++++++++++++++++++-- 1 file changed, 111 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index b13097eed..01f248327 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -12,6 +12,7 @@ from random import randint from unittest.mock import MagicMock, ANY import arrow +import pytest from telegram import Chat, Message, Update from telegram.error import NetworkError @@ -1077,19 +1078,120 @@ def test_version_handle(default_conf, update, mocker) -> None: assert '*Version:* `{}`'.format(__version__) in msg_mock.call_args_list[0][0][0] -def test_send_msg_buy_notification() -> None: - # TODO: implement me - pass +def test_send_msg_buy_notification(default_conf, mocker) -> None: + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + telegram.send_msg({ + 'type': RPCMessageType.BUY_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'limit': 1.099e-05, + 'stake_amount': 0.001, + 'stake_amount_fiat': 0.0, + 'stake_currency': 'BTC', + 'fiat_currency': 'USD' + }) + assert msg_mock.call_args[0][0] \ + == '*Bittrex:* Buying [ETH/BTC](https://bittrex.com/Market/Index?MarketName=BTC-ETH)\n' \ + 'with limit `0.00001099\n' \ + '(0.001000 BTC,0.000 USD)`' -def test_send_msg_sell_notification() -> None: - # TODO: implement me - pass +def test_send_msg_sell_notification(default_conf, mocker) -> None: + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + telegram.send_msg({ + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': 'Binance', + 'pair': 'KEY/ETH', + 'gain': 'loss', + 'market_url': 'https://www.binance.com/tradeDetail.html?symbol=KEY_ETH', + 'limit': 3.201e-05, + 'amount': 1333.3333333333335, + 'open_rate': 7.5e-05, + 'current_rate': 3.201e-05, + 'profit_amount': -0.05746268, + 'profit_percent': -0.57405275, + 'profit_fiat': -24.81204044792, + 'stake_currency': 'ETH', + 'fiat_currency': 'USD' + }) + assert msg_mock.call_args[0][0] \ + == '*Binance:* Selling [KEY/ETH]' \ + '(https://www.binance.com/tradeDetail.html?symbol=KEY_ETH)\n' \ + '*Limit:* `0.00003201`\n' \ + '*Amount:* `1333.33333333`\n' \ + '*Open Rate:* `0.00007500`\n' \ + '*Current Rate:* `0.00003201`\n' \ + '*Profit:* `-57.41%`` (loss: -0.05746268 ETH`` / -24.812 USD)`' + + msg_mock.reset_mock() + telegram.send_msg({ + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': 'Binance', + 'pair': 'KEY/ETH', + 'gain': 'loss', + 'market_url': 'https://www.binance.com/tradeDetail.html?symbol=KEY_ETH', + 'limit': 3.201e-05, + 'amount': 1333.3333333333335, + 'open_rate': 7.5e-05, + 'current_rate': 3.201e-05, + 'profit_amount': -0.05746268, + 'profit_percent': -0.57405275, + 'stake_currency': 'ETH', + }) + assert msg_mock.call_args[0][0] \ + == '*Binance:* Selling [KEY/ETH]' \ + '(https://www.binance.com/tradeDetail.html?symbol=KEY_ETH)\n' \ + '*Limit:* `0.00003201`\n' \ + '*Amount:* `1333.33333333`\n' \ + '*Open Rate:* `0.00007500`\n' \ + '*Current Rate:* `0.00003201`\n' \ + '*Profit:* `-57.41%`' -def test_send_msg_status_notification() -> None: - # TODO: implement me - pass +def test_send_msg_status_notification(default_conf, mocker) -> None: + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + telegram.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, + 'status': 'running' + }) + assert msg_mock.call_args[0][0] == '*Status:* `running`' + + +def test_send_msg_unknown_type(default_conf, mocker) -> None: + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + with pytest.raises(NotImplementedError, match=r'Unknown message type: None'): + telegram.send_msg({ + 'type': None, + }) def test__send_msg(default_conf, mocker) -> None: From 5b02b87735b6205febdb8645c8f76b3a56f25e0e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 13 Jul 2018 14:24:06 +0200 Subject: [PATCH 152/395] Update ccxt from 1.16.6 to 1.16.12 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 25c39757f..5620f31d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.6 +ccxt==1.16.12 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From 68ddd1b951e06d1686b482bbd42712b0ea37d1a4 Mon Sep 17 00:00:00 2001 From: peterkorodi Date: Sat, 14 Jul 2018 00:07:38 +0200 Subject: [PATCH 153/395] Update plotting.md Fix pairs and db-url in the doc --- docs/plotting.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/plotting.md b/docs/plotting.md index 242d80005..54a4bb4b8 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -24,7 +24,7 @@ script/plot_dataframe.py [-h] [-p pair] [--live] Example ``` -python scripts/plot_dataframe.py -p BTC_ETH +python scripts/plot_dataframe.py -p BTC/ETH ``` The `-p` pair argument, can be used to specify what @@ -34,18 +34,18 @@ pair you would like to plot. To plot the current live price use the `--live` flag: ``` -python scripts/plot_dataframe.py -p BTC_ETH --live +python scripts/plot_dataframe.py -p BTC/ETH --live ``` To plot a timerange (to zoom in): ``` -python scripts/plot_dataframe.py -p BTC_ETH --timerange=100-200 +python scripts/plot_dataframe.py -p BTC/ETH --timerange=100-200 ``` Timerange doesn't work with live data. To plot trades stored in a database use `--db-url` argument: ``` -python scripts/plot_dataframe.py --db-url tradesv3.dry_run.sqlite -p BTC_ETH +python scripts/plot_dataframe.py --db-url sqlite:///tradesv3.dry_run.sqlite -p BTC/ETH ``` To plot a test strategy the strategy should have first be backtested. From fa8b349200118d47dbd055692d2e396ce81f469c Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 14 Jul 2018 08:02:39 +0300 Subject: [PATCH 154/395] rpc: dont re-use variables with different types --- freqtrade/rpc/rpc.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 621c19fd2..9411e983b 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -227,26 +227,26 @@ class RPC(object): # doing this will utilize its caching functionallity, instead we reinitialize it here fiat = self._freqtrade.fiat_converter # Prepare data to display - profit_closed_coin = round(sum(profit_closed_coin), 8) + profit_closed_coin_sum = round(sum(profit_closed_coin), 8) profit_closed_percent = round(nan_to_num(mean(profit_closed_percent)) * 100, 2) profit_closed_fiat = fiat.convert_amount( - profit_closed_coin, + profit_closed_coin_sum, stake_currency, fiat_display_currency ) - profit_all_coin = round(sum(profit_all_coin), 8) + profit_all_coin_sum = round(sum(profit_all_coin), 8) profit_all_percent = round(nan_to_num(mean(profit_all_percent)) * 100, 2) profit_all_fiat = fiat.convert_amount( - profit_all_coin, + profit_all_coin_sum, stake_currency, fiat_display_currency ) num = float(len(durations) or 1) return { - 'profit_closed_coin': profit_closed_coin, + 'profit_closed_coin': profit_closed_coin_sum, 'profit_closed_percent': profit_closed_percent, 'profit_closed_fiat': profit_closed_fiat, - 'profit_all_coin': profit_all_coin, + 'profit_all_coin': profit_all_coin_sum, 'profit_all_percent': profit_all_percent, 'profit_all_fiat': profit_all_fiat, 'trade_count': len(trades), From 6e16c1d80d5173bc855a20f0e142805c9df004fa Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 4 Jul 2018 21:42:17 +0200 Subject: [PATCH 155/395] add webhook test file --- freqtrade/tests/rpc/test_rpc_webhook.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 freqtrade/tests/rpc/test_rpc_webhook.py diff --git a/freqtrade/tests/rpc/test_rpc_webhook.py b/freqtrade/tests/rpc/test_rpc_webhook.py new file mode 100644 index 000000000..e579c539a --- /dev/null +++ b/freqtrade/tests/rpc/test_rpc_webhook.py @@ -0,0 +1,16 @@ + + +def test__init__(): + # TODO: Implement me + pass + + +def test_send_msg(): + # TODO: Implement me + pass + + +def test__send_msg(): + """Test internal method - calling the actual api""" + # TODO: Implement me + pass From ae22af1ea3c0f3822bcd7e5463badb72f1e54171 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 5 Jul 2018 21:32:53 +0200 Subject: [PATCH 156/395] fix typo --- freqtrade/rpc/webhook.py | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 freqtrade/rpc/webhook.py diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py new file mode 100644 index 000000000..6a1abfe36 --- /dev/null +++ b/freqtrade/rpc/webhook.py @@ -0,0 +1,70 @@ +""" +This module manages webhook communication +""" +import logging +from typing import Any, Dict + +from requests import post, RequestException + +from freqtrade.rpc import RPC, RPCMessageType + + +logger = logging.getLogger(__name__) + +logger.debug('Included module rpc.webhook ...') + + +class Webhook(RPC): + """ This class handles all webhook communication """ + + def __init__(self, freqtrade) -> None: + """ + Init the Webhook class, and init the super class RPC + :param freqtrade: Instance of a freqtrade bot + :return: None + """ + super().__init__(freqtrade) + + self._config = freqtrade.config + + def cleanup(self) -> None: + """ + Cleanup pending module resources. + This will do nothing for webhooks, they will simply not be called anymore + """ + pass + + def send_msg(self, msg: Dict[str, Any]) -> None: + """ Send a message to telegram channel """ + try: + + if msg['type'] == RPCMessageType.BUY_NOTIFICATION: + valuedict = self._config['webhook'].get('webhookbuy', None) + elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: + valuedict = self._config['webhook'].get('webhooksell', None) + elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION: + valuedict = self._config['webhook'].get('webhookstatus', None) + else: + raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) + if not valuedict: + logger.info("Message type %s not configured for webhooks", msg['type']) + return + + payload = {"value1": valuedict.get('value1', '').format(**msg), + "value2": valuedict.get('value2', '').format(**msg), + "value3": valuedict.get('value3', '').format(**msg) + } + + self._send_msg(payload) + except KeyError as exc: + logger.exception("Problem calling Webhook. Please check your webhook configuration. " + "Exception: %s", exc) + + def _send_msg(self, payload: dict) -> None: + """does the actual call to the webhook""" + + try: + url = self._config['webhook']['url'] + post(url, data=payload) + except RequestException as exc: + logger.warning("Could not call webhook url. Exception: %s", exc) From fa8512789f331d5e15222d58daa13f974b6254ab Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 5 Jul 2018 21:33:01 +0200 Subject: [PATCH 157/395] add tests for webhook --- freqtrade/tests/rpc/test_rpc_webhook.py | 178 ++++++++++++++++++++++-- 1 file changed, 170 insertions(+), 8 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc_webhook.py b/freqtrade/tests/rpc/test_rpc_webhook.py index e579c539a..99876269c 100644 --- a/freqtrade/tests/rpc/test_rpc_webhook.py +++ b/freqtrade/tests/rpc/test_rpc_webhook.py @@ -1,16 +1,178 @@ +from unittest.mock import MagicMock + +import pytest +from requests import RequestException -def test__init__(): - # TODO: Implement me +from freqtrade.rpc import RPCMessageType +from freqtrade.rpc.webhook import Webhook +from freqtrade.tests.conftest import get_patched_freqtradebot, log_has + + +def get_webhook_dict() -> dict: + return { + "enabled": True, + "url": "https://maker.ifttt.com/trigger/freqtrade_test/with/key/c764udvJ5jfSlswVRukZZ2/", + "webhookbuy": { + "value1": "Buying {pair}", + "value2": "limit {limit:8f}", + "value3": "{stake_amount:8f} {stake_currency}" + }, + "webhooksell": { + "value1": "Selling {pair}", + "value2": "limit {limit:8f}", + "value3": "profit: {profit_amount:8f} {stake_currency}" + }, + "webhookstatus": { + "value1": "Status: {status}", + "value2": "", + "value3": "" + } + } + + +def test__init__(mocker, default_conf): + """ + Test __init__() method + """ + webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) + assert webhook._config == default_conf + + +def test_cleanup(default_conf, mocker) -> None: + """ + Test cleanup() method - not needed for webhook + """ pass -def test_send_msg(): - # TODO: Implement me - pass +def test_send_msg(default_conf, mocker): + """ Test send_msg for Webhook rpc class""" + default_conf["webhook"] = get_webhook_dict() + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) + msg = { + 'type': RPCMessageType.BUY_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'market_url': "http://mockedurl/ETH_BTC", + 'limit': 0.005, + 'stake_amount': 0.8, + 'stake_amount_fiat': 500, + 'stake_currency': 'BTC', + 'fiat_currency': 'EUR' + } + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + webhook.send_msg(msg=msg) + assert msg_mock.call_count == 1 + assert (msg_mock.call_args[0][0]["value1"] == + default_conf["webhook"]["webhookbuy"]["value1"].format(**msg)) + assert (msg_mock.call_args[0][0]["value2"] == + default_conf["webhook"]["webhookbuy"]["value2"].format(**msg)) + assert (msg_mock.call_args[0][0]["value3"] == + default_conf["webhook"]["webhookbuy"]["value3"].format(**msg)) + # Test sell + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + msg = { + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'gain': "profit", + 'market_url': "http://mockedurl/ETH_BTC", + 'limit': 0.005, + 'amount': 0.8, + 'open_rate': 0.004, + 'current_rate': 0.005, + 'profit_amount': 0.001, + 'profit_percent': 0.20, + 'stake_currency': 'BTC', + } + webhook.send_msg(msg=msg) + assert msg_mock.call_count == 1 + assert (msg_mock.call_args[0][0]["value1"] == + default_conf["webhook"]["webhooksell"]["value1"].format(**msg)) + assert (msg_mock.call_args[0][0]["value2"] == + default_conf["webhook"]["webhooksell"]["value2"].format(**msg)) + assert (msg_mock.call_args[0][0]["value3"] == + default_conf["webhook"]["webhooksell"]["value3"].format(**msg)) + + # Test notification + msg = { + 'type': RPCMessageType.STATUS_NOTIFICATION, + 'status': 'Unfilled sell order for BTC cancelled due to timeout' + } + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + webhook.send_msg(msg) + assert msg_mock.call_count == 1 + assert (msg_mock.call_args[0][0]["value1"] == + default_conf["webhook"]["webhookstatus"]["value1"].format(**msg)) + assert (msg_mock.call_args[0][0]["value2"] == + default_conf["webhook"]["webhookstatus"]["value2"].format(**msg)) + assert (msg_mock.call_args[0][0]["value3"] == + default_conf["webhook"]["webhookstatus"]["value3"].format(**msg)) -def test__send_msg(): +def test_exception_send_msg(default_conf, mocker, caplog): + """Test misconfigured notification""" + default_conf["webhook"] = get_webhook_dict() + default_conf["webhook"]["webhookbuy"] = None + + webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) + webhook.send_msg({'type': RPCMessageType.BUY_NOTIFICATION}) + assert log_has(f"Message type {RPCMessageType.BUY_NOTIFICATION} not configured for webhooks", + caplog.record_tuples) + + default_conf["webhook"] = get_webhook_dict() + default_conf["webhook"]["webhookbuy"]["value1"] = "{DEADBEEF:8f}" + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) + msg = { + 'type': RPCMessageType.BUY_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'market_url': "http://mockedurl/ETH_BTC", + 'limit': 0.005, + 'stake_amount': 0.8, + 'stake_amount_fiat': 500, + 'stake_currency': 'BTC', + 'fiat_currency': 'EUR' + } + webhook.send_msg(msg) + assert log_has("Problem calling Webhook. Please check your webhook configuration. " + "Exception: 'DEADBEEF'", caplog.record_tuples) + + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + msg = { + 'type': 'DEADBEEF', + 'status': 'whatever' + } + with pytest.raises(NotImplementedError): + webhook.send_msg(msg) + + +def test__send_msg(default_conf, mocker, caplog): """Test internal method - calling the actual api""" - # TODO: Implement me - pass + + default_conf["webhook"] = get_webhook_dict() + webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) + msg = {'value1': 'DEADBEEF', + 'value2': 'ALIVEBEEF', + 'value3': 'FREQTRADE'} + post = MagicMock() + mocker.patch("freqtrade.rpc.webhook.post", post) + webhook._send_msg(msg) + + assert post.call_count == 1 + assert post.call_args[1] == {'data': msg} + assert post.call_args[0] == (default_conf['webhook']['url'], ) + + post = MagicMock(side_effect=RequestException) + mocker.patch("freqtrade.rpc.webhook.post", post) + webhook._send_msg(msg) + assert log_has('Could not call webhook url. Exception: ', caplog.record_tuples) From 25250f7c105d21e1ccd5a45fff720ba2e4dd5aa0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 13:38:06 +0200 Subject: [PATCH 158/395] don't hardcode post parameters --- freqtrade/rpc/webhook.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 6a1abfe36..35631182b 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -49,11 +49,9 @@ class Webhook(RPC): if not valuedict: logger.info("Message type %s not configured for webhooks", msg['type']) return - - payload = {"value1": valuedict.get('value1', '').format(**msg), - "value2": valuedict.get('value2', '').format(**msg), - "value3": valuedict.get('value3', '').format(**msg) - } + payload = {} + for k in valuedict: + payload[k] = valuedict[k].format(**msg) self._send_msg(payload) except KeyError as exc: From a4643066a85b1f526fe1c5820784b5aff1aa4543 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 13:39:21 +0200 Subject: [PATCH 159/395] allow more flexibility in webhook --- freqtrade/constants.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index ec7765455..311f2fa72 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -100,6 +100,16 @@ CONF_SCHEMA = { }, 'required': ['enabled', 'token', 'chat_id'] }, + 'webhook': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'webhookbuy': {'type': 'object'}, + 'webhooksell': {'type': 'object'}, + 'webhookstatus': {'type': 'object'}, + 'chat_id': {'type': 'string'}, + }, + }, 'db_url': {'type': 'string'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, 'internals': { From 71df41c4ebbdfe6f0321171132c82241641fc005 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 14:10:15 +0200 Subject: [PATCH 160/395] add documentation for rpc_webhook --- docs/configuration.md | 5 +++++ docs/index.md | 1 + 2 files changed, 6 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index dd16ef6b5..ddfa9834e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -41,6 +41,11 @@ The table below will list all configuration parameters. | `telegram.enabled` | true | Yes | Enable or not the usage of Telegram. | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`. | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. +| `webhook.enabled` | false | No | Enable useage of Webhook notifications +| `webhook.url` | false | No | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. +| `webhook.webhookbuy` | false | No | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. +| `webhook.webhooksell` | false | No | Payload to send on sell. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. +| `webhook.webhookstatus` | false | No | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. | `db_url` | `sqlite:///tradesv3.sqlite` | No | Declares database URL to use. NOTE: This defaults to `sqlite://` if `dry_run` is `True`. | `initial_state` | running | No | Defines the initial application state. More information below. | `strategy` | DefaultStrategy | No | Defines Strategy class to use. diff --git a/docs/index.md b/docs/index.md index fd6bf4378..f76bb125d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -27,6 +27,7 @@ Pull-request. Do not hesitate to reach us on - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) +- [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) - [Contribute to the project](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) - [How to contribute](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) - [Run tests & Check PEP8 compliance](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) From f55df7ba636f719a4600fe95962b662bc40721a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 14:11:52 +0200 Subject: [PATCH 161/395] improve README.md formatting (styling only) --- README.md | 74 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 929d40292..c1705ff41 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,12 @@ [![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop) [![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/freqtrade/freqtrade/maintainability) - -Simple High frequency trading bot for crypto currencies designed to -support multi exchanges and be controlled via Telegram. +Simple High frequency trading bot for crypto currencies designed to support multi exchanges and be controlled via Telegram. ![freqtrade](https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docs/assets/freqtrade-screenshot.png) ## Disclaimer + This software is for educational purposes only. Do not risk money which you are afraid to lose. USE THE SOFTWARE AT YOUR OWN RISK. THE AUTHORS AND ALL AFFILIATES ASSUME NO RESPONSIBILITY FOR YOUR TRADING RESULTS. @@ -23,18 +22,18 @@ We strongly recommend you to have coding and Python knowledge. Do not hesitate to read the source code and understand the mechanism of this bot. ## Exchange marketplaces supported + - [X] [Bittrex](https://bittrex.com/) - [X] [Binance](https://www.binance.com/) - [ ] [113 others to tests](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ## Features -- [x] **Based on Python 3.6+**: For botting on any operating system - -Windows, macOS and Linux + +- [x] **Based on Python 3.6+**: For botting on any operating system - Windows, macOS and Linux - [x] **Persistence**: Persistence is achieved through sqlite - [x] **Dry-run**: Run the bot without playing money. - [x] **Backtesting**: Run a simulation of your buy/sell strategy. -- [x] **Strategy Optimization by machine learning**: Use machine learning to optimize your buy/sell -strategy parameters with real exchange data. +- [x] **Strategy Optimization by machine learning**: Use machine learning to optimize your buy/sell strategy parameters with real exchange data. - [x] **Whitelist crypto-currencies**: Select which crypto-currency you want to trade. - [x] **Blacklist crypto-currencies**: Select which crypto-currency you want to avoid. - [x] **Manageable via Telegram**: Manage the bot with Telegram @@ -43,38 +42,43 @@ strategy parameters with real exchange data. - [x] **Performance status report**: Provide a performance status of your current trades. ## Table of Contents + - [Quick start](#quick-start) - [Documentations](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) - - [Installation](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md) - - [Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) - - [Strategy Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) - - [Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - - [Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) + - [Installation](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md) + - [Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) + - [Strategy Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) + - [Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Basic Usage](#basic-usage) - [Bot commands](#bot-commands) - [Telegram RPC commands](#telegram-rpc-commands) - [Support](#support) - - [Help](#help--slack) - - [Bugs](#bugs--issues) - - [Feature Requests](#feature-requests) - - [Pull Requests](#pull-requests) + - [Help](#help--slack) + - [Bugs](#bugs--issues) + - [Feature Requests](#feature-requests) + - [Pull Requests](#pull-requests) - [Requirements](#requirements) - - [Min hardware required](#min-hardware-required) - - [Software requirements](#software-requirements) + - [Min hardware required](#min-hardware-required) + - [Software requirements](#software-requirements) ## Quick start + Freqtrade provides a Linux/macOS script to install all dependencies and help you to configure the bot. + ```bash git clone git@github.com:freqtrade/freqtrade.git git checkout develop cd freqtrade ./setup.sh --install ``` + _Windows installation is explained in [Installation doc](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md)_ - ## Documentation + We invite you to read the bot documentation to ensure you understand how the bot is working. + - [Index](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) - [Installation](https://github.com/freqtrade/freqtrade/blob/develop/docs/installation.md) - [Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md) @@ -86,7 +90,6 @@ We invite you to read the bot documentation to ensure you understand how the bot - [Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - [Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - ## Basic Usage ### Bot commands @@ -125,17 +128,15 @@ optional arguments: ``` ### Telegram RPC commands -Telegram is not mandatory. However, this is a great way to control your -bot. More details on our -[documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) + +Telegram is not mandatory. However, this is a great way to control your bot. More details on our [documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) - `/start`: Starts the trader - `/stop`: Stops the trader - `/status [table]`: Lists all open trades - `/count`: Displays number of open trades - `/profit`: Lists cumulative profit from all finished trades -- `/forcesell |all`: Instantly sells the given trade -(Ignoring `minimum_roi`). +- `/forcesell |all`: Instantly sells the given trade (Ignoring `minimum_roi`). - `/performance`: Show performance of each finished trade grouped by pair - `/balance`: Show account balance per currency - `/daily `: Shows profit or loss per day, over the last n days @@ -144,20 +145,23 @@ bot. More details on our ## Development branches -The project is currently setup in two main branches: -- `develop` - This branch has often new features, but might also cause -breaking changes. -- `master` - This branch contains the latest stable release. The bot -'should' be stable on this branch, and is generally well tested. +The project is currently setup in two main branches: + +- `develop` - This branch has often new features, but might also cause breaking changes. +- `master` - This branch contains the latest stable release. The bot 'should' be stable on this branch, and is generally well tested. ## Support + ### Help / Slack + For any questions not covered by the documentation or for further information about the bot, we encourage you to join our slack channel. + - [Click here to join Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE). ### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) + If you discover a bug in the bot, please [search our issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) first. If it hasn't been reported, please @@ -166,6 +170,7 @@ ensure you follow the template guide so that our team can assist you as quickly as possible. ### [Feature Requests](https://github.com/freqtrade/freqtrade/labels/enhancement) + Have you a great idea to improve the bot you want to share? Please, first search if this feature was not [already discussed](https://github.com/freqtrade/freqtrade/labels/enhancement). If it hasn't been requested, please @@ -174,6 +179,7 @@ and ensure you follow the template guide so that it does not get lost in the bug reports. ### [Pull Requests](https://github.com/freqtrade/freqtrade/pulls) + Feel like our bot is missing a feature? We welcome your pull requests! Please read our [Contributing document](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) @@ -181,16 +187,18 @@ to understand the requirements before sending your pull-requests. **Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. -**Important:** Always create your PR against the `develop` branch, not -`master`. +**Important:** Always create your PR against the `develop` branch, not `master`. ## Requirements ### Min hardware required + To run this bot we recommend you a cloud instance with a minimum of: -* Minimal (advised) system requirements: 2GB RAM, 1GB disk space, 2vCPU + +- Minimal (advised) system requirements: 2GB RAM, 1GB disk space, 2vCPU ### Software requirements + - [Python 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/) - [pip](https://pip.pypa.io/en/stable/installing/) - [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) From 3ca161f1968ff0845e02f5880e09c214301257d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jul 2018 14:13:49 +0200 Subject: [PATCH 162/395] Add webhook config --- docs/webhook-config.md | 74 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 docs/webhook-config.md diff --git a/docs/webhook-config.md b/docs/webhook-config.md new file mode 100644 index 000000000..71524187a --- /dev/null +++ b/docs/webhook-config.md @@ -0,0 +1,74 @@ +# Webhook usage + +This page explains how to configure your bot to talk to webhooks. + +## Configuration + +Enable webhooks by adding a webhook-section to your configuration file, and setting `webhook.enabled` to `true`. + +Sample configuration (tested using IFTTT). + +```json + "webhook": { + "enabled": true, + "url": "https://maker.ifttt.com/trigger//with/key//", + "webhookbuy": { + "value1": "Buying {pair}", + "value2": "limit {limit:8f}", + "value3": "{stake_amount:8f} {stake_currency}" + }, + "webhooksell": { + "value1": "Selling {pair}", + "value2": "limit {limit:8f}", + "value3": "profit: {profit_amount:8f} {stake_currency}" + }, + "webhookstatus": { + "value1": "Status: {status}", + "value2": "", + "value3": "" + } + }, +``` + +The url in `webhook.url` should point to the correct url for your webhook. If you're using [IFTTT](https://ifttt.com) (as shown in the sample above) please insert our event and key to the url. + +Different payloads can be configured for different events. Not all fields are necessary, but you should configure at least one of the dicts, otherwise the webhook will never be called. + +### Webhookbuy + +The fields in `webhook.webhookbuy` are filled when the bot executes a buy. Parameters are filled using string.format. +Possible parameters are: + +* exchange +* pair +* market_url +* limit +* stake_amount +* stake_amount_fiat +* stake_currency +* fiat_currency + +### Webhooksell + +The fields in `webhook.webhooksell` are filled when the bot sells a trade. Parameters are filled using string.format. +Possible parameters are: + +* exchange +* pair +* gain +* market_url +* limit +* amount +* open_rate +* current_rate +* profit_amount +* profit_percent +* profit_fiat +* stake_currency +* fiat_currency + +### Webhookstatus + +The fields in `webhook.webhookstatus` are used for regular status messages (Started / Stopped / ...). Parameters are filled using string.format. + +The only possible value here is `{status}`. From 144d308e5ed55d10103453af85108eee863d4687 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Jul 2018 19:59:17 +0200 Subject: [PATCH 163/395] Allow enabling of webhook --- freqtrade/freqtradebot.py | 3 +-- freqtrade/rpc/rpc_manager.py | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 72b5190b9..aed8f62dd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -19,8 +19,7 @@ from freqtrade.analyze import Analyze from freqtrade.exchange import Exchange from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.persistence import Trade -from freqtrade.rpc import RPCMessageType -from freqtrade.rpc import RPCManager +from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State logger = logging.getLogger(__name__) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 34094ee20..f05a1882f 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -23,6 +23,12 @@ class RPCManager(object): from freqtrade.rpc.telegram import Telegram self.registered_modules.append(Telegram(freqtrade)) + # Enable Webhook + if freqtrade.config.get('webhook', {}).get('enabled', False): + logger.info('Enabling rpc.webhook...') + from freqtrade.rpc.webhook import Webhook + self.registered_modules.append(Webhook(freqtrade)) + def cleanup(self) -> None: """ Stops all enabled rpc modules """ logger.info('Cleaning up rpc modules ...') From ee2f6ccbe95d2ec44d7b319d5cf697fa36d48409 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Jul 2018 20:17:45 +0200 Subject: [PATCH 164/395] Add test for enable_webhook --- freqtrade/rpc/rpc_manager.py | 2 +- freqtrade/tests/rpc/test_rpc_manager.py | 30 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index f05a1882f..022578378 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -25,7 +25,7 @@ class RPCManager(object): # Enable Webhook if freqtrade.config.get('webhook', {}).get('enabled', False): - logger.info('Enabling rpc.webhook...') + logger.info('Enabling rpc.webhook ...') from freqtrade.rpc.webhook import Webhook self.registered_modules.append(Webhook(freqtrade)) diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 1f9b034b9..667b3d21d 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -127,3 +127,33 @@ def test_send_msg_telegram_enabled(mocker, default_conf, caplog) -> None: assert log_has("Sending rpc message: {'type': status, 'status': 'test'}", caplog.record_tuples) assert telegram_mock.call_count == 1 + + +def test_init_webhook_disabled(mocker, default_conf, caplog) -> None: + """ Test _init() method with Webhook disabled """ + caplog.set_level(logging.DEBUG) + + conf = deepcopy(default_conf) + conf['telegram']['enabled'] = False + conf['webhook'] = {'enabled': False} + + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, conf)) + + assert not log_has('Enabling rpc.webhook ...', caplog.record_tuples) + assert rpc_manager.registered_modules == [] + + +def test_init_webhook_enabled(mocker, default_conf, caplog) -> None: + """ + Test _init() method with Webhook enabled + """ + caplog.set_level(logging.DEBUG) + default_conf['telegram']['enabled'] = False + default_conf['webhook'] = {'enabled': True} + + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) + + assert log_has('Enabling rpc.webhook ...', caplog.record_tuples) + len_modules = len(rpc_manager.registered_modules) + assert len_modules == 1 + assert 'webhook' in [mod.name for mod in rpc_manager.registered_modules] From 6336d8a0e27e41daf36df6a184486be6437ca25c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Jul 2018 21:43:52 +0200 Subject: [PATCH 165/395] remove copy leftover --- freqtrade/constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 311f2fa72..385dac1d1 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -107,7 +107,6 @@ CONF_SCHEMA = { 'webhookbuy': {'type': 'object'}, 'webhooksell': {'type': 'object'}, 'webhookstatus': {'type': 'object'}, - 'chat_id': {'type': 'string'}, }, }, 'db_url': {'type': 'string'}, From 120fc29643419ff25d5bc2ee86ea2509c5fec281 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Jul 2018 21:54:31 +0200 Subject: [PATCH 166/395] use dict comprehension --- freqtrade/rpc/webhook.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 35631182b..b3c22ee0e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -49,10 +49,8 @@ class Webhook(RPC): if not valuedict: logger.info("Message type %s not configured for webhooks", msg['type']) return - payload = {} - for k in valuedict: - payload[k] = valuedict[k].format(**msg) + payload = {key: value.format(**msg) for (key, value) in valuedict.items()} self._send_msg(payload) except KeyError as exc: logger.exception("Problem calling Webhook. Please check your webhook configuration. " From 12846272195037ee83ada4189b7ade4499446c21 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 Jul 2018 13:29:34 +0200 Subject: [PATCH 167/395] move url to private class level --- freqtrade/rpc/webhook.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index b3c22ee0e..bfc82b8d6 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -26,6 +26,7 @@ class Webhook(RPC): super().__init__(freqtrade) self._config = freqtrade.config + self._url = self._config['webhook']['url'] def cleanup(self) -> None: """ @@ -57,10 +58,9 @@ class Webhook(RPC): "Exception: %s", exc) def _send_msg(self, payload: dict) -> None: - """does the actual call to the webhook""" + """do the actual call to the webhook""" try: - url = self._config['webhook']['url'] - post(url, data=payload) + post(self._url, data=payload) except RequestException as exc: logger.warning("Could not call webhook url. Exception: %s", exc) From 278e7159bccc0f19d8becce00e5687f3e2b17183 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 14 Jul 2018 13:29:50 +0200 Subject: [PATCH 168/395] adjust webhook tests --- freqtrade/tests/rpc/test_rpc_manager.py | 2 +- freqtrade/tests/rpc/test_rpc_webhook.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 667b3d21d..4686cf5ca 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -149,7 +149,7 @@ def test_init_webhook_enabled(mocker, default_conf, caplog) -> None: """ caplog.set_level(logging.DEBUG) default_conf['telegram']['enabled'] = False - default_conf['webhook'] = {'enabled': True} + default_conf['webhook'] = {'enabled': True, 'url': "https://DEADBEEF.com"} rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) diff --git a/freqtrade/tests/rpc/test_rpc_webhook.py b/freqtrade/tests/rpc/test_rpc_webhook.py index 99876269c..b9c005d73 100644 --- a/freqtrade/tests/rpc/test_rpc_webhook.py +++ b/freqtrade/tests/rpc/test_rpc_webhook.py @@ -35,6 +35,7 @@ def test__init__(mocker, default_conf): """ Test __init__() method """ + default_conf['webhook'] = {'enabled': True, 'url': "https://DEADBEEF.com"} webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) assert webhook._config == default_conf From bc83c341189faee5a0d99125f4c48cecd2834f3d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 14 Jul 2018 14:24:07 +0200 Subject: [PATCH 169/395] Update ccxt from 1.16.12 to 1.16.16 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5620f31d2..7a6581bda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.12 +ccxt==1.16.16 SQLAlchemy==1.2.9 python-telegram-bot==10.1.0 arrow==0.12.1 From e1de988f85bb8656c41e049539ab56b7abc68522 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 14 Jul 2018 14:24:09 +0200 Subject: [PATCH 170/395] Update sqlalchemy from 1.2.9 to 1.2.10 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7a6581bda..c7ab27462 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ccxt==1.16.16 -SQLAlchemy==1.2.9 +SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 cachetools==2.1.0 From b4ba641131f9e0ab246091c4cac543dc2d551946 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Jul 2018 09:01:08 +0200 Subject: [PATCH 171/395] Update config dict with attributes loaded from strategy --- freqtrade/strategy/resolver.py | 6 ++++++ freqtrade/tests/strategy/test_strategy.py | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 10cedb073..3334d1b17 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -41,12 +41,16 @@ class StrategyResolver(object): if 'minimal_roi' in config: self.strategy.minimal_roi = config['minimal_roi'] logger.info("Override strategy \'minimal_roi\' with value in config file.") + else: + config['minimal_roi'] = self.strategy.minimal_roi if 'stoploss' in config: self.strategy.stoploss = config['stoploss'] logger.info( "Override strategy \'stoploss\' with value in config file: %s.", config['stoploss'] ) + else: + config['stoploss'] = self.strategy.stoploss if 'ticker_interval' in config: self.strategy.ticker_interval = config['ticker_interval'] @@ -54,6 +58,8 @@ class StrategyResolver(object): "Override strategy \'ticker_interval\' with value in config file: %s.", config['ticker_interval'] ) + else: + config['ticker_interval'] = self.strategy.ticker_interval # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 1e082c380..f8723ebc2 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -74,13 +74,21 @@ def test_load_not_found_strategy(): def test_strategy(result): - resolver = StrategyResolver({'strategy': 'DefaultStrategy'}) + config = {'strategy': 'DefaultStrategy'} + + resolver = StrategyResolver(config) assert hasattr(resolver.strategy, 'minimal_roi') assert resolver.strategy.minimal_roi[0] == 0.04 + assert config.get("minimal_roi")['0'] == 0.04 assert hasattr(resolver.strategy, 'stoploss') assert resolver.strategy.stoploss == -0.10 + assert config['stoploss'] == -0.10 + + assert hasattr(resolver.strategy, 'ticker_interval') + assert resolver.strategy.ticker_interval == '5m' + assert config['ticker_interval'] == '5m' assert hasattr(resolver.strategy, 'populate_indicators') assert 'adx' in resolver.strategy.populate_indicators(result) From 158226012a2fe4eb280a2eb90b2ff6b69b64cf09 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Jul 2018 09:08:14 +0200 Subject: [PATCH 172/395] consistent use of the config dict within the test --- freqtrade/tests/strategy/test_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index f8723ebc2..1aae8f3cc 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -80,7 +80,7 @@ def test_strategy(result): assert hasattr(resolver.strategy, 'minimal_roi') assert resolver.strategy.minimal_roi[0] == 0.04 - assert config.get("minimal_roi")['0'] == 0.04 + assert config["minimal_roi"]['0'] == 0.04 assert hasattr(resolver.strategy, 'stoploss') assert resolver.strategy.stoploss == -0.10 From 8f59759e97457c9bc5421fd44e07c7038be19387 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 15 Jul 2018 14:24:05 +0200 Subject: [PATCH 173/395] Update ccxt from 1.16.16 to 1.16.33 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c7ab27462..94c8421af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.16 +ccxt==1.16.33 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From a74147c472b2930d910266133d1d74609c438afa Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 9 Jul 2018 19:27:36 +0300 Subject: [PATCH 174/395] move strategy initialization outside Analyze --- freqtrade/analyze.py | 6 +++--- freqtrade/freqtradebot.py | 12 +++++++----- freqtrade/optimize/backtesting.py | 4 +++- freqtrade/tests/optimize/test_backtesting.py | 3 ++- freqtrade/tests/test_analyze.py | 5 +++-- freqtrade/tests/test_dataframe.py | 12 ++++++------ freqtrade/tests/test_freqtradebot.py | 3 +-- freqtrade/tests/test_misc.py | 3 ++- 8 files changed, 27 insertions(+), 21 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 493228e68..71d96264f 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -12,7 +12,7 @@ from pandas import DataFrame, to_datetime from freqtrade import constants from freqtrade.exchange import Exchange from freqtrade.persistence import Trade -from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from freqtrade.strategy.resolver import IStrategy logger = logging.getLogger(__name__) @@ -30,13 +30,13 @@ class Analyze(object): Analyze class contains everything the bot need to determine if the situation is good for buying or selling. """ - def __init__(self, config: dict) -> None: + def __init__(self, config: dict, strategy: IStrategy) -> None: """ Init Analyze :param config: Bot configuration (use the one from Configuration()) """ self.config = config - self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.strategy = strategy @staticmethod def parse_ticker_dataframe(ticker: list) -> DataFrame: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 72b5190b9..ad61b5533 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -22,6 +22,7 @@ from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType from freqtrade.rpc import RPCManager from freqtrade.state import State +from freqtrade.strategy.resolver import IStrategy, StrategyResolver logger = logging.getLogger(__name__) @@ -49,7 +50,8 @@ class FreqtradeBot(object): # Init objects self.config = config - self.analyze = Analyze(self.config) + self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.analyze = Analyze(self.config, self.strategy) self.fiat_converter = CryptoToFiatConverter() self.rpc: RPCManager = RPCManager(self) self.persistence = None @@ -293,8 +295,8 @@ class FreqtradeBot(object): return None amount_reserve_percent = 1 - 0.05 # reserve 5% + stoploss - if self.analyze.get_stoploss() is not None: - amount_reserve_percent += self.analyze.get_stoploss() + if self.strategy.stoploss is not None: + amount_reserve_percent += self.strategy.stoploss # it should not be more than 50% amount_reserve_percent = max(amount_reserve_percent, 0.5) return min(min_stake_amounts)/amount_reserve_percent @@ -305,7 +307,7 @@ class FreqtradeBot(object): if one pair triggers the buy_signal a new trade record gets created :return: True if a trade object has been created and persisted, False otherwise """ - interval = self.analyze.get_ticker_interval() + interval = self.strategy.ticker_interval stake_amount = self._get_trade_stake_amount() if not stake_amount: @@ -499,7 +501,7 @@ class FreqtradeBot(object): experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): (buy, sell) = self.analyze.get_signal(self.exchange, - trade.pair, self.analyze.get_ticker_interval()) + trade.pair, self.strategy.ticker_interval) if self.analyze.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): self.execute_sell(trade, current_rate) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 05bcdf4b7..67d4bb2e9 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -21,6 +21,7 @@ from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade +from freqtrade.strategy.resolver import IStrategy, StrategyResolver logger = logging.getLogger(__name__) @@ -52,7 +53,8 @@ class Backtesting(object): """ def __init__(self, config: Dict[str, Any]) -> None: self.config = config - self.analyze = Analyze(self.config) + self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.analyze = Analyze(self.config, self.strategy) self.ticker_interval = self.analyze.strategy.ticker_interval self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe self.populate_buy_trend = self.analyze.populate_buy_trend diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 6fbf71e40..89f5a0bb7 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -18,6 +18,7 @@ from freqtrade.arguments import Arguments, TimeRange from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, start) from freqtrade.tests.conftest import log_has, patch_exchange +from freqtrade.strategy.default_strategy import DefaultStrategy def get_args(args) -> List[str]: @@ -348,7 +349,7 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: assert len(data['UNITTEST/BTC']) == 99 # Load Analyze to compare the result between Backtesting function and Analyze are the same - analyze = Analyze(default_conf) + analyze = Analyze(default_conf, DefaultStrategy()) data2 = analyze.tickerdata_to_dataframe(tickerlist) assert data['UNITTEST/BTC'].equals(data2['UNITTEST/BTC']) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 6e035d842..dc7410ffc 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -14,9 +14,10 @@ from freqtrade.analyze import Analyze, SignalType from freqtrade.arguments import TimeRange from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.tests.conftest import get_patched_exchange, log_has +from freqtrade.strategy.default_strategy import DefaultStrategy # Avoid to reinit the same object again and again -_ANALYZE = Analyze({'strategy': 'DefaultStrategy'}) +_ANALYZE = Analyze({}, DefaultStrategy()) def test_signaltype_object() -> None: @@ -189,7 +190,7 @@ def test_tickerdata_to_dataframe(default_conf) -> None: """ Test Analyze.tickerdata_to_dataframe() method """ - analyze = Analyze(default_conf) + analyze = Analyze(default_conf, DefaultStrategy()) timerange = TimeRange(None, 'line', 0, -100) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index fd461a503..f4b26eb1d 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -9,26 +9,26 @@ from freqtrade.strategy.resolver import StrategyResolver _pairs = ['ETH/BTC'] -def load_dataframe_pair(pairs): +def load_dataframe_pair(pairs, strategy): ld = load_data(None, ticker_interval='5m', pairs=pairs) assert isinstance(ld, dict) assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] - analyze = Analyze({'strategy': 'DefaultStrategy'}) + analyze = Analyze({}, strategy) dataframe = analyze.analyze_ticker(dataframe) return dataframe def test_dataframe_load(): - StrategyResolver({'strategy': 'DefaultStrategy'}) - dataframe = load_dataframe_pair(_pairs) + strategy = StrategyResolver({'strategy': 'DefaultStrategy'}).strategy + dataframe = load_dataframe_pair(_pairs, strategy) assert isinstance(dataframe, pandas.core.frame.DataFrame) def test_dataframe_columns_exists(): - StrategyResolver({'strategy': 'DefaultStrategy'}) - dataframe = load_dataframe_pair(_pairs) + strategy = StrategyResolver({'strategy': 'DefaultStrategy'}).strategy + dataframe = load_dataframe_pair(_pairs, strategy) assert 'high' in dataframe.columns assert 'low' in dataframe.columns assert 'close' in dataframe.columns diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 450504f57..c628a9da3 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -316,9 +316,8 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: patch_RPCManager(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) - mocker.patch('freqtrade.freqtradebot.Analyze.get_stoploss', MagicMock(return_value=-0.05)) freqtrade = FreqtradeBot(default_conf) - + freqtrade.strategy.stoploss = -0.05 # no pair found mocker.patch( 'freqtrade.exchange.Exchange.get_markets', diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index e2ba40dee..c30225132 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -11,6 +11,7 @@ from freqtrade.analyze import Analyze from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, format_ms_time, shorten_date) from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.strategy.default_strategy import DefaultStrategy def test_shorten_date() -> None: @@ -47,7 +48,7 @@ def test_common_datearray(default_conf) -> None: Test common_datearray() :return: None """ - analyze = Analyze(default_conf) + analyze = Analyze(default_conf, DefaultStrategy()) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = analyze.tickerdata_to_dataframe(tickerlist) From 85e6c9585ad8bed469f4a47b949036c1597a50a4 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 10 Jul 2018 09:11:36 +0300 Subject: [PATCH 175/395] remove pass-through methods from Analyze --- freqtrade/analyze.py | 48 +++---------------------------- freqtrade/optimize/backtesting.py | 4 +-- freqtrade/tests/test_analyze.py | 39 +------------------------ 3 files changed, 7 insertions(+), 84 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 71d96264f..c27a31bb6 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -64,46 +64,6 @@ class Analyze(object): frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle return frame - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - - Performance Note: For the best performance be frugal on the number of indicators - you are using. Let uncomment only the indicator you are using in your strategies - or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - """ - return self.strategy.populate_indicators(dataframe=dataframe) - - def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the buy signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - return self.strategy.populate_buy_trend(dataframe=dataframe) - - def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the sell signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - return self.strategy.populate_sell_trend(dataframe=dataframe) - - def get_ticker_interval(self) -> str: - """ - Return ticker interval to use - :return: Ticker interval value to use - """ - return self.strategy.ticker_interval - - def get_stoploss(self) -> float: - """ - Return stoploss to use - :return: Strategy stoploss value to use - """ - return self.strategy.stoploss - def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: """ Parses the given ticker history and returns a populated DataFrame @@ -111,9 +71,9 @@ class Analyze(object): :return DataFrame with ticker data and indicator data """ dataframe = self.parse_ticker_dataframe(ticker_history) - dataframe = self.populate_indicators(dataframe) - dataframe = self.populate_buy_trend(dataframe) - dataframe = self.populate_sell_trend(dataframe) + dataframe = self.strategy.populate_indicators(dataframe) + dataframe = self.strategy.populate_buy_trend(dataframe) + dataframe = self.strategy.populate_sell_trend(dataframe) return dataframe def get_signal(self, exchange: Exchange, pair: str, interval: str) -> Tuple[bool, bool]: @@ -267,5 +227,5 @@ class Analyze(object): """ Creates a dataframe and populates indicators for given ticker data """ - return {pair: self.populate_indicators(self.parse_ticker_dataframe(pair_data)) + return {pair: self.strategy.populate_indicators(self.parse_ticker_dataframe(pair_data)) for pair, pair_data in tickerdata.items()} diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 67d4bb2e9..254bf80c6 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -57,8 +57,8 @@ class Backtesting(object): self.analyze = Analyze(self.config, self.strategy) self.ticker_interval = self.analyze.strategy.ticker_interval self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe - self.populate_buy_trend = self.analyze.populate_buy_trend - self.populate_sell_trend = self.analyze.populate_sell_trend + self.populate_buy_trend = self.strategy.populate_buy_trend + self.populate_sell_trend = self.strategy.populate_sell_trend # Reset keys for backtesting self.config['exchange']['key'] = '' diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index dc7410ffc..4cd68dd86 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -10,7 +10,7 @@ from unittest.mock import MagicMock import arrow from pandas import DataFrame -from freqtrade.analyze import Analyze, SignalType +from freqtrade.analyze import Analyze from freqtrade.arguments import TimeRange from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.tests.conftest import get_patched_exchange, log_has @@ -20,31 +20,6 @@ from freqtrade.strategy.default_strategy import DefaultStrategy _ANALYZE = Analyze({}, DefaultStrategy()) -def test_signaltype_object() -> None: - """ - Test the SignalType object has the mandatory Constants - :return: None - """ - assert hasattr(SignalType, 'BUY') - assert hasattr(SignalType, 'SELL') - - -def test_analyze_object() -> None: - """ - Test the Analyze object has the mandatory methods - :return: None - """ - assert hasattr(Analyze, 'parse_ticker_dataframe') - assert hasattr(Analyze, 'populate_indicators') - assert hasattr(Analyze, 'populate_buy_trend') - assert hasattr(Analyze, 'populate_sell_trend') - assert hasattr(Analyze, 'analyze_ticker') - assert hasattr(Analyze, 'get_signal') - assert hasattr(Analyze, 'should_sell') - assert hasattr(Analyze, 'min_roi_reached') - assert hasattr(Analyze, 'stop_loss_reached') - - def test_dataframe_correct_length(result): dataframe = Analyze.parse_ticker_dataframe(result) assert len(result.index) - 1 == len(dataframe.index) # last partial candle removed @@ -55,18 +30,6 @@ def test_dataframe_correct_columns(result): ['date', 'open', 'high', 'low', 'close', 'volume'] -def test_populates_buy_trend(result): - # Load the default strategy for the unit test, because this logic is done in main.py - dataframe = _ANALYZE.populate_buy_trend(_ANALYZE.populate_indicators(result)) - assert 'buy' in dataframe.columns - - -def test_populates_sell_trend(result): - # Load the default strategy for the unit test, because this logic is done in main.py - dataframe = _ANALYZE.populate_sell_trend(_ANALYZE.populate_indicators(result)) - assert 'sell' in dataframe.columns - - def test_returns_latest_buy_signal(mocker, default_conf): mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) exchange = get_patched_exchange(mocker, default_conf) From f6b8c2b40fc8c6df42cbdd45de8c26ca8b0ded8b Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 10 Jul 2018 13:04:37 +0300 Subject: [PATCH 176/395] move parse_ticker_dataframe outside Analyze class --- freqtrade/analyze.py | 56 +++++++++---------- freqtrade/tests/conftest.py | 4 +- .../tests/strategy/test_default_strategy.py | 4 +- freqtrade/tests/test_analyze.py | 6 +- freqtrade/tests/test_misc.py | 4 +- 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index c27a31bb6..7a8ba3fb0 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -17,6 +17,32 @@ from freqtrade.strategy.resolver import IStrategy logger = logging.getLogger(__name__) +def parse_ticker_dataframe(ticker: list) -> DataFrame: + """ + Analyses the trend for the given ticker history + :param ticker: See exchange.get_ticker_history + :return: DataFrame + """ + cols = ['date', 'open', 'high', 'low', 'close', 'volume'] + frame = DataFrame(ticker, columns=cols) + + frame['date'] = to_datetime(frame['date'], + unit='ms', + utc=True, + infer_datetime_format=True) + + # group by index and aggregate results to eliminate duplicate ticks + frame = frame.groupby(by='date', as_index=False, sort=True).agg({ + 'open': 'first', + 'high': 'max', + 'low': 'min', + 'close': 'last', + 'volume': 'max', + }) + frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle + return frame + + class SignalType(Enum): """ Enum to distinguish between buy and sell signals @@ -38,39 +64,13 @@ class Analyze(object): self.config = config self.strategy = strategy - @staticmethod - def parse_ticker_dataframe(ticker: list) -> DataFrame: - """ - Analyses the trend for the given ticker history - :param ticker: See exchange.get_ticker_history - :return: DataFrame - """ - cols = ['date', 'open', 'high', 'low', 'close', 'volume'] - frame = DataFrame(ticker, columns=cols) - - frame['date'] = to_datetime(frame['date'], - unit='ms', - utc=True, - infer_datetime_format=True) - - # group by index and aggregate results to eliminate duplicate ticks - frame = frame.groupby(by='date', as_index=False, sort=True).agg({ - 'open': 'first', - 'high': 'max', - 'low': 'min', - 'close': 'last', - 'volume': 'max', - }) - frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle - return frame - def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: """ Parses the given ticker history and returns a populated DataFrame add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ - dataframe = self.parse_ticker_dataframe(ticker_history) + dataframe = parse_ticker_dataframe(ticker_history) dataframe = self.strategy.populate_indicators(dataframe) dataframe = self.strategy.populate_buy_trend(dataframe) dataframe = self.strategy.populate_sell_trend(dataframe) @@ -227,5 +227,5 @@ class Analyze(object): """ Creates a dataframe and populates indicators for given ticker data """ - return {pair: self.strategy.populate_indicators(self.parse_ticker_dataframe(pair_data)) + return {pair: self.strategy.populate_indicators(parse_ticker_dataframe(pair_data)) for pair, pair_data in tickerdata.items()} diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 9c86d1ece..078dba447 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -12,7 +12,7 @@ from jsonschema import validate from telegram import Chat, Message, Update from freqtrade import constants -from freqtrade.analyze import Analyze +from freqtrade.analyze import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot @@ -616,7 +616,7 @@ def tickers(): @pytest.fixture def result(): with open('freqtrade/tests/testdata/UNITTEST_BTC-1m.json') as data_file: - return Analyze.parse_ticker_dataframe(json.load(data_file)) + return parse_ticker_dataframe(json.load(data_file)) # FIX: # Create an fixture/function diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index 900fc2234..2175dc9b3 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -3,14 +3,14 @@ import json import pytest from pandas import DataFrame -from freqtrade.analyze import Analyze +from freqtrade.analyze import parse_ticker_dataframe from freqtrade.strategy.default_strategy import DefaultStrategy @pytest.fixture def result(): with open('freqtrade/tests/testdata/ETH_BTC-1m.json') as data_file: - return Analyze.parse_ticker_dataframe(json.load(data_file)) + return parse_ticker_dataframe(json.load(data_file)) def test_default_strategy_structure(): diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 4cd68dd86..042422c5a 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -10,7 +10,7 @@ from unittest.mock import MagicMock import arrow from pandas import DataFrame -from freqtrade.analyze import Analyze +from freqtrade.analyze import Analyze, parse_ticker_dataframe from freqtrade.arguments import TimeRange from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.tests.conftest import get_patched_exchange, log_has @@ -21,7 +21,7 @@ _ANALYZE = Analyze({}, DefaultStrategy()) def test_dataframe_correct_length(result): - dataframe = Analyze.parse_ticker_dataframe(result) + dataframe = parse_ticker_dataframe(result) assert len(result.index) - 1 == len(dataframe.index) # last partial candle removed @@ -145,7 +145,7 @@ def test_parse_ticker_dataframe(ticker_history): columns = ['date', 'open', 'high', 'low', 'close', 'volume'] # Test file with BV data - dataframe = Analyze.parse_ticker_dataframe(ticker_history) + dataframe = parse_ticker_dataframe(ticker_history) assert dataframe.columns.tolist() == columns diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index c30225132..15ed1550b 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -7,7 +7,7 @@ Unit test file for misc.py import datetime from unittest.mock import MagicMock -from freqtrade.analyze import Analyze +from freqtrade.analyze import Analyze, parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, format_ms_time, shorten_date) from freqtrade.optimize.__init__ import load_tickerdata_file @@ -29,7 +29,7 @@ def test_datesarray_to_datetimearray(ticker_history): Test datesarray_to_datetimearray() function :return: None """ - dataframes = Analyze.parse_ticker_dataframe(ticker_history) + dataframes = parse_ticker_dataframe(ticker_history) dates = datesarray_to_datetimearray(dataframes['date']) assert isinstance(dates[0], datetime.datetime) From aeb4102bcbb1c0d27e3c3888c4cd0fa8005c725f Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 16 Jul 2018 08:11:17 +0300 Subject: [PATCH 177/395] refactor Analyze class methods to base Strategy class --- freqtrade/analyze.py | 198 ---------------- freqtrade/freqtradebot.py | 11 +- freqtrade/optimize/backtesting.py | 10 +- freqtrade/optimize/hyperopt.py | 6 +- freqtrade/strategy/__init__.py | 4 +- freqtrade/strategy/interface.py | 190 ++++++++++++++- freqtrade/strategy/resolver.py | 12 +- freqtrade/tests/conftest.py | 4 +- freqtrade/tests/optimize/test_backtesting.py | 10 +- freqtrade/tests/rpc/test_rpc.py | 22 +- freqtrade/tests/rpc/test_rpc_telegram.py | 51 ++-- .../tests/strategy/test_default_strategy.py | 2 +- freqtrade/tests/strategy/test_strategy.py | 21 +- freqtrade/tests/test_analyze.py | 220 +++++++++--------- freqtrade/tests/test_dataframe.py | 4 +- freqtrade/tests/test_freqtradebot.py | 161 +++++++------ freqtrade/tests/test_misc.py | 6 +- 17 files changed, 473 insertions(+), 459 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 7a8ba3fb0..254c16309 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -2,18 +2,8 @@ Functions to analyze ticker data with indicators and produce buy and sell signals """ import logging -from datetime import datetime -from enum import Enum -from typing import Dict, List, Tuple - -import arrow from pandas import DataFrame, to_datetime -from freqtrade import constants -from freqtrade.exchange import Exchange -from freqtrade.persistence import Trade -from freqtrade.strategy.resolver import IStrategy - logger = logging.getLogger(__name__) @@ -41,191 +31,3 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame: }) frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle return frame - - -class SignalType(Enum): - """ - Enum to distinguish between buy and sell signals - """ - BUY = "buy" - SELL = "sell" - - -class Analyze(object): - """ - Analyze class contains everything the bot need to determine if the situation is good for - buying or selling. - """ - def __init__(self, config: dict, strategy: IStrategy) -> None: - """ - Init Analyze - :param config: Bot configuration (use the one from Configuration()) - """ - self.config = config - self.strategy = strategy - - def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: - """ - Parses the given ticker history and returns a populated DataFrame - add several TA indicators and buy signal to it - :return DataFrame with ticker data and indicator data - """ - dataframe = parse_ticker_dataframe(ticker_history) - dataframe = self.strategy.populate_indicators(dataframe) - dataframe = self.strategy.populate_buy_trend(dataframe) - dataframe = self.strategy.populate_sell_trend(dataframe) - return dataframe - - def get_signal(self, exchange: Exchange, pair: str, interval: str) -> Tuple[bool, bool]: - """ - Calculates current signal based several technical analysis indicators - :param pair: pair in format ANT/BTC - :param interval: Interval to use (in min) - :return: (Buy, Sell) A bool-tuple indicating buy/sell signal - """ - ticker_hist = exchange.get_ticker_history(pair, interval) - if not ticker_hist: - logger.warning('Empty ticker history for pair %s', pair) - return False, False - - try: - dataframe = self.analyze_ticker(ticker_hist) - except ValueError as error: - logger.warning( - 'Unable to analyze ticker for pair %s: %s', - pair, - str(error) - ) - return False, False - except Exception as error: - logger.exception( - 'Unexpected error when analyzing ticker for pair %s: %s', - pair, - str(error) - ) - return False, False - - if dataframe.empty: - logger.warning('Empty dataframe for pair %s', pair) - return False, False - - latest = dataframe.iloc[-1] - - # Check if dataframe is out of date - signal_date = arrow.get(latest['date']) - interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] - if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))): - logger.warning( - 'Outdated history for pair %s. Last tick is %s minutes old', - pair, - (arrow.utcnow() - signal_date).seconds // 60 - ) - return False, False - - (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 - logger.debug( - 'trigger: %s (pair=%s) buy=%s sell=%s', - latest['date'], - pair, - str(buy), - str(sell) - ) - return buy, sell - - def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, sell: bool) -> bool: - """ - This function evaluate if on the condition required to trigger a sell has been reached - if the threshold is reached and updates the trade record. - :return: True if trade should be sold, False otherwise - """ - current_profit = trade.calc_profit_percent(rate) - if self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, - current_profit=current_profit): - return True - - experimental = self.config.get('experimental', {}) - - if buy and experimental.get('ignore_roi_if_buy_signal', False): - logger.debug('Buy signal still active - not selling.') - return False - - # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) - if self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date): - logger.debug('Required profit reached. Selling..') - return True - - if experimental.get('sell_profit_only', False): - logger.debug('Checking if trade is profitable..') - if trade.calc_profit(rate=rate) <= 0: - return False - if sell and not buy and experimental.get('use_sell_signal', False): - logger.debug('Sell signal received. Selling..') - return True - - return False - - def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, - current_profit: float) -> bool: - """ - Based on current profit of the trade and configured (trailing) stoploss, - decides to sell or not - """ - - trailing_stop = self.config.get('trailing_stop', False) - - trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) - - # evaluate if the stoploss was hit - if self.strategy.stoploss is not None and trade.stop_loss >= current_rate: - - if trailing_stop: - logger.debug( - f"HIT STOP: current price at {current_rate:.6f}, " - f"stop loss is {trade.stop_loss:.6f}, " - f"initial stop loss was at {trade.initial_stop_loss:.6f}, " - f"trade opened at {trade.open_rate:.6f}") - logger.debug(f"trailing stop saved {trade.stop_loss - trade.initial_stop_loss:.6f}") - - logger.debug('Stop loss hit.') - return True - - # update the stop loss afterwards, after all by definition it's supposed to be hanging - if trailing_stop: - - # check if we have a special stop loss for positive condition - # and if profit is positive - stop_loss_value = self.strategy.stoploss - if 'trailing_stop_positive' in self.config and current_profit > 0: - - # Ignore mypy error check in configuration that this is a float - stop_loss_value = self.config.get('trailing_stop_positive') # type: ignore - logger.debug(f"using positive stop loss mode: {stop_loss_value} " - f"since we have profit {current_profit}") - - trade.adjust_stop_loss(current_rate, stop_loss_value) - - return False - - def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool: - """ - Based an earlier trade and current price and ROI configuration, decides whether bot should - sell - :return True if bot should sell at current rate - """ - - # Check if time matches and current rate is above threshold - time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 - for duration, threshold in self.strategy.minimal_roi.items(): - if time_diff <= duration: - return False - if current_profit > threshold: - return True - - return False - - def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: - """ - Creates a dataframe and populates indicators for given ticker data - """ - return {pair: self.strategy.populate_indicators(parse_ticker_dataframe(pair_data)) - for pair, pair_data in tickerdata.items()} diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ad61b5533..0d916f570 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -15,7 +15,6 @@ from cachetools import TTLCache, cached from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) -from freqtrade.analyze import Analyze from freqtrade.exchange import Exchange from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.persistence import Trade @@ -51,7 +50,7 @@ class FreqtradeBot(object): # Init objects self.config = config self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.analyze = Analyze(self.config, self.strategy) +# self.analyze = Analyze(self.config, self.strategy) self.fiat_converter = CryptoToFiatConverter() self.rpc: RPCManager = RPCManager(self) self.persistence = None @@ -330,7 +329,7 @@ class FreqtradeBot(object): # Pick pair based on buy signals for _pair in whitelist: - (buy, sell) = self.analyze.get_signal(self.exchange, _pair, interval) + (buy, sell) = self.strategy.get_signal(self.exchange, _pair, interval) if buy and not sell: return self.execute_buy(_pair, stake_amount) return False @@ -500,10 +499,10 @@ class FreqtradeBot(object): (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - (buy, sell) = self.analyze.get_signal(self.exchange, - trade.pair, self.strategy.ticker_interval) + (buy, sell) = self.strategy.get_signal(self.exchange, + trade.pair, self.strategy.ticker_interval) - if self.analyze.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): + if self.strategy.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): self.execute_sell(trade, current_rate) return True logger.info('Found no sell signals for whitelisted currencies. Trying again..') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 254bf80c6..9c124f35b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -15,7 +15,6 @@ from tabulate import tabulate import freqtrade.optimize as optimize from freqtrade import DependencyException, constants -from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange @@ -54,9 +53,8 @@ class Backtesting(object): def __init__(self, config: Dict[str, Any]) -> None: self.config = config self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.analyze = Analyze(self.config, self.strategy) - self.ticker_interval = self.analyze.strategy.ticker_interval - self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe + self.ticker_interval = self.strategy.ticker_interval + self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe self.populate_buy_trend = self.strategy.populate_buy_trend self.populate_sell_trend = self.strategy.populate_sell_trend @@ -153,8 +151,8 @@ class Backtesting(object): trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1 buy_signal = sell_row.buy - if self.analyze.should_sell(trade, sell_row.open, sell_row.date, buy_signal, - sell_row.sell): + if self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, + sell_row.sell): return BacktestResult(pair=pair, profit_percent=trade.calc_profit_percent(rate=sell_row.open), diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 72bf34eb3..3b4652883 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -267,13 +267,13 @@ class Hyperopt(Backtesting): params = self.get_args(_params) if self.has_space('roi'): - self.analyze.strategy.minimal_roi = self.generate_roi_table(params) + self.strategy.minimal_roi = self.generate_roi_table(params) if self.has_space('buy'): self.populate_buy_trend = self.buy_strategy_generator(params) if self.has_space('stoploss'): - self.analyze.strategy.stoploss = params['stoploss'] + self.strategy.stoploss = params['stoploss'] processed = load(TICKERDATA_PICKLE) results = self.backtest( @@ -351,7 +351,7 @@ class Hyperopt(Backtesting): ) if self.has_space('buy'): - self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore + self.strategy.populate_indicators = Hyperopt.populate_indicators # type: ignore dump(self.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index e1dc7bb3f..283426dfa 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -7,7 +7,7 @@ from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) -def import_strategy(strategy: IStrategy) -> IStrategy: +def import_strategy(strategy: IStrategy, config: dict) -> IStrategy: """ Imports given Strategy instance to global scope of freqtrade.strategy and returns an instance of it @@ -29,4 +29,4 @@ def import_strategy(strategy: IStrategy) -> IStrategy: # Modify global scope to declare class globals()[name] = clazz - return clazz() + return clazz(config) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index f73617f46..c67870aeb 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -2,11 +2,30 @@ IStrategy interface This module defines the interface to apply for strategies """ +import logging from abc import ABC, abstractmethod -from typing import Dict +from datetime import datetime +from enum import Enum +from typing import Dict, List, Tuple +import arrow from pandas import DataFrame +from freqtrade import constants +from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.exchange import Exchange +from freqtrade.persistence import Trade + +logger = logging.getLogger(__name__) + + +class SignalType(Enum): + """ + Enum to distinguish between buy and sell signals + """ + BUY = "buy" + SELL = "sell" + class IStrategy(ABC): """ @@ -23,6 +42,9 @@ class IStrategy(ABC): stoploss: float ticker_interval: str + def __init__(self, config: dict): + self.config = config + @abstractmethod def populate_indicators(self, dataframe: DataFrame) -> DataFrame: """ @@ -46,3 +68,169 @@ class IStrategy(ABC): :param dataframe: DataFrame :return: DataFrame with sell column """ + + def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: + """ + Parses the given ticker history and returns a populated DataFrame + add several TA indicators and buy signal to it + :return DataFrame with ticker data and indicator data + """ + dataframe = parse_ticker_dataframe(ticker_history) + dataframe = self.populate_indicators(dataframe) + dataframe = self.populate_buy_trend(dataframe) + dataframe = self.populate_sell_trend(dataframe) + return dataframe + + def get_signal(self, exchange: Exchange, pair: str, interval: str) -> Tuple[bool, bool]: + """ + Calculates current signal based several technical analysis indicators + :param pair: pair in format ANT/BTC + :param interval: Interval to use (in min) + :return: (Buy, Sell) A bool-tuple indicating buy/sell signal + """ + ticker_hist = exchange.get_ticker_history(pair, interval) + if not ticker_hist: + logger.warning('Empty ticker history for pair %s', pair) + return False, False + + try: + dataframe = self.analyze_ticker(ticker_hist) + except ValueError as error: + logger.warning( + 'Unable to analyze ticker for pair %s: %s', + pair, + str(error) + ) + return False, False + except Exception as error: + logger.exception( + 'Unexpected error when analyzing ticker for pair %s: %s', + pair, + str(error) + ) + return False, False + + if dataframe.empty: + logger.warning('Empty dataframe for pair %s', pair) + return False, False + + latest = dataframe.iloc[-1] + + # Check if dataframe is out of date + signal_date = arrow.get(latest['date']) + interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] + if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))): + logger.warning( + 'Outdated history for pair %s. Last tick is %s minutes old', + pair, + (arrow.utcnow() - signal_date).seconds // 60 + ) + return False, False + + (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 + logger.debug( + 'trigger: %s (pair=%s) buy=%s sell=%s', + latest['date'], + pair, + str(buy), + str(sell) + ) + return buy, sell + + def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, sell: bool) -> bool: + """ + This function evaluate if on the condition required to trigger a sell has been reached + if the threshold is reached and updates the trade record. + :return: True if trade should be sold, False otherwise + """ + current_profit = trade.calc_profit_percent(rate) + if self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, + current_profit=current_profit): + return True + + experimental = self.config.get('experimental', {}) + + if buy and experimental.get('ignore_roi_if_buy_signal', False): + logger.debug('Buy signal still active - not selling.') + return False + + # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) + if self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date): + logger.debug('Required profit reached. Selling..') + return True + + if experimental.get('sell_profit_only', False): + logger.debug('Checking if trade is profitable..') + if trade.calc_profit(rate=rate) <= 0: + return False + if sell and not buy and experimental.get('use_sell_signal', False): + logger.debug('Sell signal received. Selling..') + return True + + return False + + def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, + current_profit: float) -> bool: + """ + Based on current profit of the trade and configured (trailing) stoploss, + decides to sell or not + """ + + trailing_stop = self.config.get('trailing_stop', False) + + trade.adjust_stop_loss(trade.open_rate, self.stoploss, initial=True) + + # evaluate if the stoploss was hit + if self.stoploss is not None and trade.stop_loss >= current_rate: + + if trailing_stop: + logger.debug( + f"HIT STOP: current price at {current_rate:.6f}, " + f"stop loss is {trade.stop_loss:.6f}, " + f"initial stop loss was at {trade.initial_stop_loss:.6f}, " + f"trade opened at {trade.open_rate:.6f}") + logger.debug(f"trailing stop saved {trade.stop_loss - trade.initial_stop_loss:.6f}") + + logger.debug('Stop loss hit.') + return True + + # update the stop loss afterwards, after all by definition it's supposed to be hanging + if trailing_stop: + + # check if we have a special stop loss for positive condition + # and if profit is positive + stop_loss_value = self.stoploss + if 'trailing_stop_positive' in self.config and current_profit > 0: + + # Ignore mypy error check in configuration that this is a float + stop_loss_value = self.config.get('trailing_stop_positive') # type: ignore + logger.debug(f"using positive stop loss mode: {stop_loss_value} " + f"since we have profit {current_profit}") + + trade.adjust_stop_loss(current_rate, stop_loss_value) + + return False + + def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool: + """ + Based an earlier trade and current price and ROI configuration, decides whether bot should + sell + :return True if bot should sell at current rate + """ + + # Check if time matches and current rate is above threshold + time_diff = (current_time.timestamp() - trade.open_date.timestamp()) / 60 + for duration, threshold in self.minimal_roi.items(): + if time_diff <= duration: + return False + if current_profit > threshold: + return True + + return False + + def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: + """ + Creates a dataframe and populates indicators for given ticker data + """ + return {pair: self.populate_indicators(parse_ticker_dataframe(pair_data)) + for pair, pair_data in tickerdata.items()} diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 10cedb073..4df713b4d 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -34,6 +34,7 @@ class StrategyResolver(object): # Verify the strategy is in the configuration, otherwise fallback to the default strategy strategy_name = config.get('strategy') or constants.DEFAULT_STRATEGY self.strategy: IStrategy = self._load_strategy(strategy_name, + config=config, extra_dir=config.get('strategy_path')) # Set attributes @@ -62,10 +63,11 @@ class StrategyResolver(object): self.strategy.stoploss = float(self.strategy.stoploss) def _load_strategy( - self, strategy_name: str, extra_dir: Optional[str] = None) -> IStrategy: + self, strategy_name: str, config: dict, extra_dir: Optional[str] = None) -> IStrategy: """ Search and loads the specified strategy. :param strategy_name: name of the module to import + :param config: configuration for the strategy :param extra_dir: additional directory to search for the given strategy :return: Strategy instance or None """ @@ -81,10 +83,10 @@ class StrategyResolver(object): for path in abs_paths: try: - strategy = self._search_strategy(path, strategy_name) + strategy = self._search_strategy(path, strategy_name=strategy_name, config=config) if strategy: logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) - return import_strategy(strategy) + return import_strategy(strategy, config=config) except FileNotFoundError: logger.warning('Path "%s" does not exist', path) @@ -114,7 +116,7 @@ class StrategyResolver(object): return next(valid_strategies_gen, None) @staticmethod - def _search_strategy(directory: str, strategy_name: str) -> Optional[IStrategy]: + def _search_strategy(directory: str, strategy_name: str, config: dict) -> Optional[IStrategy]: """ Search for the strategy_name in the given directory :param directory: relative or absolute directory path @@ -130,5 +132,5 @@ class StrategyResolver(object): os.path.abspath(os.path.join(directory, entry)), strategy_name ) if strategy: - return strategy() + return strategy(config) return None diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 078dba447..788ab5afb 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -51,13 +51,13 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: """ # mocker.patch('freqtrade.fiat_convert.Market', {'price_usd': 12345.0}) patch_coinmarketcap(mocker, {'price_usd': 12345.0}) - mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) +# mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) patch_exchange(mocker, None) mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) - mocker.patch('freqtrade.freqtradebot.Analyze.get_signal', MagicMock()) +# mocker.patch('freqtrade.freqtradebot.Analyze.get_signal', MagicMock()) return FreqtradeBot(config) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 89f5a0bb7..6e4f91891 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -13,7 +13,6 @@ import pytest from arrow import Arrow from freqtrade import DependencyException, constants, optimize -from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments, TimeRange from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, start) @@ -326,7 +325,6 @@ def test_backtesting_init(mocker, default_conf) -> None: get_fee = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5)) backtesting = Backtesting(default_conf) assert backtesting.config == default_conf - assert isinstance(backtesting.analyze, Analyze) assert backtesting.ticker_interval == '5m' assert callable(backtesting.tickerdata_to_dataframe) assert callable(backtesting.populate_buy_trend) @@ -348,9 +346,9 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: data = backtesting.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 99 - # Load Analyze to compare the result between Backtesting function and Analyze are the same - analyze = Analyze(default_conf, DefaultStrategy()) - data2 = analyze.tickerdata_to_dataframe(tickerlist) + # Load strategy to compare the result between Backtesting function and strategy are the same + strategy = DefaultStrategy(default_conf) + data2 = strategy.tickerdata_to_dataframe(tickerlist) assert data['UNITTEST/BTC'].equals(data2['UNITTEST/BTC']) @@ -413,7 +411,6 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: def get_timeframe(input1, input2): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) mocker.patch('freqtrade.exchange.Exchange.get_ticker_history') patch_exchange(mocker) @@ -454,7 +451,6 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: def get_timeframe(input1, input2): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.get_ticker_history') patch_exchange(mocker) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 6e59b4116..e6cfceae7 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -30,7 +30,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: """ Test rpc_trade_status() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -42,6 +41,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -74,7 +74,6 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: """ Test rpc_status_table() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -86,6 +85,7 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -108,7 +108,6 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, """ Test rpc_daily_profit() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -120,6 +119,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -160,7 +160,6 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, """ Test rpc_trade_statistics() method """ - patch_get_signal(mocker, (True, False)) mocker.patch.multiple( 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), @@ -176,6 +175,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -237,7 +237,6 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, """ Test rpc_trade_statistics() method """ - patch_get_signal(mocker, (True, False)) mocker.patch.multiple( 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), @@ -253,6 +252,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -309,7 +309,6 @@ def test_rpc_balance_handle(default_conf, mocker): } } - patch_get_signal(mocker, (True, False)) mocker.patch.multiple( 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), @@ -323,6 +322,7 @@ def test_rpc_balance_handle(default_conf, mocker): ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) result = rpc._rpc_balance(default_conf['fiat_display_currency']) @@ -342,7 +342,6 @@ def test_rpc_start(mocker, default_conf) -> None: """ Test rpc_start() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -352,6 +351,7 @@ def test_rpc_start(mocker, default_conf) -> None: ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -368,7 +368,6 @@ def test_rpc_stop(mocker, default_conf) -> None: """ Test rpc_stop() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -378,6 +377,7 @@ def test_rpc_stop(mocker, default_conf) -> None: ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.RUNNING @@ -395,7 +395,6 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: """ Test rpc_forcesell() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) @@ -417,6 +416,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -499,7 +499,6 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, """ Test rpc_performance() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -512,6 +511,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) # Create some test data @@ -538,7 +538,6 @@ def test_rpc_count(mocker, default_conf, ticker, fee, markets) -> None: """ Test rpc_count() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -551,6 +550,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee, markets) -> None: ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) trades = rpc._rpc_count() diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 01f248327..3336810bd 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -102,7 +102,6 @@ def test_authorized_only(default_conf, mocker, caplog) -> None: """ Test authorized_only() method when we are authorized """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) patch_exchange(mocker, None) @@ -112,7 +111,9 @@ def test_authorized_only(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - dummy = DummyCls(FreqtradeBot(conf)) + bot = FreqtradeBot(conf) + patch_get_signal(bot, (True, False)) + dummy = DummyCls(bot) dummy.dummy_handler(bot=MagicMock(), update=update) assert dummy.state['called'] is True assert log_has( @@ -133,7 +134,6 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: """ Test authorized_only() method when we are unauthorized """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) patch_exchange(mocker, None) chat = Chat(0xdeadbeef, 0) @@ -142,7 +142,9 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - dummy = DummyCls(FreqtradeBot(conf)) + bot = FreqtradeBot(conf) + patch_get_signal(bot, (True, False)) + dummy = DummyCls(bot) dummy.dummy_handler(bot=MagicMock(), update=update) assert dummy.state['called'] is False assert not log_has( @@ -163,7 +165,6 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: """ Test authorized_only() method when an exception is thrown """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) patch_exchange(mocker) @@ -172,7 +173,11 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - dummy = DummyCls(FreqtradeBot(conf)) + + bot = FreqtradeBot(conf) + patch_get_signal(bot, (True, False)) + dummy = DummyCls(bot) + dummy.dummy_exception(bot=MagicMock(), update=update) assert dummy.state['called'] is False assert not log_has( @@ -198,7 +203,6 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: conf['telegram']['enabled'] = False conf['telegram']['chat_id'] = 123 - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -233,6 +237,7 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Create some test data @@ -252,7 +257,6 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No """ Test _status() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -272,6 +276,8 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) + telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -299,7 +305,6 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) """ Test _status_table() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -320,6 +325,8 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) conf = deepcopy(default_conf) conf['stake_amount'] = 15.0 freqtradebot = FreqtradeBot(conf) + patch_get_signal(freqtradebot, (True, False)) + telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED @@ -353,7 +360,6 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, """ Test _daily() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch( 'freqtrade.fiat_convert.CryptoToFiatConverter._find_price', @@ -375,6 +381,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Create some test data @@ -427,7 +434,6 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: """ Test _daily() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -443,6 +449,7 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Try invalid data @@ -466,7 +473,6 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, """ Test _profit() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( @@ -485,6 +491,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) telegram._profit(bot=MagicMock(), update=update) @@ -568,7 +575,6 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: 'last': 0.1, } - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=mock_balance) mocker.patch('freqtrade.exchange.Exchange.get_ticker', side_effect=mock_ticker) @@ -581,6 +587,8 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) + telegram = Telegram(freqtradebot) telegram._balance(bot=MagicMock(), update=update) @@ -598,7 +606,6 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: """ Test _balance() method when the Exchange platform returns nothing """ - patch_get_signal(mocker, (True, False)) mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value={}) msg_mock = MagicMock() @@ -609,6 +616,8 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) + telegram = Telegram(freqtradebot) telegram._balance(bot=MagicMock(), update=update) @@ -732,7 +741,6 @@ def test_forcesell_handle(default_conf, update, ticker, fee, """ Test _forcesell() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) @@ -746,6 +754,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Create some test data @@ -785,7 +794,6 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, """ Test _forcesell() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) @@ -799,6 +807,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Create some test data @@ -842,7 +851,6 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker """ Test _forcesell() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) @@ -857,6 +865,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker ) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Create some test data @@ -891,7 +900,6 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: """ Test _forcesell() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) msg_mock = MagicMock() @@ -903,6 +911,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Trader is not running @@ -934,7 +943,6 @@ def test_performance_handle(default_conf, update, ticker, fee, """ Test _performance() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -951,6 +959,7 @@ def test_performance_handle(default_conf, update, ticker, fee, ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Create some test data @@ -976,7 +985,6 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: """ Test _performance() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -986,6 +994,7 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: ) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Trader is not running @@ -999,7 +1008,6 @@ def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> Non """ Test _count() method """ - patch_get_signal(mocker, (True, False)) patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -1016,6 +1024,7 @@ def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> Non ) mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index 2175dc9b3..d965c3f20 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -23,7 +23,7 @@ def test_default_strategy_structure(): def test_default_strategy(result): - strategy = DefaultStrategy() + strategy = DefaultStrategy({}) assert type(strategy.minimal_roi) is dict assert type(strategy.stoploss) is float diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 1e082c380..0f879a67b 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -12,14 +12,15 @@ from freqtrade.strategy.resolver import StrategyResolver def test_import_strategy(caplog): caplog.set_level(logging.DEBUG) + default_config = {} - strategy = DefaultStrategy() + strategy = DefaultStrategy(default_config) strategy.some_method = lambda *args, **kwargs: 42 assert strategy.__module__ == 'freqtrade.strategy.default_strategy' assert strategy.some_method() == 42 - imported_strategy = import_strategy(strategy) + imported_strategy = import_strategy(strategy, default_config) assert dir(strategy) == dir(imported_strategy) @@ -35,13 +36,23 @@ def test_import_strategy(caplog): def test_search_strategy(): + default_config = {} default_location = os.path.join(os.path.dirname( os.path.realpath(__file__)), '..', '..', 'strategy' ) assert isinstance( - StrategyResolver._search_strategy(default_location, 'DefaultStrategy'), IStrategy + StrategyResolver._search_strategy( + default_location, + config=default_config, + strategy_name='DefaultStrategy' + ), + IStrategy ) - assert StrategyResolver._search_strategy(default_location, 'NotFoundStrategy') is None + assert StrategyResolver._search_strategy( + default_location, + config=default_config, + strategy_name='NotFoundStrategy' + ) is None def test_load_strategy(result): @@ -70,7 +81,7 @@ def test_load_not_found_strategy(): with pytest.raises(ImportError, match=r'Impossible to load Strategy \'NotFoundStrategy\'.' r' This class does not exist or contains Python code errors'): - strategy._load_strategy('NotFoundStrategy') + strategy._load_strategy(strategy_name='NotFoundStrategy', config={}) def test_strategy(result): diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 042422c5a..577fc3553 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -10,14 +10,14 @@ from unittest.mock import MagicMock import arrow from pandas import DataFrame -from freqtrade.analyze import Analyze, parse_ticker_dataframe +from freqtrade.analyze import parse_ticker_dataframe from freqtrade.arguments import TimeRange from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.tests.conftest import get_patched_exchange, log_has from freqtrade.strategy.default_strategy import DefaultStrategy # Avoid to reinit the same object again and again -_ANALYZE = Analyze({}, DefaultStrategy()) +#_ANALYZE = Analyze({}, DefaultStrategy()) def test_dataframe_correct_length(result): @@ -30,133 +30,133 @@ def test_dataframe_correct_columns(result): ['date', 'open', 'high', 'low', 'close', 'volume'] -def test_returns_latest_buy_signal(mocker, default_conf): - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) - exchange = get_patched_exchange(mocker, default_conf) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) - ) - ) - assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) +# def test_returns_latest_buy_signal(mocker, default_conf): +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) +# exchange = get_patched_exchange(mocker, default_conf) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) +# ) +# ) +# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) - ) - ) - assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) +# ) +# ) +# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) -def test_returns_latest_sell_signal(mocker, default_conf): - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) - exchange = get_patched_exchange(mocker, default_conf) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) - ) - ) +# def test_returns_latest_sell_signal(mocker, default_conf): +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) +# exchange = get_patched_exchange(mocker, default_conf) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) +# ) +# ) - assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) +# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) - ) - ) - assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) +# ) +# ) +# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) -def test_get_signal_empty(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=None) - exchange = get_patched_exchange(mocker, default_conf) - assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) - assert log_has('Empty ticker history for pair foo', caplog.record_tuples) +# def test_get_signal_empty(default_conf, mocker, caplog): +# caplog.set_level(logging.INFO) +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=None) +# exchange = get_patched_exchange(mocker, default_conf) +# assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) +# assert log_has('Empty ticker history for pair foo', caplog.record_tuples) -def test_get_signal_exception_valueerror(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) - exchange = get_patched_exchange(mocker, default_conf) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - side_effect=ValueError('xyz') - ) - ) - assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) - assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples) +# def test_get_signal_exception_valueerror(default_conf, mocker, caplog): +# caplog.set_level(logging.INFO) +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) +# exchange = get_patched_exchange(mocker, default_conf) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# side_effect=ValueError('xyz') +# ) +# ) +# assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) +# assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples) -def test_get_signal_empty_dataframe(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) - exchange = get_patched_exchange(mocker, default_conf) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - return_value=DataFrame([]) - ) - ) - assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) - assert log_has('Empty dataframe for pair xyz', caplog.record_tuples) +# def test_get_signal_empty_dataframe(default_conf, mocker, caplog): +# caplog.set_level(logging.INFO) +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) +# exchange = get_patched_exchange(mocker, default_conf) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# return_value=DataFrame([]) +# ) +# ) +# assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) +# assert log_has('Empty dataframe for pair xyz', caplog.record_tuples) -def test_get_signal_old_dataframe(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) - exchange = get_patched_exchange(mocker, default_conf) - # default_conf defines a 5m interval. we check interval * 2 + 5m - # this is necessary as the last candle is removed (partial candles) by default - oldtime = arrow.utcnow().shift(minutes=-16) - ticks = DataFrame([{'buy': 1, 'date': oldtime}]) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - return_value=DataFrame(ticks) - ) - ) - assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) - assert log_has( - 'Outdated history for pair xyz. Last tick is 16 minutes old', - caplog.record_tuples - ) +# def test_get_signal_old_dataframe(default_conf, mocker, caplog): +# caplog.set_level(logging.INFO) +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) +# exchange = get_patched_exchange(mocker, default_conf) +# # default_conf defines a 5m interval. we check interval * 2 + 5m +# # this is necessary as the last candle is removed (partial candles) by default +# oldtime = arrow.utcnow().shift(minutes=-16) +# ticks = DataFrame([{'buy': 1, 'date': oldtime}]) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# return_value=DataFrame(ticks) +# ) +# ) +# assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) +# assert log_has( +# 'Outdated history for pair xyz. Last tick is 16 minutes old', +# caplog.record_tuples +# ) -def test_get_signal_handles_exceptions(mocker, default_conf): - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) - exchange = get_patched_exchange(mocker, default_conf) - mocker.patch.multiple( - 'freqtrade.analyze.Analyze', - analyze_ticker=MagicMock( - side_effect=Exception('invalid ticker history ') - ) - ) +# def test_get_signal_handles_exceptions(mocker, default_conf): +# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) +# exchange = get_patched_exchange(mocker, default_conf) +# mocker.patch.multiple( +# 'freqtrade.analyze.Analyze', +# analyze_ticker=MagicMock( +# side_effect=Exception('invalid ticker history ') +# ) +# ) - assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, False) +# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, False) -def test_parse_ticker_dataframe(ticker_history): - columns = ['date', 'open', 'high', 'low', 'close', 'volume'] +# def test_parse_ticker_dataframe(ticker_history): +# columns = ['date', 'open', 'high', 'low', 'close', 'volume'] - # Test file with BV data - dataframe = parse_ticker_dataframe(ticker_history) - assert dataframe.columns.tolist() == columns +# # Test file with BV data +# dataframe = parse_ticker_dataframe(ticker_history) +# assert dataframe.columns.tolist() == columns -def test_tickerdata_to_dataframe(default_conf) -> None: - """ - Test Analyze.tickerdata_to_dataframe() method - """ - analyze = Analyze(default_conf, DefaultStrategy()) +# def test_tickerdata_to_dataframe(default_conf) -> None: +# """ +# Test Analyze.tickerdata_to_dataframe() method +# """ +# analyze = Analyze(default_conf, DefaultStrategy()) - timerange = TimeRange(None, 'line', 0, -100) - tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) - tickerlist = {'UNITTEST/BTC': tick} - data = analyze.tickerdata_to_dataframe(tickerlist) - assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed +# timerange = TimeRange(None, 'line', 0, -100) +# tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) +# tickerlist = {'UNITTEST/BTC': tick} +# data = analyze.tickerdata_to_dataframe(tickerlist) +# assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index f4b26eb1d..019587af1 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -2,7 +2,6 @@ import pandas -from freqtrade.analyze import Analyze from freqtrade.optimize import load_data from freqtrade.strategy.resolver import StrategyResolver @@ -15,8 +14,7 @@ def load_dataframe_pair(pairs, strategy): assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] - analyze = Analyze({}, strategy) - dataframe = analyze.analyze_ticker(dataframe) + dataframe = strategy.analyze_ticker(dataframe) return dataframe diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index c628a9da3..b7ae96048 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -31,7 +31,6 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: :param config: Config to pass to the bot :return: None """ - mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) patch_exchange(mocker) @@ -40,17 +39,13 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: return FreqtradeBot(config) -def patch_get_signal(mocker, value=(True, False)) -> None: +def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: """ - - :param mocker: mocker to patch Analyze class - :param value: which value Analyze.get_signal() must return + :param mocker: mocker to patch IStrategy class + :param value: which value IStrategy.get_signal() must return :return: None """ - mocker.patch( - 'freqtrade.freqtradebot.Analyze.get_signal', - side_effect=lambda e, s, t: value - ) + freqtrade.strategy.get_signal = lambda e, s, t: value def patch_RPCManager(mocker) -> MagicMock: @@ -267,7 +262,6 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, """ Test get_trade_stake_amount() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -285,6 +279,7 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, conf['max_open_trades'] = 2 freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) # no open trades, order amount should be 'balance / max_open_trades' result = freqtrade._get_trade_stake_amount() @@ -452,7 +447,6 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocke """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -467,6 +461,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocke # Save state of current whitelist whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) freqtrade.create_trade() trade = Trade.query.first() @@ -490,7 +485,6 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -503,6 +497,7 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, get_markets=markets ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) with pytest.raises(DependencyException, match=r'.*stake amount.*'): freqtrade.create_trade() @@ -513,7 +508,6 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) @@ -529,6 +523,7 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, conf = deepcopy(default_conf) conf['stake_amount'] = 0.0005 freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) freqtrade.create_trade() rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2] @@ -540,7 +535,6 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) @@ -556,6 +550,7 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord conf = deepcopy(default_conf) conf['stake_amount'] = 0.000000005 freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) result = freqtrade.create_trade() assert result is False @@ -566,7 +561,6 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -583,6 +577,7 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) assert freqtrade.create_trade() is False assert freqtrade._get_trade_stake_amount() is None @@ -592,7 +587,6 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, marke """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -608,6 +602,7 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, marke conf['exchange']['pair_whitelist'] = ["ETH/BTC"] conf['exchange']['pair_blacklist'] = ["ETH/BTC"] freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) freqtrade.create_trade() @@ -620,7 +615,6 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, """ Test create_trade() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -636,6 +630,7 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, conf['exchange']['pair_whitelist'] = ["ETH/BTC"] conf['exchange']['pair_blacklist'] = ["ETH/BTC"] freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) freqtrade.create_trade() @@ -650,7 +645,6 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: conf = deepcopy(default_conf) conf['dry_run'] = True - patch_get_signal(mocker, value=(False, False)) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -664,6 +658,7 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: conf = deepcopy(default_conf) conf['stake_amount'] = 10 freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade, value=(False, False)) Trade.query = MagicMock() Trade.query.filter = MagicMock() @@ -675,7 +670,6 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, """ Test the trade creation in _process() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -688,6 +682,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert not trades @@ -716,7 +711,6 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non """ Test _process() method when a RequestException happens """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -729,6 +723,8 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + result = freqtrade._process() assert result is False assert sleep_mock.has_calls() @@ -738,7 +734,6 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) -> """ Test _process() method when an OperationalException happens """ - patch_get_signal(mocker) msg_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -749,6 +744,8 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) -> buy=MagicMock(side_effect=OperationalException) ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + assert freqtrade.state == State.RUNNING result = freqtrade._process() @@ -762,7 +759,6 @@ def test_process_trade_handling( """ Test _process() """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -775,6 +771,7 @@ def test_process_trade_handling( get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert not trades @@ -913,7 +910,6 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, """ Test check_handle() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -931,6 +927,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) freqtrade.create_trade() @@ -941,7 +938,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, trade.update(limit_buy_order) assert trade.is_open is True - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True assert trade.open_order_id == limit_sell_order['id'] @@ -962,10 +959,8 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, conf = deepcopy(default_conf) conf.update({'experimental': {'use_sell_signal': True}}) - patch_get_signal(mocker, value=(True, True)) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -976,6 +971,8 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, ) freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade, value=(True, True)) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() @@ -985,7 +982,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, assert nb_trades == 0 # Buy is triggering, so buying ... - patch_get_signal(mocker, value=(True, False)) + patch_get_signal(freqtrade, value=(True, False)) freqtrade.create_trade() trades = Trade.query.all() nb_trades = len(trades) @@ -993,7 +990,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, assert trades[0].is_open is True # Buy and Sell are not triggering, so doing nothing ... - patch_get_signal(mocker, value=(False, False)) + patch_get_signal(freqtrade, value=(False, False)) assert freqtrade.handle_trade(trades[0]) is False trades = Trade.query.all() nb_trades = len(trades) @@ -1001,7 +998,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, assert trades[0].is_open is True # Buy and Sell are triggering, so doing nothing ... - patch_get_signal(mocker, value=(True, True)) + patch_get_signal(freqtrade, value=(True, True)) assert freqtrade.handle_trade(trades[0]) is False trades = Trade.query.all() nb_trades = len(trades) @@ -1009,7 +1006,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, assert trades[0].is_open is True # Sell is triggering, guess what : we are Selling! - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) trades = Trade.query.all() assert freqtrade.handle_trade(trades[0]) is True @@ -1023,7 +1020,6 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, conf = deepcopy(default_conf) conf.update({'experimental': {'use_sell_signal': True}}) - patch_get_signal(mocker, value=(True, False)) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1035,8 +1031,10 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, get_markets=markets ) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade, value=(True, False)) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.create_trade() trade = Trade.query.first() @@ -1047,7 +1045,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, # we might just want to check if we are in a sell condition without # executing # if ROI is reached we must sell - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) assert log_has('Required profit reached. Selling..', caplog.record_tuples) @@ -1061,7 +1059,6 @@ def test_handle_trade_experimental( conf = deepcopy(default_conf) conf.update({'experimental': {'use_sell_signal': True}}) - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1072,18 +1069,19 @@ def test_handle_trade_experimental( get_fee=fee, get_markets=markets ) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() trade = Trade.query.first() trade.is_open = True - patch_get_signal(mocker, value=(False, False)) + patch_get_signal(freqtrade, value=(False, False)) assert not freqtrade.handle_trade(trade) - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) assert log_has('Sell signal received. Selling..', caplog.record_tuples) @@ -1093,7 +1091,6 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, """ Test check_handle() method """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1105,6 +1102,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, get_markets=markets ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) # Create trade and sell it freqtrade.create_trade() @@ -1345,7 +1343,6 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc """ Test execute_sell() method with a ticker going UP """ - patch_get_signal(mocker) rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1357,6 +1354,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc ) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) # Create some test data freqtrade.create_trade() @@ -1397,7 +1395,6 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, """ Test execute_sell() method with a ticker going DOWN """ - patch_get_signal(mocker) rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) @@ -1409,6 +1406,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, get_markets=markets ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) # Create some test data freqtrade.create_trade() @@ -1450,7 +1448,6 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, """ Test execute_sell() method with a ticker going DOWN and with a bot config empty """ - patch_get_signal(mocker) rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -1461,6 +1458,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, get_markets=markets ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) # Create some test data freqtrade.create_trade() @@ -1500,7 +1498,6 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, """ Test execute_sell() method with a ticker going DOWN and with a bot config empty """ - patch_get_signal(mocker) rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -1511,6 +1508,7 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, get_markets=markets ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) # Create some test data freqtrade.create_trade() @@ -1550,10 +1548,8 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, """ Test sell_profit_only feature when enabled """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1572,11 +1568,14 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, 'sell_profit_only': True, } freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True @@ -1585,10 +1584,8 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, """ Test sell_profit_only feature when disabled """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1607,11 +1604,13 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, 'sell_profit_only': False, } freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True @@ -1619,10 +1618,8 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market """ Test sell_profit_only feature when enabled and we have a loss """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.stop_loss_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1641,11 +1638,14 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market 'sell_profit_only': True, } freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.stop_loss_reached = \ + lambda current_rate, trade, current_time, current_profit: False freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is False @@ -1653,10 +1653,8 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke """ Test sell_profit_only feature when enabled and we have a loss """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1677,11 +1675,14 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke } freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True @@ -1689,10 +1690,8 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m """ Test sell_profit_only feature when enabled and we have a loss """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1712,15 +1711,18 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m } freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) - patch_get_signal(mocker, value=(True, True)) + patch_get_signal(freqtrade, value=(True, True)) assert freqtrade.handle_trade(trade) is False # Test if buy-signal is absent (should sell due to roi = true) - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True @@ -1728,10 +1730,8 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) """ Test sell_profit_only feature when enabled and we have a loss """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1748,6 +1748,9 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) conf['trailing_stop'] = True print(limit_buy_order) freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.create_trade() trade = Trade.query.first() @@ -1765,10 +1768,8 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, Test sell_profit_only feature when enabled and we have a loss """ buy_price = limit_buy_order['price'] - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1785,6 +1786,8 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, conf['trailing_stop'] = True conf['trailing_stop_positive'] = 0.01 freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() trade = Trade.query.first() @@ -1826,10 +1829,8 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, """ Test sell_profit_only feature when enabled and we have a loss """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1849,16 +1850,19 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, } freqtrade = FreqtradeBot(conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) # Sell due to min_roi_reached - patch_get_signal(mocker, value=(True, True)) + patch_get_signal(freqtrade, value=(True, True)) assert freqtrade.handle_trade(trade) is True # Test if buy-signal is absent - patch_get_signal(mocker, value=(False, True)) + patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True @@ -1869,7 +1873,6 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -1882,6 +1885,8 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1896,7 +1901,6 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -1909,6 +1913,8 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1922,7 +1928,6 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, mo """ trades_for_order[0]['fee']['currency'] = 'ETH' - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -1936,6 +1941,8 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, mo open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -1948,7 +1955,6 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock trades_for_order[0]['fee']['currency'] = 'BNB' trades_for_order[0]['fee']['cost'] = 0.00094518 - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -1962,6 +1968,8 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -1971,7 +1979,6 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c Test get_real_amount with split trades (multiple trades for this order) """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -1985,6 +1992,8 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -1999,7 +2008,6 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'} - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -2014,6 +2022,8 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004 assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' @@ -2028,7 +2038,6 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004} - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -2042,6 +2051,8 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + # Amount does not change assert freqtrade.get_real_amount(trade, limit_buy_order) == amount @@ -2053,7 +2064,6 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, # Remove "Currency" from fee dict trades_for_order[0]['fee'] = {'cost': 0.008} - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -2067,6 +2077,7 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, open_order_id="123456" ) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -2075,7 +2086,6 @@ def test_get_real_amount_open_trade(default_conf, mocker): """ Test get_real_amount condition trade.fee_open == 0 or order['status'] == 'open' """ - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -2093,4 +2103,5 @@ def test_get_real_amount_open_trade(default_conf, mocker): 'status': 'open', } freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) assert freqtrade.get_real_amount(trade, order) == amount diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 15ed1550b..1e9b5c181 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -7,7 +7,7 @@ Unit test file for misc.py import datetime from unittest.mock import MagicMock -from freqtrade.analyze import Analyze, parse_ticker_dataframe +from freqtrade.analyze import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, format_ms_time, shorten_date) from freqtrade.optimize.__init__ import load_tickerdata_file @@ -48,10 +48,10 @@ def test_common_datearray(default_conf) -> None: Test common_datearray() :return: None """ - analyze = Analyze(default_conf, DefaultStrategy()) + strategy = DefaultStrategy(default_conf) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} - dataframes = analyze.tickerdata_to_dataframe(tickerlist) + dataframes = strategy.tickerdata_to_dataframe(tickerlist) dates = common_datearray(dataframes) From 5c87c420c7d4f0f1f60367a2c85b13b8ceb62957 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 16 Jul 2018 08:59:14 +0300 Subject: [PATCH 178/395] restore one analyze test --- freqtrade/tests/test_analyze.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 577fc3553..8411fa250 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -141,12 +141,12 @@ def test_dataframe_correct_columns(result): # assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, False) -# def test_parse_ticker_dataframe(ticker_history): -# columns = ['date', 'open', 'high', 'low', 'close', 'volume'] +def test_parse_ticker_dataframe(ticker_history): + columns = ['date', 'open', 'high', 'low', 'close', 'volume'] -# # Test file with BV data -# dataframe = parse_ticker_dataframe(ticker_history) -# assert dataframe.columns.tolist() == columns + # Test file with BV data + dataframe = parse_ticker_dataframe(ticker_history) + assert dataframe.columns.tolist() == columns # def test_tickerdata_to_dataframe(default_conf) -> None: From 62f4d734b9e07b798f20256a870a239c25eaac93 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 16 Jul 2018 14:24:06 +0200 Subject: [PATCH 179/395] Update ccxt from 1.16.33 to 1.16.36 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 94c8421af..fb99e3b1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.33 +ccxt==1.16.36 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 4f957728bf4c2d35d009ff2558de6da35fd2336a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 16 Jul 2018 14:24:07 +0200 Subject: [PATCH 180/395] Update scikit-learn from 0.19.1 to 0.19.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fb99e3b1e..a2eec5d3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ requests==2.19.1 urllib3==1.22 wrapt==1.10.11 pandas==0.23.3 -scikit-learn==0.19.1 +scikit-learn==0.19.2 scipy==1.1.0 jsonschema==2.6.0 numpy==1.14.5 From 78af4bc78506e00a2b1d563db8ca8e0b44e114bb Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 10:21:53 +0300 Subject: [PATCH 181/395] move and fix tests from Analyze to interface of strategy --- freqtrade/tests/strategy/test_interface.py | 126 +++++++++++++++++++ freqtrade/tests/test_analyze.py | 137 --------------------- 2 files changed, 126 insertions(+), 137 deletions(-) create mode 100644 freqtrade/tests/strategy/test_interface.py diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py new file mode 100644 index 000000000..a016b7f96 --- /dev/null +++ b/freqtrade/tests/strategy/test_interface.py @@ -0,0 +1,126 @@ +# pragma pylint: disable=missing-docstring, C0103 + +""" +Unit test file for analyse.py +""" + +import logging +from unittest.mock import MagicMock + +import arrow +from pandas import DataFrame + +from freqtrade.arguments import TimeRange +from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.tests.conftest import get_patched_exchange, log_has +from freqtrade.strategy.default_strategy import DefaultStrategy + +# Avoid to reinit the same object again and again +_STRATEGY = DefaultStrategy(config={}) + + +def test_returns_latest_buy_signal(mocker, default_conf): + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) + ) + assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) + + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) + ) + assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) + + +def test_returns_latest_sell_signal(mocker, default_conf): + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) + ) + + assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) + + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) + ) + assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) + + +def test_get_signal_empty(default_conf, mocker, caplog): + caplog.set_level(logging.INFO) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=None) + exchange = get_patched_exchange(mocker, default_conf) + assert (False, False) == _STRATEGY.get_signal(exchange, 'foo', default_conf['ticker_interval']) + assert log_has('Empty ticker history for pair foo', caplog.record_tuples) + + +def test_get_signal_exception_valueerror(default_conf, mocker, caplog): + caplog.set_level(logging.INFO) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + side_effect=ValueError('xyz') + ) + assert (False, False) == _STRATEGY.get_signal(exchange, 'foo', default_conf['ticker_interval']) + assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples) + + +def test_get_signal_empty_dataframe(default_conf, mocker, caplog): + caplog.set_level(logging.INFO) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + return_value=DataFrame([]) + ) + assert (False, False) == _STRATEGY.get_signal(exchange, 'xyz', default_conf['ticker_interval']) + assert log_has('Empty dataframe for pair xyz', caplog.record_tuples) + + +def test_get_signal_old_dataframe(default_conf, mocker, caplog): + caplog.set_level(logging.INFO) + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) + exchange = get_patched_exchange(mocker, default_conf) + # default_conf defines a 5m interval. we check interval * 2 + 5m + # this is necessary as the last candle is removed (partial candles) by default + oldtime = arrow.utcnow().shift(minutes=-16) + ticks = DataFrame([{'buy': 1, 'date': oldtime}]) + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + return_value=DataFrame(ticks) + ) + assert (False, False) == _STRATEGY.get_signal(exchange, 'xyz', default_conf['ticker_interval']) + assert log_has( + 'Outdated history for pair xyz. Last tick is 16 minutes old', + caplog.record_tuples + ) + + +def test_get_signal_handles_exceptions(mocker, default_conf): + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch.object( + _STRATEGY, 'analyze_ticker', + side_effect=Exception('invalid ticker history ') + ) + assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (False, False) + + +def test_tickerdata_to_dataframe(default_conf) -> None: + """ + Test Analyze.tickerdata_to_dataframe() method + """ + strategy = DefaultStrategy(default_conf) + + timerange = TimeRange(None, 'line', 0, -100) + tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) + tickerlist = {'UNITTEST/BTC': tick} + data = strategy.tickerdata_to_dataframe(tickerlist) + assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 8411fa250..fdd83809c 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -4,20 +4,7 @@ Unit test file for analyse.py """ -import logging -from unittest.mock import MagicMock - -import arrow -from pandas import DataFrame - from freqtrade.analyze import parse_ticker_dataframe -from freqtrade.arguments import TimeRange -from freqtrade.optimize.__init__ import load_tickerdata_file -from freqtrade.tests.conftest import get_patched_exchange, log_has -from freqtrade.strategy.default_strategy import DefaultStrategy - -# Avoid to reinit the same object again and again -#_ANALYZE = Analyze({}, DefaultStrategy()) def test_dataframe_correct_length(result): @@ -30,133 +17,9 @@ def test_dataframe_correct_columns(result): ['date', 'open', 'high', 'low', 'close', 'volume'] -# def test_returns_latest_buy_signal(mocker, default_conf): -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) -# exchange = get_patched_exchange(mocker, default_conf) -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) -# ) -# ) -# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) - -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) -# ) -# ) -# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) - - -# def test_returns_latest_sell_signal(mocker, default_conf): -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) -# exchange = get_patched_exchange(mocker, default_conf) -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) -# ) -# ) - -# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) - -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) -# ) -# ) -# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) - - -# def test_get_signal_empty(default_conf, mocker, caplog): -# caplog.set_level(logging.INFO) -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=None) -# exchange = get_patched_exchange(mocker, default_conf) -# assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) -# assert log_has('Empty ticker history for pair foo', caplog.record_tuples) - - -# def test_get_signal_exception_valueerror(default_conf, mocker, caplog): -# caplog.set_level(logging.INFO) -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) -# exchange = get_patched_exchange(mocker, default_conf) -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# side_effect=ValueError('xyz') -# ) -# ) -# assert (False, False) == _ANALYZE.get_signal(exchange, 'foo', default_conf['ticker_interval']) -# assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples) - - -# def test_get_signal_empty_dataframe(default_conf, mocker, caplog): -# caplog.set_level(logging.INFO) -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) -# exchange = get_patched_exchange(mocker, default_conf) -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# return_value=DataFrame([]) -# ) -# ) -# assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) -# assert log_has('Empty dataframe for pair xyz', caplog.record_tuples) - - -# def test_get_signal_old_dataframe(default_conf, mocker, caplog): -# caplog.set_level(logging.INFO) -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) -# exchange = get_patched_exchange(mocker, default_conf) -# # default_conf defines a 5m interval. we check interval * 2 + 5m -# # this is necessary as the last candle is removed (partial candles) by default -# oldtime = arrow.utcnow().shift(minutes=-16) -# ticks = DataFrame([{'buy': 1, 'date': oldtime}]) -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# return_value=DataFrame(ticks) -# ) -# ) -# assert (False, False) == _ANALYZE.get_signal(exchange, 'xyz', default_conf['ticker_interval']) -# assert log_has( -# 'Outdated history for pair xyz. Last tick is 16 minutes old', -# caplog.record_tuples -# ) - - -# def test_get_signal_handles_exceptions(mocker, default_conf): -# mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) -# exchange = get_patched_exchange(mocker, default_conf) -# mocker.patch.multiple( -# 'freqtrade.analyze.Analyze', -# analyze_ticker=MagicMock( -# side_effect=Exception('invalid ticker history ') -# ) -# ) - -# assert _ANALYZE.get_signal(exchange, 'ETH/BTC', '5m') == (False, False) - - def test_parse_ticker_dataframe(ticker_history): columns = ['date', 'open', 'high', 'low', 'close', 'volume'] # Test file with BV data dataframe = parse_ticker_dataframe(ticker_history) assert dataframe.columns.tolist() == columns - - -# def test_tickerdata_to_dataframe(default_conf) -> None: -# """ -# Test Analyze.tickerdata_to_dataframe() method -# """ -# analyze = Analyze(default_conf, DefaultStrategy()) - -# timerange = TimeRange(None, 'line', 0, -100) -# tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) -# tickerlist = {'UNITTEST/BTC': tick} -# data = analyze.tickerdata_to_dataframe(tickerlist) -# assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed From dbc3874b4f41ff7d4b99cf257dc2d706f4679316 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 10:47:15 +0300 Subject: [PATCH 182/395] __init__ must return None to please mypy --- freqtrade/strategy/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index c67870aeb..e9d5b904d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -42,7 +42,7 @@ class IStrategy(ABC): stoploss: float ticker_interval: str - def __init__(self, config: dict): + def __init__(self, config: dict) -> None: self.config = config @abstractmethod From 084264669fc936ec6ba22a28d4cb5fd255fd8159 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 11:02:07 +0300 Subject: [PATCH 183/395] fix the last failing unit test --- freqtrade/tests/strategy/test_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 0f879a67b..52021475a 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -64,7 +64,7 @@ def test_load_strategy(result): def test_load_strategy_invalid_directory(result, caplog): resolver = StrategyResolver() extra_dir = os.path.join('some', 'path') - resolver._load_strategy('TestStrategy', extra_dir) + resolver._load_strategy('TestStrategy', config={}, extra_dir=extra_dir) assert ( 'freqtrade.strategy.resolver', From 06d024cc46f56a118a459537bc1e85c8adf3546c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 11:07:27 +0300 Subject: [PATCH 184/395] make pytest ignore this file --- user_data/strategies/test_strategy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 34f496e38..c04f4935f 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -12,6 +12,7 @@ import numpy # noqa # This class is a sample. Feel free to customize it. class TestStrategy(IStrategy): + __test__ = False # pytest expects to find tests here because of the name """ This is a test strategy to inspire you. More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md From e11ec2896255d9e0ac52548170a46a2316017d5a Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 11:13:35 +0300 Subject: [PATCH 185/395] remove leftover commented-out code --- freqtrade/freqtradebot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0d916f570..222e6ec96 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -50,7 +50,6 @@ class FreqtradeBot(object): # Init objects self.config = config self.strategy: IStrategy = StrategyResolver(self.config).strategy -# self.analyze = Analyze(self.config, self.strategy) self.fiat_converter = CryptoToFiatConverter() self.rpc: RPCManager = RPCManager(self) self.persistence = None From 50b15b8052b7c096e7623d681486d5c83ebf55e3 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 11:41:21 +0300 Subject: [PATCH 186/395] fix plot_dataframe to use strategy instead of Analyze --- scripts/plot_dataframe.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 3b86afc9e..11f1f85d5 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -40,11 +40,11 @@ from plotly.offline import plot import freqtrade.optimize as optimize from freqtrade import persistence -from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments, TimeRange from freqtrade.exchange import Exchange from freqtrade.optimize.backtesting import setup_configuration from freqtrade.persistence import Trade +from freqtrade.strategy.resolver import StrategyResolver logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} @@ -122,7 +122,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: # Load the strategy try: - analyze = Analyze(_CONF) + strategy = StrategyResolver(_CONF).strategy exchange = Exchange(_CONF) except AttributeError: logger.critical( @@ -132,7 +132,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: exit() # Set the ticker to use - tick_interval = analyze.get_ticker_interval() + tick_interval = strategy.ticker_interval # Load pair tickers tickers = {} @@ -156,11 +156,11 @@ def plot_analyzed_dataframe(args: Namespace) -> None: # Get trades already made from the DB trades = load_trades(args, pair, timerange) - dataframes = analyze.tickerdata_to_dataframe(tickers) + dataframes = strategy.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] - dataframe = analyze.populate_buy_trend(dataframe) - dataframe = analyze.populate_sell_trend(dataframe) + dataframe = strategy.populate_buy_trend(dataframe) + dataframe = strategy.populate_sell_trend(dataframe) if len(dataframe.index) > args.plot_limit: logger.warning('Ticker contained more than %s candles as defined ' From 4a26eb34eab381c1a7b57cbd8148439509256c6c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 11:47:09 +0300 Subject: [PATCH 187/395] fix plot_profit to use strategy instead of Analyze --- scripts/plot_profit.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 012446065..9c3468c74 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -26,9 +26,8 @@ import plotly.graph_objs as go from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration -from freqtrade.analyze import Analyze from freqtrade import constants - +from freqtrade.strategy.resolver import StrategyResolver import freqtrade.optimize as optimize import freqtrade.misc as misc @@ -87,7 +86,8 @@ def plot_profit(args: Namespace) -> None: # Init strategy try: - analyze = Analyze({'strategy': config.get('strategy')}) + strategy = StrategyResolver({'strategy': config.get('strategy')}).strategy + except AttributeError: logger.critical( 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', @@ -113,7 +113,7 @@ def plot_profit(args: Namespace) -> None: else: filter_pairs = config['exchange']['pair_whitelist'] - tick_interval = analyze.strategy.ticker_interval + tick_interval = strategy.ticker_interval pairs = config['exchange']['pair_whitelist'] if filter_pairs: @@ -127,7 +127,7 @@ def plot_profit(args: Namespace) -> None: refresh_pairs=False, timerange=timerange ) - dataframes = analyze.tickerdata_to_dataframe(tickers) + dataframes = strategy.tickerdata_to_dataframe(tickers) # NOTE: the dataframes are of unequal length, # 'dates' is an merged date array of them all. From e021d22c7fc820e8d049167bfadd763c802115ce Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 17 Jul 2018 14:24:09 +0200 Subject: [PATCH 188/395] Update ccxt from 1.16.36 to 1.16.50 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a2eec5d3f..c1ff711df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.36 +ccxt==1.16.50 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 85fd4dd3ff580e15e3b489753005000842dcc017 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 17 Jul 2018 12:12:25 +0300 Subject: [PATCH 189/395] rename analyze.py to exchange_helpers.py --- freqtrade/{analyze.py => exchange/exchange_helpers.py} | 0 freqtrade/strategy/interface.py | 2 +- freqtrade/tests/conftest.py | 6 ++---- .../{test_analyze.py => exchange/test_exchange_helpers.py} | 4 ++-- freqtrade/tests/strategy/test_default_strategy.py | 2 +- freqtrade/tests/test_misc.py | 2 +- 6 files changed, 7 insertions(+), 9 deletions(-) rename freqtrade/{analyze.py => exchange/exchange_helpers.py} (100%) rename freqtrade/tests/{test_analyze.py => exchange/test_exchange_helpers.py} (85%) diff --git a/freqtrade/analyze.py b/freqtrade/exchange/exchange_helpers.py similarity index 100% rename from freqtrade/analyze.py rename to freqtrade/exchange/exchange_helpers.py diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index e9d5b904d..fb8bcd31d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -12,7 +12,7 @@ import arrow from pandas import DataFrame from freqtrade import constants -from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.persistence import Trade diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 788ab5afb..a9ed92765 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -12,7 +12,7 @@ from jsonschema import validate from telegram import Chat, Message, Update from freqtrade import constants -from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot @@ -20,7 +20,7 @@ logging.getLogger('').setLevel(logging.INFO) def log_has(line, logs): - # caplog mocker returns log as a tuple: ('freqtrade.analyze', logging.WARNING, 'foobar') + # caplog mocker returns log as a tuple: ('freqtrade.something', logging.WARNING, 'foobar') # and we want to match line against foobar in the tuple return reduce(lambda a, b: a or b, filter(lambda x: x[2] == line, logs), @@ -51,13 +51,11 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: """ # mocker.patch('freqtrade.fiat_convert.Market', {'price_usd': 12345.0}) patch_coinmarketcap(mocker, {'price_usd': 12345.0}) -# mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) patch_exchange(mocker, None) mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) -# mocker.patch('freqtrade.freqtradebot.Analyze.get_signal', MagicMock()) return FreqtradeBot(config) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/exchange/test_exchange_helpers.py similarity index 85% rename from freqtrade/tests/test_analyze.py rename to freqtrade/tests/exchange/test_exchange_helpers.py index fdd83809c..6a3bc9eb6 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/exchange/test_exchange_helpers.py @@ -1,10 +1,10 @@ # pragma pylint: disable=missing-docstring, C0103 """ -Unit test file for analyse.py +Unit test file for exchange_helpers.py """ -from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe def test_dataframe_correct_length(result): diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index d965c3f20..37df1748f 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -3,7 +3,7 @@ import json import pytest from pandas import DataFrame -from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.strategy.default_strategy import DefaultStrategy diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 1e9b5c181..76290c6ca 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -7,7 +7,7 @@ Unit test file for misc.py import datetime from unittest.mock import MagicMock -from freqtrade.analyze import parse_ticker_dataframe +from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, format_ms_time, shorten_date) from freqtrade.optimize.__init__ import load_tickerdata_file From e17618407bc3abb75eaed957b2faf350e45088a9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Jul 2018 20:26:59 +0200 Subject: [PATCH 190/395] Rename --realistic-simulation to --enable-position-stacking --- docs/bot-usage.md | 8 +++---- freqtrade/arguments.py | 7 ++++--- freqtrade/configuration.py | 11 +++++----- freqtrade/optimize/backtesting.py | 13 ++++++------ freqtrade/optimize/hyperopt.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 22 ++++++++++---------- freqtrade/tests/test_configuration.py | 10 ++++----- 7 files changed, 38 insertions(+), 35 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 25fc78f0a..a02403160 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -127,8 +127,8 @@ optional arguments: -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) --realistic-simulation - uses max_open_trades from config to simulate real - world limitations + Disables buying the same pair multiple times to + simulate real world limitations --timerange TIMERANGE specify what timerange of data to use. -l, --live using live data @@ -173,8 +173,8 @@ optional arguments: -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) --realistic-simulation - uses max_open_trades from config to simulate real - world limitations + Disables buying the same pair multiple times to + simulate real world limitations --timerange TIMERANGE specify what timerange of data to use. -e INT, --epochs INT specify number of epochs (default: 100) -s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...], --spaces {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...] diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 731c5d88c..2c1d05070 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -178,10 +178,11 @@ class Arguments(object): type=str, ) parser.add_argument( - '--realistic-simulation', - help='uses max_open_trades from config to simulate real world limitations', + '--enable-position-stacking', + help='Allow buying the same pair twice (position stacking)', action='store_true', - dest='realistic_simulation', + dest='position_stacking', + default=False ) parser.add_argument( '--timerange', diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 582b2889c..276156b8d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -142,10 +142,11 @@ class Configuration(object): config.update({'live': True}) logger.info('Parameter -l/--live detected ...') - # If --realistic-simulation is used we add it to the configuration - if 'realistic_simulation' in self.args and self.args.realistic_simulation: - config.update({'realistic_simulation': True}) - logger.info('Parameter --realistic-simulation detected ...') + # If --enable-position-stacking is used we add it to the configuration + if 'position_stacking' in self.args and self.args.position_stacking: + config.update({'position_stacking': True}) + logger.info('Parameter --enable-position-stacking detected ...') + logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) # If --timerange is used we add it to the configuration @@ -182,7 +183,7 @@ class Configuration(object): Extract information for sys.argv and load Hyperopt configuration :return: configuration as dictionary """ - # If --realistic-simulation is used we add it to the configuration + # If --epochs is used we add it to the configuration if 'epochs' in self.args and self.args.epochs: config.update({'epochs': self.args.epochs}) logger.info('Parameter --epochs detected ...') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 05bcdf4b7..9c336db6d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -198,13 +198,13 @@ class Backtesting(object): stake_amount: btc amount to use for each trade processed: a processed dictionary with format {pair, data} max_open_trades: maximum number of concurrent trades (default: 0, disabled) - realistic: do we try to simulate realistic trades? (default: True) + position_stacking: do we allow position stacking? (default: False) :return: DataFrame """ headers = ['date', 'buy', 'open', 'close', 'sell'] processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) - realistic = args.get('realistic', False) + position_stacking = args.get('position_stacking', False) trades = [] trade_count_lock: Dict = {} for pair, pair_data in processed.items(): @@ -228,7 +228,7 @@ class Backtesting(object): if row.buy == 0 or row.sell == 1: continue # skip rows where no buy signal or that would immediately sell off - if realistic: + if not position_stacking: if lock_pair_until is not None and row.date <= lock_pair_until: continue if max_open_trades > 0: @@ -283,10 +283,11 @@ class Backtesting(object): logger.critical("No data found. Terminating.") return # Ignore max_open_trades in backtesting, except realistic flag was passed - if self.config.get('realistic_simulation', False): + # TODO: this is not position stacking!! + if self.config.get('position_stacking', False): max_open_trades = self.config['max_open_trades'] else: - logger.info('Ignoring max_open_trades (realistic_simulation not set) ...') + logger.info('Ignoring max_open_trades (position_stacking not set) ...') max_open_trades = 0 preprocessed = self.tickerdata_to_dataframe(data) @@ -306,7 +307,7 @@ class Backtesting(object): 'stake_amount': self.config.get('stake_amount'), 'processed': preprocessed, 'max_open_trades': max_open_trades, - 'realistic': self.config.get('realistic_simulation', False), + 'position_stacking': self.config.get('position_stacking', False), } ) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 72bf34eb3..73b74e6ad 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -280,7 +280,7 @@ class Hyperopt(Backtesting): { 'stake_amount': self.config['stake_amount'], 'processed': processed, - 'realistic': self.config.get('realistic_simulation', False), + 'position_stacking': self.config.get('position_stacking', False), } ) result_explanation = self.format_results(results) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 6fbf71e40..2c559d68e 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -96,7 +96,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None: 'stake_amount': config['stake_amount'], 'processed': processed, 'max_open_trades': 1, - 'realistic': True + 'position_stacking': False } ) # results :: @@ -127,7 +127,7 @@ def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): 'stake_amount': conf['stake_amount'], 'processed': backtesting.tickerdata_to_dataframe(data), 'max_open_trades': 10, - 'realistic': True, + 'position_stacking': False, 'record': record } @@ -193,8 +193,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'live' not in config assert not log_has('Parameter -l/--live detected ...', caplog.record_tuples) - assert 'realistic_simulation' not in config - assert not log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + assert 'position_stacking' not in config + assert not log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples) assert 'refresh_pairs' not in config assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) @@ -218,7 +218,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non 'backtesting', '--ticker-interval', '1m', '--live', - '--realistic-simulation', + '--enable-position-stacking', '--refresh-pairs-cached', '--timerange', ':100', '--export', '/bar/foo', @@ -246,8 +246,8 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'live' in config assert log_has('Parameter -l/--live detected ...', caplog.record_tuples) - assert 'realistic_simulation' in config - assert log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + assert 'position_stacking' in config + assert log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples) assert log_has('Using max_open_trades: 1 ...', caplog.record_tuples) assert 'refresh_pairs' in config @@ -495,7 +495,7 @@ def test_backtest(default_conf, fee, mocker) -> None: 'stake_amount': default_conf['stake_amount'], 'processed': data_processed, 'max_open_trades': 10, - 'realistic': True + 'position_stacking': False } ) assert not results.empty @@ -543,7 +543,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: 'stake_amount': default_conf['stake_amount'], 'processed': backtesting.tickerdata_to_dataframe(data), 'max_open_trades': 1, - 'realistic': True + 'position_stacking': False } ) assert not results.empty @@ -718,7 +718,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): '--ticker-interval', '1m', '--live', '--timerange', '-100', - '--realistic-simulation' + '--enable-position-stacking' ] args = get_args(args) start(args) @@ -734,7 +734,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): 'Using stake_amount: 0.001 ...', 'Downloading data for all pairs in whitelist ...', 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..', - 'Parameter --realistic-simulation detected ...' + 'Parameter --enable-position-stacking detected ...' ] for line in exists: diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index e64e1b486..d29e4e9d9 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -275,8 +275,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'live' not in config assert not log_has('Parameter -l/--live detected ...', caplog.record_tuples) - assert 'realistic_simulation' not in config - assert not log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + assert 'position_stacking' not in config + assert not log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples) assert 'refresh_pairs' not in config assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) @@ -300,7 +300,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non 'backtesting', '--ticker-interval', '1m', '--live', - '--realistic-simulation', + '--enable-position-stacking', '--refresh-pairs-cached', '--timerange', ':100', '--export', '/bar/foo' @@ -330,8 +330,8 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'live' in config assert log_has('Parameter -l/--live detected ...', caplog.record_tuples) - assert 'realistic_simulation'in config - assert log_has('Parameter --realistic-simulation detected ...', caplog.record_tuples) + assert 'position_stacking'in config + assert log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples) assert log_has('Using max_open_trades: 1 ...', caplog.record_tuples) assert 'refresh_pairs'in config From b29eed32ca2c7ee2c1116c5d018b6879f9069fc5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Jul 2018 20:29:53 +0200 Subject: [PATCH 191/395] update documentation --- docs/backtesting.md | 8 ++++---- docs/bot-usage.md | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 172969ae2..5044c9243 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -29,25 +29,25 @@ The backtesting is very easy with freqtrade. #### With 5 min tickers (Per default) ```bash -python3 ./freqtrade/main.py backtesting --realistic-simulation +python3 ./freqtrade/main.py backtesting ``` #### With 1 min tickers ```bash -python3 ./freqtrade/main.py backtesting --realistic-simulation --ticker-interval 1m +python3 ./freqtrade/main.py backtesting --ticker-interval 1m ``` #### Update cached pairs with the latest data ```bash -python3 ./freqtrade/main.py backtesting --realistic-simulation --refresh-pairs-cached +python3 ./freqtrade/main.py backtesting --refresh-pairs-cached ``` #### With live data (do not alter your testdata files) ```bash -python3 ./freqtrade/main.py backtesting --realistic-simulation --live +python3 ./freqtrade/main.py backtesting --live ``` #### Using a different on-disk ticker-data source diff --git a/docs/bot-usage.md b/docs/bot-usage.md index a02403160..1b83ff8e7 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -126,7 +126,7 @@ optional arguments: -h, --help show this help message and exit -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) - --realistic-simulation + --enable-position-tracking Disables buying the same pair multiple times to simulate real world limitations --timerange TIMERANGE @@ -164,7 +164,7 @@ To optimize your strategy, you can use hyperopt parameter hyperoptimization to find optimal parameter values for your stategy. ``` -usage: main.py hyperopt [-h] [-i TICKER_INTERVAL] [--realistic-simulation] +usage: main.py hyperopt [-h] [-i TICKER_INTERVAL] [--enable-position-tracking] [--timerange TIMERANGE] [-e INT] [-s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...]] @@ -172,7 +172,7 @@ optional arguments: -h, --help show this help message and exit -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) - --realistic-simulation + --enable-position-tracking Disables buying the same pair multiple times to simulate real world limitations --timerange TIMERANGE specify what timerange of data to use. From c82276ecbeaa056204ebbac40575f425dc98ee17 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Jul 2018 21:05:03 +0200 Subject: [PATCH 192/395] add --disable-max-market-positions --- freqtrade/arguments.py | 10 ++++++++++ freqtrade/configuration.py | 8 +++++++- freqtrade/optimize/backtesting.py | 7 +++---- freqtrade/tests/optimize/test_backtesting.py | 11 ++++++++--- freqtrade/tests/test_configuration.py | 6 +++++- 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 2c1d05070..3b10b90df 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -184,6 +184,16 @@ class Arguments(object): dest='position_stacking', default=False ) + + parser.add_argument( + '--disable-max-market-positions', + help='Disable applying `max_open_trades` during backtest ' + '(same as setting `max_open_trades` to a very high number)', + action='store_false', + dest='use_max_market_positions', + default=True + ) + parser.add_argument( '--timerange', help='specify what timerange of data to use.', diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 276156b8d..f5c1c398d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -147,7 +147,13 @@ class Configuration(object): config.update({'position_stacking': True}) logger.info('Parameter --enable-position-stacking detected ...') - logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) + # If --disable-max-market-positions is used we add it to the configuration + if 'use_max_market_positions' in self.args and not self.args.use_max_market_positions: + config.update({'use_max_market_positions': False}) + logger.info('Parameter --disable-max-market-positions detected ...') + logger.info('max_open_trades set to unlimited ...') + else: + logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) # If --timerange is used we add it to the configuration if 'timerange' in self.args and self.args.timerange: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9c336db6d..b050af8b5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -282,12 +282,11 @@ class Backtesting(object): if not data: logger.critical("No data found. Terminating.") return - # Ignore max_open_trades in backtesting, except realistic flag was passed - # TODO: this is not position stacking!! - if self.config.get('position_stacking', False): + # Use max_open_trades in backtesting, except --disable-max-market-positions is set + if self.config.get('use_max_market_positions', True): max_open_trades = self.config['max_open_trades'] else: - logger.info('Ignoring max_open_trades (position_stacking not set) ...') + logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 preprocessed = self.tickerdata_to_dataframe(data) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 2c559d68e..dc4b667df 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -219,6 +219,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--ticker-interval', '1m', '--live', '--enable-position-stacking', + '--disable-max-market-positions', '--refresh-pairs-cached', '--timerange', ':100', '--export', '/bar/foo', @@ -248,7 +249,10 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'position_stacking' in config assert log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples) - assert log_has('Using max_open_trades: 1 ...', caplog.record_tuples) + + assert 'use_max_market_positions' in config + assert log_has('Parameter --disable-max-market-positions detected ...', caplog.record_tuples) + assert log_has('max_open_trades set to unlimited ...', caplog.record_tuples) assert 'refresh_pairs' in config assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) @@ -718,7 +722,8 @@ def test_backtest_start_live(default_conf, mocker, caplog): '--ticker-interval', '1m', '--live', '--timerange', '-100', - '--enable-position-stacking' + '--enable-position-stacking', + '--disable-max-market-positions' ] args = get_args(args) start(args) @@ -727,7 +732,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): 'Parameter -i/--ticker-interval detected ...', 'Using ticker_interval: 1m ...', 'Parameter -l/--live detected ...', - 'Using max_open_trades: 1 ...', + 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Parameter --timerange detected: -100 ...', 'Using data folder: freqtrade/tests/testdata ...', 'Using stake_currency: BTC ...', diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index d29e4e9d9..da8d3bebf 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -301,6 +301,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--ticker-interval', '1m', '--live', '--enable-position-stacking', + '--disable-max-market-positions', '--refresh-pairs-cached', '--timerange', ':100', '--export', '/bar/foo' @@ -332,7 +333,10 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'position_stacking'in config assert log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples) - assert log_has('Using max_open_trades: 1 ...', caplog.record_tuples) + + assert 'use_max_market_positions' in config + assert log_has('Parameter --disable-max-market-positions detected ...', caplog.record_tuples) + assert log_has('max_open_trades set to unlimited ...', caplog.record_tuples) assert 'refresh_pairs'in config assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) From a290286fef14450c0c5f69084aad0e39242ab28f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Jul 2018 21:05:31 +0200 Subject: [PATCH 193/395] update documentation --- docs/bot-usage.md | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 1b83ff8e7..d5c76f4a4 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -117,18 +117,22 @@ python3 ./freqtrade/main.py -c config.json --db-url sqlite:///tradesv3.dry_run.s Backtesting also uses the config specified via `-c/--config`. ``` -usage: main.py backtesting [-h] [-i TICKER_INTERVAL] [--realistic-simulation] - [--timerange TIMERANGE] [-l] [-r] [--export EXPORT] - [--export-filename EXPORTFILENAME] - +usage: main.py backtesting [-h] [-i TICKER_INTERVAL] + [--enable-position-stacking] + [--disable-max-market-positions] + [--timerange TIMERANGE] [-l] [-r] + [--export EXPORT] [--export-filename PATH] optional arguments: -h, --help show this help message and exit -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) - --enable-position-tracking - Disables buying the same pair multiple times to - simulate real world limitations + --enable-position-stacking + Allow buying the same pair twice (position stacking) + --disable-max-market-positions + Disable applying `max_open_trades` during backtest + (same as setting `max_open_trades` to a very high + number) --timerange TIMERANGE specify what timerange of data to use. -l, --live using live data @@ -138,11 +142,13 @@ optional arguments: run your backtesting with up-to-date data. --export EXPORT export backtest results, argument are: trades Example --export=trades - --export-filename EXPORTFILENAME + --export-filename PATH Save backtest results to this filename requires --export to be set as well Example --export- - filename=backtest_today.json (default: backtest- - result.json + filename=user_data/backtest_data/backtest_today.json + (default: user_data/backtest_data/backtest- + result.json) + ``` ### How to use --refresh-pairs-cached parameter? @@ -164,22 +170,29 @@ To optimize your strategy, you can use hyperopt parameter hyperoptimization to find optimal parameter values for your stategy. ``` -usage: main.py hyperopt [-h] [-i TICKER_INTERVAL] [--enable-position-tracking] - [--timerange TIMERANGE] [-e INT] - [-s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...]] +usage: freqtrade hyperopt [-h] [-i TICKER_INTERVAL] + [--enable-position-stacking] + [--disable-max-market-positions] + [--timerange TIMERANGE] [-e INT] + [-s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...]] optional arguments: -h, --help show this help message and exit -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) - --enable-position-tracking - Disables buying the same pair multiple times to - simulate real world limitations - --timerange TIMERANGE specify what timerange of data to use. + --enable-position-stacking + Allow buying the same pair twice (position stacking) + --disable-max-market-positions + Disable applying `max_open_trades` during backtest + (same as setting `max_open_trades` to a very high + number) + --timerange TIMERANGE + specify what timerange of data to use. -e INT, --epochs INT specify number of epochs (default: 100) -s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...], --spaces {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...] Specify which parameters to hyperopt. Space separate list. Default: all + ``` ## A parameter missing in the configuration? From 3df79b85429c6f8bddbbcad37cc96c9eab396ecf Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Jul 2018 21:12:05 +0200 Subject: [PATCH 194/395] fix hanging intend --- freqtrade/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 3b10b90df..f315ea05b 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -188,7 +188,7 @@ class Arguments(object): parser.add_argument( '--disable-max-market-positions', help='Disable applying `max_open_trades` during backtest ' - '(same as setting `max_open_trades` to a very high number)', + '(same as setting `max_open_trades` to a very high number)', action='store_false', dest='use_max_market_positions', default=True From ee8e890f5090564ecb9a451fd4fee5872665881c Mon Sep 17 00:00:00 2001 From: Luis Felipe Diaz Chica Date: Wed, 18 Jul 2018 01:36:39 -0500 Subject: [PATCH 195/395] Add docs to get_trade_stake_amount function --- freqtrade/freqtradebot.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 72b5190b9..e0839bb1c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -248,6 +248,11 @@ class FreqtradeBot(object): return ticker['ask'] + balance * (ticker['last'] - ticker['ask']) def _get_trade_stake_amount(self) -> Optional[float]: + """ + Check if stake amount can be fulfilled with the available balance + for the stake currency + :return: float: Stake Amount + """ stake_amount = self.config['stake_amount'] avaliable_amount = self.exchange.get_balance(self.config['stake_currency']) From 08237abe20f650a093ab10dad983823ed63ef514 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 09:06:12 +0200 Subject: [PATCH 196/395] Fix wrong backtest duration identified in #1038 --- freqtrade/optimize/backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 05bcdf4b7..1ca2dacec 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -159,7 +159,7 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date).seconds // 60, + trade_duration=(sell_row.date - buy_row.date).total_seconds() // 60, open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=False, @@ -174,7 +174,7 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date).seconds // 60, + trade_duration=(sell_row.date - buy_row.date)..total_seconds() // 60, open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=True, From 8e4d2abd4e4f522bdfa33c35478c9c12d9a56123 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 09:10:17 +0200 Subject: [PATCH 197/395] Fix typo --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1ca2dacec..0605799b1 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -174,7 +174,7 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date)..total_seconds() // 60, + trade_duration=(sell_row.date - buy_row.date).total_seconds() // 60, open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=True, From f9f6a3bd04bc95e7aad3f99db6af805630af86aa Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 09:29:51 +0200 Subject: [PATCH 198/395] cast to int to keep exports constant --- freqtrade/optimize/backtesting.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0605799b1..d5039fe7d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -159,7 +159,8 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date).total_seconds() // 60, + trade_duration=int(( + sell_row.date - buy_row.date).total_seconds() // 60), open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=False, @@ -174,7 +175,8 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=(sell_row.date - buy_row.date).total_seconds() // 60, + trade_duration=int(( + sell_row.date - buy_row.date).total_seconds() // 60), open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=True, From a374f95687689f414e2fa0475532dcb3b550b7eb Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 18 Jul 2018 14:24:07 +0200 Subject: [PATCH 199/395] Update ccxt from 1.16.50 to 1.16.57 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c1ff711df..f22eae180 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.50 +ccxt==1.16.57 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 79b10304359d24dab0eb9382692640072269a573 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 20:08:55 +0200 Subject: [PATCH 200/395] output duration in a more readable way --- freqtrade/optimize/backtesting.py | 10 ++++++---- freqtrade/tests/optimize/test_backtesting.py | 9 ++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d5039fe7d..d60ce9a54 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,7 +6,7 @@ This module contains the backtesting logic import logging import operator from argparse import Namespace -from datetime import datetime +from datetime import datetime, timedelta from typing import Any, Dict, List, NamedTuple, Optional, Tuple import arrow @@ -88,7 +88,7 @@ class Backtesting(object): """ stake_currency = str(self.config.get('stake_currency')) - floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.1f') + floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.1f', '.1f', '.1f') tabular_data = [] headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] @@ -100,7 +100,8 @@ class Backtesting(object): result.profit_percent.mean() * 100.0, result.profit_percent.sum() * 100.0, result.profit_abs.sum(), - result.trade_duration.mean(), + str(timedelta( + minutes=round(result.trade_duration.mean()))) if len(result) else 'nan', len(result[result.profit_abs > 0]), len(result[result.profit_abs < 0]) ]) @@ -112,7 +113,8 @@ class Backtesting(object): results.profit_percent.mean() * 100.0, results.profit_percent.sum() * 100.0, results.profit_abs.sum(), - results.trade_duration.mean(), + str(timedelta( + minutes=round(results.trade_duration.mean()))) if len(results) else 'nan', len(results[results.profit_abs > 0]), len(results[results.profit_abs < 0]) ]) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 6fbf71e40..103137818 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -392,15 +392,14 @@ def test_generate_text_table(default_conf, mocker): result_str = ( '| pair | buy count | avg profit % | cum profit % | ' - 'total profit BTC | avg duration | profit | loss |\n' + 'total profit BTC | avg duration | profit | loss |\n' '|:--------|------------:|---------------:|---------------:|' - '-------------------:|---------------:|---------:|-------:|\n' + '-------------------:|:---------------|---------:|-------:|\n' '| ETH/BTC | 2 | 15.00 | 30.00 | ' - '0.60000000 | 20.0 | 2 | 0 |\n' + '0.60000000 | 0:20:00 | 2 | 0 |\n' '| TOTAL | 2 | 15.00 | 30.00 | ' - '0.60000000 | 20.0 | 2 | 0 |' + '0.60000000 | 0:20:00 | 2 | 0 |' ) - print(result_str) assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str From 789b98015fb0fd7a586e5ac316df521e49e1802a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 22:52:57 +0200 Subject: [PATCH 201/395] Allow different loglevels --- freqtrade/arguments.py | 8 +++----- freqtrade/configuration.py | 17 +++++++++++++++-- freqtrade/main.py | 12 +----------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 731c5d88c..b43755c6f 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -3,7 +3,6 @@ This module contains the argument manager class """ import argparse -import logging import os import re from typing import List, NamedTuple, Optional @@ -64,11 +63,10 @@ class Arguments(object): """ self.parser.add_argument( '-v', '--verbose', - help='be verbose', - action='store_const', + help='verbose mode (-vv for more, -vvv to get all messages)', + action='count', dest='loglevel', - const=logging.DEBUG, - default=logging.INFO, + default=0, ) self.parser.add_argument( '--version', diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 582b2889c..7e3cf3e69 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -12,10 +12,22 @@ from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match from freqtrade import OperationalException, constants - logger = logging.getLogger(__name__) +def set_loggers(log_level: int = 0) -> None: + """ + Set the logger level for Third party libs + :return: None + """ + + logging.getLogger('requests').setLevel(logging.INFO if log_level <= 1 else logging.DEBUG) + logging.getLogger("urllib3").setLevel(logging.INFO if log_level <= 1 else logging.DEBUG) + logging.getLogger('ccxt.base.exchange').setLevel( + logging.INFO if log_level <= 2 else logging.DEBUG) + logging.getLogger('telegram').setLevel(logging.INFO) + + class Configuration(object): """ Class to read and init the bot configuration @@ -81,9 +93,10 @@ class Configuration(object): if 'loglevel' in self.args and self.args.loglevel: config.update({'loglevel': self.args.loglevel}) logging.basicConfig( - level=config['loglevel'], + level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', ) + set_loggers(config['loglevel']) logger.info('Log level set to %s', logging.getLevelName(config['loglevel'])) # Add dynamic_whitelist if found diff --git a/freqtrade/main.py b/freqtrade/main.py index 977212faf..3ed478ec3 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -10,7 +10,7 @@ from typing import List from freqtrade import OperationalException from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration +from freqtrade.configuration import Configuration, set_loggers from freqtrade.freqtradebot import FreqtradeBot from freqtrade.state import State from freqtrade.rpc import RPCMessageType @@ -84,16 +84,6 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot: return freqtrade -def set_loggers() -> None: - """ - Set the logger level for Third party libs - :return: None - """ - logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) - logging.getLogger('ccxt.base.exchange').setLevel(logging.INFO) - logging.getLogger('telegram').setLevel(logging.INFO) - - if __name__ == '__main__': set_loggers() main(sys.argv[1:]) From 1ab7f5fb6d4cfbe445c05d7ded427ae539e0d715 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 22:53:44 +0200 Subject: [PATCH 202/395] add tests for more debug levels --- freqtrade/tests/test_arguments.py | 11 ++++---- freqtrade/tests/test_configuration.py | 38 ++++++++++++++++++++++++++- freqtrade/tests/test_main.py | 25 +++--------------- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 8a41e3379..07018c79e 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -5,7 +5,6 @@ Unit test file for arguments.py """ import argparse -import logging import pytest @@ -35,7 +34,7 @@ def test_parse_args_defaults() -> None: args = Arguments([], '').get_parsed_arg() assert args.config == 'config.json' assert args.dynamic_whitelist is None - assert args.loglevel == logging.INFO + assert args.loglevel == 0 def test_parse_args_config() -> None: @@ -53,10 +52,10 @@ def test_parse_args_db_url() -> None: def test_parse_args_verbose() -> None: args = Arguments(['-v'], '').get_parsed_arg() - assert args.loglevel == logging.DEBUG + assert args.loglevel == 1 args = Arguments(['--verbose'], '').get_parsed_arg() - assert args.loglevel == logging.DEBUG + assert args.loglevel == 1 def test_scripts_options() -> None: @@ -153,7 +152,7 @@ def test_parse_args_backtesting_custom() -> None: call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.live is True - assert call_args.loglevel == logging.INFO + assert call_args.loglevel == 0 assert call_args.subparser == 'backtesting' assert call_args.func is not None assert call_args.ticker_interval == '1m' @@ -170,7 +169,7 @@ def test_parse_args_hyperopt_custom() -> None: call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.epochs == 20 - assert call_args.loglevel == logging.INFO + assert call_args.loglevel == 0 assert call_args.subparser == 'hyperopt' assert call_args.spaces == ['buy'] assert call_args.func is not None diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index e64e1b486..5fa8e2184 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -6,6 +6,7 @@ Unit test file for configuration.py import json from argparse import Namespace from copy import deepcopy +import logging from unittest.mock import MagicMock import pytest @@ -13,7 +14,7 @@ from jsonschema import ValidationError from freqtrade import OperationalException from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration +from freqtrade.configuration import Configuration, set_loggers from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.tests.conftest import log_has @@ -402,3 +403,38 @@ def test_check_exchange(default_conf) -> None: match=r'.*Exchange "unknown_exchange" not supported.*' ): configuration.check_exchange(conf) + + +def test_set_loggers() -> None: + """ + Test set_loggers() update the logger level for third-party libraries + """ + previous_value1 = logging.getLogger('requests').level + previous_value2 = logging.getLogger('ccxt.base.exchange').level + previous_value3 = logging.getLogger('telegram').level + + set_loggers() + + value1 = logging.getLogger('requests').level + assert previous_value1 is not value1 + assert value1 is logging.INFO + + value2 = logging.getLogger('ccxt.base.exchange').level + assert previous_value2 is not value2 + assert value2 is logging.INFO + + value3 = logging.getLogger('telegram').level + assert previous_value3 is not value3 + assert value3 is logging.INFO + + set_loggers(log_level=2) + + assert logging.getLogger('requests').level is logging.DEBUG + assert logging.getLogger('ccxt.base.exchange').level is logging.INFO + assert logging.getLogger('telegram').level is logging.INFO + + set_loggers(log_level=3) + + assert logging.getLogger('requests').level is logging.DEBUG + assert logging.getLogger('ccxt.base.exchange').level is logging.DEBUG + assert logging.getLogger('telegram').level is logging.INFO diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 20a02eedc..446945a07 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -2,7 +2,6 @@ Unit test file for main.py """ -import logging from copy import deepcopy from unittest.mock import MagicMock @@ -11,7 +10,7 @@ import pytest from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.main import main, reconfigure, set_loggers +from freqtrade.main import main, reconfigure from freqtrade.state import State from freqtrade.tests.conftest import log_has, patch_exchange @@ -27,7 +26,7 @@ def test_parse_args_backtesting(mocker) -> None: call_args = backtesting_mock.call_args[0][0] assert call_args.config == 'config.json' assert call_args.live is False - assert call_args.loglevel == 20 + assert call_args.loglevel == 0 assert call_args.subparser == 'backtesting' assert call_args.func is not None assert call_args.ticker_interval is None @@ -42,29 +41,11 @@ def test_main_start_hyperopt(mocker) -> None: assert hyperopt_mock.call_count == 1 call_args = hyperopt_mock.call_args[0][0] assert call_args.config == 'config.json' - assert call_args.loglevel == 20 + assert call_args.loglevel == 0 assert call_args.subparser == 'hyperopt' assert call_args.func is not None -def test_set_loggers() -> None: - """ - Test set_loggers() update the logger level for third-party libraries - """ - previous_value1 = logging.getLogger('requests.packages.urllib3').level - previous_value2 = logging.getLogger('telegram').level - - set_loggers() - - value1 = logging.getLogger('requests.packages.urllib3').level - assert previous_value1 is not value1 - assert value1 is logging.INFO - - value2 = logging.getLogger('telegram').level - assert previous_value2 is not value2 - assert value2 is logging.INFO - - def test_main_fatal_exception(mocker, default_conf, caplog) -> None: """ Test main() function From 75c0a476f8e1c9e7a749fd6f8331398c11921eaa Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 23:36:25 +0200 Subject: [PATCH 203/395] Test setting verbosity in commandline --- freqtrade/tests/test_configuration.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 5fa8e2184..3e088195f 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -405,6 +405,25 @@ def test_check_exchange(default_conf) -> None: configuration.check_exchange(conf) +def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: + """ + Test Configuration.load_config() with cli params used + """ + + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf))) + # Prevent setting loggers + mocker.patch('freqtrade.configuration.set_loggers', MagicMock) + arglist = ['-vvv'] + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + validated_conf = configuration.load_config() + + assert validated_conf.get('loglevel') == 3 + assert log_has('Log level set to Level 3', caplog.record_tuples) + + def test_set_loggers() -> None: """ Test set_loggers() update the logger level for third-party libraries From aa69177436c42b09237a60199116c105931c7454 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Jul 2018 13:14:21 +0200 Subject: [PATCH 204/395] Properly check emptyness and adjust floatfmt --- freqtrade/optimize/backtesting.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d60ce9a54..83b543136 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -88,7 +88,7 @@ class Backtesting(object): """ stake_currency = str(self.config.get('stake_currency')) - floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.1f', '.1f', '.1f') + floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f') tabular_data = [] headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] @@ -101,7 +101,7 @@ class Backtesting(object): result.profit_percent.sum() * 100.0, result.profit_abs.sum(), str(timedelta( - minutes=round(result.trade_duration.mean()))) if len(result) else 'nan', + minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00', len(result[result.profit_abs > 0]), len(result[result.profit_abs < 0]) ]) @@ -114,7 +114,7 @@ class Backtesting(object): results.profit_percent.sum() * 100.0, results.profit_abs.sum(), str(timedelta( - minutes=round(results.trade_duration.mean()))) if len(results) else 'nan', + minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', len(results[results.profit_abs > 0]), len(results[results.profit_abs < 0]) ]) From 8f254031c655fa297aef4eb782aea790d6ba0a44 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Jul 2018 13:19:36 +0200 Subject: [PATCH 205/395] Add short form for parameters, change default for hyperopt --- freqtrade/arguments.py | 6 +++--- freqtrade/optimize/hyperopt.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index f315ea05b..16a01396b 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -178,15 +178,15 @@ class Arguments(object): type=str, ) parser.add_argument( - '--enable-position-stacking', - help='Allow buying the same pair twice (position stacking)', + '--eps', '--enable-position-stacking', + help='Allow buying the same pair multiple times (position stacking)', action='store_true', dest='position_stacking', default=False ) parser.add_argument( - '--disable-max-market-positions', + '--dmmp', '--disable-max-market-positions', help='Disable applying `max_open_trades` during backtest ' '(same as setting `max_open_trades` to a very high number)', action='store_false', diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 73b74e6ad..addfe19ee 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -280,7 +280,7 @@ class Hyperopt(Backtesting): { 'stake_amount': self.config['stake_amount'], 'processed': processed, - 'position_stacking': self.config.get('position_stacking', False), + 'position_stacking': self.config.get('position_stacking', True), } ) result_explanation = self.format_results(results) From 71100a67c952e9910320a1805318ce283ac6dd5b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Jul 2018 13:20:15 +0200 Subject: [PATCH 206/395] update documentation with new options --- docs/bot-usage.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index d5c76f4a4..4e479adac 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -117,9 +117,7 @@ python3 ./freqtrade/main.py -c config.json --db-url sqlite:///tradesv3.dry_run.s Backtesting also uses the config specified via `-c/--config`. ``` -usage: main.py backtesting [-h] [-i TICKER_INTERVAL] - [--enable-position-stacking] - [--disable-max-market-positions] +usage: main.py backtesting [-h] [-i TICKER_INTERVAL] [--eps] [--dmmp] [--timerange TIMERANGE] [-l] [-r] [--export EXPORT] [--export-filename PATH] @@ -127,9 +125,10 @@ optional arguments: -h, --help show this help message and exit -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) - --enable-position-stacking - Allow buying the same pair twice (position stacking) - --disable-max-market-positions + --eps, --enable-position-stacking + Allow buying the same pair multiple times (position + stacking) + --dmmp, --disable-max-market-positions Disable applying `max_open_trades` during backtest (same as setting `max_open_trades` to a very high number) @@ -170,9 +169,7 @@ To optimize your strategy, you can use hyperopt parameter hyperoptimization to find optimal parameter values for your stategy. ``` -usage: freqtrade hyperopt [-h] [-i TICKER_INTERVAL] - [--enable-position-stacking] - [--disable-max-market-positions] +usage: freqtrade hyperopt [-h] [-i TICKER_INTERVAL] [--eps] [--dmmp] [--timerange TIMERANGE] [-e INT] [-s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...]] @@ -180,9 +177,10 @@ optional arguments: -h, --help show this help message and exit -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL specify ticker interval (1m, 5m, 30m, 1h, 1d) - --enable-position-stacking - Allow buying the same pair twice (position stacking) - --disable-max-market-positions + --eps, --enable-position-stacking + Allow buying the same pair multiple times (position + stacking) + --dmmp, --disable-max-market-positions Disable applying `max_open_trades` during backtest (same as setting `max_open_trades` to a very high number) From c0a7725c1fdcf8dadae42ba8a7af4c2dda657f8f Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Jul 2018 21:23:35 +0200 Subject: [PATCH 207/395] Add stoploss offset --- freqtrade/constants.py | 1 + freqtrade/strategy/interface.py | 10 +++-- freqtrade/tests/test_freqtradebot.py | 66 +++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 4 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 385dac1d1..a2b69b2d7 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -63,6 +63,7 @@ CONF_SCHEMA = { 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, 'trailing_stop': {'type': 'boolean'}, 'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1}, + 'trailing_stop_positive_offset': {'type': 'number', 'minimum': 0, 'maximum': 1}, 'unfilledtimeout': { 'type': 'object', 'properties': { diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index fb8bcd31d..7cf990cf5 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -174,6 +174,7 @@ class IStrategy(ABC): """ Based on current profit of the trade and configured (trailing) stoploss, decides to sell or not + :param current_profit: current profit in percent """ trailing_stop = self.config.get('trailing_stop', False) @@ -199,13 +200,16 @@ class IStrategy(ABC): # check if we have a special stop loss for positive condition # and if profit is positive - stop_loss_value = self.stoploss - if 'trailing_stop_positive' in self.config and current_profit > 0: + stop_loss_value = self.strategy.stoploss + sl_offset = self.config.get('trailing_stop_positive_offset', 0.0) + + if 'trailing_stop_positive' in self.config and current_profit > sl_offset: # Ignore mypy error check in configuration that this is a float stop_loss_value = self.config.get('trailing_stop_positive') # type: ignore logger.debug(f"using positive stop loss mode: {stop_loss_value} " - f"since we have profit {current_profit}") + f"with offset {sl_offset:.4g} " + f"since we have profit {current_profit:.4f}%") trade.adjust_stop_loss(current_rate, stop_loss_value) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index b7ae96048..61907a321 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1805,7 +1805,71 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, })) # stop-loss not reached, adjusted stoploss assert freqtrade.handle_trade(trade) is False - assert log_has(f'using positive stop loss mode: 0.01 since we have profit 0.26662643', + assert log_has(f'using positive stop loss mode: 0.01 with offset 0 ' + f'since we have profit 0.2666%', + caplog.record_tuples) + assert log_has(f'adjusted stop loss', caplog.record_tuples) + assert trade.stop_loss == 0.0000138501 + + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(return_value={ + 'bid': buy_price + 0.000002, + 'ask': buy_price + 0.000002, + 'last': buy_price + 0.000002 + })) + # Lower price again (but still positive) + assert freqtrade.handle_trade(trade) is True + assert log_has( + f'HIT STOP: current price at {buy_price + 0.000002:.6f}, ' + f'stop loss is {trade.stop_loss:.6f}, ' + f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) + + +def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, caplog, mocker) -> None: + """ + Test sell_profit_only feature when enabled and we have a loss + """ + buy_price = limit_buy_order['price'] + patch_get_signal(mocker) + patch_RPCManager(mocker) + patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': buy_price - 0.000001, + 'ask': buy_price - 0.000001, + 'last': buy_price - 0.000001 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + + conf = deepcopy(default_conf) + conf['trailing_stop'] = True + conf['trailing_stop_positive'] = 0.01 + conf['trailing_stop_positive_offset'] = 0.011 + freqtrade = FreqtradeBot(conf) + freqtrade.create_trade() + + trade = Trade.query.first() + trade.update(limit_buy_order) + caplog.set_level(logging.DEBUG) + # stop-loss not reached + assert freqtrade.handle_trade(trade) is False + + # Raise ticker above buy price + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(return_value={ + 'bid': buy_price + 0.000003, + 'ask': buy_price + 0.000003, + 'last': buy_price + 0.000003 + })) + # stop-loss not reached, adjusted stoploss + assert freqtrade.handle_trade(trade) is False + assert log_has(f'using positive stop loss mode: 0.01 with offset 0.011 ' + f'since we have profit 0.2666%', caplog.record_tuples) assert log_has(f'adjusted stop loss', caplog.record_tuples) assert trade.stop_loss == 0.0000138501 From 6a3c8e3933a911ba7df6d9694be235a78579a78a Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Jul 2018 21:23:45 +0200 Subject: [PATCH 208/395] update docs for trailing stoploss offset --- docs/configuration.md | 1 + docs/stoploss.md | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index ddfa9834e..d4ccedf84 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,6 +27,7 @@ The table below will list all configuration parameters. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). | `trailing_stoploss_positve` | 0 | No | Changes stop-loss once profit has been reached. +| `trailing_stoploss_positve_offset` | 0 | No | Offset on when to apply `trailing_stoploss_positive`. Percentage value which should be positive. | `unfilledtimeout.buy` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. | `unfilledtimeout.sell` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. diff --git a/docs/stoploss.md b/docs/stoploss.md index db4433a02..9740672cd 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -35,14 +35,17 @@ basically what this means is that your stop loss will be adjusted to be always b ### Custom positive loss -Due to demand, it is possible to have a default stop loss, when you are in the red with your buy, but once your buy turns positive, -the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you are in the -black, it will be changed to be only a 1% stop loss +Due to demand, it is possible to have a default stop loss, when you are in the red with your buy, but once your profit surpasses a certain percentage, +the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you have 1.1% profit, +it will be changed to be only a 1% stop loss, which trails the green candles until it goes below them. -This can be configured in the main configuration file and requires `"trailing_stop": true` to be set to true. +Both values can be configured in the main configuration file and requires `"trailing_stop": true` to be set to true. ``` json "trailing_stop_positive": 0.01, + "trailing_stop_positive_offset": 0.011, ``` -The 0.01 would translate to a 1% stop loss, once you hit profit. +The 0.01 would translate to a 1% stop loss, once you hit 1.1% profit. + +You should also make sure to have this value higher than your minimal ROI, otherwise minimal ROI will apply first and sell your trade. From 365ba98131c57a90280f8bf615e18d79c67eee2f Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Jul 2018 21:30:50 +0200 Subject: [PATCH 209/395] add option to full_json example --- config_full.json.example | 1 + 1 file changed, 1 insertion(+) diff --git a/config_full.json.example b/config_full.json.example index 4003b1c5c..b0714535f 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -7,6 +7,7 @@ "ticker_interval": "5m", "trailing_stop": false, "trailing_stop_positive": 0.005, + "trailing_stop_positive_offset": 0.0051, "minimal_roi": { "40": 0.0, "30": 0.01, From 6bb7167b56d525f9c68ad1bbc0b154d4fe2a695f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Jul 2018 19:22:34 +0200 Subject: [PATCH 210/395] Add sellType enum --- freqtrade/strategy/interface.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index fb8bcd31d..811f3232e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -27,6 +27,16 @@ class SignalType(Enum): SELL = "sell" +class SellType(Enum): + """ + Enum to distinguish between sell reasons + """ + ROI = "roi" + STOP_LOSS = "stop_loss" + TRAILING_STOP_LOSS = "trailing_stop_loss" + SELL_SIGNAL = "sell_signal" + + class IStrategy(ABC): """ Interface for freqtrade strategies From f991109b0a35df2423b35cf27b55a5020d503b71 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Jul 2018 19:57:01 +0200 Subject: [PATCH 211/395] Add sell-reason to sell-tree --- freqtrade/freqtradebot.py | 10 +++++++--- freqtrade/persistence.py | 6 ++++-- freqtrade/rpc/rpc.py | 3 ++- freqtrade/strategy/interface.py | 32 ++++++++++++++++++-------------- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 35c0a1705..65dab15dd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -20,6 +20,7 @@ from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State +from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver logger = logging.getLogger(__name__) @@ -505,8 +506,9 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(self.exchange, trade.pair, self.strategy.ticker_interval) - if self.strategy.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): - self.execute_sell(trade, current_rate) + should_sell = self.strategy.should_sell(trade, current_rate, datetime.utcnow(), buy, sell) + if should_sell[0]: + self.execute_sell(trade, current_rate, should_sell[1]) return True logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False @@ -607,17 +609,19 @@ class FreqtradeBot(object): # TODO: figure out how to handle partially complete sell orders return False - def execute_sell(self, trade: Trade, limit: float) -> None: + def execute_sell(self, trade: Trade, limit: float, sellreason: SellType) -> None: """ Executes a limit sell for the given trade and limit :param trade: Trade instance :param limit: limit rate for the sell order + :param sellrason: Reaseon the sell was triggered :return: None """ # Execute sell and update trade record order_id = self.exchange.sell(str(trade.pair), limit, trade.amount)['id'] trade.open_order_id = order_id trade.close_rate_requested = limit + trade.sell_reason = sellreason.value profit_trade = trade.calc_profit(rate=limit) current_rate = self.exchange.get_ticker(trade.pair)['bid'] diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 0e0b22e82..086e8c2ea 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -88,6 +88,7 @@ def check_migrate(engine) -> None: stop_loss = get_column_def(cols, 'stop_loss', '0.0') initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') max_rate = get_column_def(cols, 'max_rate', '0.0') + sell_reason = get_column_def(cols, 'sell_reason', 'null') # Schema migration necessary engine.execute(f"alter table trades rename to {table_back_name}") @@ -99,7 +100,7 @@ def check_migrate(engine) -> None: (id, exchange, pair, is_open, fee_open, fee_close, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id, - stop_loss, initial_stop_loss, max_rate + stop_loss, initial_stop_loss, max_rate, sell_reason ) select id, lower(exchange), case @@ -114,7 +115,7 @@ def check_migrate(engine) -> None: {close_rate_requested} close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id, {stop_loss} stop_loss, {initial_stop_loss} initial_stop_loss, - {max_rate} max_rate + {max_rate} max_rate, {sell_reason} sell_reason from {table_back_name} """) @@ -170,6 +171,7 @@ class Trade(_DECL_BASE): initial_stop_loss = Column(Float, nullable=True, default=0.0) # absolute value of the highest reached price max_rate = Column(Float, nullable=True, default=0.0) + sell_reason = Column(String, nullable=True) def __repr__(self): open_since = arrow.get(self.open_date).humanize() if self.is_open else 'closed' diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 9411e983b..96556df78 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -13,6 +13,7 @@ import sqlalchemy as sql from numpy import mean, nan_to_num from pandas import DataFrame +from freqtrade.analyze import SellType from freqtrade.misc import shorten_date from freqtrade.persistence import Trade from freqtrade.state import State @@ -344,7 +345,7 @@ class RPC(object): # Get current rate and execute sell current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] - self._freqtrade.execute_sell(trade, current_rate) + self._freqtrade.execute_sell(trade, current_rate, SellType.FORCE_SELL) # ---- EOF def _exec_forcesell ---- if self._freqtrade.state != State.RUNNING: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 811f3232e..c5d58dd46 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -6,7 +6,7 @@ import logging from abc import ABC, abstractmethod from datetime import datetime from enum import Enum -from typing import Dict, List, Tuple +from typing import Dict, List, Tuple, Optional import arrow from pandas import DataFrame @@ -35,6 +35,7 @@ class SellType(Enum): STOP_LOSS = "stop_loss" TRAILING_STOP_LOSS = "trailing_stop_loss" SELL_SIGNAL = "sell_signal" + FORCE_SELL = "force_sell" class IStrategy(ABC): @@ -147,40 +148,42 @@ class IStrategy(ABC): ) return buy, sell - def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, sell: bool) -> bool: + def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, + sell: bool) -> Tuple[bool, Optional[SellType]]: """ This function evaluate if on the condition required to trigger a sell has been reached if the threshold is reached and updates the trade record. :return: True if trade should be sold, False otherwise """ current_profit = trade.calc_profit_percent(rate) - if self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, - current_profit=current_profit): - return True + stoplossflag = self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, + current_profit=current_profit) + if stoplossflag[0]: + return (True, stoplossflag[1]) experimental = self.config.get('experimental', {}) if buy and experimental.get('ignore_roi_if_buy_signal', False): logger.debug('Buy signal still active - not selling.') - return False + return (False, None) # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) if self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date): logger.debug('Required profit reached. Selling..') - return True + return (True, SellType.ROI) if experimental.get('sell_profit_only', False): logger.debug('Checking if trade is profitable..') if trade.calc_profit(rate=rate) <= 0: - return False + return (False, None) if sell and not buy and experimental.get('use_sell_signal', False): logger.debug('Sell signal received. Selling..') - return True + return (True, SellType.SELL_SIGNAL) - return False + return (False, None) def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, - current_profit: float) -> bool: + current_profit: float) -> Tuple[bool, Optional[SellType]]: """ Based on current profit of the trade and configured (trailing) stoploss, decides to sell or not @@ -192,8 +195,9 @@ class IStrategy(ABC): # evaluate if the stoploss was hit if self.stoploss is not None and trade.stop_loss >= current_rate: - + selltype = SellType.STOP_LOSS if trailing_stop: + selltype = SellType.TRAILING_STOP_LOSS logger.debug( f"HIT STOP: current price at {current_rate:.6f}, " f"stop loss is {trade.stop_loss:.6f}, " @@ -202,7 +206,7 @@ class IStrategy(ABC): logger.debug(f"trailing stop saved {trade.stop_loss - trade.initial_stop_loss:.6f}") logger.debug('Stop loss hit.') - return True + return (True, selltype) # update the stop loss afterwards, after all by definition it's supposed to be hanging if trailing_stop: @@ -219,7 +223,7 @@ class IStrategy(ABC): trade.adjust_stop_loss(current_rate, stop_loss_value) - return False + return (False, None) def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool: """ From f2bfc9ccc2939fa262b4812a98f7f8e8a45c3263 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 19 Jul 2018 14:24:07 +0200 Subject: [PATCH 212/395] Update ccxt from 1.16.57 to 1.16.68 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f22eae180..576185f06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.57 +ccxt==1.16.68 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 49a7c7f08e4e0f6d4cb522dd9538430d6aaabb76 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Jul 2018 19:57:20 +0200 Subject: [PATCH 213/395] fix tests --- freqtrade/tests/test_freqtradebot.py | 11 +++++++---- freqtrade/tests/test_persistence.py | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index b7ae96048..d37b2aaf6 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -16,6 +16,7 @@ import requests from freqtrade import (DependencyException, OperationalException, TemporaryError, constants) +from freqtrade.analyze import SellType from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType @@ -1369,7 +1370,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc get_ticker=ticker_sell_up ) - freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid']) + freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sellreason=SellType.ROI) assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] @@ -1421,7 +1422,8 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, get_ticker=ticker_sell_down ) - freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid']) + freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'], + sellreason=SellType.STOP_LOSS) assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] @@ -1474,7 +1476,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, ) freqtrade.config = {} - freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid']) + freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sellreason=SellType.ROI) assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] @@ -1524,7 +1526,8 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, ) freqtrade.config = {} - freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid']) + freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'], + sellreason=SellType.STOP_LOSS) assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index b24f2dd6c..d2a736f73 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -465,6 +465,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert trade.max_rate == 0.0 assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 + assert trade.sell_reason is None assert log_has("trying trades_bak1", caplog.record_tuples) assert log_has("trying trades_bak2", caplog.record_tuples) From 0147b1631aaa78e25037f7e6378325aaec56a0b4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Jul 2018 19:59:30 +0200 Subject: [PATCH 214/395] remove optional from selltype --- freqtrade/strategy/interface.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index c5d58dd46..cb8b8873d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -36,6 +36,7 @@ class SellType(Enum): TRAILING_STOP_LOSS = "trailing_stop_loss" SELL_SIGNAL = "sell_signal" FORCE_SELL = "force_sell" + NONE = "" class IStrategy(ABC): @@ -149,7 +150,7 @@ class IStrategy(ABC): return buy, sell def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, - sell: bool) -> Tuple[bool, Optional[SellType]]: + sell: bool) -> Tuple[bool, SellType]: """ This function evaluate if on the condition required to trigger a sell has been reached if the threshold is reached and updates the trade record. @@ -165,7 +166,7 @@ class IStrategy(ABC): if buy and experimental.get('ignore_roi_if_buy_signal', False): logger.debug('Buy signal still active - not selling.') - return (False, None) + return (False, SellType.NONE) # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) if self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date): @@ -175,15 +176,15 @@ class IStrategy(ABC): if experimental.get('sell_profit_only', False): logger.debug('Checking if trade is profitable..') if trade.calc_profit(rate=rate) <= 0: - return (False, None) + return (False, SellType.NONE) if sell and not buy and experimental.get('use_sell_signal', False): logger.debug('Sell signal received. Selling..') return (True, SellType.SELL_SIGNAL) - return (False, None) + return (False, SellType.NONE) def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, - current_profit: float) -> Tuple[bool, Optional[SellType]]: + current_profit: float) -> Tuple[bool, SellType]: """ Based on current profit of the trade and configured (trailing) stoploss, decides to sell or not @@ -223,7 +224,7 @@ class IStrategy(ABC): trade.adjust_stop_loss(current_rate, stop_loss_value) - return (False, None) + return (False, SellType.NONE) def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool: """ From cbffd3650b85f93ecc7fca1613102285e3e4b7dd Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Jul 2018 20:03:40 +0200 Subject: [PATCH 215/395] add sell_reason to backtesting --- freqtrade/optimize/backtesting.py | 13 +++++++++---- freqtrade/tests/optimize/test_backtesting.py | 5 ++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9c124f35b..da73b7648 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -20,6 +20,7 @@ from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade +from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver logger = logging.getLogger(__name__) @@ -40,6 +41,7 @@ class BacktestResult(NamedTuple): open_at_end: bool open_rate: float close_rate: float + sell_reason: SellType class Backtesting(object): @@ -151,8 +153,9 @@ class Backtesting(object): trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1 buy_signal = sell_row.buy - if self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, - sell_row.sell): + sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, + sell_row.sell) + if sell[0]: return BacktestResult(pair=pair, profit_percent=trade.calc_profit_percent(rate=sell_row.open), @@ -164,7 +167,8 @@ class Backtesting(object): close_index=sell_row.Index, open_at_end=False, open_rate=buy_row.open, - close_rate=sell_row.open + close_rate=sell_row.open, + sell_reason=sell[1] ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period @@ -179,7 +183,8 @@ class Backtesting(object): close_index=sell_row.Index, open_at_end=True, open_rate=buy_row.open, - close_rate=sell_row.open + close_rate=sell_row.open, + sell_reason=SellType.FORCE_SELL ) logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair, btr.profit_percent, btr.profit_abs) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 6e4f91891..6d1f7391d 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -17,6 +17,7 @@ from freqtrade.arguments import Arguments, TimeRange from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, start) from freqtrade.tests.conftest import log_has, patch_exchange +from freqtrade.strategy.interface import SellType from freqtrade.strategy.default_strategy import DefaultStrategy @@ -511,7 +512,9 @@ def test_backtest(default_conf, fee, mocker) -> None: 'trade_duration': [240, 50], 'open_at_end': [False, False], 'open_rate': [0.104445, 0.10302485], - 'close_rate': [0.105, 0.10359999]}) + 'close_rate': [0.105, 0.10359999], + 'sell_reason': [SellType.ROI, SellType.ROI] + }) pd.testing.assert_frame_equal(results, expected) data_pair = data_processed[pair] for _, t in results.iterrows(): From 838b0e7b76c3d02ccf4c82528216645706fad19f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Jul 2018 20:12:27 +0200 Subject: [PATCH 216/395] Remove unused import --- freqtrade/strategy/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index cb8b8873d..dde1df614 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -6,7 +6,7 @@ import logging from abc import ABC, abstractmethod from datetime import datetime from enum import Enum -from typing import Dict, List, Tuple, Optional +from typing import Dict, List, Tuple import arrow from pandas import DataFrame From 8c0b19f80c67040181045af52c48e0a1259e7f15 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Jul 2018 20:20:44 +0200 Subject: [PATCH 217/395] Check sell-reason for sell-reason-specific tests --- freqtrade/tests/test_freqtradebot.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index d37b2aaf6..7ac421bbb 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1580,6 +1580,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, trade.update(limit_buy_order) patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True + assert trade.sell_reason == SellType.SELL_SIGNAL.value def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, @@ -1615,6 +1616,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, trade.update(limit_buy_order) patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True + assert trade.sell_reason == SellType.SELL_SIGNAL.value def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, markets, mocker) -> None: @@ -1687,6 +1689,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke trade.update(limit_buy_order) patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True + assert trade.sell_reason == SellType.SELL_SIGNAL.value def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: @@ -1727,6 +1730,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m # Test if buy-signal is absent (should sell due to roi = true) patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True + assert trade.sell_reason == SellType.ROI.value def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) -> None: @@ -1764,6 +1768,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) assert log_has( f'HIT STOP: current price at 0.000001, stop loss is {trade.stop_loss:.6f}, ' f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) + assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, mocker) -> None: @@ -1825,6 +1830,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, f'HIT STOP: current price at {buy_price + 0.000002:.6f}, ' f'stop loss is {trade.stop_loss:.6f}, ' f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) + assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, @@ -1867,6 +1873,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, # Test if buy-signal is absent patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True + assert trade.sell_reason == SellType.STOP_LOSS.value def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, caplog, mocker): From 2a6162901449e063e93b828f2b3afb400c5a97bf Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Jul 2018 20:32:56 +0200 Subject: [PATCH 218/395] Export sell_reason from backtest --- freqtrade/optimize/backtesting.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index da73b7648..46b2aac19 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -124,7 +124,7 @@ class Backtesting(object): records = [(t.pair, t.profit_percent, t.open_time.timestamp(), t.close_time.timestamp(), t.open_index - 1, t.trade_duration, - t.open_rate, t.close_rate, t.open_at_end) + t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value) for index, t in results.iterrows()] if records: diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 6d1f7391d..c71930782 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -660,7 +660,9 @@ def test_backtest_record(default_conf, fee, mocker): "open_index": [1, 119, 153, 185], "close_index": [118, 151, 184, 199], "trade_duration": [123, 34, 31, 14], - "open_at_end": [False, False, False, True] + "open_at_end": [False, False, False, True], + "sell_reason": [SellType.ROI, SellType.STOP_LOSS, + SellType.ROI, SellType.FORCE_SELL] }) backtesting._store_backtest_result("backtest-result.json", results) assert len(results) == 4 @@ -673,7 +675,7 @@ def test_backtest_record(default_conf, fee, mocker): # Below follows just a typecheck of the schema/type of trade-records oix = None for (pair, profit, date_buy, date_sell, buy_index, dur, - openr, closer, open_at_end) in records: + openr, closer, open_at_end, sell_reason) in records: assert pair == 'UNITTEST/BTC' assert isinstance(profit, float) # FIX: buy/sell should be converted to ints @@ -682,6 +684,7 @@ def test_backtest_record(default_conf, fee, mocker): assert isinstance(openr, float) assert isinstance(closer, float) assert isinstance(open_at_end, bool) + assert isinstance(sell_reason, str) isinstance(buy_index, pd._libs.tslib.Timestamp) if oix: assert buy_index > oix From 4059871c286275afe97dc4d0931d90a828379cc2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Jul 2018 20:38:14 +0200 Subject: [PATCH 219/395] Add get_strategy_name --- freqtrade/strategy/interface.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index dde1df614..5d1cecdb3 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -80,6 +80,13 @@ class IStrategy(ABC): :param dataframe: DataFrame :return: DataFrame with sell column """ + return self.strategy.populate_sell_trend(dataframe=dataframe) + + def get_strategy_name(self) -> str: + """ + Returns strategy class name + """ + return self.strategy.__class__.__name__ def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: """ From 426c25f63105befaf9fef999188dc6c936a90c71 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Jul 2018 20:38:57 +0200 Subject: [PATCH 220/395] record ticker_interval and strategyname --- freqtrade/freqtradebot.py | 5 +++-- freqtrade/persistence.py | 10 ++++++++-- freqtrade/tests/test_persistence.py | 2 ++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 65dab15dd..8c71119ae 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -54,7 +54,6 @@ class FreqtradeBot(object): self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = Exchange(self.config) - self._init_modules() def _init_modules(self) -> None: @@ -393,7 +392,9 @@ class FreqtradeBot(object): open_rate_requested=buy_limit, open_date=datetime.utcnow(), exchange=self.exchange.id, - open_order_id=order_id + open_order_id=order_id, + strategy=self.analyze.get_strategy_name(), + ticker_interval=constants.TICKER_INTERVAL_MINUTES[self.analyze.get_ticker_interval()] ) Trade.session.add(trade) Trade.session.flush() diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 086e8c2ea..7544ca8db 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -89,6 +89,8 @@ def check_migrate(engine) -> None: initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') max_rate = get_column_def(cols, 'max_rate', '0.0') sell_reason = get_column_def(cols, 'sell_reason', 'null') + strategy = get_column_def(cols, 'strategy', 'null') + ticker_interval = get_column_def(cols, 'ticker_interval', 'null') # Schema migration necessary engine.execute(f"alter table trades rename to {table_back_name}") @@ -100,7 +102,8 @@ def check_migrate(engine) -> None: (id, exchange, pair, is_open, fee_open, fee_close, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id, - stop_loss, initial_stop_loss, max_rate, sell_reason + stop_loss, initial_stop_loss, max_rate, sell_reason, strategy, + ticker_interval ) select id, lower(exchange), case @@ -115,7 +118,8 @@ def check_migrate(engine) -> None: {close_rate_requested} close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id, {stop_loss} stop_loss, {initial_stop_loss} initial_stop_loss, - {max_rate} max_rate, {sell_reason} sell_reason + {max_rate} max_rate, {sell_reason} sell_reason, {strategy} strategy, + {ticker_interval} ticker_interval from {table_back_name} """) @@ -172,6 +176,8 @@ class Trade(_DECL_BASE): # absolute value of the highest reached price max_rate = Column(Float, nullable=True, default=0.0) sell_reason = Column(String, nullable=True) + strategy = Column(String, nullable=True) + ticker_interval = Column(Integer, nullable=True) def __repr__(self): open_since = arrow.get(self.open_date).humanize() if self.is_open else 'closed' diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index d2a736f73..bb6747739 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -466,6 +466,8 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 assert trade.sell_reason is None + assert trade.strategy is None + assert trade.ticker_interval is None assert log_has("trying trades_bak1", caplog.record_tuples) assert log_has("trying trades_bak2", caplog.record_tuples) From 506aa0e3d355873b8e0a2e3d19a70f0eb34414e3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Jul 2018 21:19:43 +0200 Subject: [PATCH 221/395] Add print_sales table and test --- freqtrade/optimize/backtesting.py | 34 ++++++++++++++++---- freqtrade/tests/optimize/test_backtesting.py | 29 +++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 46b2aac19..3f867df22 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -120,6 +120,16 @@ class Backtesting(object): ]) return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") + def _generate_text_table_sell_reason(self, data: Dict[str, Dict], results: DataFrame) -> str: + """ + Generate small table outlining Backtest results + """ + tabular_data = [] + headers = ['Sell Reason', 'Count'] + for reason, count in results['sell_reason'].value_counts().iteritems(): + tabular_data.append([reason.value, count]) + return tabulate(tabular_data, headers=headers, tablefmt="pipe") + def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: records = [(t.pair, t.profit_percent, t.open_time.timestamp(), @@ -319,20 +329,32 @@ class Backtesting(object): self._store_backtest_result(self.config.get('exportfilename'), results) logger.info( - '\n================================================= ' - 'BACKTESTING REPORT' - ' ==================================================\n' + '\n' + '=' * 49 + + ' BACKTESTING REPORT ' + + '=' * 50 + '\n' '%s', self._generate_text_table( data, results ) ) + # logger.info( + # results[['sell_reason']].groupby('sell_reason').count() + # ) logger.info( - '\n=============================================== ' - 'LEFT OPEN TRADES REPORT' - ' ===============================================\n' + '\n' + '=' * 4 + + ' SELL READON STATS ' + + '=' * 4 + '\n' + '%s \n', + self._generate_text_table_sell_reason(data, results) + + ) + + logger.info( + '\n' + '=' * 47 + + ' LEFT OPEN TRADES REPORT ' + + '=' * 47 + '\n' '%s', self._generate_text_table( data, diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index c71930782..d3fc233ce 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -404,6 +404,35 @@ def test_generate_text_table(default_conf, mocker): assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str +def test_generate_text_table_sell_reason(default_conf, mocker): + """ + Test Backtesting.generate_text_table_sell_reason() method + """ + patch_exchange(mocker) + backtesting = Backtesting(default_conf) + + results = pd.DataFrame( + { + 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], + 'profit_percent': [0.1, 0.2, 0.3], + 'profit_abs': [0.2, 0.4, 0.5], + 'trade_duration': [10, 30, 10], + 'profit': [2, 0, 0], + 'loss': [0, 0, 1], + 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] + } + ) + + result_str = ( + '| Sell Reason | Count |\n' + '|:--------------|--------:|\n' + '| roi | 2 |\n' + '| stop_loss | 1 |' + ) + assert backtesting._generate_text_table_sell_reason( + data={'ETH/BTC': {}}, results=results) == result_str + + def test_backtesting_start(default_conf, mocker, caplog) -> None: """ Test Backtesting.start() method From ad98c62329457ba441dcaaccefa975456d8e4e38 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Jul 2018 21:37:54 +0200 Subject: [PATCH 222/395] update backtest anlaysis cheatsheet --- docs/backtesting.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 172969ae2..0a7675848 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -83,7 +83,7 @@ with filename.open() as file: data = json.load(file) columns = ["pair", "profit", "opents", "closets", "index", "duration", - "open_rate", "close_rate", "open_at_end"] + "open_rate", "close_rate", "open_at_end", "sell_reason"] df = pd.DataFrame(data, columns=columns) df['opents'] = pd.to_datetime(df['opents'], @@ -98,6 +98,8 @@ df['closets'] = pd.to_datetime(df['closets'], ) ``` +If you have some ideas for interresting / helpfull backtest data analysis, feel free to submit a PR so the community can benefit from it. + #### Exporting trades to file specifying a custom filename ```bash From a452864b419733292e321e0311989e01fc4e057b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 12 Jul 2018 22:21:52 +0200 Subject: [PATCH 223/395] Use namedtuple for sell_return --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/optimize/backtesting.py | 6 +++--- freqtrade/rpc/rpc.py | 2 +- freqtrade/strategy/interface.py | 32 +++++++++++++++++----------- freqtrade/tests/test_freqtradebot.py | 4 +++- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8c71119ae..566b0670f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -508,8 +508,8 @@ class FreqtradeBot(object): trade.pair, self.strategy.ticker_interval) should_sell = self.strategy.should_sell(trade, current_rate, datetime.utcnow(), buy, sell) - if should_sell[0]: - self.execute_sell(trade, current_rate, should_sell[1]) + if should_sell.sell_flag: + self.execute_sell(trade, current_rate, should_sell.sell_type) return True logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3f867df22..391e05b83 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -164,8 +164,8 @@ class Backtesting(object): buy_signal = sell_row.buy sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, - sell_row.sell) - if sell[0]: + sell_row.sell) + if sell.sell_flag: return BacktestResult(pair=pair, profit_percent=trade.calc_profit_percent(rate=sell_row.open), @@ -178,7 +178,7 @@ class Backtesting(object): open_at_end=False, open_rate=buy_row.open, close_rate=sell_row.open, - sell_reason=sell[1] + sell_reason=sell.sell_type ) if partial_ticker: # no sell condition found - trade stil open at end of backtest period diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 96556df78..27ec7ea7a 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -13,10 +13,10 @@ import sqlalchemy as sql from numpy import mean, nan_to_num from pandas import DataFrame -from freqtrade.analyze import SellType from freqtrade.misc import shorten_date from freqtrade.persistence import Trade from freqtrade.state import State +from freqtrade.strategy.interface import SellType logger = logging.getLogger(__name__) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 5d1cecdb3..7b8d91010 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -6,7 +6,7 @@ import logging from abc import ABC, abstractmethod from datetime import datetime from enum import Enum -from typing import Dict, List, Tuple +from typing import Dict, List, NamedTuple, Tuple import arrow from pandas import DataFrame @@ -39,6 +39,14 @@ class SellType(Enum): NONE = "" +class SellCheckTuple(NamedTuple): + """ + NamedTuple for Sell type + reason + """ + sell_flag: bool + sell_type: SellType + + class IStrategy(ABC): """ Interface for freqtrade strategies @@ -157,7 +165,7 @@ class IStrategy(ABC): return buy, sell def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, - sell: bool) -> Tuple[bool, SellType]: + sell: bool) -> SellCheckTuple: """ This function evaluate if on the condition required to trigger a sell has been reached if the threshold is reached and updates the trade record. @@ -166,32 +174,32 @@ class IStrategy(ABC): current_profit = trade.calc_profit_percent(rate) stoplossflag = self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, current_profit=current_profit) - if stoplossflag[0]: - return (True, stoplossflag[1]) + if stoplossflag.sell_flag: + return stoplossflag experimental = self.config.get('experimental', {}) if buy and experimental.get('ignore_roi_if_buy_signal', False): logger.debug('Buy signal still active - not selling.') - return (False, SellType.NONE) + return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) if self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date): logger.debug('Required profit reached. Selling..') - return (True, SellType.ROI) + return SellCheckTuple(sell_flag=True, sell_type=SellType.ROI) if experimental.get('sell_profit_only', False): logger.debug('Checking if trade is profitable..') if trade.calc_profit(rate=rate) <= 0: - return (False, SellType.NONE) + return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) if sell and not buy and experimental.get('use_sell_signal', False): logger.debug('Sell signal received. Selling..') - return (True, SellType.SELL_SIGNAL) + return SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL) - return (False, SellType.NONE) + return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, - current_profit: float) -> Tuple[bool, SellType]: + current_profit: float) -> SellCheckTuple: """ Based on current profit of the trade and configured (trailing) stoploss, decides to sell or not @@ -214,7 +222,7 @@ class IStrategy(ABC): logger.debug(f"trailing stop saved {trade.stop_loss - trade.initial_stop_loss:.6f}") logger.debug('Stop loss hit.') - return (True, selltype) + return SellCheckTuple(sell_flag=True, sell_type=selltype) # update the stop loss afterwards, after all by definition it's supposed to be hanging if trailing_stop: @@ -231,7 +239,7 @@ class IStrategy(ABC): trade.adjust_stop_loss(current_rate, stop_loss_value) - return (False, SellType.NONE) + return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool: """ diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 7ac421bbb..65cd99689 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -16,7 +16,7 @@ import requests from freqtrade import (DependencyException, OperationalException, TemporaryError, constants) -from freqtrade.analyze import SellType +from freqtrade.strategy.interface.IStrategy import SellType, SellCheckTuple from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType @@ -1625,6 +1625,8 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) + mocker.patch('freqtrade.freqtradebot.strategy.interface.stop_loss_reached', + return_value=SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), From 760c79c5e95bb3eeec23c366d986f97a8a78f08a Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Jul 2018 20:19:32 +0200 Subject: [PATCH 224/395] Use `.center()` to output trades header line --- freqtrade/optimize/backtesting.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 391e05b83..73899f9a2 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -343,19 +343,17 @@ class Backtesting(object): # ) logger.info( - '\n' + '=' * 4 + - ' SELL READON STATS ' + - '=' * 4 + '\n' - '%s \n', + '\n' + + ' SELL READON STATS '.center(119, '=') + + '\n%s \n', self._generate_text_table_sell_reason(data, results) ) logger.info( - '\n' + '=' * 47 + - ' LEFT OPEN TRADES REPORT ' + - '=' * 47 + '\n' - '%s', + '\n' + + ' LEFT OPEN TRADES REPORT '.center(119, '=') + + '\n%s', self._generate_text_table( data, results.loc[results.open_at_end] From 4fb9823cfb6e09ed16ca97f40b76b2647abd3c7f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Jul 2018 19:41:42 +0200 Subject: [PATCH 225/395] fix rebase problem --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/optimize/backtesting.py | 2 +- freqtrade/strategy/interface.py | 3 +-- freqtrade/tests/test_freqtradebot.py | 7 +++---- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 566b0670f..f6bdcdf52 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -393,8 +393,8 @@ class FreqtradeBot(object): open_date=datetime.utcnow(), exchange=self.exchange.id, open_order_id=order_id, - strategy=self.analyze.get_strategy_name(), - ticker_interval=constants.TICKER_INTERVAL_MINUTES[self.analyze.get_ticker_interval()] + strategy=self.strategy.get_strategy_name(), + ticker_interval=constants.TICKER_INTERVAL_MINUTES[self.config['ticker_interval']] ) Trade.session.add(trade) Trade.session.flush() diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 73899f9a2..db8e923b2 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -164,7 +164,7 @@ class Backtesting(object): buy_signal = sell_row.buy sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, - sell_row.sell) + sell_row.sell) if sell.sell_flag: return BacktestResult(pair=pair, diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 7b8d91010..9120d3e04 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -88,13 +88,12 @@ class IStrategy(ABC): :param dataframe: DataFrame :return: DataFrame with sell column """ - return self.strategy.populate_sell_trend(dataframe=dataframe) def get_strategy_name(self) -> str: """ Returns strategy class name """ - return self.strategy.__class__.__name__ + return self.__class__.__name__ def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: """ diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 65cd99689..223e7318d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -16,11 +16,11 @@ import requests from freqtrade import (DependencyException, OperationalException, TemporaryError, constants) -from freqtrade.strategy.interface.IStrategy import SellType, SellCheckTuple from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType from freqtrade.state import State +from freqtrade.strategy.interface import SellType, SellCheckTuple from freqtrade.tests.conftest import log_has, patch_coinmarketcap, patch_exchange @@ -1625,8 +1625,6 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.strategy.interface.stop_loss_reached', - return_value=SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1647,7 +1645,8 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market freqtrade = FreqtradeBot(conf) patch_get_signal(freqtrade) freqtrade.strategy.stop_loss_reached = \ - lambda current_rate, trade, current_time, current_profit: False + lambda current_rate, trade, current_time, current_profit: SellCheckTuple( + sell_flag=False, sell_type=SellType.NONE) freqtrade.create_trade() trade = Trade.query.first() From 060469fefc774b0dd0110bfbd76106b354c6fce2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Jul 2018 20:12:20 +0200 Subject: [PATCH 226/395] Add stuff after rebase --- freqtrade/strategy/interface.py | 2 +- freqtrade/tests/test_freqtradebot.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 7cf990cf5..61ad41891 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -200,7 +200,7 @@ class IStrategy(ABC): # check if we have a special stop loss for positive condition # and if profit is positive - stop_loss_value = self.strategy.stoploss + stop_loss_value = self.stoploss sl_offset = self.config.get('trailing_stop_positive_offset', 0.0) if 'trailing_stop_positive' in self.config and current_profit > sl_offset: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 61907a321..cf1f35e3d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -45,7 +45,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: :param value: which value IStrategy.get_signal() must return :return: None """ - freqtrade.strategy.get_signal = lambda e, s, t: value + freqtrade.get_signal = lambda e, s, t: value def patch_RPCManager(mocker) -> MagicMock: @@ -1833,7 +1833,6 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, caplog, m patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1851,6 +1850,8 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, caplog, m conf['trailing_stop_positive'] = 0.01 conf['trailing_stop_positive_offset'] = 0.011 freqtrade = FreqtradeBot(conf) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.create_trade() trade = Trade.query.first() From 1b2bfad34827298dd3f9b3bf46bd97dab8c4b83a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Jul 2018 20:36:49 +0200 Subject: [PATCH 227/395] Fix wrong test --- freqtrade/tests/test_freqtradebot.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index cf1f35e3d..9448f8c68 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -45,7 +45,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: :param value: which value IStrategy.get_signal() must return :return: None """ - freqtrade.get_signal = lambda e, s, t: value + freqtrade.strategy.get_signal = lambda e, s, t: value def patch_RPCManager(mocker) -> MagicMock: @@ -1830,7 +1830,6 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, caplog, m Test sell_profit_only feature when enabled and we have a loss """ buy_price = limit_buy_order['price'] - patch_get_signal(mocker) patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1850,8 +1849,8 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, caplog, m conf['trailing_stop_positive'] = 0.01 conf['trailing_stop_positive_offset'] = 0.011 freqtrade = FreqtradeBot(conf) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True - + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() trade = Trade.query.first() From 90915b6b2f794f4d1e635b1aa0ae9d94a006dbb6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Jul 2018 20:43:41 +0200 Subject: [PATCH 228/395] Revert "Add more verbosity levels" --- freqtrade/arguments.py | 8 ++-- freqtrade/configuration.py | 17 +------- freqtrade/main.py | 12 +++++- freqtrade/tests/test_arguments.py | 11 +++--- freqtrade/tests/test_configuration.py | 57 +-------------------------- freqtrade/tests/test_main.py | 25 ++++++++++-- 6 files changed, 47 insertions(+), 83 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 022a2c739..16a01396b 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -3,6 +3,7 @@ This module contains the argument manager class """ import argparse +import logging import os import re from typing import List, NamedTuple, Optional @@ -63,10 +64,11 @@ class Arguments(object): """ self.parser.add_argument( '-v', '--verbose', - help='verbose mode (-vv for more, -vvv to get all messages)', - action='count', + help='be verbose', + action='store_const', dest='loglevel', - default=0, + const=logging.DEBUG, + default=logging.INFO, ) self.parser.add_argument( '--version', diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 76faa9819..f5c1c398d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -12,22 +12,10 @@ from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match from freqtrade import OperationalException, constants + logger = logging.getLogger(__name__) -def set_loggers(log_level: int = 0) -> None: - """ - Set the logger level for Third party libs - :return: None - """ - - logging.getLogger('requests').setLevel(logging.INFO if log_level <= 1 else logging.DEBUG) - logging.getLogger("urllib3").setLevel(logging.INFO if log_level <= 1 else logging.DEBUG) - logging.getLogger('ccxt.base.exchange').setLevel( - logging.INFO if log_level <= 2 else logging.DEBUG) - logging.getLogger('telegram').setLevel(logging.INFO) - - class Configuration(object): """ Class to read and init the bot configuration @@ -93,10 +81,9 @@ class Configuration(object): if 'loglevel' in self.args and self.args.loglevel: config.update({'loglevel': self.args.loglevel}) logging.basicConfig( - level=logging.DEBUG, + level=config['loglevel'], format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', ) - set_loggers(config['loglevel']) logger.info('Log level set to %s', logging.getLevelName(config['loglevel'])) # Add dynamic_whitelist if found diff --git a/freqtrade/main.py b/freqtrade/main.py index 3ed478ec3..977212faf 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -10,7 +10,7 @@ from typing import List from freqtrade import OperationalException from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration, set_loggers +from freqtrade.configuration import Configuration from freqtrade.freqtradebot import FreqtradeBot from freqtrade.state import State from freqtrade.rpc import RPCMessageType @@ -84,6 +84,16 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot: return freqtrade +def set_loggers() -> None: + """ + Set the logger level for Third party libs + :return: None + """ + logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) + logging.getLogger('ccxt.base.exchange').setLevel(logging.INFO) + logging.getLogger('telegram').setLevel(logging.INFO) + + if __name__ == '__main__': set_loggers() main(sys.argv[1:]) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 07018c79e..8a41e3379 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -5,6 +5,7 @@ Unit test file for arguments.py """ import argparse +import logging import pytest @@ -34,7 +35,7 @@ def test_parse_args_defaults() -> None: args = Arguments([], '').get_parsed_arg() assert args.config == 'config.json' assert args.dynamic_whitelist is None - assert args.loglevel == 0 + assert args.loglevel == logging.INFO def test_parse_args_config() -> None: @@ -52,10 +53,10 @@ def test_parse_args_db_url() -> None: def test_parse_args_verbose() -> None: args = Arguments(['-v'], '').get_parsed_arg() - assert args.loglevel == 1 + assert args.loglevel == logging.DEBUG args = Arguments(['--verbose'], '').get_parsed_arg() - assert args.loglevel == 1 + assert args.loglevel == logging.DEBUG def test_scripts_options() -> None: @@ -152,7 +153,7 @@ def test_parse_args_backtesting_custom() -> None: call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.live is True - assert call_args.loglevel == 0 + assert call_args.loglevel == logging.INFO assert call_args.subparser == 'backtesting' assert call_args.func is not None assert call_args.ticker_interval == '1m' @@ -169,7 +170,7 @@ def test_parse_args_hyperopt_custom() -> None: call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.epochs == 20 - assert call_args.loglevel == 0 + assert call_args.loglevel == logging.INFO assert call_args.subparser == 'hyperopt' assert call_args.spaces == ['buy'] assert call_args.func is not None diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index e4ef9d8f5..da8d3bebf 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -6,7 +6,6 @@ Unit test file for configuration.py import json from argparse import Namespace from copy import deepcopy -import logging from unittest.mock import MagicMock import pytest @@ -14,7 +13,7 @@ from jsonschema import ValidationError from freqtrade import OperationalException from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration, set_loggers +from freqtrade.configuration import Configuration from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.tests.conftest import log_has @@ -407,57 +406,3 @@ def test_check_exchange(default_conf) -> None: match=r'.*Exchange "unknown_exchange" not supported.*' ): configuration.check_exchange(conf) - - -def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: - """ - Test Configuration.load_config() with cli params used - """ - - mocker.patch('freqtrade.configuration.open', mocker.mock_open( - read_data=json.dumps(default_conf))) - # Prevent setting loggers - mocker.patch('freqtrade.configuration.set_loggers', MagicMock) - arglist = ['-vvv'] - args = Arguments(arglist, '').get_parsed_arg() - - configuration = Configuration(args) - validated_conf = configuration.load_config() - - assert validated_conf.get('loglevel') == 3 - assert log_has('Log level set to Level 3', caplog.record_tuples) - - -def test_set_loggers() -> None: - """ - Test set_loggers() update the logger level for third-party libraries - """ - previous_value1 = logging.getLogger('requests').level - previous_value2 = logging.getLogger('ccxt.base.exchange').level - previous_value3 = logging.getLogger('telegram').level - - set_loggers() - - value1 = logging.getLogger('requests').level - assert previous_value1 is not value1 - assert value1 is logging.INFO - - value2 = logging.getLogger('ccxt.base.exchange').level - assert previous_value2 is not value2 - assert value2 is logging.INFO - - value3 = logging.getLogger('telegram').level - assert previous_value3 is not value3 - assert value3 is logging.INFO - - set_loggers(log_level=2) - - assert logging.getLogger('requests').level is logging.DEBUG - assert logging.getLogger('ccxt.base.exchange').level is logging.INFO - assert logging.getLogger('telegram').level is logging.INFO - - set_loggers(log_level=3) - - assert logging.getLogger('requests').level is logging.DEBUG - assert logging.getLogger('ccxt.base.exchange').level is logging.DEBUG - assert logging.getLogger('telegram').level is logging.INFO diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 446945a07..20a02eedc 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -2,6 +2,7 @@ Unit test file for main.py """ +import logging from copy import deepcopy from unittest.mock import MagicMock @@ -10,7 +11,7 @@ import pytest from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.main import main, reconfigure +from freqtrade.main import main, reconfigure, set_loggers from freqtrade.state import State from freqtrade.tests.conftest import log_has, patch_exchange @@ -26,7 +27,7 @@ def test_parse_args_backtesting(mocker) -> None: call_args = backtesting_mock.call_args[0][0] assert call_args.config == 'config.json' assert call_args.live is False - assert call_args.loglevel == 0 + assert call_args.loglevel == 20 assert call_args.subparser == 'backtesting' assert call_args.func is not None assert call_args.ticker_interval is None @@ -41,11 +42,29 @@ def test_main_start_hyperopt(mocker) -> None: assert hyperopt_mock.call_count == 1 call_args = hyperopt_mock.call_args[0][0] assert call_args.config == 'config.json' - assert call_args.loglevel == 0 + assert call_args.loglevel == 20 assert call_args.subparser == 'hyperopt' assert call_args.func is not None +def test_set_loggers() -> None: + """ + Test set_loggers() update the logger level for third-party libraries + """ + previous_value1 = logging.getLogger('requests.packages.urllib3').level + previous_value2 = logging.getLogger('telegram').level + + set_loggers() + + value1 = logging.getLogger('requests.packages.urllib3').level + assert previous_value1 is not value1 + assert value1 is logging.INFO + + value2 = logging.getLogger('telegram').level + assert previous_value2 is not value2 + assert value2 is logging.INFO + + def test_main_fatal_exception(mocker, default_conf, caplog) -> None: """ Test main() function From dd1290e38e3f01cccf1959ea5155a30725a73814 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Jul 2018 21:12:27 +0200 Subject: [PATCH 229/395] Add multiple verbosity levels --- freqtrade/arguments.py | 8 ++-- freqtrade/configuration.py | 29 +++++++++--- freqtrade/main.py | 12 +---- freqtrade/tests/test_arguments.py | 11 +++-- freqtrade/tests/test_configuration.py | 63 ++++++++++++++++++++++++++- freqtrade/tests/test_main.py | 25 ++--------- 6 files changed, 96 insertions(+), 52 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 16a01396b..022a2c739 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -3,7 +3,6 @@ This module contains the argument manager class """ import argparse -import logging import os import re from typing import List, NamedTuple, Optional @@ -64,11 +63,10 @@ class Arguments(object): """ self.parser.add_argument( '-v', '--verbose', - help='be verbose', - action='store_const', + help='verbose mode (-vv for more, -vvv to get all messages)', + action='count', dest='loglevel', - const=logging.DEBUG, - default=logging.INFO, + default=0, ) self.parser.add_argument( '--version', diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index f5c1c398d..dcc6e4332 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -12,10 +12,22 @@ from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match from freqtrade import OperationalException, constants - logger = logging.getLogger(__name__) +def set_loggers(log_level: int = 0) -> None: + """ + Set the logger level for Third party libs + :return: None + """ + + logging.getLogger('requests').setLevel(logging.INFO if log_level <= 1 else logging.DEBUG) + logging.getLogger("urllib3").setLevel(logging.INFO if log_level <= 1 else logging.DEBUG) + logging.getLogger('ccxt.base.exchange').setLevel( + logging.INFO if log_level <= 2 else logging.DEBUG) + logging.getLogger('telegram').setLevel(logging.INFO) + + class Configuration(object): """ Class to read and init the bot configuration @@ -79,12 +91,15 @@ class Configuration(object): # Log level if 'loglevel' in self.args and self.args.loglevel: - config.update({'loglevel': self.args.loglevel}) - logging.basicConfig( - level=config['loglevel'], - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - ) - logger.info('Log level set to %s', logging.getLevelName(config['loglevel'])) + config.update({'verbosity': self.args.loglevel}) + else: + config.update({'verbosity': 0}) + logging.basicConfig( + level=logging.INFO if config['verbosity'] < 1 else logging.DEBUG, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + ) + set_loggers(config['verbosity']) + logger.info('Verbosity set to %s', config['verbosity']) # Add dynamic_whitelist if found if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: diff --git a/freqtrade/main.py b/freqtrade/main.py index 977212faf..3ed478ec3 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -10,7 +10,7 @@ from typing import List from freqtrade import OperationalException from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration +from freqtrade.configuration import Configuration, set_loggers from freqtrade.freqtradebot import FreqtradeBot from freqtrade.state import State from freqtrade.rpc import RPCMessageType @@ -84,16 +84,6 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot: return freqtrade -def set_loggers() -> None: - """ - Set the logger level for Third party libs - :return: None - """ - logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO) - logging.getLogger('ccxt.base.exchange').setLevel(logging.INFO) - logging.getLogger('telegram').setLevel(logging.INFO) - - if __name__ == '__main__': set_loggers() main(sys.argv[1:]) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 8a41e3379..07018c79e 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -5,7 +5,6 @@ Unit test file for arguments.py """ import argparse -import logging import pytest @@ -35,7 +34,7 @@ def test_parse_args_defaults() -> None: args = Arguments([], '').get_parsed_arg() assert args.config == 'config.json' assert args.dynamic_whitelist is None - assert args.loglevel == logging.INFO + assert args.loglevel == 0 def test_parse_args_config() -> None: @@ -53,10 +52,10 @@ def test_parse_args_db_url() -> None: def test_parse_args_verbose() -> None: args = Arguments(['-v'], '').get_parsed_arg() - assert args.loglevel == logging.DEBUG + assert args.loglevel == 1 args = Arguments(['--verbose'], '').get_parsed_arg() - assert args.loglevel == logging.DEBUG + assert args.loglevel == 1 def test_scripts_options() -> None: @@ -153,7 +152,7 @@ def test_parse_args_backtesting_custom() -> None: call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.live is True - assert call_args.loglevel == logging.INFO + assert call_args.loglevel == 0 assert call_args.subparser == 'backtesting' assert call_args.func is not None assert call_args.ticker_interval == '1m' @@ -170,7 +169,7 @@ def test_parse_args_hyperopt_custom() -> None: call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.epochs == 20 - assert call_args.loglevel == logging.INFO + assert call_args.loglevel == 0 assert call_args.subparser == 'hyperopt' assert call_args.spaces == ['buy'] assert call_args.func is not None diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index da8d3bebf..a8a2c5fce 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -6,6 +6,7 @@ Unit test file for configuration.py import json from argparse import Namespace from copy import deepcopy +import logging from unittest.mock import MagicMock import pytest @@ -13,7 +14,7 @@ from jsonschema import ValidationError from freqtrade import OperationalException from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration +from freqtrade.configuration import Configuration, set_loggers from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.tests.conftest import log_has @@ -406,3 +407,63 @@ def test_check_exchange(default_conf) -> None: match=r'.*Exchange "unknown_exchange" not supported.*' ): configuration.check_exchange(conf) + + +def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: + """ + Test Configuration.load_config() with cli params used + """ + + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf))) + # Prevent setting loggers + mocker.patch('freqtrade.configuration.set_loggers', MagicMock) + arglist = ['-vvv'] + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + validated_conf = configuration.load_config() + + assert validated_conf.get('verbosity') == 3 + assert log_has('Verbosity set to 3', caplog.record_tuples) + + +def test_set_loggers() -> None: + """ + Test set_loggers() update the logger level for third-party libraries + """ + # Reset Logging to Debug, otherwise this fails randomly as it's set globally + logging.getLogger('requests').setLevel(logging.DEBUG) + logging.getLogger("urllib3").setLevel(logging.DEBUG) + logging.getLogger('ccxt.base.exchange').setLevel(logging.DEBUG) + logging.getLogger('telegram').setLevel(logging.DEBUG) + + previous_value1 = logging.getLogger('requests').level + previous_value2 = logging.getLogger('ccxt.base.exchange').level + previous_value3 = logging.getLogger('telegram').level + + set_loggers() + + value1 = logging.getLogger('requests').level + assert previous_value1 is not value1 + assert value1 is logging.INFO + + value2 = logging.getLogger('ccxt.base.exchange').level + assert previous_value2 is not value2 + assert value2 is logging.INFO + + value3 = logging.getLogger('telegram').level + assert previous_value3 is not value3 + assert value3 is logging.INFO + + set_loggers(log_level=2) + + assert logging.getLogger('requests').level is logging.DEBUG + assert logging.getLogger('ccxt.base.exchange').level is logging.INFO + assert logging.getLogger('telegram').level is logging.INFO + + set_loggers(log_level=3) + + assert logging.getLogger('requests').level is logging.DEBUG + assert logging.getLogger('ccxt.base.exchange').level is logging.DEBUG + assert logging.getLogger('telegram').level is logging.INFO diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 20a02eedc..446945a07 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -2,7 +2,6 @@ Unit test file for main.py """ -import logging from copy import deepcopy from unittest.mock import MagicMock @@ -11,7 +10,7 @@ import pytest from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.main import main, reconfigure, set_loggers +from freqtrade.main import main, reconfigure from freqtrade.state import State from freqtrade.tests.conftest import log_has, patch_exchange @@ -27,7 +26,7 @@ def test_parse_args_backtesting(mocker) -> None: call_args = backtesting_mock.call_args[0][0] assert call_args.config == 'config.json' assert call_args.live is False - assert call_args.loglevel == 20 + assert call_args.loglevel == 0 assert call_args.subparser == 'backtesting' assert call_args.func is not None assert call_args.ticker_interval is None @@ -42,29 +41,11 @@ def test_main_start_hyperopt(mocker) -> None: assert hyperopt_mock.call_count == 1 call_args = hyperopt_mock.call_args[0][0] assert call_args.config == 'config.json' - assert call_args.loglevel == 20 + assert call_args.loglevel == 0 assert call_args.subparser == 'hyperopt' assert call_args.func is not None -def test_set_loggers() -> None: - """ - Test set_loggers() update the logger level for third-party libraries - """ - previous_value1 = logging.getLogger('requests.packages.urllib3').level - previous_value2 = logging.getLogger('telegram').level - - set_loggers() - - value1 = logging.getLogger('requests.packages.urllib3').level - assert previous_value1 is not value1 - assert value1 is logging.INFO - - value2 = logging.getLogger('telegram').level - assert previous_value2 is not value2 - assert value2 is logging.INFO - - def test_main_fatal_exception(mocker, default_conf, caplog) -> None: """ Test main() function From 7f6c79eb76c3c90f986edcd31975b9153cb7ba6a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 20 Jul 2018 14:24:06 +0200 Subject: [PATCH 230/395] Update ccxt from 1.16.68 to 1.16.75 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 576185f06..e54952c23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.68 +ccxt==1.16.75 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From ab3478a7421fe83b28dd355194a6db57426fb664 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 21 Jul 2018 14:24:05 +0200 Subject: [PATCH 231/395] Update ccxt from 1.16.75 to 1.16.80 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e54952c23..4fa30014e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.75 +ccxt==1.16.80 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 66af41192ac419dcfb79ce46192771a9d9721d3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Jul 2018 19:50:38 +0200 Subject: [PATCH 232/395] Catch all exceptions from fiat-convert api calls --- freqtrade/fiat_convert.py | 5 ++--- freqtrade/tests/test_fiat_convert.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/freqtrade/fiat_convert.py b/freqtrade/fiat_convert.py index 2e1a7cac8..6812bf77f 100644 --- a/freqtrade/fiat_convert.py +++ b/freqtrade/fiat_convert.py @@ -7,7 +7,6 @@ import logging import time from typing import Dict, List -from requests.exceptions import RequestException from coinmarketcap import Market from freqtrade.constants import SUPPORTED_FIAT @@ -90,10 +89,10 @@ class CryptoToFiatConverter(object): coinlistings = self._coinmarketcap.listings() self._cryptomap = dict(map(lambda coin: (coin["symbol"], str(coin["id"])), coinlistings["data"])) - except (ValueError, RequestException) as exception: + except (BaseException) as exception: logger.error( "Could not load FIAT Cryptocurrency map for the following problem: %s", - exception + type(exception).__name__ ) def convert_amount(self, crypto_amount: float, crypto_symbol: str, fiat_symbol: str) -> float: diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index 5af85d268..8fd3b66b4 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -183,6 +183,24 @@ def test_fiat_convert_without_network(mocker): CryptoToFiatConverter._coinmarketcap = cmc_temp +def test_fiat_invalid_response(mocker, caplog): + # Because CryptoToFiatConverter is a Singleton we reset the listings + listmock = MagicMock(return_value="{'novalidjson':DEADBEEFf}") + mocker.patch.multiple( + 'freqtrade.fiat_convert.Market', + listings=listmock, + ) + # with pytest.raises(RequestEsxception): + fiat_convert = CryptoToFiatConverter() + fiat_convert._cryptomap = {} + fiat_convert._load_cryptomap() + + length_cryptomap = len(fiat_convert._cryptomap) + assert length_cryptomap == 0 + assert log_has('Could not load FIAT Cryptocurrency map for the following problem: TypeError', + caplog.record_tuples) + + def test_convert_amount(mocker): patch_coinmarketcap(mocker) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter.get_price', return_value=12345.0) From 9467461160907f4b96252a8d60520cf2c911d069 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Jul 2018 20:13:32 +0200 Subject: [PATCH 233/395] only init FIATConvert when telegram is enabled --- freqtrade/freqtradebot.py | 16 ---------------- freqtrade/rpc/telegram.py | 7 +++++++ freqtrade/tests/test_main.py | 5 ----- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 35c0a1705..4296334f1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -16,7 +16,6 @@ from cachetools import TTLCache, cached from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange -from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State @@ -49,7 +48,6 @@ class FreqtradeBot(object): # Init objects self.config = config self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.fiat_converter = CryptoToFiatConverter() self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = Exchange(self.config) @@ -363,12 +361,6 @@ class FreqtradeBot(object): order_id = self.exchange.buy(pair, buy_limit, amount)['id'] - stake_amount_fiat = self.fiat_converter.convert_amount( - stake_amount, - stake_currency, - fiat_currency - ) - self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, 'exchange': self.exchange.name.capitalize(), @@ -376,7 +368,6 @@ class FreqtradeBot(object): 'market_url': pair_url, 'limit': buy_limit, 'stake_amount': stake_amount, - 'stake_amount_fiat': stake_amount_fiat, 'stake_currency': stake_currency, 'fiat_currency': fiat_currency }) @@ -643,14 +634,7 @@ class FreqtradeBot(object): if 'stake_currency' in self.config and 'fiat_display_currency' in self.config: stake_currency = self.config['stake_currency'] fiat_currency = self.config['fiat_display_currency'] - fiat_converter = CryptoToFiatConverter() - profit_fiat = fiat_converter.convert_amount( - profit_trade, - stake_currency, - fiat_currency, - ) msg.update({ - 'profit_fiat': profit_fiat, 'stake_currency': stake_currency, 'fiat_currency': fiat_currency, }) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 02b74358e..ea978bf8d 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -12,6 +12,7 @@ from telegram.error import NetworkError, TelegramError from telegram.ext import CommandHandler, Updater from freqtrade.__init__ import __version__ +from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.rpc import RPC, RPCException, RPCMessageType logger = logging.getLogger(__name__) @@ -66,6 +67,7 @@ class Telegram(RPC): self._updater: Updater = None self._config = freqtrade.config self._init() + self._fiat_converter = CryptoToFiatConverter() def _init(self) -> None: """ @@ -114,6 +116,9 @@ class Telegram(RPC): """ Send a message to telegram channel """ if msg['type'] == RPCMessageType.BUY_NOTIFICATION: + msg['stake_amount_fiat'] = self._fiat_converter.convert_amount( + msg['stake_amount'], msg['stake_currency'], msg['fiat_currency']) + message = "*{exchange}:* Buying [{pair}]({market_url})\n" \ "with limit `{limit:.8f}\n" \ "({stake_amount:.6f} {stake_currency}," \ @@ -135,6 +140,8 @@ class Telegram(RPC): # This might not be the case if the message origin is triggered by /forcesell if all(prop in msg for prop in ['gain', 'profit_fiat', 'fiat_currency', 'stake_currency']): + msg['profit_fiat'] = self._fiat_converter.convert_amount( + msg['profit_amount'], msg['stake_currency'], msg['fiat_currency']) message += '` ({gain}: {profit_amount:.8f} {stake_currency}`' \ '` / {profit_fiat:.3f} {fiat_currency})`'.format(**msg) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 446945a07..80f5367b8 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -62,7 +62,6 @@ def test_main_fatal_exception(mocker, default_conf, caplog) -> None: 'freqtrade.configuration.Configuration._load_config_file', lambda *args, **kwargs: default_conf ) - mocker.patch('freqtrade.freqtradebot.CryptoToFiatConverter', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) args = ['-c', 'config.json.example'] @@ -90,7 +89,6 @@ def test_main_keyboard_interrupt(mocker, default_conf, caplog) -> None: 'freqtrade.configuration.Configuration._load_config_file', lambda *args, **kwargs: default_conf ) - mocker.patch('freqtrade.freqtradebot.CryptoToFiatConverter', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) args = ['-c', 'config.json.example'] @@ -118,7 +116,6 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None: 'freqtrade.configuration.Configuration._load_config_file', lambda *args, **kwargs: default_conf ) - mocker.patch('freqtrade.freqtradebot.CryptoToFiatConverter', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) args = ['-c', 'config.json.example'] @@ -146,7 +143,6 @@ def test_main_reload_conf(mocker, default_conf, caplog) -> None: 'freqtrade.configuration.Configuration._load_config_file', lambda *args, **kwargs: default_conf ) - mocker.patch('freqtrade.freqtradebot.CryptoToFiatConverter', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) # Raise exception as side effect to avoid endless loop @@ -174,7 +170,6 @@ def test_reconfigure(mocker, default_conf) -> None: 'freqtrade.configuration.Configuration._load_config_file', lambda *args, **kwargs: default_conf ) - mocker.patch('freqtrade.freqtradebot.CryptoToFiatConverter', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) freqtrade = FreqtradeBot(default_conf) From be3f04775ae476c406b733efafb66c96d801a513 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Jul 2018 20:21:00 +0200 Subject: [PATCH 234/395] remove unnecessary mocks - add mocks which went to exchange --- freqtrade/tests/test_freqtradebot.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index b7ae96048..b5223b4b3 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1352,7 +1352,6 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc get_fee=fee, get_markets=markets ) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -1385,7 +1384,6 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc 'current_rate': 1.172e-05, 'profit_amount': 6.126e-05, 'profit_percent': 0.06110514, - 'profit_fiat': 0.9189, 'stake_currency': 'BTC', 'fiat_currency': 'USD', } == last_msg @@ -1397,7 +1395,6 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, """ rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1437,7 +1434,6 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, 'current_rate': 1.044e-05, 'profit_amount': -5.492e-05, 'profit_percent': -0.05478343, - 'profit_fiat': -0.8238000000000001, 'stake_currency': 'BTC', 'fiat_currency': 'USD', } == last_msg @@ -1726,7 +1722,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m assert freqtrade.handle_trade(trade) is True -def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) -> None: +def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ @@ -1742,6 +1738,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets, ) conf = deepcopy(default_conf) @@ -1763,7 +1760,8 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) -def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, mocker) -> None: +def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets, + caplog, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ @@ -1780,6 +1778,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets, ) conf = deepcopy(default_conf) From 0681a806cc934c974ad5b9c0842cd368edc80697 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Jul 2018 20:44:38 +0200 Subject: [PATCH 235/395] move cryptofiatconvert to rpc --- freqtrade/rpc/rpc.py | 24 +++++++++++++++--------- freqtrade/tests/rpc/test_rpc.py | 9 ++++++--- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 9411e983b..0e643b734 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -13,8 +13,10 @@ import sqlalchemy as sql from numpy import mean, nan_to_num from pandas import DataFrame +from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.misc import shorten_date from freqtrade.persistence import Trade + from freqtrade.state import State logger = logging.getLogger(__name__) @@ -48,6 +50,9 @@ class RPC(object): """ RPC class can be used to have extra feature, like bot data, and access to DB data """ + # Initialize _fiat_converter if needed in each RPC handler + _fiat_converter: CryptoToFiatConverter = None + def __init__(self, freqtrade) -> None: """ Initializes all enabled rpc modules @@ -141,7 +146,7 @@ class RPC(object): if not (isinstance(timescale, int) and timescale > 0): raise RPCException('timescale must be an integer greater than 0') - fiat = self._freqtrade.fiat_converter + fiat = self._fiat_converter for day in range(0, timescale): profitday = today - timedelta(days=day) trades = Trade.query \ @@ -168,7 +173,7 @@ class RPC(object): value['amount'], stake_currency, fiat_display_currency - ), + ) if self._fiat_converter else 0, symbol=fiat_display_currency ), '{value} trade{s}'.format( @@ -225,22 +230,23 @@ class RPC(object): # FIX: we want to keep fiatconverter in a state/environment, # doing this will utilize its caching functionallity, instead we reinitialize it here - fiat = self._freqtrade.fiat_converter # Prepare data to display profit_closed_coin_sum = round(sum(profit_closed_coin), 8) profit_closed_percent = round(nan_to_num(mean(profit_closed_percent)) * 100, 2) - profit_closed_fiat = fiat.convert_amount( + profit_closed_fiat = self._fiat_converter.convert_amount( profit_closed_coin_sum, stake_currency, fiat_display_currency - ) + ) if self._fiat_converter else 0 + profit_all_coin_sum = round(sum(profit_all_coin), 8) profit_all_percent = round(nan_to_num(mean(profit_all_percent)) * 100, 2) - profit_all_fiat = fiat.convert_amount( + profit_all_fiat = self._fiat_converter.convert_amount( profit_all_coin_sum, stake_currency, fiat_display_currency - ) + ) if self._fiat_converter else 0 + num = float(len(durations) or 1) return { 'profit_closed_coin': profit_closed_coin_sum, @@ -284,9 +290,9 @@ class RPC(object): if total == 0.0: raise RPCException('all balances are zero') - fiat = self._freqtrade.fiat_converter symbol = fiat_display_currency - value = fiat.convert_amount(total, 'BTC', symbol) + value = self._fiat_converter.convert_amount(total, 'BTC', + symbol) if self._fiat_converter else 0 return { 'currencies': output, 'total': total, diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index e6cfceae7..e3530a149 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -9,6 +9,7 @@ from unittest.mock import MagicMock, ANY import pytest +from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException @@ -124,7 +125,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, fiat_display_currency = default_conf['fiat_display_currency'] rpc = RPC(freqtradebot) - + rpc._fiat_converter = CryptoToFiatConverter() # Create some test data freqtradebot.create_trade() trade = Trade.query.first() @@ -164,7 +165,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), ) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -180,6 +181,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, fiat_display_currency = default_conf['fiat_display_currency'] rpc = RPC(freqtradebot) + rpc._fiat_converter = CryptoToFiatConverter() with pytest.raises(RPCException, match=r'.*no closed trade*'): rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) @@ -313,7 +315,7 @@ def test_rpc_balance_handle(default_conf, mocker): 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), ) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -324,6 +326,7 @@ def test_rpc_balance_handle(default_conf, mocker): freqtradebot = FreqtradeBot(default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) + rpc._fiat_converter = CryptoToFiatConverter() result = rpc._rpc_balance(default_conf['fiat_display_currency']) assert prec_satoshi(result['total'], 12) From f297d22edbc202d34a3e388979a705bfeff00086 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Jul 2018 20:49:57 +0200 Subject: [PATCH 236/395] fix some tests in rpc_telegram --- freqtrade/tests/rpc/test_rpc_telegram.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 3336810bd..61cf52839 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -17,6 +17,7 @@ from telegram import Chat, Message, Update from telegram.error import NetworkError from freqtrade import __version__ +from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType @@ -362,7 +363,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, """ patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch( - 'freqtrade.fiat_convert.CryptoToFiatConverter._find_price', + 'freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0 ) mocker.patch.multiple( @@ -474,7 +475,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, Test _profit() method """ patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -742,7 +743,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, Test _forcesell() method """ patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) mocker.patch.multiple( @@ -783,7 +784,6 @@ def test_forcesell_handle(default_conf, update, ticker, fee, 'current_rate': 1.172e-05, 'profit_amount': 6.126e-05, 'profit_percent': 0.06110514, - 'profit_fiat': 0.9189, 'stake_currency': 'BTC', 'fiat_currency': 'USD', } == last_msg @@ -1134,7 +1134,6 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'current_rate': 3.201e-05, 'profit_amount': -0.05746268, 'profit_percent': -0.57405275, - 'profit_fiat': -24.81204044792, 'stake_currency': 'ETH', 'fiat_currency': 'USD' }) From 5ab1e669783b4d817ae98d8cb355b794fb8d864b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 22 Jul 2018 14:24:05 +0200 Subject: [PATCH 237/395] Update ccxt from 1.16.80 to 1.16.86 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4fa30014e..bc52980fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.80 +ccxt==1.16.86 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 6cc0a72bcad370ad9aea0bff9ed84666485636d3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Jul 2018 14:35:29 +0200 Subject: [PATCH 238/395] ADd optional to class _fiat_convert --- freqtrade/rpc/rpc.py | 7 +++---- freqtrade/rpc/telegram.py | 11 +++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 0e643b734..1fc99fd98 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -6,7 +6,7 @@ from abc import abstractmethod from datetime import timedelta, datetime, date from decimal import Decimal from enum import Enum -from typing import Dict, Any, List +from typing import Dict, Any, List, Optional import arrow import sqlalchemy as sql @@ -51,7 +51,7 @@ class RPC(object): RPC class can be used to have extra feature, like bot data, and access to DB data """ # Initialize _fiat_converter if needed in each RPC handler - _fiat_converter: CryptoToFiatConverter = None + _fiat_converter: Optional[CryptoToFiatConverter] = None def __init__(self, freqtrade) -> None: """ @@ -146,7 +146,6 @@ class RPC(object): if not (isinstance(timescale, int) and timescale > 0): raise RPCException('timescale must be an integer greater than 0') - fiat = self._fiat_converter for day in range(0, timescale): profitday = today - timedelta(days=day) trades = Trade.query \ @@ -169,7 +168,7 @@ class RPC(object): symbol=stake_currency ), '{value:.3f} {symbol}'.format( - value=fiat.convert_amount( + value=self._fiat_converter.convert_amount( value['amount'], stake_currency, fiat_display_currency diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ea978bf8d..61f5863df 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -116,8 +116,11 @@ class Telegram(RPC): """ Send a message to telegram channel """ if msg['type'] == RPCMessageType.BUY_NOTIFICATION: - msg['stake_amount_fiat'] = self._fiat_converter.convert_amount( - msg['stake_amount'], msg['stake_currency'], msg['fiat_currency']) + if self._fiat_converter: + msg['stake_amount_fiat'] = self._fiat_converter.convert_amount( + msg['stake_amount'], msg['stake_currency'], msg['fiat_currency']) + else: + msg['stake_amount_fiat'] = 0 message = "*{exchange}:* Buying [{pair}]({market_url})\n" \ "with limit `{limit:.8f}\n" \ @@ -138,8 +141,8 @@ class Telegram(RPC): # Check if all sell properties are available. # This might not be the case if the message origin is triggered by /forcesell - if all(prop in msg for prop in ['gain', 'profit_fiat', - 'fiat_currency', 'stake_currency']): + if (all(prop in msg for prop in ['gain', 'fiat_currency', 'stake_currency']) + and self._fiat_converter): msg['profit_fiat'] = self._fiat_converter.convert_amount( msg['profit_amount'], msg['stake_currency'], msg['fiat_currency']) message += '` ({gain}: {profit_amount:.8f} {stake_currency}`' \ From 2b297869a10be08ae0add38d5f75d986b984052c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Jul 2018 14:35:59 +0200 Subject: [PATCH 239/395] adjust checks to fit new functionality --- freqtrade/tests/rpc/test_rpc_telegram.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 61cf52839..b7b20ff92 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -17,7 +17,6 @@ from telegram import Chat, Message, Update from telegram.error import NetworkError from freqtrade import __version__ -from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType @@ -841,7 +840,6 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, 'current_rate': 1.044e-05, 'profit_amount': -5.492e-05, 'profit_percent': -0.05478343, - 'profit_fiat': -0.8238000000000001, 'stake_currency': 'BTC', 'fiat_currency': 'USD', } == last_msg @@ -890,7 +888,6 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker 'current_rate': 1.098e-05, 'profit_amount': -5.91e-06, 'profit_percent': -0.00589292, - 'profit_fiat': -0.08865, 'stake_currency': 'BTC', 'fiat_currency': 'USD', } == msg @@ -1122,6 +1119,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) + telegram._fiat_converter.convert_amount = lambda a, b, c: -24.812 telegram.send_msg({ 'type': RPCMessageType.SELL_NOTIFICATION, 'exchange': 'Binance', From fae4c3a4e3e45a7d9c803d509fb806347e9322da Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Jul 2018 14:48:06 +0200 Subject: [PATCH 240/395] only init if stake_currency is set --- freqtrade/rpc/telegram.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 61f5863df..c01aa2701 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -67,7 +67,8 @@ class Telegram(RPC): self._updater: Updater = None self._config = freqtrade.config self._init() - self._fiat_converter = CryptoToFiatConverter() + if self._config.get('stake_currency', None): + self._fiat_converter = CryptoToFiatConverter() def _init(self) -> None: """ From 4d864df59ed856599a64f230425bbb1683c2c4c8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Jul 2018 14:49:07 +0200 Subject: [PATCH 241/395] Add tests for no_fiat functionality --- freqtrade/tests/rpc/test_rpc_telegram.py | 62 ++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index b7b20ff92..b82268a7b 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1200,6 +1200,68 @@ def test_send_msg_unknown_type(default_conf, mocker) -> None: }) +def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None: + del default_conf['stake_currency'] + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + telegram.send_msg({ + 'type': RPCMessageType.BUY_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'limit': 1.099e-05, + 'stake_amount': 0.001, + 'stake_amount_fiat': 0.0, + 'stake_currency': 'BTC', + 'fiat_currency': 'USD' + }) + assert msg_mock.call_args[0][0] \ + == '*Bittrex:* Buying [ETH/BTC](https://bittrex.com/Market/Index?MarketName=BTC-ETH)\n' \ + 'with limit `0.00001099\n' \ + '(0.001000 BTC,0.000 USD)`' + + +def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: + del default_conf['stake_currency'] + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + telegram.send_msg({ + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': 'Binance', + 'pair': 'KEY/ETH', + 'gain': 'loss', + 'market_url': 'https://www.binance.com/tradeDetail.html?symbol=KEY_ETH', + 'limit': 3.201e-05, + 'amount': 1333.3333333333335, + 'open_rate': 7.5e-05, + 'current_rate': 3.201e-05, + 'profit_amount': -0.05746268, + 'profit_percent': -0.57405275, + 'stake_currency': 'ETH', + 'fiat_currency': 'USD' + }) + assert msg_mock.call_args[0][0] \ + == '*Binance:* Selling [KEY/ETH]' \ + '(https://www.binance.com/tradeDetail.html?symbol=KEY_ETH)\n' \ + '*Limit:* `0.00003201`\n' \ + '*Amount:* `1333.33333333`\n' \ + '*Open Rate:* `0.00007500`\n' \ + '*Current Rate:* `0.00003201`\n' \ + '*Profit:* `-57.41%`' + + def test__send_msg(default_conf, mocker) -> None: """ Test send_msg() method From bd2771b8f990fa60fa9efaee7b786e821fcda656 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Jul 2018 14:52:58 +0200 Subject: [PATCH 242/395] use correct property --- freqtrade/rpc/telegram.py | 2 +- freqtrade/tests/rpc/test_rpc_telegram.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c01aa2701..166979d25 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -67,7 +67,7 @@ class Telegram(RPC): self._updater: Updater = None self._config = freqtrade.config self._init() - if self._config.get('stake_currency', None): + if self._config.get('fiat_display_currency', None): self._fiat_converter = CryptoToFiatConverter() def _init(self) -> None: diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index b82268a7b..da4279a06 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1201,7 +1201,7 @@ def test_send_msg_unknown_type(default_conf, mocker) -> None: def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None: - del default_conf['stake_currency'] + del default_conf['fiat_display_currency'] msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -1228,7 +1228,7 @@ def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None: def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: - del default_conf['stake_currency'] + del default_conf['fiat_display_currency'] msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', From 4c8411537f6e7e9d66686ae383d375bbf52e0495 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Jul 2018 14:53:46 +0200 Subject: [PATCH 243/395] Don't require fiat-currency --- freqtrade/constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 385dac1d1..c27cd875c 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -153,7 +153,6 @@ CONF_SCHEMA = { 'max_open_trades', 'stake_currency', 'stake_amount', - 'fiat_display_currency', 'dry_run', 'bid_strategy', 'telegram' From f54ac5a8de6ebfdbf2d1f5b0f71f77fe5de14bb8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Jul 2018 17:05:22 +0100 Subject: [PATCH 244/395] revert bugfix done in it's own branch --- freqtrade/tests/test_freqtradebot.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index b5223b4b3..e87bbc4a4 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1722,7 +1722,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m assert freqtrade.handle_trade(trade) is True -def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, mocker) -> None: +def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ @@ -1738,7 +1738,6 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - get_markets=markets, ) conf = deepcopy(default_conf) @@ -1760,8 +1759,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) -def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets, - caplog, mocker) -> None: +def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ @@ -1778,7 +1776,6 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - get_markets=markets, ) conf = deepcopy(default_conf) From 23fe0db2dff9a8a59afc502f742005b5b21204a7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Jul 2018 14:24:41 +0200 Subject: [PATCH 245/395] Add missing mock --- freqtrade/tests/test_freqtradebot.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index b7ae96048..54b67b558 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1726,7 +1726,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m assert freqtrade.handle_trade(trade) is True -def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) -> None: +def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ @@ -1742,6 +1742,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets, ) conf = deepcopy(default_conf) @@ -1763,7 +1764,8 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) -def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, mocker) -> None: +def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets, + caplog, mocker) -> None: """ Test sell_profit_only feature when enabled and we have a loss """ @@ -1780,6 +1782,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets, ) conf = deepcopy(default_conf) From 0775a371fe6e6f56bace96e1ce4406fee5dccad4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Jul 2018 00:54:20 +0100 Subject: [PATCH 246/395] rename sellreason to sell_Reason, fix typos --- docs/backtesting.md | 2 +- freqtrade/freqtradebot.py | 6 +++--- freqtrade/tests/test_freqtradebot.py | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 0a7675848..255359a6a 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -98,7 +98,7 @@ df['closets'] = pd.to_datetime(df['closets'], ) ``` -If you have some ideas for interresting / helpfull backtest data analysis, feel free to submit a PR so the community can benefit from it. +If you have some ideas for interesting / helpful backtest data analysis, feel free to submit a PR so the community can benefit from it. #### Exporting trades to file specifying a custom filename diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f6bdcdf52..c0263c6fb 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -610,19 +610,19 @@ class FreqtradeBot(object): # TODO: figure out how to handle partially complete sell orders return False - def execute_sell(self, trade: Trade, limit: float, sellreason: SellType) -> None: + def execute_sell(self, trade: Trade, limit: float, sell_reason: SellType) -> None: """ Executes a limit sell for the given trade and limit :param trade: Trade instance :param limit: limit rate for the sell order - :param sellrason: Reaseon the sell was triggered + :param sellreason: Reason the sell was triggered :return: None """ # Execute sell and update trade record order_id = self.exchange.sell(str(trade.pair), limit, trade.amount)['id'] trade.open_order_id = order_id trade.close_rate_requested = limit - trade.sell_reason = sellreason.value + trade.sell_reason = sell_reason.value profit_trade = trade.calc_profit(rate=limit) current_rate = self.exchange.get_ticker(trade.pair)['bid'] diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 223e7318d..277193464 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1370,7 +1370,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc get_ticker=ticker_sell_up ) - freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sellreason=SellType.ROI) + freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI) assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] @@ -1423,7 +1423,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, ) freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'], - sellreason=SellType.STOP_LOSS) + sell_reason=SellType.STOP_LOSS) assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] @@ -1476,7 +1476,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, ) freqtrade.config = {} - freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sellreason=SellType.ROI) + freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI) assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] @@ -1527,7 +1527,7 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, freqtrade.config = {} freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'], - sellreason=SellType.STOP_LOSS) + sell_reason=SellType.STOP_LOSS) assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] From 643de58c4db92ecc67f0868b7fd380a75dc40135 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Jul 2018 09:09:56 +0100 Subject: [PATCH 247/395] Add test to check for a mid-migrated database (not old but not new) --- freqtrade/tests/test_persistence.py | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index b24f2dd6c..7ec87304e 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -469,6 +469,65 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert log_has("trying trades_bak2", caplog.record_tuples) +def test_migrate_mid_state(mocker, default_conf, fee, caplog): + """ + Test Database migration (starting with new pairformat) + """ + amount = 103.223 + create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( + id INTEGER NOT NULL, + exchange VARCHAR NOT NULL, + pair VARCHAR NOT NULL, + is_open BOOLEAN NOT NULL, + fee_open FLOAT NOT NULL, + fee_close FLOAT NOT NULL, + open_rate FLOAT, + close_rate FLOAT, + close_profit FLOAT, + stake_amount FLOAT NOT NULL, + amount FLOAT, + open_date DATETIME NOT NULL, + close_date DATETIME, + open_order_id VARCHAR, + PRIMARY KEY (id), + CHECK (is_open IN (0, 1)) + );""" + insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee_open, fee_close, + open_rate, stake_amount, amount, open_date) + VALUES ('binance', 'ETC/BTC', 1, {fee}, {fee}, + 0.00258580, {stake}, {amount}, + '2019-11-28 12:44:24.000000') + """.format(fee=fee.return_value, + stake=default_conf.get("stake_amount"), + amount=amount + ) + engine = create_engine('sqlite://') + mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) + + # Create table using the old format + engine.execute(create_table_old) + engine.execute(insert_table_old) + + # Run init to test migration + init(default_conf) + + assert len(Trade.query.filter(Trade.id == 1).all()) == 1 + trade = Trade.query.filter(Trade.id == 1).first() + assert trade.fee_open == fee.return_value + assert trade.fee_close == fee.return_value + assert trade.open_rate_requested is None + assert trade.close_rate_requested is None + assert trade.is_open == 1 + assert trade.amount == amount + assert trade.stake_amount == default_conf.get("stake_amount") + assert trade.pair == "ETC/BTC" + assert trade.exchange == "binance" + assert trade.max_rate == 0.0 + assert trade.stop_loss == 0.0 + assert trade.initial_stop_loss == 0.0 + assert log_has("trying trades_bak0", caplog.record_tuples) + + def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): trade = Trade( pair='ETH/BTC', From 10fc2c67c7066b0c5359e3b5e40bdaa2a7c0c5b6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Jul 2018 09:10:37 +0100 Subject: [PATCH 248/395] Fix bug causing a database-migration to fail from aspecific state --- freqtrade/persistence.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 0e0b22e82..5baa5834d 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -83,6 +83,8 @@ def check_migrate(engine) -> None: # Check for latest column if not has_column(cols, 'max_rate'): + fee_open = get_column_def(cols, 'fee_open', 'fee') + fee_close = get_column_def(cols, 'fee_close', 'fee') open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') stop_loss = get_column_def(cols, 'stop_loss', '0.0') @@ -109,7 +111,7 @@ def check_migrate(engine) -> None: else pair end pair, - is_open, fee fee_open, fee fee_close, + is_open, {fee_open} fee_open, {fee_close} fee_close, open_rate, {open_rate_requested} open_rate_requested, close_rate, {close_rate_requested} close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id, From 4575919d78f77395468ed9bb0f68f05a61b7d532 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 23 Jul 2018 14:24:05 +0200 Subject: [PATCH 249/395] Update ccxt from 1.16.86 to 1.16.88 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bc52980fe..b97f612cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.86 +ccxt==1.16.88 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 456e49fe355b4defc9f954c70467fddc7eb8a63a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Jul 2018 00:01:51 +0100 Subject: [PATCH 250/395] default fiat_currency to none --- freqtrade/rpc/telegram.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 166979d25..a63422970 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -224,7 +224,7 @@ class Telegram(RPC): :return: None """ stake_cur = self._config['stake_currency'] - fiat_disp_cur = self._config['fiat_display_currency'] + fiat_disp_cur = self._config.get('fiat_display_currency', None) try: timescale = int(update.message.text.replace('/daily', '').strip()) except (TypeError, ValueError): @@ -257,7 +257,7 @@ class Telegram(RPC): :return: None """ stake_cur = self._config['stake_currency'] - fiat_disp_cur = self._config['fiat_display_currency'] + fiat_disp_cur = self._config.get('fiat_display_currency', None) try: stats = self._rpc_trade_statistics( @@ -296,7 +296,7 @@ class Telegram(RPC): def _balance(self, bot: Bot, update: Update) -> None: """ Handler for /balance """ try: - result = self._rpc_balance(self._config['fiat_display_currency']) + result = self._rpc_balance(self._config.get('fiat_display_currency', None)) output = '' for currency in result['currencies']: output += "*{currency}:*\n" \ From 1a9ead45ebb75f360789316e2cca7072ce790dcc Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Jul 2018 08:00:56 +0100 Subject: [PATCH 251/395] fix missed fiat_display_currency config value --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4296334f1..25c6ece0d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -344,7 +344,7 @@ class FreqtradeBot(object): pair_s = pair.replace('_', '/') pair_url = self.exchange.get_pair_detail_url(pair) stake_currency = self.config['stake_currency'] - fiat_currency = self.config['fiat_display_currency'] + fiat_currency = self.config.get('fiat_display_currency', None) # Calculate amount buy_limit = self.get_target_bid(self.exchange.get_ticker(pair)) From 30b72ad98a21f1131e1b7ddb2311ec93007e3915 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Jul 2018 08:20:32 +0100 Subject: [PATCH 252/395] don't show fiat-currency if not set --- freqtrade/rpc/telegram.py | 8 +++++--- freqtrade/tests/rpc/test_rpc_telegram.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a63422970..f11fa4232 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -125,9 +125,11 @@ class Telegram(RPC): message = "*{exchange}:* Buying [{pair}]({market_url})\n" \ "with limit `{limit:.8f}\n" \ - "({stake_amount:.6f} {stake_currency}," \ - "{stake_amount_fiat:.3f} {fiat_currency})`" \ - .format(**msg) + "({stake_amount:.6f} {stake_currency}".format(**msg) + + if msg.get('fiat_currency', None): + message += ",{stake_amount_fiat:.3f} {fiat_currency}".format(**msg) + message += ")`" elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: msg['amount'] = round(msg['amount'], 8) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index da4279a06..1ce8a26f1 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1219,12 +1219,12 @@ def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None: 'stake_amount': 0.001, 'stake_amount_fiat': 0.0, 'stake_currency': 'BTC', - 'fiat_currency': 'USD' + 'fiat_currency': None }) assert msg_mock.call_args[0][0] \ == '*Bittrex:* Buying [ETH/BTC](https://bittrex.com/Market/Index?MarketName=BTC-ETH)\n' \ 'with limit `0.00001099\n' \ - '(0.001000 BTC,0.000 USD)`' + '(0.001000 BTC)`' def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: From 4928686af91dcb4f0cc052e9d9cbabfc6dd3b03f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Jul 2018 09:35:05 +0100 Subject: [PATCH 253/395] Remove currency from daily table --- freqtrade/rpc/rpc.py | 2 +- freqtrade/rpc/telegram.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 1fc99fd98..1e83f5eb9 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -173,7 +173,7 @@ class RPC(object): stake_currency, fiat_display_currency ) if self._fiat_converter else 0, - symbol=fiat_display_currency + symbol=fiat_display_currency if fiat_display_currency else '' ), '{value} trade{s}'.format( value=value['trades'], diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index f11fa4232..de0c50ab6 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -241,7 +241,7 @@ class Telegram(RPC): headers=[ 'Day', f'Profit {stake_cur}', - f'Profit {fiat_disp_cur}' + f'Profit {fiat_disp_cur if fiat_disp_cur else ""}' ], tablefmt='simple') message = f'Daily Profit over the last {timescale} days:\n
{stats}
' From cf6e229729fb6bf218a1a38c622e2684df5f245d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 24 Jul 2018 14:24:06 +0200 Subject: [PATCH 254/395] Update ccxt from 1.16.88 to 1.16.89 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b97f612cb..793450dd0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.88 +ccxt==1.16.89 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 7feea8c7a648d59b84aab76ae687ac2deb963d95 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 24 Jul 2018 14:24:08 +0200 Subject: [PATCH 255/395] Update numpy from 1.14.5 to 1.15.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 793450dd0..07e8c5834 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ pandas==0.23.3 scikit-learn==0.19.2 scipy==1.1.0 jsonschema==2.6.0 -numpy==1.14.5 +numpy==1.15.0 TA-Lib==0.4.17 pytest==3.6.3 pytest-mock==1.10.0 From ff6435948e3105439e3e6e4e5dc417d130fda85a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Jul 2018 22:53:10 +0100 Subject: [PATCH 256/395] Fix random test failure --- freqtrade/tests/rpc/test_rpc_telegram.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 1ce8a26f1..63ef3ca91 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1119,6 +1119,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) + old_convamount = telegram._fiat_converter.convert_amount telegram._fiat_converter.convert_amount = lambda a, b, c: -24.812 telegram.send_msg({ 'type': RPCMessageType.SELL_NOTIFICATION, @@ -1167,7 +1168,8 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: '*Open Rate:* `0.00007500`\n' \ '*Current Rate:* `0.00003201`\n' \ '*Profit:* `-57.41%`' - + # Reset singleton function to avoid random breaks + telegram._fiat_converter.convert_amount = old_convamount def test_send_msg_status_notification(default_conf, mocker) -> None: msg_mock = MagicMock() From dc1ad3cbf6eb9846b8db4a3ff57f5081237c431c Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Jul 2018 23:08:40 +0100 Subject: [PATCH 257/395] whitespace issues --- freqtrade/rpc/rpc.py | 3 +-- freqtrade/tests/rpc/test_rpc_telegram.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 1e83f5eb9..2b48f956e 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -16,7 +16,6 @@ from pandas import DataFrame from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.misc import shorten_date from freqtrade.persistence import Trade - from freqtrade.state import State logger = logging.getLogger(__name__) @@ -50,7 +49,7 @@ class RPC(object): """ RPC class can be used to have extra feature, like bot data, and access to DB data """ - # Initialize _fiat_converter if needed in each RPC handler + # Bind _fiat_converter if needed in each RPC handler _fiat_converter: Optional[CryptoToFiatConverter] = None def __init__(self, freqtrade) -> None: diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 63ef3ca91..b2cab6d37 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1171,6 +1171,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: # Reset singleton function to avoid random breaks telegram._fiat_converter.convert_amount = old_convamount + def test_send_msg_status_notification(default_conf, mocker) -> None: msg_mock = MagicMock() mocker.patch.multiple( From 4f4daf40710bb8a05fc1f3652830b0b48400f4db Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 25 Jul 2018 14:24:07 +0200 Subject: [PATCH 258/395] Update ccxt from 1.16.89 to 1.17.11 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 07e8c5834..16bd20e99 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.16.89 +ccxt==1.17.11 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 4b38c8b11d82bcf89a64716c039e26dd9a8b414c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 25 Jul 2018 17:04:25 +0300 Subject: [PATCH 259/395] use pandas own min and max for column sorting --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 7bdc3d93a..78cbe6d33 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -77,7 +77,7 @@ class Backtesting(object): :return: tuple containing min_date, max_date """ timeframe = [ - (arrow.get(min(frame.date)), arrow.get(max(frame.date))) + (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) for frame in data.values() ] return min(timeframe, key=operator.itemgetter(0))[0], \ From 7b49f746d1d1dc49da4887be0b1e6b9b61e990ca Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Jul 2018 22:47:20 +0100 Subject: [PATCH 260/395] remove #FIX which was fixed --- freqtrade/rpc/rpc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 2b48f956e..ab80a9ab2 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -226,8 +226,6 @@ class RPC(object): bp_pair, bp_rate = best_pair - # FIX: we want to keep fiatconverter in a state/environment, - # doing this will utilize its caching functionallity, instead we reinitialize it here # Prepare data to display profit_closed_coin_sum = round(sum(profit_closed_coin), 8) profit_closed_percent = round(nan_to_num(mean(profit_closed_percent)) * 100, 2) From 452a1cad9d9306c2b34e40fbd219997ca8286e9c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 Jul 2018 07:26:23 +0100 Subject: [PATCH 261/395] don't default fiat_convert to None for outputs --- freqtrade/rpc/rpc.py | 2 +- freqtrade/rpc/telegram.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index ab80a9ab2..bbb1ea9e5 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -172,7 +172,7 @@ class RPC(object): stake_currency, fiat_display_currency ) if self._fiat_converter else 0, - symbol=fiat_display_currency if fiat_display_currency else '' + symbol=fiat_display_currency ), '{value} trade{s}'.format( value=value['trades'], diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index de0c50ab6..3b5ce3f74 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -226,7 +226,7 @@ class Telegram(RPC): :return: None """ stake_cur = self._config['stake_currency'] - fiat_disp_cur = self._config.get('fiat_display_currency', None) + fiat_disp_cur = self._config.get('fiat_display_currency', '') try: timescale = int(update.message.text.replace('/daily', '').strip()) except (TypeError, ValueError): @@ -241,7 +241,7 @@ class Telegram(RPC): headers=[ 'Day', f'Profit {stake_cur}', - f'Profit {fiat_disp_cur if fiat_disp_cur else ""}' + f'Profit {fiat_disp_cur}' ], tablefmt='simple') message = f'Daily Profit over the last {timescale} days:\n
{stats}
' @@ -259,7 +259,7 @@ class Telegram(RPC): :return: None """ stake_cur = self._config['stake_currency'] - fiat_disp_cur = self._config.get('fiat_display_currency', None) + fiat_disp_cur = self._config.get('fiat_display_currency', '') try: stats = self._rpc_trade_statistics( @@ -298,7 +298,7 @@ class Telegram(RPC): def _balance(self, bot: Bot, update: Update) -> None: """ Handler for /balance """ try: - result = self._rpc_balance(self._config.get('fiat_display_currency', None)) + result = self._rpc_balance(self._config.get('fiat_display_currency', '')) output = '' for currency in result['currencies']: output += "*{currency}:*\n" \ From 0c7ceadb27bf2ded2cddf798ad94a2b8ffbc88b1 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 26 Jul 2018 14:24:05 +0200 Subject: [PATCH 262/395] Update ccxt from 1.17.11 to 1.17.20 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 16bd20e99..6a3c8d1ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.11 +ccxt==1.17.20 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 484103b957974e2091c619626b1359074902b258 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 Jul 2018 18:22:23 +0100 Subject: [PATCH 263/395] extract get_history_data from get_signal --- freqtrade/freqtradebot.py | 11 ++++++++--- freqtrade/strategy/interface.py | 3 +-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3527fba81..08eda2843 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -327,13 +327,18 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') + th = {} + for _pair in whitelist: + th[_pair] = self.exchange.get_ticker_history(_pair, interval) # Pick pair based on buy signals + bought_at_least_one = False for _pair in whitelist: - (buy, sell) = self.strategy.get_signal(self.exchange, _pair, interval) + (buy, sell) = self.strategy.get_signal(_pair, interval, th[_pair]) + if buy and not sell: - return self.execute_buy(_pair, stake_amount) - return False + bought_at_least_one |= self.execute_buy(_pair, stake_amount) + return bought_at_least_one def execute_buy(self, pair: str, stake_amount: float) -> bool: """ diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 9120d3e04..23b0563a2 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -107,14 +107,13 @@ class IStrategy(ABC): dataframe = self.populate_sell_trend(dataframe) return dataframe - def get_signal(self, exchange: Exchange, pair: str, interval: str) -> Tuple[bool, bool]: + def get_signal(self, pair: str, interval: str, ticker_hist: List[Dict]) -> Tuple[bool, bool]: """ Calculates current signal based several technical analysis indicators :param pair: pair in format ANT/BTC :param interval: Interval to use (in min) :return: (Buy, Sell) A bool-tuple indicating buy/sell signal """ - ticker_hist = exchange.get_ticker_history(pair, interval) if not ticker_hist: logger.warning('Empty ticker history for pair %s', pair) return False, False From 3324cdfcbe78a110e28ae0024b72c8c1cf46a537 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 Jul 2018 18:58:49 +0100 Subject: [PATCH 264/395] add mock for get_history in patch_get_signal --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/tests/test_freqtradebot.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 08eda2843..824607a2d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -337,8 +337,8 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(_pair, interval, th[_pair]) if buy and not sell: - bought_at_least_one |= self.execute_buy(_pair, stake_amount) - return bought_at_least_one + return self.execute_buy(_pair, stake_amount) + return False def execute_buy(self, pair: str, stake_amount: float) -> bool: """ diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 72f11abf9..fb6687a2c 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -47,6 +47,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: :return: None """ freqtrade.strategy.get_signal = lambda e, s, t: value + freqtrade.exchange.get_ticker_history = lambda p, i: None def patch_RPCManager(mocker) -> MagicMock: From f2a9be36847e9eb98300da6d1992383cb35902cb Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 Jul 2018 19:06:25 +0100 Subject: [PATCH 265/395] Adjust tests and remove legacy variable --- freqtrade/freqtradebot.py | 1 - freqtrade/tests/strategy/test_interface.py | 30 +++++++--------------- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 824607a2d..3317a7b68 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -332,7 +332,6 @@ class FreqtradeBot(object): th[_pair] = self.exchange.get_ticker_history(_pair, interval) # Pick pair based on buy signals - bought_at_least_one = False for _pair in whitelist: (buy, sell) = self.strategy.get_signal(_pair, interval, th[_pair]) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index a016b7f96..de11a9d2c 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -20,74 +20,62 @@ _STRATEGY = DefaultStrategy(config={}) def test_returns_latest_buy_signal(mocker, default_conf): - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) - exchange = get_patched_exchange(mocker, default_conf) mocker.patch.object( _STRATEGY, 'analyze_ticker', return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) ) - assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) + assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (True, False) mocker.patch.object( _STRATEGY, 'analyze_ticker', return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) ) - assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) + assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (False, True) def test_returns_latest_sell_signal(mocker, default_conf): - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) - exchange = get_patched_exchange(mocker, default_conf) mocker.patch.object( _STRATEGY, 'analyze_ticker', return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) ) - assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (False, True) + assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (False, True) mocker.patch.object( _STRATEGY, 'analyze_ticker', return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) ) - assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (True, False) + assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (True, False) def test_get_signal_empty(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=None) - exchange = get_patched_exchange(mocker, default_conf) - assert (False, False) == _STRATEGY.get_signal(exchange, 'foo', default_conf['ticker_interval']) + assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'], + None) assert log_has('Empty ticker history for pair foo', caplog.record_tuples) def test_get_signal_exception_valueerror(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) - exchange = get_patched_exchange(mocker, default_conf) mocker.patch.object( _STRATEGY, 'analyze_ticker', side_effect=ValueError('xyz') ) - assert (False, False) == _STRATEGY.get_signal(exchange, 'foo', default_conf['ticker_interval']) + assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'], 1) assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples) def test_get_signal_empty_dataframe(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) - exchange = get_patched_exchange(mocker, default_conf) mocker.patch.object( _STRATEGY, 'analyze_ticker', return_value=DataFrame([]) ) - assert (False, False) == _STRATEGY.get_signal(exchange, 'xyz', default_conf['ticker_interval']) + assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'], 1) assert log_has('Empty dataframe for pair xyz', caplog.record_tuples) def test_get_signal_old_dataframe(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=1) - exchange = get_patched_exchange(mocker, default_conf) # default_conf defines a 5m interval. we check interval * 2 + 5m # this is necessary as the last candle is removed (partial candles) by default oldtime = arrow.utcnow().shift(minutes=-16) @@ -96,7 +84,7 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog): _STRATEGY, 'analyze_ticker', return_value=DataFrame(ticks) ) - assert (False, False) == _STRATEGY.get_signal(exchange, 'xyz', default_conf['ticker_interval']) + assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'], 1) assert log_has( 'Outdated history for pair xyz. Last tick is 16 minutes old', caplog.record_tuples From df3e76a65df49448315786f23af73d637a76889d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 Jul 2018 19:11:29 +0100 Subject: [PATCH 266/395] Remove legacy code, fix missed call --- freqtrade/freqtradebot.py | 11 ++++++----- freqtrade/strategy/interface.py | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3317a7b68..d7c5657c3 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -327,13 +327,13 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') - th = {} + thistory = {} for _pair in whitelist: - th[_pair] = self.exchange.get_ticker_history(_pair, interval) + thistory[_pair] = self.exchange.get_ticker_history(_pair, interval) # Pick pair based on buy signals for _pair in whitelist: - (buy, sell) = self.strategy.get_signal(_pair, interval, th[_pair]) + (buy, sell) = self.strategy.get_signal(_pair, interval, thistory[_pair]) if buy and not sell: return self.execute_buy(_pair, stake_amount) @@ -499,8 +499,9 @@ class FreqtradeBot(object): (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - (buy, sell) = self.strategy.get_signal(self.exchange, - trade.pair, self.strategy.ticker_interval) + ticker = self.exchange.get_ticker_history(trade.pair, self.strategy.ticker_interval) + (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, + ticker) should_sell = self.strategy.should_sell(trade, current_rate, datetime.utcnow(), buy, sell) if should_sell.sell_flag: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 23b0563a2..a5145aabb 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -13,7 +13,6 @@ from pandas import DataFrame from freqtrade import constants from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe -from freqtrade.exchange import Exchange from freqtrade.persistence import Trade logger = logging.getLogger(__name__) From 48cd468b6ca10438d367644ae970de35bc4ce16b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Jul 2018 07:40:27 +0100 Subject: [PATCH 267/395] Don't do all network calls at once without async --- freqtrade/freqtradebot.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d7c5657c3..46fbb3a38 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -327,13 +327,11 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') - thistory = {} - for _pair in whitelist: - thistory[_pair] = self.exchange.get_ticker_history(_pair, interval) # Pick pair based on buy signals for _pair in whitelist: - (buy, sell) = self.strategy.get_signal(_pair, interval, thistory[_pair]) + thistory = self.exchange.get_ticker_history(_pair, interval) + (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) if buy and not sell: return self.execute_buy(_pair, stake_amount) From d23b3ccc5e8c01825a2e3c0fde77d6552495e731 Mon Sep 17 00:00:00 2001 From: creslinux Date: Fri, 27 Jul 2018 08:55:36 +0000 Subject: [PATCH 268/395] odd cut and paste error fixed. --- freqtrade/constants.py | 1 + freqtrade/exchange/__init__.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index c27cd875c..ecaf158ec 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -124,6 +124,7 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'name': {'type': 'string'}, + 'sandbox': {'type': 'boolean'}, 'key': {'type': 'string'}, 'secret': {'type': 'string'}, 'pair_whitelist': { diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 972ff49ca..35cf3d6d5 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -95,6 +95,11 @@ class Exchange(object): except (KeyError, AttributeError): raise OperationalException(f'Exchange {name} is not supported') + # check if config requests sanbox, if so use ['test'] from url + if (exchange_config.get('sandbox')): + api.urls['api'] = api.urls['test']; + # exchange.urls['api'] = exchange.urls['test']; + return api @property From 7efa81073a5dca5ffc89f469b2c4dcf774a433c4 Mon Sep 17 00:00:00 2001 From: creslinux Date: Fri, 27 Jul 2018 09:10:09 +0000 Subject: [PATCH 269/395] Removed ; at line end. --- freqtrade/exchange/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 35cf3d6d5..64ada5ef6 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -97,8 +97,7 @@ class Exchange(object): # check if config requests sanbox, if so use ['test'] from url if (exchange_config.get('sandbox')): - api.urls['api'] = api.urls['test']; - # exchange.urls['api'] = exchange.urls['test']; + api.urls['api'] = api.urls['test'] return api From c47253133a895a3d256ecd9df0208fe7c7c187aa Mon Sep 17 00:00:00 2001 From: creslinux Date: Fri, 27 Jul 2018 12:07:07 +0000 Subject: [PATCH 270/395] have to begin before we can stop --- freqtrade/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index c27cd875c..1611903ab 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -126,6 +126,7 @@ CONF_SCHEMA = { 'name': {'type': 'string'}, 'key': {'type': 'string'}, 'secret': {'type': 'string'}, + 'password': {'type': 'string'}, 'pair_whitelist': { 'type': 'array', 'items': { From 40ae250193009997e9e94d26c9c7f037684ec1b3 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Fri, 27 Jul 2018 12:19:01 +0000 Subject: [PATCH 271/395] Update constants.py Adding UID also, as itll get ran into in future on an exchange that needs it. --- freqtrade/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 1611903ab..24de36f3d 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -127,6 +127,7 @@ CONF_SCHEMA = { 'key': {'type': 'string'}, 'secret': {'type': 'string'}, 'password': {'type': 'string'}, + 'uid': {'type': 'string'}, 'pair_whitelist': { 'type': 'array', 'items': { From 4547ae930aa75f34a8f9ac7c5f8a3ea53b049df0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 27 Jul 2018 14:24:06 +0200 Subject: [PATCH 272/395] Update ccxt from 1.17.20 to 1.17.29 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6a3c8d1ba..4255e2c38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.20 +ccxt==1.17.29 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From ca0d658f15960a3664fea2a3af817948328d47e0 Mon Sep 17 00:00:00 2001 From: Sandoche ADITTANE Date: Fri, 27 Jul 2018 15:28:06 +0200 Subject: [PATCH 273/395] Error fixed in the quickstart documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c1705ff41..b5715f743 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,8 @@ Freqtrade provides a Linux/macOS script to install all dependencies and help you ```bash git clone git@github.com:freqtrade/freqtrade.git -git checkout develop cd freqtrade +git checkout develop ./setup.sh --install ``` From 243b63e39ca5ec3523d03dd0067ef301bd85e0c3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Jul 2018 21:12:48 +0100 Subject: [PATCH 274/395] fix rpc test going to network (unsuitable for flights...) --- freqtrade/tests/rpc/test_rpc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index e3530a149..2b4b71e05 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -165,6 +165,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), ) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -315,6 +316,7 @@ def test_rpc_balance_handle(default_conf, mocker): 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), ) + patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( From b2b81c8b2dbb7db6cae0ccf28069c9ef633bd526 Mon Sep 17 00:00:00 2001 From: creslinux Date: Fri, 27 Jul 2018 20:18:12 +0000 Subject: [PATCH 275/395] Update documentation with hot to sandbox test. Allowing end-to-end GDAX API use without risking real money. --- README.md | 2 + docs/index.md | 1 + docs/sandbox-testing.md | 142 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 docs/sandbox-testing.md diff --git a/README.md b/README.md index c1705ff41..3d4c81580 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ hesitate to read the source code and understand the mechanism of this bot. - [Strategy Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) - [Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - [Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) + - [Sandbox Testing](https://github.com/freqtrade/freqtrade/blob/develop/docs/sandbox-testing.md) - [Basic Usage](#basic-usage) - [Bot commands](#bot-commands) - [Telegram RPC commands](#telegram-rpc-commands) @@ -61,6 +62,7 @@ hesitate to read the source code and understand the mechanism of this bot. - [Requirements](#requirements) - [Min hardware required](#min-hardware-required) - [Software requirements](#software-requirements) + ## Quick start diff --git a/docs/index.md b/docs/index.md index f76bb125d..730f1095e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -33,3 +33,4 @@ Pull-request. Do not hesitate to reach us on - [Run tests & Check PEP8 compliance](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) - [FAQ](https://github.com/freqtrade/freqtrade/blob/develop/docs/faq.md) - [SQL cheatsheet](https://github.com/freqtrade/freqtrade/blob/develop/docs/sql_cheatsheet.md) +- [Sandbox Testing](https://github.com/freqtrade/freqtrade/blob/develop/docs/sandbox-testing.md)) diff --git a/docs/sandbox-testing.md b/docs/sandbox-testing.md new file mode 100644 index 000000000..57e40e623 --- /dev/null +++ b/docs/sandbox-testing.md @@ -0,0 +1,142 @@ +# Sandbox API testing +Where an exchange provides a sandbox for risk-free integration, or end-to-end, testing CCXT provides access to these. + +This document is a *light overview of configuring Freqtrade and GDAX sandbox. +This can be useful to developers and trader alike as Freqtrade is quite customisable. + +When testing your API connectivity, make sure to use the following URLs. +***Website** +https://public.sandbox.gdax.com +***REST API** +https://api-public.sandbox.gdax.com + +--- +# Configure a Sandbox account on Gdax +Aim of this document section +- An sanbox account +- create 2FA (needed to create an API) +- Add test 50BTC to account +- Create : +- - API-KEY +- - API-Secret +- - API Password + +## Acccount + +This link will redirect to the sandbox main page to login / create account dialogues: +https://public.sandbox.pro.coinbase.com/orders/ + +After registration and Email confimation you wil be redirected into your sanbox account. It is easy to verify youre in sandbox by checking the URL bar. +> https://public.sandbox.pro.coinbase.com/ + +## Enable 2Fa (a prerequisite to creating sandbox API Keys) +From within sand box site select your profile, top right. +>Or as a direct link: https://public.sandbox.pro.coinbase.com/profile + +From the menu panel to the left of the screen select +> Security: "*View or Update*" + +In the new site select "enable authenticator" as typical google auth. +- open Google Authenticator on your phone +- scan barcode +- enter your generated 2fa + +## Enable API Access +From within sandbox select profile>api>create api-keys +>or as a direct link: https://public.sandbox.pro.coinbase.com/profile/api + +Ensure **view** and **trade** are "checked" and sumbit your 2Fa +- COPY AND PASTE THE PASSPHRASE into a notepade this will be needed later +- COPY AND PASTE THE API SECRET popup into a notepad this will needed later +- COPY AND PASTE THE API KEY into a notepad this will needed later + +## Add 50 BTC test funds +To add funds, use the web interface deposit and withdraw buttons. +Select Wallets. +> Or as a direct link: https://public.sandbox.pro.coinbase.com/wallets + +- Deposits (bottom left of screen) +- - Deposit Funds Bitcoin +- - - Coinbase BTC Wallet +- - - - MAx (50 BTC) +- - - - - Deposit + +--- +# Configure Freqtrade to use Gax Sandbox + +The aim of this document section + - enable sandbox URLs in Freqtrade + - Configure API + - - secret + - - key + - - passphrase + +## Sandbox URLs +Freqtrade makes use of CCXT which in turn provides a list of URLs to Freqtrade. +These incldue ['test'] and ['api']. +- [Test] if available will point to an Exchanges sandbox. +- [Api] normally used, and resolves to live API target on the exchange + +To make use of sandbox / test add "sandbox": true, to your config.json +``` + "exchange": { + "name": "gdax", + "sandbox": true, + "key": "5wowfxemogxeowo;heiohgmd", + "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==", + "password": "1bkjfkhfhfu6sr", + "pair_whitelist": [ + "BTC/USD" +``` +Also insert your +- api-key (noted earlier) +- api-secret (noted earlier) +- password (the passphrase - noted earlier) + +--- +## You should now be ready to test your sandbox! +Ensure Freqtrade logs show the sandbox URL, and trades made are shown in sandbox. +** Typically the BTC/USD has the most activity in sandbox to test against. + +## GDAX - Old Candles problem +It is my experience that GDAX sandbox candles may be 20+- minutes out of date. This can cause trades to fail as one of Freqtrades safety checks + +To disable this check, edit: +>strategy/interface.py +Look for the following section: +``` + # Check if dataframe is out of date + signal_date = arrow.get(latest['date']) + interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] + if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))): + logger.warning( + 'Outdated history for pair %s. Last tick is %s minutes old', + pair, + (arrow.utcnow() - signal_date).seconds // 60 + ) + return False, False +``` + +And Hash out as follows: +``` + # # Check if dataframe is out of date + # signal_date = arrow.get(latest['date']) + # interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] + # if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))): + # logger.warning( + # 'Outdated history for pair %s. Last tick is %s minutes old', + # pair, + # (arrow.utcnow() - signal_date).seconds // 60 + # ) + # return False, False + ``` + + + + + + + + + + From 099e7020c890a4cc08bc1b320f6a297b8a9cf2ed Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 28 Jul 2018 14:24:06 +0200 Subject: [PATCH 276/395] Update ccxt from 1.17.29 to 1.17.39 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4255e2c38..7407f5184 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.29 +ccxt==1.17.39 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 8648ac9da23176f7075c0ecedf084cbca830efba Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 28 Jul 2018 17:42:56 +0000 Subject: [PATCH 277/395] Update documentation with hot to sandbox test. Allowing end-to-end GDAX API use without risking real money. --- docs/sandbox-testing.md | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/docs/sandbox-testing.md b/docs/sandbox-testing.md index 57e40e623..86bb30799 100644 --- a/docs/sandbox-testing.md +++ b/docs/sandbox-testing.md @@ -26,7 +26,7 @@ Aim of this document section This link will redirect to the sandbox main page to login / create account dialogues: https://public.sandbox.pro.coinbase.com/orders/ -After registration and Email confimation you wil be redirected into your sanbox account. It is easy to verify youre in sandbox by checking the URL bar. +After registration and Email confimation you wil be redirected into your sanbox account. It is easy to verify you're in sandbox by checking the URL bar. > https://public.sandbox.pro.coinbase.com/ ## Enable 2Fa (a prerequisite to creating sandbox API Keys) @@ -46,9 +46,9 @@ From within sandbox select profile>api>create api-keys >or as a direct link: https://public.sandbox.pro.coinbase.com/profile/api Ensure **view** and **trade** are "checked" and sumbit your 2Fa -- COPY AND PASTE THE PASSPHRASE into a notepade this will be needed later -- COPY AND PASTE THE API SECRET popup into a notepad this will needed later -- COPY AND PASTE THE API KEY into a notepad this will needed later +- **Copy and paste the Passphase** into a notepade this will be needed later +- **Copy and paste the API Secret** popup into a notepad this will needed later +- **Copy and paste the API Key** into a notepad this will needed later ## Add 50 BTC test funds To add funds, use the web interface deposit and withdraw buttons. @@ -58,7 +58,7 @@ Select Wallets. - Deposits (bottom left of screen) - - Deposit Funds Bitcoin - - - Coinbase BTC Wallet -- - - - MAx (50 BTC) +- - - - Max (50 BTC) - - - - - Deposit --- @@ -130,13 +130,4 @@ And Hash out as follows: # ) # return False, False ``` - - - - - - - - - - + \ No newline at end of file From cdd8cc551c8dbdc031ca45d85240eea836e6ebf6 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 28 Jul 2018 21:56:11 +0300 Subject: [PATCH 278/395] backtesting: try to load data with ujson if it exists --- freqtrade/optimize/__init__.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index e806ff2b8..5aa47725d 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,7 +1,11 @@ # pragma pylint: disable=missing-docstring import gzip -import json +try: + import ujson as json +except ImportError: + import json +import inspect import logging import os from typing import Optional, List, Dict, Tuple, Any @@ -14,6 +18,14 @@ from freqtrade.arguments import TimeRange logger = logging.getLogger(__name__) +def json_load(data): + """Try to load data with ujson""" + if inspect.getfullargspec(json.load)[5].get('precise_float'): + return json.load(data, precise_float=True) + else: + return json.load(data) + + def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: if not tickerlist: return tickerlist @@ -63,11 +75,11 @@ def load_tickerdata_file( if os.path.isfile(gzipfile): logger.debug('Loading ticker data from file %s', gzipfile) with gzip.open(gzipfile) as tickerdata: - pairdata = json.load(tickerdata) + pairdata = json_load(tickerdata) elif os.path.isfile(file): logger.debug('Loading ticker data from file %s', file) with open(file) as tickerdata: - pairdata = json.load(tickerdata) + pairdata = json_load(tickerdata) else: return None @@ -163,7 +175,7 @@ def load_cached_data_for_updating(filename: str, # read the cached file if os.path.isfile(filename): with open(filename, "rt") as file: - data = json.load(file) + data = json_load(file) # remove the last item, because we are not sure if it is correct # it could be fetched when the candle was incompleted if data: From cb2fff89095d1f3102d4517239483837f6c15792 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 28 Jul 2018 22:06:26 +0300 Subject: [PATCH 279/395] mypy doesn't handle common idiomacy so disable the line (see the open issue more details) --- freqtrade/optimize/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 5aa47725d..5c1bd06ab 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -4,7 +4,8 @@ import gzip try: import ujson as json except ImportError: - import json + # see mypy/issues/1153 + import json # type: ignore import inspect import logging import os From 0a059662b3c46994f25fb2689bdd1926aaa72ec5 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 28 Jul 2018 20:32:10 +0000 Subject: [PATCH 280/395] Submitting with unit test for the working scenario. Strongly recommend core team check the unit test is even targetting the correct code in exchange/__init__.py I have a real knowledge gap on mocker, in so far as how tests map to what they're targeting. --- freqtrade/exchange/__init__.py | 12 ++++++--- freqtrade/tests/exchange/test_exchange.py | 30 +++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 64ada5ef6..670d9d3a5 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -95,9 +95,7 @@ class Exchange(object): except (KeyError, AttributeError): raise OperationalException(f'Exchange {name} is not supported') - # check if config requests sanbox, if so use ['test'] from url - if (exchange_config.get('sandbox')): - api.urls['api'] = api.urls['test'] + self.set_sandbox(api, exchange_config, name) return api @@ -111,6 +109,14 @@ class Exchange(object): """exchange ccxt id""" return self._api.id + def set_sandbox(self, api, exchange_config: dict, name: str): + if exchange_config.get('sandbox'): + if api.urls.get('test'): + api.urls['api'] = api.urls['test'] + else: + logger.warning(self, "No Sandbox URL in CCXT, exiting. Please check your config.json") + raise OperationalException(f'Exchange {name} does not provide a sandbox api') + def validate_pairs(self, pairs: List[str]) -> None: """ Checks if all given pairs are tradable on the current exchange. diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 282d8ef01..1a0287857 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -51,6 +51,36 @@ def test_init_exception(default_conf, mocker): mocker.patch("ccxt.binance", MagicMock(side_effect=AttributeError)) Exchange(default_conf) +def test_set_sandbox(default_conf, mocker): + """ + Test working scenario + """ + api_mock = MagicMock() + api_mock.load_markets = MagicMock(return_value={ + 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' + }) + url_mock = PropertyMock(return_value={'test': "api-public.sandbox.gdax.com", 'api': 'https://api.gdax.com'}) + type(api_mock).urls = url_mock + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + Exchange(default_conf) + +# def test_set_sandbox_exception(default_conf, mocker): +# """ +# Test Fail scenario +# """ +# api_mock = MagicMock() +# api_mock.load_markets = MagicMock(return_value={ +# 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' +# }) +# url_mock = PropertyMock(return_value={'api': 'https://api.gdax.com'}) +# type(api_mock).urls = url_mock +# +# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) +# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) +# +# with pytest.raises(OperationalException, match=r'does not provide a sandbox api'): +# Exchange(default_conf) def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() From b3df1b1ba70d476d7558bf43c82ffdc86cd2aaf3 Mon Sep 17 00:00:00 2001 From: Gert Date: Sat, 28 Jul 2018 21:31:20 -0700 Subject: [PATCH 281/395] added documentation: --- docs/configuration.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index dd16ef6b5..57c7a0bec 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -191,6 +191,33 @@ you run it in production mode. ``` If you have not your Bittrex API key yet, [see our tutorial](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md). + +### Embedding Strategies + +FreqTrade provides you with with an easy way to embed the strategy into your configuration file. +This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, +in your chosen config file. + +##### Encoding a string as BASE64 + +This is a quick example, how to generate the BASE64 string in python + +```python +from base64 import urlsafe_b64encode + +with open(file, 'r') as f: + content = f.read() +content = urlsafe_b64encode(content.encode('utf-8')) +``` + +The variable 'content', will contain the strategy file in a BASE64 encoded form. Which can now be set in your configurations file as following + +```json +"strategy": "NameOfStrategy:BASE64String" +``` + +Please ensure that 'NameOfStrategy' is identical to the strategy name! + ## Next step Now you have configured your config.json, the next step is to [start your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md). From fc06d028b8b857368f202495dbf18f44d8945eac Mon Sep 17 00:00:00 2001 From: creslinux Date: Sun, 29 Jul 2018 08:02:04 +0000 Subject: [PATCH 282/395] Unit tests for sandbox pass / fail scenarios Big Wave of appreciation to xmatthias for the guidence on how Mocker works --- freqtrade/tests/exchange/test_exchange.py | 43 ++++++++++++++--------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 1a0287857..862229bea 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -63,24 +63,33 @@ def test_set_sandbox(default_conf, mocker): type(api_mock).urls = url_mock mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) - Exchange(default_conf) -# def test_set_sandbox_exception(default_conf, mocker): -# """ -# Test Fail scenario -# """ -# api_mock = MagicMock() -# api_mock.load_markets = MagicMock(return_value={ -# 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' -# }) -# url_mock = PropertyMock(return_value={'api': 'https://api.gdax.com'}) -# type(api_mock).urls = url_mock -# -# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) -# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) -# -# with pytest.raises(OperationalException, match=r'does not provide a sandbox api'): -# Exchange(default_conf) + exchange = Exchange(default_conf) + liveurl = exchange._api.urls['api'] + default_conf['exchange']['sandbox'] = True + exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname') + assert exchange._api.urls['api'] != liveurl + +def test_set_sandbox_exception(default_conf, mocker): + """ + Test Fail scenario + """ + api_mock = MagicMock() + api_mock.load_markets = MagicMock(return_value={ + 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' + }) + url_mock = PropertyMock(return_value={'api': 'https://api.gdax.com'}) + type(api_mock).urls = url_mock + + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + + with pytest.raises(OperationalException, match=r'does not provide a sandbox api'): + exchange = Exchange(default_conf) + default_conf['exchange']['sandbox'] = True + exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname') + assert exchange._api.urls.get('test') == False + def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() From 1e804c0df582589ed6e500d7a1b6a172a60e6ed4 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sun, 29 Jul 2018 08:10:55 +0000 Subject: [PATCH 283/395] flake 8 --- freqtrade/exchange/__init__.py | 3 ++- freqtrade/tests/exchange/test_exchange.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 670d9d3a5..139b1c667 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -114,7 +114,8 @@ class Exchange(object): if api.urls.get('test'): api.urls['api'] = api.urls['test'] else: - logger.warning(self, "No Sandbox URL in CCXT, exiting. Please check your config.json") + logger.warning(self, "No Sandbox URL in CCXT, exiting. " + "Please check your config.json") raise OperationalException(f'Exchange {name} does not provide a sandbox api') def validate_pairs(self, pairs: List[str]) -> None: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 862229bea..2c58d928c 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -51,6 +51,7 @@ def test_init_exception(default_conf, mocker): mocker.patch("ccxt.binance", MagicMock(side_effect=AttributeError)) Exchange(default_conf) + def test_set_sandbox(default_conf, mocker): """ Test working scenario @@ -59,7 +60,8 @@ def test_set_sandbox(default_conf, mocker): api_mock.load_markets = MagicMock(return_value={ 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' }) - url_mock = PropertyMock(return_value={'test': "api-public.sandbox.gdax.com", 'api': 'https://api.gdax.com'}) + url_mock = PropertyMock(return_value={'test': "api-public.sandbox.gdax.com", + 'api': 'https://api.gdax.com'}) type(api_mock).urls = url_mock mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) @@ -70,6 +72,7 @@ def test_set_sandbox(default_conf, mocker): exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname') assert exchange._api.urls['api'] != liveurl + def test_set_sandbox_exception(default_conf, mocker): """ Test Fail scenario @@ -88,7 +91,6 @@ def test_set_sandbox_exception(default_conf, mocker): exchange = Exchange(default_conf) default_conf['exchange']['sandbox'] = True exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname') - assert exchange._api.urls.get('test') == False def test_validate_pairs(default_conf, mocker): From c85c7a3a77a9cbf60b89ed8c2c2e18511c9d23af Mon Sep 17 00:00:00 2001 From: creslinux Date: Sun, 29 Jul 2018 09:12:05 +0000 Subject: [PATCH 284/395] Documentation fixes. --- docs/sandbox-testing.md | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/docs/sandbox-testing.md b/docs/sandbox-testing.md index 86bb30799..572fbccef 100644 --- a/docs/sandbox-testing.md +++ b/docs/sandbox-testing.md @@ -36,7 +36,7 @@ From within sand box site select your profile, top right. From the menu panel to the left of the screen select > Security: "*View or Update*" -In the new site select "enable authenticator" as typical google auth. +In the new site select "enable authenticator" as typical google Authenticator. - open Google Authenticator on your phone - scan barcode - enter your generated 2fa @@ -45,14 +45,16 @@ In the new site select "enable authenticator" as typical google auth. From within sandbox select profile>api>create api-keys >or as a direct link: https://public.sandbox.pro.coinbase.com/profile/api -Ensure **view** and **trade** are "checked" and sumbit your 2Fa +Click on "create one" and ensure **view** and **trade** are "checked" and sumbit your 2Fa - **Copy and paste the Passphase** into a notepade this will be needed later - **Copy and paste the API Secret** popup into a notepad this will needed later - **Copy and paste the API Key** into a notepad this will needed later ## Add 50 BTC test funds To add funds, use the web interface deposit and withdraw buttons. -Select Wallets. + + +To begin select 'Wallets' from the top menu. > Or as a direct link: https://public.sandbox.pro.coinbase.com/wallets - Deposits (bottom left of screen) @@ -61,11 +63,12 @@ Select Wallets. - - - - Max (50 BTC) - - - - - Deposit +*This process may be repeated for other currencies, ETH as example* --- # Configure Freqtrade to use Gax Sandbox The aim of this document section - - enable sandbox URLs in Freqtrade + - Enable sandbox URLs in Freqtrade - Configure API - - secret - - key @@ -73,9 +76,9 @@ The aim of this document section ## Sandbox URLs Freqtrade makes use of CCXT which in turn provides a list of URLs to Freqtrade. -These incldue ['test'] and ['api']. -- [Test] if available will point to an Exchanges sandbox. -- [Api] normally used, and resolves to live API target on the exchange +These include `['test']` and `['api']`. +- `[Test]` if available will point to an Exchanges sandbox. +- `[Api]` normally used, and resolves to live API target on the exchange To make use of sandbox / test add "sandbox": true, to your config.json ``` @@ -117,7 +120,7 @@ Look for the following section: return False, False ``` -And Hash out as follows: +You could Hash out the entire check as follows: ``` # # Check if dataframe is out of date # signal_date = arrow.get(latest['date']) @@ -130,4 +133,19 @@ And Hash out as follows: # ) # return False, False ``` - \ No newline at end of file + + Or inrease the timeout to offer a level of protection/alignment of this test to freqtrade in live. + + As example, to allow an additional 30 minutes. "(interval_minutes * 2 + 5 + 30)" + ``` + # Check if dataframe is out of date + signal_date = arrow.get(latest['date']) + interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] + if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5 + 30))): + logger.warning( + 'Outdated history for pair %s. Last tick is %s minutes old', + pair, + (arrow.utcnow() - signal_date).seconds // 60 + ) + return False, False +``` \ No newline at end of file From dd71071740bb6017fce4dad3bd94403d92564b24 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sun, 29 Jul 2018 09:15:13 +0000 Subject: [PATCH 285/395] Added logger.info when Sandbox is enabled. --- freqtrade/exchange/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 139b1c667..c57986658 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -113,6 +113,7 @@ class Exchange(object): if exchange_config.get('sandbox'): if api.urls.get('test'): api.urls['api'] = api.urls['test'] + logger.info("Enabled Sandbox API on %s", name) else: logger.warning(self, "No Sandbox URL in CCXT, exiting. " "Please check your config.json") From 7f27beff4baca0f6a05c09a788133a240858e7cb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 13:23:11 +0200 Subject: [PATCH 286/395] Revert "backtesting: try to load data with ujson if it exists" --- freqtrade/optimize/__init__.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 5c1bd06ab..e806ff2b8 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,12 +1,7 @@ # pragma pylint: disable=missing-docstring import gzip -try: - import ujson as json -except ImportError: - # see mypy/issues/1153 - import json # type: ignore -import inspect +import json import logging import os from typing import Optional, List, Dict, Tuple, Any @@ -19,14 +14,6 @@ from freqtrade.arguments import TimeRange logger = logging.getLogger(__name__) -def json_load(data): - """Try to load data with ujson""" - if inspect.getfullargspec(json.load)[5].get('precise_float'): - return json.load(data, precise_float=True) - else: - return json.load(data) - - def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: if not tickerlist: return tickerlist @@ -76,11 +63,11 @@ def load_tickerdata_file( if os.path.isfile(gzipfile): logger.debug('Loading ticker data from file %s', gzipfile) with gzip.open(gzipfile) as tickerdata: - pairdata = json_load(tickerdata) + pairdata = json.load(tickerdata) elif os.path.isfile(file): logger.debug('Loading ticker data from file %s', file) with open(file) as tickerdata: - pairdata = json_load(tickerdata) + pairdata = json.load(tickerdata) else: return None @@ -176,7 +163,7 @@ def load_cached_data_for_updating(filename: str, # read the cached file if os.path.isfile(filename): with open(filename, "rt") as file: - data = json_load(file) + data = json.load(file) # remove the last item, because we are not sure if it is correct # it could be fetched when the candle was incompleted if data: From ebfcc0fc13e66b54daa521c3ce7d1b8910012254 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 14:01:50 +0200 Subject: [PATCH 287/395] install numpy before ta-lib to fix build errors --- Dockerfile | 3 ++- docs/installation.md | 14 ++++++++++---- setup.py | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index d2c2b1b22..309763d2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,8 @@ WORKDIR /freqtrade # Install dependencies COPY requirements.txt /freqtrade/ -RUN pip install -r requirements.txt +RUN pip install numpy \ + && pip install -r requirements.txt # Install and execute COPY . /freqtrade/ diff --git a/docs/installation.md b/docs/installation.md index e5724a7dc..7a7719fc0 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -56,23 +56,29 @@ Reset parameter will hard reset your branch (only if you are on `master` or `dev Config parameter is a `config.json` configurator. This script will ask you questions to setup your bot and create your `config.json`. - ## Manual installation - Linux/MacOS + The following steps are made for Linux/MacOS environment -**1. Clone the repo** +### 1. Clone the repo + ```bash git clone git@github.com:freqtrade/freqtrade.git git checkout develop cd freqtrade ``` -**2. Create the config file** + +### 2. Create the config file + Switch `"dry_run": true,` + ```bash cp config.json.example config.json vi config.json ``` -**3. Build your docker image and run it** + +### 3. Build your docker image and run it + ```bash docker build -t freqtrade . docker run --rm -v /etc/localtime:/etc/localtime:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade diff --git a/setup.py b/setup.py index cd0574fa2..8853ef7f8 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup(name='freqtrade', license='GPLv3', packages=['freqtrade'], scripts=['bin/freqtrade'], - setup_requires=['pytest-runner'], + setup_requires=['pytest-runner', 'numpy'], tests_require=['pytest', 'pytest-mock', 'pytest-cov'], install_requires=[ 'ccxt', From 9c7f53d90dceb25f26f056e0f7c66d361134bc67 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 29 Jul 2018 14:24:06 +0200 Subject: [PATCH 288/395] Update ccxt from 1.17.39 to 1.17.45 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7407f5184..21dfe2948 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.39 +ccxt==1.17.45 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 2ef35400c9620885b07832ba3539a3788635bbd0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 29 Jul 2018 14:24:08 +0200 Subject: [PATCH 289/395] Update pytest from 3.6.3 to 3.6.4 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 21dfe2948..3a00111ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.0 TA-Lib==0.4.17 -pytest==3.6.3 +pytest==3.6.4 pytest-mock==1.10.0 pytest-cov==2.5.1 tabulate==0.8.2 From 1bbb86c621be1892a57c223c367ab900f6a54446 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 29 Jul 2018 16:23:17 +0300 Subject: [PATCH 290/395] remove nonsense asserts --- freqtrade/tests/rpc/test_rpc_manager.py | 6 ------ freqtrade/tests/strategy/test_strategy.py | 12 ----------- freqtrade/tests/test_arguments.py | 12 ----------- freqtrade/tests/test_configuration.py | 13 ------------ freqtrade/tests/test_constants.py | 25 ----------------------- freqtrade/tests/test_freqtradebot.py | 16 --------------- freqtrade/tests/test_state.py | 14 ------------- 7 files changed, 98 deletions(-) delete mode 100644 freqtrade/tests/test_constants.py delete mode 100644 freqtrade/tests/test_state.py diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 4686cf5ca..0280f9cc3 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -10,12 +10,6 @@ from freqtrade.rpc import RPCMessageType, RPCManager from freqtrade.tests.conftest import log_has, get_patched_freqtradebot -def test_rpc_manager_object() -> None: - """ Test the Arguments object has the mandatory methods """ - assert hasattr(RPCManager, 'send_msg') - assert hasattr(RPCManager, 'cleanup') - - def test__init__(mocker, default_conf) -> None: """ Test __init__() method """ conf = deepcopy(default_conf) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 2f9221467..e25738775 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -57,7 +57,6 @@ def test_search_strategy(): def test_load_strategy(result): resolver = StrategyResolver({'strategy': 'TestStrategy'}) - assert hasattr(resolver.strategy, 'populate_indicators') assert 'adx' in resolver.strategy.populate_indicators(result) @@ -71,8 +70,6 @@ def test_load_strategy_invalid_directory(result, caplog): logging.WARNING, 'Path "{}" does not exist'.format(extra_dir), ) in caplog.record_tuples - - assert hasattr(resolver.strategy, 'populate_indicators') assert 'adx' in resolver.strategy.populate_indicators(result) @@ -89,26 +86,20 @@ def test_strategy(result): resolver = StrategyResolver(config) - assert hasattr(resolver.strategy, 'minimal_roi') assert resolver.strategy.minimal_roi[0] == 0.04 assert config["minimal_roi"]['0'] == 0.04 - assert hasattr(resolver.strategy, 'stoploss') assert resolver.strategy.stoploss == -0.10 assert config['stoploss'] == -0.10 - assert hasattr(resolver.strategy, 'ticker_interval') assert resolver.strategy.ticker_interval == '5m' assert config['ticker_interval'] == '5m' - assert hasattr(resolver.strategy, 'populate_indicators') assert 'adx' in resolver.strategy.populate_indicators(result) - assert hasattr(resolver.strategy, 'populate_buy_trend') dataframe = resolver.strategy.populate_buy_trend(resolver.strategy.populate_indicators(result)) assert 'buy' in dataframe.columns - assert hasattr(resolver.strategy, 'populate_sell_trend') dataframe = resolver.strategy.populate_sell_trend(resolver.strategy.populate_indicators(result)) assert 'sell' in dataframe.columns @@ -123,7 +114,6 @@ def test_strategy_override_minimal_roi(caplog): } resolver = StrategyResolver(config) - assert hasattr(resolver.strategy, 'minimal_roi') assert resolver.strategy.minimal_roi[0] == 0.5 assert ('freqtrade.strategy.resolver', logging.INFO, @@ -139,7 +129,6 @@ def test_strategy_override_stoploss(caplog): } resolver = StrategyResolver(config) - assert hasattr(resolver.strategy, 'stoploss') assert resolver.strategy.stoploss == -0.5 assert ('freqtrade.strategy.resolver', logging.INFO, @@ -156,7 +145,6 @@ def test_strategy_override_ticker_interval(caplog): } resolver = StrategyResolver(config) - assert hasattr(resolver.strategy, 'ticker_interval') assert resolver.strategy.ticker_interval == 60 assert ('freqtrade.strategy.resolver', logging.INFO, diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 07018c79e..c7740ce47 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -11,23 +11,11 @@ import pytest from freqtrade.arguments import Arguments, TimeRange -def test_arguments_object() -> None: - """ - Test the Arguments object has the mandatory methods - :return: None - """ - assert hasattr(Arguments, 'get_parsed_arg') - assert hasattr(Arguments, 'parse_args') - assert hasattr(Arguments, 'parse_timerange') - assert hasattr(Arguments, 'scripts_options') - - # Parse common command-line-arguments. Used for all tools def test_parse_args_none() -> None: arguments = Arguments([], '') assert isinstance(arguments, Arguments) assert isinstance(arguments.parser, argparse.ArgumentParser) - assert isinstance(arguments.parser, argparse.ArgumentParser) def test_parse_args_defaults() -> None: diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index a8a2c5fce..46b08a3d4 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -19,19 +19,6 @@ from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.tests.conftest import log_has -def test_configuration_object() -> None: - """ - Test the Constants object has the mandatory Constants - """ - assert hasattr(Configuration, 'load_config') - assert hasattr(Configuration, '_load_config_file') - assert hasattr(Configuration, '_validate_config') - assert hasattr(Configuration, '_load_common_config') - assert hasattr(Configuration, '_load_backtesting_config') - assert hasattr(Configuration, '_load_hyperopt_config') - assert hasattr(Configuration, 'get_config') - - def test_load_config_invalid_pair(default_conf) -> None: """ Test the configuration validator with an invalid PAIR format diff --git a/freqtrade/tests/test_constants.py b/freqtrade/tests/test_constants.py deleted file mode 100644 index 541c6e533..000000000 --- a/freqtrade/tests/test_constants.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -Unit test file for constants.py -""" - -from freqtrade import constants - - -def test_constant_object() -> None: - """ - Test the Constants object has the mandatory Constants - """ - assert hasattr(constants, 'CONF_SCHEMA') - assert hasattr(constants, 'DYNAMIC_WHITELIST') - assert hasattr(constants, 'PROCESS_THROTTLE_SECS') - assert hasattr(constants, 'TICKER_INTERVAL') - assert hasattr(constants, 'HYPEROPT_EPOCH') - assert hasattr(constants, 'RETRY_TIMEOUT') - assert hasattr(constants, 'DEFAULT_STRATEGY') - - -def test_conf_schema() -> None: - """ - Test the CONF_SCHEMA is from the right type - """ - assert isinstance(constants.CONF_SCHEMA, dict) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index a1683ae6a..b1e08383b 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -62,22 +62,6 @@ def patch_RPCManager(mocker) -> MagicMock: # Unit tests -def test_freqtradebot_object() -> None: - """ - Test the FreqtradeBot object has the mandatory public methods - """ - assert hasattr(FreqtradeBot, 'worker') - assert hasattr(FreqtradeBot, 'cleanup') - assert hasattr(FreqtradeBot, 'create_trade') - assert hasattr(FreqtradeBot, 'get_target_bid') - assert hasattr(FreqtradeBot, 'process_maybe_execute_buy') - assert hasattr(FreqtradeBot, 'process_maybe_execute_sell') - assert hasattr(FreqtradeBot, 'handle_trade') - assert hasattr(FreqtradeBot, 'check_handle_timedout') - assert hasattr(FreqtradeBot, 'handle_timedout_limit_buy') - assert hasattr(FreqtradeBot, 'handle_timedout_limit_sell') - assert hasattr(FreqtradeBot, 'execute_sell') - def test_freqtradebot(mocker, default_conf) -> None: """ diff --git a/freqtrade/tests/test_state.py b/freqtrade/tests/test_state.py deleted file mode 100644 index 51fa06cc2..000000000 --- a/freqtrade/tests/test_state.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Unit test file for constants.py -""" - -from freqtrade.state import State - - -def test_state_object() -> None: - """ - Test the State object has the mandatory states - :return: None - """ - assert hasattr(State, 'RUNNING') - assert hasattr(State, 'STOPPED') From f832edf5bcad168c2da4eccb62de91ed4e9dab0c Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Sun, 29 Jul 2018 17:09:44 +0300 Subject: [PATCH 291/395] remove useless docstrings from tests --- freqtrade/tests/exchange/test_exchange.py | 2 +- .../tests/exchange/test_exchange_helpers.py | 4 - freqtrade/tests/optimize/test_hyperopt.py | 19 ---- freqtrade/tests/optimize/test_optimize.py | 14 --- freqtrade/tests/rpc/test_rpc.py | 35 +------ freqtrade/tests/rpc/test_rpc_manager.py | 36 +------ freqtrade/tests/rpc/test_rpc_telegram.py | 97 +------------------ freqtrade/tests/rpc/test_rpc_webhook.py | 17 +--- freqtrade/tests/strategy/test_interface.py | 4 - freqtrade/tests/test_acl_pair.py | 2 - freqtrade/tests/test_configuration.py | 66 +------------ freqtrade/tests/test_indicator_helpers.py | 2 + freqtrade/tests/test_main.py | 24 +---- freqtrade/tests/test_misc.py | 24 ----- 14 files changed, 16 insertions(+), 330 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 282d8ef01..814f56acc 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -173,7 +173,7 @@ def test_validate_timeframes_not_in_config(default_conf, mocker): Exchange(default_conf) -def test_exchangehas(default_conf, mocker): +def test_exchange_has(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) assert not exchange.exchange_has('ASDFASDF') api_mock = MagicMock() diff --git a/freqtrade/tests/exchange/test_exchange_helpers.py b/freqtrade/tests/exchange/test_exchange_helpers.py index 6a3bc9eb6..82525e805 100644 --- a/freqtrade/tests/exchange/test_exchange_helpers.py +++ b/freqtrade/tests/exchange/test_exchange_helpers.py @@ -1,9 +1,5 @@ # pragma pylint: disable=missing-docstring, C0103 -""" -Unit test file for exchange_helpers.py -""" - from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 72a102c22..dd7bf7da0 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -77,9 +77,6 @@ def test_start(mocker, default_conf, caplog) -> None: def test_loss_calculation_prefer_correct_trade_count(init_hyperopt) -> None: - """ - Test Hyperopt.calculate_loss() - """ hyperopt = _HYPEROPT StrategyResolver({'strategy': 'DefaultStrategy'}) @@ -91,9 +88,6 @@ def test_loss_calculation_prefer_correct_trade_count(init_hyperopt) -> None: def test_loss_calculation_prefer_shorter_trades(init_hyperopt) -> None: - """ - Test Hyperopt.calculate_loss() - """ hyperopt = _HYPEROPT shorter = hyperopt.calculate_loss(1, 100, 20) @@ -240,9 +234,6 @@ def test_format_results(init_hyperopt): def test_has_space(init_hyperopt): - """ - Test Hyperopt.has_space() method - """ _HYPEROPT.config.update({'spaces': ['buy', 'roi']}) assert _HYPEROPT.has_space('roi') assert _HYPEROPT.has_space('buy') @@ -253,9 +244,6 @@ def test_has_space(init_hyperopt): def test_populate_indicators(init_hyperopt) -> None: - """ - Test Hyperopt.populate_indicators() - """ tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = _HYPEROPT.tickerdata_to_dataframe(tickerlist) @@ -268,9 +256,6 @@ def test_populate_indicators(init_hyperopt) -> None: def test_buy_strategy_generator(init_hyperopt) -> None: - """ - Test Hyperopt.buy_strategy_generator() - """ tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = _HYPEROPT.tickerdata_to_dataframe(tickerlist) @@ -296,9 +281,6 @@ def test_buy_strategy_generator(init_hyperopt) -> None: def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: - """ - Test Hyperopt.generate_optimizer() function - """ conf = deepcopy(default_conf) conf.update({'config': 'config.json.example'}) conf.update({'timerange': None}) @@ -335,7 +317,6 @@ def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: 'roi_p3': 0.1, 'stoploss': -0.4, } - response_expected = { 'loss': 1.9840569076926293, 'result': ' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC ' diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 4ab32343a..eef79bef3 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -53,9 +53,6 @@ def _clean_test_file(file: str) -> None: def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - """ - Test load_data() with 30 min ticker - """ mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json') _backup_file(file, copy_file=True) @@ -66,9 +63,6 @@ def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - """ - Test load_data() with 5 min ticker - """ mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json') @@ -80,11 +74,7 @@ def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: - """ - Test load_data() with 1 min ticker - """ mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) - file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') _backup_file(file, copy_file=True) optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) @@ -421,10 +411,6 @@ def test_trim_tickerlist() -> None: def test_file_dump_json() -> None: - """ - Test file_dump_json() - :return: None - """ file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'test_{id}.json'.format(id=str(uuid.uuid4()))) data = {'bar': 'foo'} diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 2b4b71e05..63624db85 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -1,9 +1,6 @@ +# pragma pylint: disable=missing-docstring, C0103 # pragma pylint: disable=invalid-sequence-index, invalid-name, too-many-arguments -""" -Unit test file for rpc/rpc.py -""" - from datetime import datetime from unittest.mock import MagicMock, ANY @@ -28,9 +25,6 @@ def prec_satoshi(a, b) -> float: # Unit tests def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: - """ - Test rpc_trade_status() method - """ patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -72,9 +66,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: - """ - Test rpc_status_table() method - """ patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -106,9 +97,6 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: def test_rpc_daily_profit(default_conf, update, ticker, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: - """ - Test rpc_daily_profit() method - """ patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -158,9 +146,6 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: - """ - Test rpc_trade_statistics() method - """ mocker.patch.multiple( 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), @@ -237,9 +222,6 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # trade.open_rate (it is set to None) def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, ticker_sell_up, limit_buy_order, limit_sell_order): - """ - Test rpc_trade_statistics() method - """ mocker.patch.multiple( 'freqtrade.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), @@ -344,9 +326,6 @@ def test_rpc_balance_handle(default_conf, mocker): def test_rpc_start(mocker, default_conf) -> None: - """ - Test rpc_start() method - """ patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -370,9 +349,6 @@ def test_rpc_start(mocker, default_conf) -> None: def test_rpc_stop(mocker, default_conf) -> None: - """ - Test rpc_stop() method - """ patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -397,9 +373,6 @@ def test_rpc_stop(mocker, default_conf) -> None: def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: - """ - Test rpc_forcesell() method - """ patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) @@ -501,9 +474,6 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: def test_performance_handle(default_conf, ticker, limit_buy_order, fee, limit_sell_order, markets, mocker) -> None: - """ - Test rpc_performance() method - """ patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -540,9 +510,6 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, def test_rpc_count(mocker, default_conf, ticker, fee, markets) -> None: - """ - Test rpc_count() method - """ patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 0280f9cc3..c4f27787b 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -1,6 +1,4 @@ -""" -Unit test file for rpc/rpc_manager.py -""" +# pragma pylint: disable=missing-docstring, C0103 import logging from copy import deepcopy @@ -11,7 +9,6 @@ from freqtrade.tests.conftest import log_has, get_patched_freqtradebot def test__init__(mocker, default_conf) -> None: - """ Test __init__() method """ conf = deepcopy(default_conf) conf['telegram']['enabled'] = False @@ -20,12 +17,9 @@ def test__init__(mocker, default_conf) -> None: def test_init_telegram_disabled(mocker, default_conf, caplog) -> None: - """ Test _init() method with Telegram disabled """ caplog.set_level(logging.DEBUG) - conf = deepcopy(default_conf) conf['telegram']['enabled'] = False - rpc_manager = RPCManager(get_patched_freqtradebot(mocker, conf)) assert not log_has('Enabling rpc.telegram ...', caplog.record_tuples) @@ -33,12 +27,8 @@ def test_init_telegram_disabled(mocker, default_conf, caplog) -> None: def test_init_telegram_enabled(mocker, default_conf, caplog) -> None: - """ - Test _init() method with Telegram enabled - """ caplog.set_level(logging.DEBUG) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) - rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) assert log_has('Enabling rpc.telegram ...', caplog.record_tuples) @@ -48,12 +38,8 @@ def test_init_telegram_enabled(mocker, default_conf, caplog) -> None: def test_cleanup_telegram_disabled(mocker, default_conf, caplog) -> None: - """ - Test cleanup() method with Telegram disabled - """ caplog.set_level(logging.DEBUG) telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.cleanup', MagicMock()) - conf = deepcopy(default_conf) conf['telegram']['enabled'] = False @@ -66,9 +52,6 @@ def test_cleanup_telegram_disabled(mocker, default_conf, caplog) -> None: def test_cleanup_telegram_enabled(mocker, default_conf, caplog) -> None: - """ - Test cleanup() method with Telegram enabled - """ caplog.set_level(logging.DEBUG) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.cleanup', MagicMock()) @@ -86,11 +69,7 @@ def test_cleanup_telegram_enabled(mocker, default_conf, caplog) -> None: def test_send_msg_telegram_disabled(mocker, default_conf, caplog) -> None: - """ - Test send_msg() method with Telegram disabled - """ telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) - conf = deepcopy(default_conf) conf['telegram']['enabled'] = False @@ -106,9 +85,6 @@ def test_send_msg_telegram_disabled(mocker, default_conf, caplog) -> None: def test_send_msg_telegram_enabled(mocker, default_conf, caplog) -> None: - """ - Test send_msg() method with Telegram disabled - """ telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) @@ -124,13 +100,10 @@ def test_send_msg_telegram_enabled(mocker, default_conf, caplog) -> None: def test_init_webhook_disabled(mocker, default_conf, caplog) -> None: - """ Test _init() method with Webhook disabled """ caplog.set_level(logging.DEBUG) - conf = deepcopy(default_conf) conf['telegram']['enabled'] = False conf['webhook'] = {'enabled': False} - rpc_manager = RPCManager(get_patched_freqtradebot(mocker, conf)) assert not log_has('Enabling rpc.webhook ...', caplog.record_tuples) @@ -138,16 +111,11 @@ def test_init_webhook_disabled(mocker, default_conf, caplog) -> None: def test_init_webhook_enabled(mocker, default_conf, caplog) -> None: - """ - Test _init() method with Webhook enabled - """ caplog.set_level(logging.DEBUG) default_conf['telegram']['enabled'] = False default_conf['webhook'] = {'enabled': True, 'url': "https://DEADBEEF.com"} - rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) assert log_has('Enabling rpc.webhook ...', caplog.record_tuples) - len_modules = len(rpc_manager.registered_modules) - assert len_modules == 1 + assert len(rpc_manager.registered_modules) == 1 assert 'webhook' in [mod.name for mod in rpc_manager.registered_modules] diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index b2cab6d37..ceb8a7808 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1,10 +1,7 @@ +# pragma pylint: disable=missing-docstring, C0103 # pragma pylint: disable=protected-access, unused-argument, invalid-name # pragma pylint: disable=too-many-lines, too-many-arguments -""" -Unit test file for rpc/telegram.py -""" - import re from copy import deepcopy from datetime import datetime @@ -55,9 +52,6 @@ class DummyCls(Telegram): def test__init__(default_conf, mocker) -> None: - """ - Test __init__() method - """ mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) @@ -67,7 +61,6 @@ def test__init__(default_conf, mocker) -> None: def test_init(default_conf, mocker, caplog) -> None: - """ Test _init() method """ start_polling = MagicMock() mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock(return_value=start_polling)) @@ -86,9 +79,6 @@ def test_init(default_conf, mocker, caplog) -> None: def test_cleanup(default_conf, mocker) -> None: - """ - Test cleanup() method - """ updater_mock = MagicMock() updater_mock.stop = MagicMock() mocker.patch('freqtrade.rpc.telegram.Updater', updater_mock) @@ -99,9 +89,6 @@ def test_cleanup(default_conf, mocker) -> None: def test_authorized_only(default_conf, mocker, caplog) -> None: - """ - Test authorized_only() method when we are authorized - """ patch_coinmarketcap(mocker) patch_exchange(mocker, None) @@ -131,9 +118,6 @@ def test_authorized_only(default_conf, mocker, caplog) -> None: def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: - """ - Test authorized_only() method when we are unauthorized - """ patch_coinmarketcap(mocker) patch_exchange(mocker, None) chat = Chat(0xdeadbeef, 0) @@ -162,9 +146,6 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: def test_authorized_only_exception(default_conf, mocker, caplog) -> None: - """ - Test authorized_only() method when an exception is thrown - """ patch_coinmarketcap(mocker) patch_exchange(mocker) @@ -195,9 +176,6 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: - """ - Test _status() method - """ update.message.chat.id = 123 conf = deepcopy(default_conf) conf['telegram']['enabled'] = False @@ -254,9 +232,6 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> None: - """ - Test _status() method - """ patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -302,9 +277,6 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) -> None: - """ - Test _status_table() method - """ patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -357,9 +329,6 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, limit_sell_order, markets, mocker) -> None: - """ - Test _daily() method - """ patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch( 'freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', @@ -431,9 +400,6 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: - """ - Test _daily() method - """ patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -470,9 +436,6 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: - """ - Test _profit() method - """ patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( @@ -531,10 +494,6 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, def test_telegram_balance_handle(default_conf, update, mocker) -> None: - """ - Test _balance() method - """ - mock_balance = { 'BTC': { 'total': 12.0, @@ -559,9 +518,6 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: } def mock_ticker(symbol, refresh): - """ - Mock Bittrex.get_ticker() response - """ if symbol == 'BTC/USDT': return { 'bid': 10000.00, @@ -602,10 +558,7 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: assert 'BTC: 14.00000000' in result -def test_zero_balance_handle(default_conf, update, mocker) -> None: - """ - Test _balance() method when the Exchange platform returns nothing - """ +def test_balance_handle_empty_response(default_conf, update, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value={}) msg_mock = MagicMock() @@ -627,9 +580,6 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None: def test_start_handle(default_conf, update, mocker) -> None: - """ - Test _start() method - """ msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -648,9 +598,6 @@ def test_start_handle(default_conf, update, mocker) -> None: def test_start_handle_already_running(default_conf, update, mocker) -> None: - """ - Test _start() method - """ msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -670,9 +617,6 @@ def test_start_handle_already_running(default_conf, update, mocker) -> None: def test_stop_handle(default_conf, update, mocker) -> None: - """ - Test _stop() method - """ patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -693,9 +637,6 @@ def test_stop_handle(default_conf, update, mocker) -> None: def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: - """ - Test _stop() method - """ patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -716,7 +657,6 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: def test_reload_conf_handle(default_conf, update, mocker) -> None: - """ Test _reload_conf() method """ patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -738,9 +678,6 @@ def test_reload_conf_handle(default_conf, update, mocker) -> None: def test_forcesell_handle(default_conf, update, ticker, fee, ticker_sell_up, markets, mocker) -> None: - """ - Test _forcesell() method - """ patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) @@ -790,9 +727,6 @@ def test_forcesell_handle(default_conf, update, ticker, fee, def test_forcesell_down_handle(default_conf, update, ticker, fee, ticker_sell_down, markets, mocker) -> None: - """ - Test _forcesell() method - """ patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) @@ -846,9 +780,6 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker) -> None: - """ - Test _forcesell() method - """ patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) @@ -894,9 +825,6 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: - """ - Test _forcesell() method - """ patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) msg_mock = MagicMock() @@ -937,9 +865,6 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: def test_performance_handle(default_conf, update, ticker, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: - """ - Test _performance() method - """ patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -979,9 +904,6 @@ def test_performance_handle(default_conf, update, ticker, fee, def test_performance_handle_invalid(default_conf, update, mocker) -> None: - """ - Test _performance() method - """ patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -1002,9 +924,6 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> None: - """ - Test _count() method - """ patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -1046,9 +965,6 @@ def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> Non def test_help_handle(default_conf, update, mocker) -> None: - """ - Test _help() method - """ patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -1066,9 +982,6 @@ def test_help_handle(default_conf, update, mocker) -> None: def test_version_handle(default_conf, update, mocker) -> None: - """ - Test _version() method - """ patch_coinmarketcap(mocker) msg_mock = MagicMock() mocker.patch.multiple( @@ -1266,9 +1179,6 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: def test__send_msg(default_conf, mocker) -> None: - """ - Test send_msg() method - """ patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) conf = deepcopy(default_conf) @@ -1282,9 +1192,6 @@ def test__send_msg(default_conf, mocker) -> None: def test__send_msg_network_error(default_conf, mocker, caplog) -> None: - """ - Test send_msg() method - """ patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) conf = deepcopy(default_conf) diff --git a/freqtrade/tests/rpc/test_rpc_webhook.py b/freqtrade/tests/rpc/test_rpc_webhook.py index b9c005d73..bfe0416b0 100644 --- a/freqtrade/tests/rpc/test_rpc_webhook.py +++ b/freqtrade/tests/rpc/test_rpc_webhook.py @@ -1,9 +1,10 @@ +# pragma pylint: disable=missing-docstring, C0103, protected-access + from unittest.mock import MagicMock import pytest from requests import RequestException - from freqtrade.rpc import RPCMessageType from freqtrade.rpc.webhook import Webhook from freqtrade.tests.conftest import get_patched_freqtradebot, log_has @@ -32,23 +33,12 @@ def get_webhook_dict() -> dict: def test__init__(mocker, default_conf): - """ - Test __init__() method - """ default_conf['webhook'] = {'enabled': True, 'url': "https://DEADBEEF.com"} webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) assert webhook._config == default_conf -def test_cleanup(default_conf, mocker) -> None: - """ - Test cleanup() method - not needed for webhook - """ - pass - - def test_send_msg(default_conf, mocker): - """ Test send_msg for Webhook rpc class""" default_conf["webhook"] = get_webhook_dict() msg_mock = MagicMock() mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) @@ -118,7 +108,6 @@ def test_send_msg(default_conf, mocker): def test_exception_send_msg(default_conf, mocker, caplog): - """Test misconfigured notification""" default_conf["webhook"] = get_webhook_dict() default_conf["webhook"]["webhookbuy"] = None @@ -158,8 +147,6 @@ def test_exception_send_msg(default_conf, mocker, caplog): def test__send_msg(default_conf, mocker, caplog): - """Test internal method - calling the actual api""" - default_conf["webhook"] = get_webhook_dict() webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) msg = {'value1': 'DEADBEEF', diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index de11a9d2c..1099f4b5f 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -1,9 +1,5 @@ # pragma pylint: disable=missing-docstring, C0103 -""" -Unit test file for analyse.py -""" - import logging from unittest.mock import MagicMock diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 094c166b8..535684b22 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -11,7 +11,6 @@ import freqtrade.tests.conftest as tt # test tools def whitelist_conf(): config = tt.default_conf() - config['stake_currency'] = 'BTC' config['exchange']['pair_whitelist'] = [ 'ETH/BTC', @@ -20,7 +19,6 @@ def whitelist_conf(): 'SWT/BTC', 'BCC/BTC' ] - config['exchange']['pair_blacklist'] = [ 'BLK/BTC' ] diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 46b08a3d4..d4f9f46e1 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -1,8 +1,5 @@ -# pragma pylint: disable=protected-access, invalid-name +# pragma pylint: disable=missing-docstring, protected-access, invalid-name -""" -Unit test file for configuration.py -""" import json from argparse import Namespace from copy import deepcopy @@ -20,9 +17,6 @@ from freqtrade.tests.conftest import log_has def test_load_config_invalid_pair(default_conf) -> None: - """ - Test the configuration validator with an invalid PAIR format - """ conf = deepcopy(default_conf) conf['exchange']['pair_whitelist'].append('ETH-BTC') @@ -32,9 +26,6 @@ def test_load_config_invalid_pair(default_conf) -> None: def test_load_config_missing_attributes(default_conf) -> None: - """ - Test the configuration validator with a missing attribute - """ conf = deepcopy(default_conf) conf.pop('exchange') @@ -44,9 +35,6 @@ def test_load_config_missing_attributes(default_conf) -> None: def test_load_config_incorrect_stake_amount(default_conf) -> None: - """ - Test the configuration validator with a missing attribute - """ conf = deepcopy(default_conf) conf['stake_amount'] = 'fake' @@ -56,9 +44,6 @@ def test_load_config_incorrect_stake_amount(default_conf) -> None: def test_load_config_file(default_conf, mocker, caplog) -> None: - """ - Test Configuration._load_config_file() method - """ file_mock = mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) @@ -72,9 +57,6 @@ def test_load_config_file(default_conf, mocker, caplog) -> None: def test_load_config_max_open_trades_zero(default_conf, mocker, caplog) -> None: - """ - Test Configuration._load_config_file() method - """ conf = deepcopy(default_conf) conf['max_open_trades'] = 0 file_mock = mocker.patch('freqtrade.configuration.open', mocker.mock_open( @@ -87,9 +69,6 @@ def test_load_config_max_open_trades_zero(default_conf, mocker, caplog) -> None: def test_load_config_file_exception(mocker) -> None: - """ - Test Configuration._load_config_file() method - """ mocker.patch( 'freqtrade.configuration.open', MagicMock(side_effect=FileNotFoundError('File not found')) @@ -101,9 +80,6 @@ def test_load_config_file_exception(mocker) -> None: def test_load_config(default_conf, mocker) -> None: - """ - Test Configuration.load_config() without any cli params - """ mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) @@ -118,13 +94,9 @@ def test_load_config(default_conf, mocker) -> None: def test_load_config_with_params(default_conf, mocker) -> None: - """ - Test Configuration.load_config() with cli params used - """ mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) - arglist = [ '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', @@ -132,7 +104,6 @@ def test_load_config_with_params(default_conf, mocker) -> None: '--db-url', 'sqlite:///someurl', ] args = Arguments(arglist, '').get_parsed_arg() - configuration = Configuration(args) validated_conf = configuration.load_config() @@ -149,10 +120,10 @@ def test_load_config_with_params(default_conf, mocker) -> None: )) arglist = [ - '--dynamic-whitelist', '10', - '--strategy', 'TestStrategy', - '--strategy-path', '/some/path' - ] + '--dynamic-whitelist', '10', + '--strategy', 'TestStrategy', + '--strategy-path', '/some/path' + ] args = Arguments(arglist, '').get_parsed_arg() configuration = Configuration(args) @@ -180,9 +151,6 @@ def test_load_config_with_params(default_conf, mocker) -> None: def test_load_custom_strategy(default_conf, mocker) -> None: - """ - Test Configuration.load_config() without any cli params - """ custom_conf = deepcopy(default_conf) custom_conf.update({ 'strategy': 'CustomStrategy', @@ -201,13 +169,9 @@ def test_load_custom_strategy(default_conf, mocker) -> None: def test_show_info(default_conf, mocker, caplog) -> None: - """ - Test Configuration.show_info() - """ mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) - arglist = [ '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', @@ -224,19 +188,14 @@ def test_show_info(default_conf, mocker, caplog) -> None: '(not applicable with Backtesting and Hyperopt)', caplog.record_tuples ) - assert log_has('Using DB: "sqlite:///tmp/testdb"', caplog.record_tuples) assert log_has('Dry run is enabled', caplog.record_tuples) def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: - """ - Test setup_configuration() function - """ mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) - arglist = [ '--config', 'config.json', '--strategy', 'DefaultStrategy', @@ -274,9 +233,6 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None: - """ - Test setup_configuration() function - """ mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) @@ -342,19 +298,14 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: - """ - Test setup_configuration() function - """ mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) - arglist = [ 'hyperopt', '--epochs', '10', '--spaces', 'all', ] - args = Arguments(arglist, '').get_parsed_arg() configuration = Configuration(args) @@ -397,10 +348,6 @@ def test_check_exchange(default_conf) -> None: def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: - """ - Test Configuration.load_config() with cli params used - """ - mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf))) # Prevent setting loggers @@ -416,9 +363,6 @@ def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: def test_set_loggers() -> None: - """ - Test set_loggers() update the logger level for third-party libraries - """ # Reset Logging to Debug, otherwise this fails randomly as it's set globally logging.getLogger('requests').setLevel(logging.DEBUG) logging.getLogger("urllib3").setLevel(logging.DEBUG) diff --git a/freqtrade/tests/test_indicator_helpers.py b/freqtrade/tests/test_indicator_helpers.py index f3d34ec0b..99d6cd79c 100644 --- a/freqtrade/tests/test_indicator_helpers.py +++ b/freqtrade/tests/test_indicator_helpers.py @@ -1,3 +1,5 @@ +# pragma pylint: disable=missing-docstring + import pandas as pd from freqtrade.indicator_helpers import went_down, went_up diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 80f5367b8..7aae98ebe 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -1,6 +1,4 @@ -""" -Unit test file for main.py -""" +# pragma pylint: disable=missing-docstring from copy import deepcopy from unittest.mock import MagicMock @@ -33,9 +31,6 @@ def test_parse_args_backtesting(mocker) -> None: def test_main_start_hyperopt(mocker) -> None: - """ - Test that main() can start hyperopt - """ hyperopt_mock = mocker.patch('freqtrade.optimize.hyperopt.start', MagicMock()) main(['hyperopt']) assert hyperopt_mock.call_count == 1 @@ -47,10 +42,6 @@ def test_main_start_hyperopt(mocker) -> None: def test_main_fatal_exception(mocker, default_conf, caplog) -> None: - """ - Test main() function - In this test we are skipping the while True loop by throwing an exception. - """ patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', @@ -74,10 +65,6 @@ def test_main_fatal_exception(mocker, default_conf, caplog) -> None: def test_main_keyboard_interrupt(mocker, default_conf, caplog) -> None: - """ - Test main() function - In this test we are skipping the while True loop by throwing an exception. - """ patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', @@ -101,10 +88,6 @@ def test_main_keyboard_interrupt(mocker, default_conf, caplog) -> None: def test_main_operational_exception(mocker, default_conf, caplog) -> None: - """ - Test main() function - In this test we are skipping the while True loop by throwing an exception. - """ patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', @@ -128,10 +111,6 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None: def test_main_reload_conf(mocker, default_conf, caplog) -> None: - """ - Test main() function - In this test we are skipping the while True loop by throwing an exception. - """ patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', @@ -158,7 +137,6 @@ def test_main_reload_conf(mocker, default_conf, caplog) -> None: def test_reconfigure(mocker, default_conf) -> None: - """ Test recreate() function """ patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 76290c6ca..26e0c5ee6 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -1,9 +1,5 @@ # pragma pylint: disable=missing-docstring,C0103 -""" -Unit test file for misc.py -""" - import datetime from unittest.mock import MagicMock @@ -15,20 +11,12 @@ from freqtrade.strategy.default_strategy import DefaultStrategy def test_shorten_date() -> None: - """ - Test shorten_date() function - :return: None - """ str_data = '1 day, 2 hours, 3 minutes, 4 seconds ago' str_shorten_data = '1 d, 2 h, 3 min, 4 sec ago' assert shorten_date(str_data) == str_shorten_data def test_datesarray_to_datetimearray(ticker_history): - """ - Test datesarray_to_datetimearray() function - :return: None - """ dataframes = parse_ticker_dataframe(ticker_history) dates = datesarray_to_datetimearray(dataframes['date']) @@ -44,10 +32,6 @@ def test_datesarray_to_datetimearray(ticker_history): def test_common_datearray(default_conf) -> None: - """ - Test common_datearray() - :return: None - """ strategy = DefaultStrategy(default_conf) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} @@ -61,10 +45,6 @@ def test_common_datearray(default_conf) -> None: def test_file_dump_json(mocker) -> None: - """ - Test file_dump_json() - :return: None - """ file_open = mocker.patch('freqtrade.misc.open', MagicMock()) json_dump = mocker.patch('json.dump', MagicMock()) file_dump_json('somefile', [1, 2, 3]) @@ -78,10 +58,6 @@ def test_file_dump_json(mocker) -> None: def test_format_ms_time() -> None: - """ - test format_ms_time() - :return: None - """ # Date 2018-04-10 18:02:01 date_in_epoch_ms = 1523383321000 date = format_ms_time(date_in_epoch_ms) From 296d3d8bbe10f562b9e4d14d8379ca04bb864674 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Thu, 14 Jun 2018 20:27:41 -0700 Subject: [PATCH 292/395] working on refacturing of the strategy class --- freqtrade/strategy/interface.py | 54 +++++++++++++++++++++++++++++++-- freqtrade/strategy/resolver.py | 2 ++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index dbe6435b7..e6987d114 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -10,6 +10,7 @@ from typing import Dict, List, NamedTuple, Tuple import arrow from pandas import DataFrame +import warnings from freqtrade import constants from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe @@ -57,36 +58,52 @@ class IStrategy(ABC): ticker_interval -> str: value of the ticker interval to use for the strategy """ + # associated minimal roi minimal_roi: Dict + + # associated stoploss stoploss: float + + # associated ticker interval ticker_interval: str def __init__(self, config: dict) -> None: self.config = config - @abstractmethod def populate_indicators(self, dataframe: DataFrame) -> DataFrame: """ Populate indicators that will be used in the Buy and Sell strategy :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() :return: a Dataframe with all mandatory indicators for the strategies """ + warnings.warn("deprecated - please replace this method with advise_indicators!", DeprecationWarning) + return dataframe - @abstractmethod def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame :return: DataFrame with buy column """ + warnings.warn("deprecated - please replace this method with advise_buy!", DeprecationWarning) + dataframe.loc[ + ( + ), + 'buy'] = 0 + return dataframe - @abstractmethod def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame :return: DataFrame with sell column """ + warnings.warn("deprecated - please replace this method with advise_sell!", DeprecationWarning) + dataframe.loc[ + ( + ), + 'sell'] = 0 + return dataframe def get_strategy_name(self) -> str: """ @@ -265,3 +282,34 @@ class IStrategy(ABC): """ return {pair: self.populate_indicators(parse_ticker_dataframe(pair_data)) for pair, pair_data in tickerdata.items()} + + def advise_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: + """ + + This wraps around the internal method + + Populate indicators that will be used in the Buy and Sell strategy + :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :param pair: The currently traded pair + :return: a Dataframe with all mandatory indicators for the strategies + """ + return self.populate_indicators(dataframe) + + def advise_buy(self, dataframe: DataFrame, pair: str) -> DataFrame: + """ + Based on TA indicators, populates the buy signal for the given dataframe + :param dataframe: DataFrame + :param pair: The currently traded pair + :return: DataFrame with buy column + """ + + return self.populate_buy_trend(dataframe) + + def advise_sell(self, dataframe: DataFrame, pair: str) -> DataFrame: + """ + Based on TA indicators, populates the sell signal for the given dataframe + :param dataframe: DataFrame + :param pair: The currently traded pair + :return: DataFrame with sell column + """ + return self.populate_sell_trend(dataframe) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 3360cd44a..882501f65 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -37,6 +37,8 @@ class StrategyResolver(object): config=config, extra_dir=config.get('strategy_path')) + self.strategy.config = config + # Set attributes # Check if we need to override configuration if 'minimal_roi' in config: From 57f683697db206348dfa4a0485af3dded77bde9a Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Fri, 15 Jun 2018 09:59:34 -0700 Subject: [PATCH 293/395] revised code --- freqtrade/strategy/interface.py | 12 +++--------- freqtrade/strategy/resolver.py | 1 - 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index e6987d114..23055cf88 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -7,10 +7,10 @@ from abc import ABC, abstractmethod from datetime import datetime from enum import Enum from typing import Dict, List, NamedTuple, Tuple +import warnings import arrow from pandas import DataFrame -import warnings from freqtrade import constants from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe @@ -86,10 +86,7 @@ class IStrategy(ABC): :return: DataFrame with buy column """ warnings.warn("deprecated - please replace this method with advise_buy!", DeprecationWarning) - dataframe.loc[ - ( - ), - 'buy'] = 0 + dataframe.loc[(), 'buy'] = 0 return dataframe def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: @@ -99,10 +96,7 @@ class IStrategy(ABC): :return: DataFrame with sell column """ warnings.warn("deprecated - please replace this method with advise_sell!", DeprecationWarning) - dataframe.loc[ - ( - ), - 'sell'] = 0 + dataframe.loc[(), 'sell'] = 0 return dataframe def get_strategy_name(self) -> str: diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 882501f65..0361bd91f 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -37,7 +37,6 @@ class StrategyResolver(object): config=config, extra_dir=config.get('strategy_path')) - self.strategy.config = config # Set attributes # Check if we need to override configuration From 19b996641771a2bfa530985472bd8b7de7f3ce47 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Tue, 19 Jun 2018 10:00:41 -0700 Subject: [PATCH 294/395] satisfied flake8 again --- freqtrade/strategy/interface.py | 9 ++++++--- freqtrade/strategy/resolver.py | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 23055cf88..94098a63a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -76,7 +76,8 @@ class IStrategy(ABC): :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() :return: a Dataframe with all mandatory indicators for the strategies """ - warnings.warn("deprecated - please replace this method with advise_indicators!", DeprecationWarning) + warnings.warn("deprecated - please replace this method with advise_indicators!", + DeprecationWarning) return dataframe def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: @@ -85,7 +86,8 @@ class IStrategy(ABC): :param dataframe: DataFrame :return: DataFrame with buy column """ - warnings.warn("deprecated - please replace this method with advise_buy!", DeprecationWarning) + warnings.warn("deprecated - please replace this method with advise_buy!", + DeprecationWarning) dataframe.loc[(), 'buy'] = 0 return dataframe @@ -95,7 +97,8 @@ class IStrategy(ABC): :param dataframe: DataFrame :return: DataFrame with sell column """ - warnings.warn("deprecated - please replace this method with advise_sell!", DeprecationWarning) + warnings.warn("deprecated - please replace this method with advise_sell!", + DeprecationWarning) dataframe.loc[(), 'sell'] = 0 return dataframe diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 0361bd91f..3360cd44a 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -37,7 +37,6 @@ class StrategyResolver(object): config=config, extra_dir=config.get('strategy_path')) - # Set attributes # Check if we need to override configuration if 'minimal_roi' in config: From 2e6e5029ba91f81daa323bf4846e33590e96c23d Mon Sep 17 00:00:00 2001 From: xmatthias Date: Tue, 26 Jun 2018 07:08:09 +0200 Subject: [PATCH 295/395] fix mypy and tests --- freqtrade/optimize/backtesting.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 78cbe6d33..852759c12 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -230,7 +230,7 @@ class Backtesting(object): pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run ticker_data = self.populate_sell_trend( - self.populate_buy_trend(pair_data))[headers].copy() + self.populate_buy_trend(pair_data, pair), pair)[headers].copy() # to avoid using data from future, we buy/sell with signal from previous candle ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 836c7c302..6f578d079 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -146,7 +146,7 @@ def _trend(signals, buy_value, sell_value): return signals -def _trend_alternate(dataframe=None): +def _trend_alternate(dataframe=None, pair=None): signals = dataframe low = signals['low'] n = len(low) @@ -623,7 +623,7 @@ def test_backtest_ticks(default_conf, fee, mocker): def test_backtest_clash_buy_sell(mocker, default_conf): # Override the default buy trend function in our default_strategy - def fun(dataframe=None): + def fun(dataframe=None, pair=None): buy_value = 1 sell_value = 1 return _trend(dataframe, buy_value, sell_value) @@ -638,7 +638,7 @@ def test_backtest_clash_buy_sell(mocker, default_conf): def test_backtest_only_sell(mocker, default_conf): # Override the default buy trend function in our default_strategy - def fun(dataframe=None): + def fun(dataframe=None, pair=None): buy_value = 0 sell_value = 1 return _trend(dataframe, buy_value, sell_value) From 58714888587a3c9ec44cf55da8f07f62973e4609 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Mon, 16 Jul 2018 20:40:17 -0700 Subject: [PATCH 296/395] fixed errors and making flake pass --- freqtrade/strategy/interface.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 94098a63a..213546497 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -70,6 +70,7 @@ class IStrategy(ABC): def __init__(self, config: dict) -> None: self.config = config + @abstractmethod def populate_indicators(self, dataframe: DataFrame) -> DataFrame: """ Populate indicators that will be used in the Buy and Sell strategy @@ -80,6 +81,7 @@ class IStrategy(ABC): DeprecationWarning) return dataframe + @abstractmethod def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe @@ -91,6 +93,7 @@ class IStrategy(ABC): dataframe.loc[(), 'buy'] = 0 return dataframe + @abstractmethod def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe From abc55a6e6b727005d5c4196f58453d3e238cca83 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Mon, 16 Jul 2018 21:03:41 -0700 Subject: [PATCH 297/395] fixing? hyperopt --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 59cc0f296..a4b3a6af7 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -75,7 +75,7 @@ class Hyperopt(Backtesting): return arg_dict @staticmethod - def populate_indicators(dataframe: DataFrame) -> DataFrame: + def populate_indicators(dataframe: DataFrame, pair:str) -> DataFrame: dataframe['adx'] = ta.ADX(dataframe) macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] From 3dd7d209e90144fca52291a88ce4e27097a03059 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Mon, 16 Jul 2018 21:22:59 -0700 Subject: [PATCH 298/395] more test fixes --- freqtrade/optimize/hyperopt.py | 6 ++++-- freqtrade/tests/optimize/test_hyperopt.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a4b3a6af7..1cbfa592a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -40,6 +40,7 @@ class Hyperopt(Backtesting): hyperopt = Hyperopt(config) hyperopt.start() """ + def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) # set TARGET_TRADES to suit your number concurrent trades so its realistic @@ -75,7 +76,7 @@ class Hyperopt(Backtesting): return arg_dict @staticmethod - def populate_indicators(dataframe: DataFrame, pair:str) -> DataFrame: + def populate_indicators(dataframe: DataFrame, pair: str) -> DataFrame: dataframe['adx'] = ta.ADX(dataframe) macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -228,6 +229,7 @@ class Hyperopt(Backtesting): """ Define the buy strategy parameters to be used by hyperopt """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: """ Buy strategy Hyperopt will build and use @@ -360,7 +362,7 @@ class Hyperopt(Backtesting): logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') opt = self.get_optimizer(cpus) - EVALS = max(self.total_tries//cpus, 1) + EVALS = max(self.total_tries // cpus, 1) try: with Parallel(n_jobs=cpus) as parallel: for i in range(EVALS): diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index dd7bf7da0..6326814d7 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -117,7 +117,7 @@ def test_log_results_if_loss_improves(init_hyperopt, capsys) -> None: } ) out, err = capsys.readouterr() - assert ' 1/2: foo. Loss 1.00000'in out + assert ' 1/2: foo. Loss 1.00000' in out def test_no_log_if_loss_does_not_improve(init_hyperopt, caplog) -> None: @@ -247,7 +247,7 @@ def test_populate_indicators(init_hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = _HYPEROPT.tickerdata_to_dataframe(tickerlist) - dataframe = _HYPEROPT.populate_indicators(dataframes['UNITTEST/BTC']) + dataframe = _HYPEROPT.populate_indicators(dataframes['UNITTEST/BTC'], 'UNITTEST/BTC') # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe @@ -259,7 +259,7 @@ def test_buy_strategy_generator(init_hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = _HYPEROPT.tickerdata_to_dataframe(tickerlist) - dataframe = _HYPEROPT.populate_indicators(dataframes['UNITTEST/BTC']) + dataframe = _HYPEROPT.populate_indicators(dataframes['UNITTEST/BTC'], 'UNITTEST/BTC') populate_buy_trend = _HYPEROPT.buy_strategy_generator( { From 0dcaa82c3b9d512ae91318903acafe00b2e1b3d4 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Mon, 16 Jul 2018 21:33:49 -0700 Subject: [PATCH 299/395] fixed test? --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 1cbfa592a..2df38a5ef 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -230,7 +230,7 @@ class Hyperopt(Backtesting): Define the buy strategy parameters to be used by hyperopt """ - def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + def populate_buy_trend(dataframe: DataFrame, pair: str) -> DataFrame: """ Buy strategy Hyperopt will build and use """ From 921f645623c21ca4c681d1172f7664cd0e703b2a Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Mon, 16 Jul 2018 21:41:42 -0700 Subject: [PATCH 300/395] fixing tests... --- freqtrade/tests/optimize/test_hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 6326814d7..9b7d301cc 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -274,7 +274,7 @@ def test_buy_strategy_generator(init_hyperopt) -> None: 'trigger': 'bb_lower' } ) - result = populate_buy_trend(dataframe) + result = populate_buy_trend(dataframe, 'UNITTEST/BTC') # Check if some indicators are generated. We will not test all of them assert 'buy' in result assert 1 in result['buy'] From 7300c0a0feaebaddc9ba697399dcffc4181a9b70 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 21:31:19 +0200 Subject: [PATCH 301/395] remove @abstractmethod as this method may not be present in new strategies --- freqtrade/strategy/interface.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 213546497..94098a63a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -70,7 +70,6 @@ class IStrategy(ABC): def __init__(self, config: dict) -> None: self.config = config - @abstractmethod def populate_indicators(self, dataframe: DataFrame) -> DataFrame: """ Populate indicators that will be used in the Buy and Sell strategy @@ -81,7 +80,6 @@ class IStrategy(ABC): DeprecationWarning) return dataframe - @abstractmethod def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe @@ -93,7 +91,6 @@ class IStrategy(ABC): dataframe.loc[(), 'buy'] = 0 return dataframe - @abstractmethod def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe From 2f905cb6968423ffb927492304714dace60a4ef6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 21:34:04 +0200 Subject: [PATCH 302/395] Update test-strategy with new methods --- user_data/strategies/test_strategy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index c04f4935f..26aa46a76 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -44,7 +44,7 @@ class TestStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + def advise_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Adds several different TA indicators to the given DataFrame @@ -211,7 +211,7 @@ class TestStrategy(IStrategy): return dataframe - def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + def advise_buy(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame @@ -227,7 +227,7 @@ class TestStrategy(IStrategy): return dataframe - def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: + def advise_sell(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame From c9a97bccb7e62244d1326016156cf9ea491e2bb3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 21:45:04 +0200 Subject: [PATCH 303/395] Add tests for deprecation --- freqtrade/tests/strategy/test_strategy.py | 41 ++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index e25738775..76a1d2b35 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import logging import os +import warnings import pytest @@ -8,6 +9,7 @@ from freqtrade.strategy import import_strategy from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.resolver import StrategyResolver +from freqtrade.tests.conftest import log_has def test_import_strategy(caplog): @@ -57,7 +59,8 @@ def test_search_strategy(): def test_load_strategy(result): resolver = StrategyResolver({'strategy': 'TestStrategy'}) - assert 'adx' in resolver.strategy.populate_indicators(result) + pair = 'ETH/BTC' + assert 'adx' in resolver.strategy.advise_indicators(result, pair=pair) def test_load_strategy_invalid_directory(result, caplog): @@ -150,3 +153,39 @@ def test_strategy_override_ticker_interval(caplog): logging.INFO, 'Override strategy \'ticker_interval\' with value in config file: 60.' ) in caplog.record_tuples + + +def test_deprecate_populate_indicators(result): + resolver = StrategyResolver({'strategy': 'TestStrategy'}) + with warnings.catch_warnings(record=True) as w: + # Cause all warnings to always be triggered. + warnings.simplefilter("always") + resolver.strategy.populate_indicators(result) + assert len(w) == 1 + assert issubclass(w[-1].category, DeprecationWarning) + assert "deprecated - please replace this method with advise_indicators!" in str( + w[-1].message) + + +def test_deprecate_populate_buy_trend(result): + resolver = StrategyResolver({'strategy': 'TestStrategy'}) + with warnings.catch_warnings(record=True) as w: + # Cause all warnings to always be triggered. + warnings.simplefilter("always") + resolver.strategy.populate_buy_trend(result) + assert len(w) == 1 + assert issubclass(w[-1].category, DeprecationWarning) + assert "deprecated - please replace this method with advise_buy!" in str( + w[-1].message) + + +def test_deprecate_populate_sell_trend(result): + resolver = StrategyResolver({'strategy': 'TestStrategy'}) + with warnings.catch_warnings(record=True) as w: + # Cause all warnings to always be triggered. + warnings.simplefilter("always") + resolver.strategy.populate_sell_trend(result) + assert len(w) == 1 + assert issubclass(w[-1].category, DeprecationWarning) + assert "deprecated - please replace this method with advise_sell!" in str( + w[-1].message) From fa48b8a535bb2e5fcf58ffcdb33ac481ff20c08a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 21:52:40 +0200 Subject: [PATCH 304/395] Update documentation with advise-* methods --- docs/bot-optimization.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index d7b628fd4..62ab24070 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -61,22 +61,23 @@ file as reference.** ### Buy strategy -Edit the method `populate_buy_trend()` into your strategy file to +Edit the method `advise_buy()` into your strategy file to update your buy strategy. Sample from `user_data/strategies/test_strategy.py`: ```python -def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: +def advise_buy(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe - :param dataframe: DataFrame + :param dataframe: DataFrame populated with indicators + :param pair: Pair currently analyzed :return: DataFrame with buy column """ dataframe.loc[ ( (dataframe['adx'] > 30) & - (dataframe['tema'] <= dataframe['blower']) & + (dataframe['tema'] <= dataframe['bb_middleband']) & (dataframe['tema'] > dataframe['tema'].shift(1)) ), 'buy'] = 1 @@ -86,21 +87,23 @@ def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: ### Sell strategy -Edit the method `populate_sell_trend()` into your strategy file to update your sell strategy. +Edit the method `advise_sell()` into your strategy file to update your sell strategy. +Please note that the sell-signal is only used if `use_sell_signal` is set to true in the configuration. Sample from `user_data/strategies/test_strategy.py`: ```python -def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: +def advise_sell(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe - :param dataframe: DataFrame + :param dataframe: DataFrame populated with indicators + :param pair: Pair currently analyzed :return: DataFrame with buy column """ dataframe.loc[ ( (dataframe['adx'] > 70) & - (dataframe['tema'] > dataframe['blower']) & + (dataframe['tema'] > dataframe['bb_middleband']) & (dataframe['tema'] < dataframe['tema'].shift(1)) ), 'sell'] = 1 @@ -109,14 +112,14 @@ def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: ## Add more Indicator -As you have seen, buy and sell strategies need indicators. You can add -more indicators by extending the list contained in -the method `populate_indicators()` from your strategy file. +As you have seen, buy and sell strategies need indicators. You can add more indicators by extending the list contained in the method `advise_indicators()` from your strategy file. + +You should only add the indicators used in either `advise_buy()`, `advise_sell()`, or to populate another indicator, otherwise performance may suffer. Sample: ```python -def populate_indicators(dataframe: DataFrame) -> DataFrame: +def advise_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Adds several different TA indicators to the given DataFrame """ From 4ebd706cb8fc050353c21cf6f086e9303b74a34e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 21:53:03 +0200 Subject: [PATCH 305/395] improve comments --- user_data/strategies/test_strategy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 26aa46a76..56dc1b6a8 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -214,7 +214,8 @@ class TestStrategy(IStrategy): def advise_buy(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe - :param dataframe: DataFrame + :param dataframe: DataFrame populated with indicators + :param pair: Pair currently analyzed :return: DataFrame with buy column """ dataframe.loc[ @@ -230,7 +231,8 @@ class TestStrategy(IStrategy): def advise_sell(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe - :param dataframe: DataFrame + :param dataframe: DataFrame populated with indicators + :param pair: Pair currently analyzed :return: DataFrame with buy column """ dataframe.loc[ From aa772c28adbc973f803139b16328eb2d0bffe649 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 22:04:34 +0200 Subject: [PATCH 306/395] Add tests for advise_indicator methods --- freqtrade/strategy/default_strategy.py | 8 +++++--- freqtrade/strategy/interface.py | 2 +- freqtrade/tests/strategy/test_strategy.py | 13 +++++++------ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 22689f17a..60dabd431 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -28,7 +28,7 @@ class DefaultStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + def advise_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Adds several different TA indicators to the given DataFrame @@ -196,10 +196,11 @@ class DefaultStrategy(IStrategy): return dataframe - def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + def advise_buy(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame + :param pair: Pair currently analyzed :return: DataFrame with buy column """ dataframe.loc[ @@ -217,10 +218,11 @@ class DefaultStrategy(IStrategy): return dataframe - def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: + def advise_sell(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame + :param pair: Pair currently analyzed :return: DataFrame with buy column """ dataframe.loc[ diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 94098a63a..4d1e135fd 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -3,7 +3,7 @@ IStrategy interface This module defines the interface to apply for strategies """ import logging -from abc import ABC, abstractmethod +from abc import ABC from datetime import datetime from enum import Enum from typing import Dict, List, NamedTuple, Tuple diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 76a1d2b35..03ab884d0 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -9,7 +9,6 @@ from freqtrade.strategy import import_strategy from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.resolver import StrategyResolver -from freqtrade.tests.conftest import log_has def test_import_strategy(caplog): @@ -73,7 +72,8 @@ def test_load_strategy_invalid_directory(result, caplog): logging.WARNING, 'Path "{}" does not exist'.format(extra_dir), ) in caplog.record_tuples - assert 'adx' in resolver.strategy.populate_indicators(result) + + assert 'adx' in resolver.strategy.advise_indicators(result, 'ETH/BTC') def test_load_not_found_strategy(): @@ -88,7 +88,7 @@ def test_strategy(result): config = {'strategy': 'DefaultStrategy'} resolver = StrategyResolver(config) - + pair = 'ETH/BTC' assert resolver.strategy.minimal_roi[0] == 0.04 assert config["minimal_roi"]['0'] == 0.04 @@ -98,12 +98,13 @@ def test_strategy(result): assert resolver.strategy.ticker_interval == '5m' assert config['ticker_interval'] == '5m' - assert 'adx' in resolver.strategy.populate_indicators(result) + df_indicators = resolver.strategy.advise_indicators(result, pair=pair) + assert 'adx' in df_indicators - dataframe = resolver.strategy.populate_buy_trend(resolver.strategy.populate_indicators(result)) + dataframe = resolver.strategy.advise_buy(df_indicators, pair=pair) assert 'buy' in dataframe.columns - dataframe = resolver.strategy.populate_sell_trend(resolver.strategy.populate_indicators(result)) + dataframe = resolver.strategy.advise_sell(df_indicators, pair='ETH/BTC') assert 'sell' in dataframe.columns From 0eff6719c2178f65c6c1c23cad2551c7766986dc Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Jul 2018 23:23:30 +0200 Subject: [PATCH 307/395] improve tests for legacy-strategy loading --- freqtrade/tests/strategy/legacy_strategy.py | 242 ++++++++++++++++++++ freqtrade/tests/strategy/test_strategy.py | 34 ++- 2 files changed, 270 insertions(+), 6 deletions(-) create mode 100644 freqtrade/tests/strategy/legacy_strategy.py diff --git a/freqtrade/tests/strategy/legacy_strategy.py b/freqtrade/tests/strategy/legacy_strategy.py new file mode 100644 index 000000000..cb97bd63b --- /dev/null +++ b/freqtrade/tests/strategy/legacy_strategy.py @@ -0,0 +1,242 @@ + +# --- Do not remove these libs --- +from freqtrade.strategy.interface import IStrategy +from pandas import DataFrame +# -------------------------------- + +# Add your lib to import here +import talib.abstract as ta +import freqtrade.vendor.qtpylib.indicators as qtpylib +import numpy # noqa + + +# This class is a sample. Feel free to customize it. +class TestStrategyLegacy(IStrategy): + """ + This is a test strategy to inspire you. + More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md + + You can: + - Rename the class name (Do not forget to update class_name) + - Add any methods you want to build your strategy + - Add any lib you need to build your strategy + + You must keep: + - the lib in the section "Do not remove these libs" + - the prototype for the methods: minimal_roi, stoploss, populate_indicators, populate_buy_trend, + populate_sell_trend, hyperopt_space, buy_strategy_generator + """ + + # Minimal ROI designed for the strategy. + # This attribute will be overridden if the config file contains "minimal_roi" + minimal_roi = { + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 + } + + # Optimal stoploss designed for the strategy + # This attribute will be overridden if the config file contains "stoploss" + stoploss = -0.10 + + # Optimal ticker interval for the strategy + ticker_interval = '5m' + + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + + Performance Note: For the best performance be frugal on the number of indicators + you are using. Let uncomment only the indicator you are using in your strategies + or your hyperopt configuration, otherwise you will waste your memory and CPU usage. + """ + + # Momentum Indicator + # ------------------------------------ + + # ADX + dataframe['adx'] = ta.ADX(dataframe) + + """ + # Awesome oscillator + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + + # Commodity Channel Index: values Oversold:<-100, Overbought:>100 + dataframe['cci'] = ta.CCI(dataframe) + + # MACD + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + + # MFI + dataframe['mfi'] = ta.MFI(dataframe) + + # Minus Directional Indicator / Movement + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # Plus Directional Indicator / Movement + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # ROC + dataframe['roc'] = ta.ROC(dataframe) + + # RSI + dataframe['rsi'] = ta.RSI(dataframe) + + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + """ + + # Overlap Studies + # ------------------------------------ + + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + + """ + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + + # SAR Parabol + dataframe['sar'] = ta.SAR(dataframe) + + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + """ + + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + + # Cycle Indicator + # ------------------------------------ + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] + + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ + + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ + + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ + + # Chart type + # ------------------------------------ + """ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] + """ + + return dataframe + + def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + """ + Based on TA indicators, populates the buy signal for the given dataframe + :param dataframe: DataFrame + :return: DataFrame with buy column + """ + dataframe.loc[ + ( + (dataframe['adx'] > 30) & + (dataframe['tema'] <= dataframe['bb_middleband']) & + (dataframe['tema'] > dataframe['tema'].shift(1)) + ), + 'buy'] = 1 + + return dataframe + + def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: + """ + Based on TA indicators, populates the sell signal for the given dataframe + :param dataframe: DataFrame + :return: DataFrame with buy column + """ + dataframe.loc[ + ( + (dataframe['adx'] > 70) & + (dataframe['tema'] > dataframe['bb_middleband']) & + (dataframe['tema'] < dataframe['tema'].shift(1)) + ), + 'sell'] = 1 + return dataframe diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 03ab884d0..6c11f0092 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import logging -import os +from os import path +from unittest.mock import MagicMock import warnings import pytest @@ -37,9 +38,8 @@ def test_import_strategy(caplog): def test_search_strategy(): - default_config = {} - default_location = os.path.join(os.path.dirname( - os.path.realpath(__file__)), '..', '..', 'strategy' + default_location = path.join(path.dirname( + path.realpath(__file__)), '..', '..', 'strategy' ) assert isinstance( StrategyResolver._search_strategy( @@ -64,8 +64,8 @@ def test_load_strategy(result): def test_load_strategy_invalid_directory(result, caplog): resolver = StrategyResolver() - extra_dir = os.path.join('some', 'path') - resolver._load_strategy('TestStrategy', config={}, extra_dir=extra_dir) + extra_dir = path.join('some', 'path') + resolver._load_strategy('TestStrategy', extra_dir) assert ( 'freqtrade.strategy.resolver', @@ -190,3 +190,25 @@ def test_deprecate_populate_sell_trend(result): assert issubclass(w[-1].category, DeprecationWarning) assert "deprecated - please replace this method with advise_sell!" in str( w[-1].message) + + +def test_call_deprecated_function(result, monkeypatch): + default_location = path.join(path.dirname(path.realpath(__file__))) + resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', + 'strategy_path': default_location}) + pair = 'ETH/BTC' + indicators_mock = MagicMock() + buy_trend_mock = MagicMock() + sell_trend_mock = MagicMock() + + monkeypatch.setattr(resolver.strategy, 'populate_indicators', indicators_mock) + resolver.strategy.advise_indicators(result, pair=pair) + assert indicators_mock.call_count == 1 + + monkeypatch.setattr(resolver.strategy, 'populate_buy_trend', buy_trend_mock) + resolver.strategy.advise_buy(result, pair=pair) + assert buy_trend_mock.call_count == 1 + + monkeypatch.setattr(resolver.strategy, 'populate_sell_trend', sell_trend_mock) + resolver.strategy.advise_sell(result, pair=pair) + assert sell_trend_mock.call_count == 1 From df8700ead086470aabba675c7d0eb658a283c71d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 20 Jul 2018 20:56:44 +0200 Subject: [PATCH 308/395] Adapt after merge from develop --- freqtrade/optimize/backtesting.py | 8 ++++---- freqtrade/strategy/interface.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 852759c12..e3c3974be 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -57,8 +57,8 @@ class Backtesting(object): self.strategy: IStrategy = StrategyResolver(self.config).strategy self.ticker_interval = self.strategy.ticker_interval self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.populate_buy_trend = self.strategy.populate_buy_trend - self.populate_sell_trend = self.strategy.populate_sell_trend + self.advise_buy = self.strategy.advise_buy + self.advise_sell = self.strategy.advise_sell # Reset keys for backtesting self.config['exchange']['key'] = '' @@ -229,8 +229,8 @@ class Backtesting(object): for pair, pair_data in processed.items(): pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - ticker_data = self.populate_sell_trend( - self.populate_buy_trend(pair_data, pair), pair)[headers].copy() + ticker_data = self.advise_sell( + self.advise_buy(pair_data, pair), pair)[headers].copy() # to avoid using data from future, we buy/sell with signal from previous candle ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 4d1e135fd..45a131c5e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -277,7 +277,7 @@ class IStrategy(ABC): """ Creates a dataframe and populates indicators for given ticker data """ - return {pair: self.populate_indicators(parse_ticker_dataframe(pair_data)) + return {pair: self.advise_indicators(parse_ticker_dataframe(pair_data), pair) for pair, pair_data in tickerdata.items()} def advise_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: From f12167f0dcecdc28f3caece83e8d434b16f17164 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 20 Jul 2018 20:59:56 +0200 Subject: [PATCH 309/395] Fix backtesting test --- freqtrade/tests/optimize/test_backtesting.py | 22 ++++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 6f578d079..91ea2eee1 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -332,8 +332,8 @@ def test_backtesting_init(mocker, default_conf) -> None: assert backtesting.config == default_conf assert backtesting.ticker_interval == '5m' assert callable(backtesting.tickerdata_to_dataframe) - assert callable(backtesting.populate_buy_trend) - assert callable(backtesting.populate_sell_trend) + assert callable(backtesting.advise_buy) + assert callable(backtesting.advise_sell) get_fee.assert_called() assert backtesting.fee == 0.5 @@ -611,12 +611,12 @@ def test_backtest_ticks(default_conf, fee, mocker): mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) patch_exchange(mocker) ticks = [1, 5] - fun = Backtesting(default_conf).populate_buy_trend + fun = Backtesting(default_conf).advise_buy for _ in ticks: backtest_conf = _make_backtest_conf(mocker, conf=default_conf) backtesting = Backtesting(default_conf) - backtesting.populate_buy_trend = fun # Override - backtesting.populate_sell_trend = fun # Override + backtesting.advise_buy = fun # Override + backtesting.advise_sell = fun # Override results = backtesting.backtest(backtest_conf) assert not results.empty @@ -630,8 +630,8 @@ def test_backtest_clash_buy_sell(mocker, default_conf): backtest_conf = _make_backtest_conf(mocker, conf=default_conf) backtesting = Backtesting(default_conf) - backtesting.populate_buy_trend = fun # Override - backtesting.populate_sell_trend = fun # Override + backtesting.advise_buy = fun # Override + backtesting.advise_sell = fun # Override results = backtesting.backtest(backtest_conf) assert results.empty @@ -645,8 +645,8 @@ def test_backtest_only_sell(mocker, default_conf): backtest_conf = _make_backtest_conf(mocker, conf=default_conf) backtesting = Backtesting(default_conf) - backtesting.populate_buy_trend = fun # Override - backtesting.populate_sell_trend = fun # Override + backtesting.advise_buy = fun # Override + backtesting.advise_sell = fun # Override results = backtesting.backtest(backtest_conf) assert results.empty @@ -655,8 +655,8 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) backtest_conf = _make_backtest_conf(mocker, conf=default_conf, pair='UNITTEST/BTC') backtesting = Backtesting(default_conf) - backtesting.populate_buy_trend = _trend_alternate # Override - backtesting.populate_sell_trend = _trend_alternate # Override + backtesting.advise_buy = _trend_alternate # Override + backtesting.advise_sell = _trend_alternate # Override results = backtesting.backtest(backtest_conf) backtesting._store_backtest_result("test_.json", results) assert len(results) == 4 From 18b8f20f1c533512a6e9212c019de87e6928f6a2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 20 Jul 2018 21:48:59 +0200 Subject: [PATCH 310/395] fix small test bug --- freqtrade/tests/strategy/test_strategy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 6c11f0092..9a62c8c73 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -38,6 +38,7 @@ def test_import_strategy(caplog): def test_search_strategy(): + default_config = {} default_location = path.join(path.dirname( path.realpath(__file__)), '..', '..', 'strategy' ) @@ -65,7 +66,7 @@ def test_load_strategy(result): def test_load_strategy_invalid_directory(result, caplog): resolver = StrategyResolver() extra_dir = path.join('some', 'path') - resolver._load_strategy('TestStrategy', extra_dir) + resolver._load_strategy('TestStrategy', config={}, extra_dir=extra_dir) assert ( 'freqtrade.strategy.resolver', From 8a9c54ed61c5a2ea2d84b2ee21d6c36df4232d7c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 20 Jul 2018 22:02:17 +0200 Subject: [PATCH 311/395] use new methods --- freqtrade/strategy/interface.py | 10 +++++----- freqtrade/tests/test_dataframe.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 45a131c5e..40db21cc2 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -108,16 +108,16 @@ class IStrategy(ABC): """ return self.__class__.__name__ - def analyze_ticker(self, ticker_history: List[Dict]) -> DataFrame: + def analyze_ticker(self, ticker_history: List[Dict], pair: str) -> DataFrame: """ Parses the given ticker history and returns a populated DataFrame add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ dataframe = parse_ticker_dataframe(ticker_history) - dataframe = self.populate_indicators(dataframe) - dataframe = self.populate_buy_trend(dataframe) - dataframe = self.populate_sell_trend(dataframe) + dataframe = self.advise_indicators(dataframe, pair) + dataframe = self.advise_buy(dataframe, pair) + dataframe = self.advise_sell(dataframe, pair) return dataframe def get_signal(self, pair: str, interval: str, ticker_hist: List[Dict]) -> Tuple[bool, bool]: @@ -132,7 +132,7 @@ class IStrategy(ABC): return False, False try: - dataframe = self.analyze_ticker(ticker_hist) + dataframe = self.analyze_ticker(ticker_hist, pair) except ValueError as error: logger.warning( 'Unable to analyze ticker for pair %s: %s', diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index 019587af1..ce144e118 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -14,7 +14,7 @@ def load_dataframe_pair(pairs, strategy): assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] - dataframe = strategy.analyze_ticker(dataframe) + dataframe = strategy.analyze_ticker(dataframe, pairs[0]) return dataframe From 791c5ff0710dba50eadf3e708bfc0e9857f69b20 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 20 Jul 2018 22:06:15 +0200 Subject: [PATCH 312/395] update comments to explain what advise methods do --- freqtrade/strategy/interface.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 40db21cc2..5051e9398 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -282,10 +282,8 @@ class IStrategy(ABC): def advise_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: """ - - This wraps around the internal method - Populate indicators that will be used in the Buy and Sell strategy + If not overridden, calls the legacy method `populate_indicators to keep strategies working :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() :param pair: The currently traded pair :return: a Dataframe with all mandatory indicators for the strategies @@ -295,6 +293,7 @@ class IStrategy(ABC): def advise_buy(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe + If not overridden, calls the legacy method `populate_buy_trend to keep strategies working :param dataframe: DataFrame :param pair: The currently traded pair :return: DataFrame with buy column @@ -305,6 +304,7 @@ class IStrategy(ABC): def advise_sell(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe + If not overridden, calls the legacy method `populate_sell_trend to keep strategies working :param dataframe: DataFrame :param pair: The currently traded pair :return: DataFrame with sell column From cf83416d6985666a7ead9216aca55126ae5b968a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 20 Jul 2018 22:07:18 +0200 Subject: [PATCH 313/395] update script to use new method --- scripts/plot_dataframe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 11f1f85d5..06e1cd1d8 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -159,8 +159,8 @@ def plot_analyzed_dataframe(args: Namespace) -> None: dataframes = strategy.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] - dataframe = strategy.populate_buy_trend(dataframe) - dataframe = strategy.populate_sell_trend(dataframe) + dataframe = strategy.advise_buy(dataframe, pair) + dataframe = strategy.advise_sell(dataframe, pair) if len(dataframe.index) > args.plot_limit: logger.warning('Ticker contained more than %s candles as defined ' From 98665dcef4afc1a6f69583dedebcd2c8d4c394a9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 20 Jul 2018 22:15:32 +0200 Subject: [PATCH 314/395] revert inadvertent wihtespace changes --- freqtrade/optimize/hyperopt.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2df38a5ef..af41d799f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -40,7 +40,6 @@ class Hyperopt(Backtesting): hyperopt = Hyperopt(config) hyperopt.start() """ - def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) # set TARGET_TRADES to suit your number concurrent trades so its realistic @@ -229,7 +228,6 @@ class Hyperopt(Backtesting): """ Define the buy strategy parameters to be used by hyperopt """ - def populate_buy_trend(dataframe: DataFrame, pair: str) -> DataFrame: """ Buy strategy Hyperopt will build and use From f286ba6b8786eb7670aa9c5b98839ddc81a88b16 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Jul 2018 17:39:35 +0200 Subject: [PATCH 315/395] overload populate_indicators to work with and without pair argumen all while not breaking users strategies --- freqtrade/strategy/default_strategy.py | 6 +- freqtrade/strategy/interface.py | 53 +++++++------ .../tests/strategy/test_default_strategy.py | 7 +- freqtrade/tests/strategy/test_strategy.py | 75 ++++++++++--------- user_data/strategies/test_strategy.py | 6 +- 5 files changed, 82 insertions(+), 65 deletions(-) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 60dabd431..6285a483e 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -28,7 +28,7 @@ class DefaultStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' - def advise_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Adds several different TA indicators to the given DataFrame @@ -196,7 +196,7 @@ class DefaultStrategy(IStrategy): return dataframe - def advise_buy(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_buy_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame @@ -218,7 +218,7 @@ class DefaultStrategy(IStrategy): return dataframe - def advise_sell(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_sell_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 5051e9398..e80ee0e0e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -3,7 +3,7 @@ IStrategy interface This module defines the interface to apply for strategies """ import logging -from abc import ABC +from abc import ABC, abstractmethod from datetime import datetime from enum import Enum from typing import Dict, List, NamedTuple, Tuple @@ -70,37 +70,32 @@ class IStrategy(ABC): def __init__(self, config: dict) -> None: self.config = config - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + @abstractmethod + def populate_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Populate indicators that will be used in the Buy and Sell strategy :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :param pair: Pair currently analyzed :return: a Dataframe with all mandatory indicators for the strategies """ - warnings.warn("deprecated - please replace this method with advise_indicators!", - DeprecationWarning) - return dataframe - def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + @abstractmethod + def populate_buy_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame + :param pair: Pair currently analyzed :return: DataFrame with buy column """ - warnings.warn("deprecated - please replace this method with advise_buy!", - DeprecationWarning) - dataframe.loc[(), 'buy'] = 0 - return dataframe - def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: + @abstractmethod + def populate_sell_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame + :param pair: Pair currently analyzed :return: DataFrame with sell column """ - warnings.warn("deprecated - please replace this method with advise_sell!", - DeprecationWarning) - dataframe.loc[(), 'sell'] = 0 - return dataframe def get_strategy_name(self) -> str: """ @@ -283,30 +278,44 @@ class IStrategy(ABC): def advise_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Populate indicators that will be used in the Buy and Sell strategy - If not overridden, calls the legacy method `populate_indicators to keep strategies working + This method should not be overridden. :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() :param pair: The currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ - return self.populate_indicators(dataframe) + if len(self.populate_indicators.__annotations__) == 2: + warnings.warn("deprecated - check out the Sample strategy to see " + "the current function headers!", DeprecationWarning) + return self.populate_indicators(dataframe) # type: ignore + else: + return self.populate_indicators(dataframe, pair) def advise_buy(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe - If not overridden, calls the legacy method `populate_buy_trend to keep strategies working + This method should not be overridden. :param dataframe: DataFrame :param pair: The currently traded pair :return: DataFrame with buy column """ - - return self.populate_buy_trend(dataframe) + if len(self.populate_buy_trend.__annotations__) == 2: + warnings.warn("deprecated - check out the Sample strategy to see " + "the current function headers!", DeprecationWarning) + return self.populate_buy_trend(dataframe) # type: ignore + else: + return self.populate_buy_trend(dataframe, pair) def advise_sell(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe - If not overridden, calls the legacy method `populate_sell_trend to keep strategies working + This method should not be overridden. :param dataframe: DataFrame :param pair: The currently traded pair :return: DataFrame with sell column """ - return self.populate_sell_trend(dataframe) + if len(self.populate_sell_trend.__annotations__) == 2: + warnings.warn("deprecated - check out the Sample strategy to see " + "the current function headers!", DeprecationWarning) + return self.populate_sell_trend(dataframe) # type: ignore + else: + return self.populate_sell_trend(dataframe, pair) diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index 37df1748f..2b10e9023 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -25,10 +25,11 @@ def test_default_strategy_structure(): def test_default_strategy(result): strategy = DefaultStrategy({}) + pair = 'ETH/BTC' assert type(strategy.minimal_roi) is dict assert type(strategy.stoploss) is float assert type(strategy.ticker_interval) is str - indicators = strategy.populate_indicators(result) + indicators = strategy.populate_indicators(result, pair) assert type(indicators) is DataFrame - assert type(strategy.populate_buy_trend(indicators)) is DataFrame - assert type(strategy.populate_sell_trend(indicators)) is DataFrame + assert type(strategy.populate_buy_trend(indicators, pair)) is DataFrame + assert type(strategy.populate_sell_trend(indicators, pair)) is DataFrame diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 9a62c8c73..c90271506 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,10 +1,10 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import logging from os import path -from unittest.mock import MagicMock import warnings import pytest +from pandas import DataFrame from freqtrade.strategy import import_strategy from freqtrade.strategy.default_strategy import DefaultStrategy @@ -60,6 +60,9 @@ def test_search_strategy(): def test_load_strategy(result): resolver = StrategyResolver({'strategy': 'TestStrategy'}) pair = 'ETH/BTC' + assert len(resolver.strategy.populate_indicators.__annotations__) == 3 + assert 'dataframe' in resolver.strategy.populate_indicators.__annotations__ + assert 'pair' in resolver.strategy.populate_indicators.__annotations__ assert 'adx' in resolver.strategy.advise_indicators(result, pair=pair) @@ -158,39 +161,35 @@ def test_strategy_override_ticker_interval(caplog): def test_deprecate_populate_indicators(result): - resolver = StrategyResolver({'strategy': 'TestStrategy'}) + default_location = path.join(path.dirname(path.realpath(__file__))) + resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', + 'strategy_path': default_location}) with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") - resolver.strategy.populate_indicators(result) + indicators = resolver.strategy.advise_indicators(result, 'ETH/BTC') assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) - assert "deprecated - please replace this method with advise_indicators!" in str( - w[-1].message) + assert "deprecated - check out the Sample strategy to see the current function headers!" \ + in str(w[-1].message) + with warnings.catch_warnings(record=True) as w: + # Cause all warnings to always be triggered. + warnings.simplefilter("always") + resolver.strategy.advise_buy(indicators, 'ETH/BTC') + assert len(w) == 1 + assert issubclass(w[-1].category, DeprecationWarning) + assert "deprecated - check out the Sample strategy to see the current function headers!" \ + in str(w[-1].message) -def test_deprecate_populate_buy_trend(result): - resolver = StrategyResolver({'strategy': 'TestStrategy'}) with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") - resolver.strategy.populate_buy_trend(result) + resolver.strategy.advise_sell(indicators, 'ETH_BTC') assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) - assert "deprecated - please replace this method with advise_buy!" in str( - w[-1].message) - - -def test_deprecate_populate_sell_trend(result): - resolver = StrategyResolver({'strategy': 'TestStrategy'}) - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - resolver.strategy.populate_sell_trend(result) - assert len(w) == 1 - assert issubclass(w[-1].category, DeprecationWarning) - assert "deprecated - please replace this method with advise_sell!" in str( - w[-1].message) + assert "deprecated - check out the Sample strategy to see the current function headers!" \ + in str(w[-1].message) def test_call_deprecated_function(result, monkeypatch): @@ -198,18 +197,26 @@ def test_call_deprecated_function(result, monkeypatch): resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', 'strategy_path': default_location}) pair = 'ETH/BTC' - indicators_mock = MagicMock() - buy_trend_mock = MagicMock() - sell_trend_mock = MagicMock() - monkeypatch.setattr(resolver.strategy, 'populate_indicators', indicators_mock) - resolver.strategy.advise_indicators(result, pair=pair) - assert indicators_mock.call_count == 1 + # Make sure we are using a legacy function + assert len(resolver.strategy.populate_indicators.__annotations__) == 2 + assert 'dataframe' in resolver.strategy.populate_indicators.__annotations__ + assert 'pair' not in resolver.strategy.populate_indicators.__annotations__ + assert len(resolver.strategy.populate_buy_trend.__annotations__) == 2 + assert 'dataframe' in resolver.strategy.populate_buy_trend.__annotations__ + assert 'pair' not in resolver.strategy.populate_buy_trend.__annotations__ + assert len(resolver.strategy.populate_sell_trend.__annotations__) == 2 + assert 'dataframe' in resolver.strategy.populate_sell_trend.__annotations__ + assert 'pair' not in resolver.strategy.populate_sell_trend.__annotations__ - monkeypatch.setattr(resolver.strategy, 'populate_buy_trend', buy_trend_mock) - resolver.strategy.advise_buy(result, pair=pair) - assert buy_trend_mock.call_count == 1 + indicator_df = resolver.strategy.advise_indicators(result, pair=pair) + assert type(indicator_df) is DataFrame + assert 'adx' in indicator_df.columns - monkeypatch.setattr(resolver.strategy, 'populate_sell_trend', sell_trend_mock) - resolver.strategy.advise_sell(result, pair=pair) - assert sell_trend_mock.call_count == 1 + buydf = resolver.strategy.advise_buy(result, pair=pair) + assert type(buydf) is DataFrame + assert 'buy' in buydf.columns + + selldf = resolver.strategy.advise_sell(result, pair=pair) + assert type(selldf) is DataFrame + assert 'sell' in selldf diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 56dc1b6a8..96d1e0bfd 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -44,7 +44,7 @@ class TestStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' - def advise_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Adds several different TA indicators to the given DataFrame @@ -211,7 +211,7 @@ class TestStrategy(IStrategy): return dataframe - def advise_buy(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_buy_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame populated with indicators @@ -228,7 +228,7 @@ class TestStrategy(IStrategy): return dataframe - def advise_sell(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_sell_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame populated with indicators From 39cf0deccebf2a9081116b5ef05e57e2e49b1215 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Jul 2018 17:38:21 +0100 Subject: [PATCH 316/395] don't use __annotate__ it is only present when typehints are used which cannot be guaranteed for userdefined classes --- freqtrade/strategy/interface.py | 9 ++++++--- freqtrade/strategy/resolver.py | 7 +++++++ freqtrade/tests/strategy/test_strategy.py | 12 +++--------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index e80ee0e0e..887c1d583 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -58,6 +58,9 @@ class IStrategy(ABC): ticker_interval -> str: value of the ticker interval to use for the strategy """ + _populate_fun_len: int = 0 + _buy_fun_len: int = 0 + _sell_fun_len: int = 0 # associated minimal roi minimal_roi: Dict @@ -283,7 +286,7 @@ class IStrategy(ABC): :param pair: The currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ - if len(self.populate_indicators.__annotations__) == 2: + if self._populate_fun_len == 2: warnings.warn("deprecated - check out the Sample strategy to see " "the current function headers!", DeprecationWarning) return self.populate_indicators(dataframe) # type: ignore @@ -298,7 +301,7 @@ class IStrategy(ABC): :param pair: The currently traded pair :return: DataFrame with buy column """ - if len(self.populate_buy_trend.__annotations__) == 2: + if self._buy_fun_len == 2: warnings.warn("deprecated - check out the Sample strategy to see " "the current function headers!", DeprecationWarning) return self.populate_buy_trend(dataframe) # type: ignore @@ -313,7 +316,7 @@ class IStrategy(ABC): :param pair: The currently traded pair :return: DataFrame with sell column """ - if len(self.populate_sell_trend.__annotations__) == 2: + if self._sell_fun_len == 2: warnings.warn("deprecated - check out the Sample strategy to see " "the current function headers!", DeprecationWarning) return self.populate_sell_trend(dataframe) # type: ignore diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 3360cd44a..ea887e43e 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -92,6 +92,13 @@ class StrategyResolver(object): strategy = self._search_strategy(path, strategy_name=strategy_name, config=config) if strategy: logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) + strategy._populate_fun_len = len( + inspect.getfullargspec(strategy.populate_indicators).args) + strategy._buy_fun_len = len( + inspect.getfullargspec(strategy.populate_buy_trend).args) + strategy._sell_fun_len = len( + inspect.getfullargspec(strategy.populate_sell_trend).args) + return import_strategy(strategy, config=config) except FileNotFoundError: logger.warning('Path "%s" does not exist', path) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index c90271506..02eea312f 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -199,15 +199,9 @@ def test_call_deprecated_function(result, monkeypatch): pair = 'ETH/BTC' # Make sure we are using a legacy function - assert len(resolver.strategy.populate_indicators.__annotations__) == 2 - assert 'dataframe' in resolver.strategy.populate_indicators.__annotations__ - assert 'pair' not in resolver.strategy.populate_indicators.__annotations__ - assert len(resolver.strategy.populate_buy_trend.__annotations__) == 2 - assert 'dataframe' in resolver.strategy.populate_buy_trend.__annotations__ - assert 'pair' not in resolver.strategy.populate_buy_trend.__annotations__ - assert len(resolver.strategy.populate_sell_trend.__annotations__) == 2 - assert 'dataframe' in resolver.strategy.populate_sell_trend.__annotations__ - assert 'pair' not in resolver.strategy.populate_sell_trend.__annotations__ + assert resolver.strategy._populate_fun_len == 2 + assert resolver.strategy._buy_fun_len == 2 + assert resolver.strategy._sell_fun_len == 2 indicator_df = resolver.strategy.advise_indicators(result, pair=pair) assert type(indicator_df) is DataFrame From 5fbce13830cb193878c64da6cb140ad39b94832b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Jul 2018 23:29:54 +0100 Subject: [PATCH 317/395] update hyperopt to use new methods --- freqtrade/optimize/hyperopt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index af41d799f..f0d81e3ff 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -270,7 +270,7 @@ class Hyperopt(Backtesting): self.strategy.minimal_roi = self.generate_roi_table(params) if self.has_space('buy'): - self.populate_buy_trend = self.buy_strategy_generator(params) + self.advise_buy = self.buy_strategy_generator(params) if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] @@ -351,7 +351,7 @@ class Hyperopt(Backtesting): ) if self.has_space('buy'): - self.strategy.populate_indicators = Hyperopt.populate_indicators # type: ignore + self.strategy.advise_indicators = Hyperopt.populate_indicators # type: ignore dump(self.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() From 82680ac6aa80b0aea35e4c2df2f15323eb9d5282 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Jul 2018 07:54:01 +0100 Subject: [PATCH 318/395] improve docstrings for strategy --- freqtrade/strategy/default_strategy.py | 3 +++ user_data/strategies/test_strategy.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 6285a483e..9eaf093ae 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -35,6 +35,9 @@ class DefaultStrategy(IStrategy): Performance Note: For the best performance be frugal on the number of indicators you are using. Let uncomment only the indicator you are using in your strategies or your hyperopt configuration, otherwise you will waste your memory and CPU usage. + :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :param pair: Pair currently analyzed + :return: a Dataframe with all mandatory indicators for the strategies """ # Momentum Indicator diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 96d1e0bfd..65e06558c 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -18,6 +18,7 @@ class TestStrategy(IStrategy): More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md You can: + :return: a Dataframe with all mandatory indicators for the strategies - Rename the class name (Do not forget to update class_name) - Add any methods you want to build your strategy - Add any lib you need to build your strategy @@ -51,6 +52,9 @@ class TestStrategy(IStrategy): Performance Note: For the best performance be frugal on the number of indicators you are using. Let uncomment only the indicator you are using in your strategies or your hyperopt configuration, otherwise you will waste your memory and CPU usage. + :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :param pair: Pair currently analyzed + :return: a Dataframe with all mandatory indicators for the strategies """ # Momentum Indicator From 941879dc190ac5d73ab543148cb36f3d799aed07 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Jul 2018 07:54:17 +0100 Subject: [PATCH 319/395] revert docs to use populate_* functions --- docs/bot-optimization.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 62ab24070..ebd8cbe8c 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -61,13 +61,13 @@ file as reference.** ### Buy strategy -Edit the method `advise_buy()` into your strategy file to +Edit the method `populate_buy_trend()` into your strategy file to update your buy strategy. Sample from `user_data/strategies/test_strategy.py`: ```python -def advise_buy(self, dataframe: DataFrame, pair: str) -> DataFrame: +def populate_buy_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame populated with indicators @@ -87,13 +87,13 @@ def advise_buy(self, dataframe: DataFrame, pair: str) -> DataFrame: ### Sell strategy -Edit the method `advise_sell()` into your strategy file to update your sell strategy. +Edit the method `populate_sell_trend()` into your strategy file to update your sell strategy. Please note that the sell-signal is only used if `use_sell_signal` is set to true in the configuration. Sample from `user_data/strategies/test_strategy.py`: ```python -def advise_sell(self, dataframe: DataFrame, pair: str) -> DataFrame: +def populate_sell_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame populated with indicators @@ -112,14 +112,14 @@ def advise_sell(self, dataframe: DataFrame, pair: str) -> DataFrame: ## Add more Indicator -As you have seen, buy and sell strategies need indicators. You can add more indicators by extending the list contained in the method `advise_indicators()` from your strategy file. +As you have seen, buy and sell strategies need indicators. You can add more indicators by extending the list contained in the method `populate_indicators()` from your strategy file. -You should only add the indicators used in either `advise_buy()`, `advise_sell()`, or to populate another indicator, otherwise performance may suffer. +You should only add the indicators used in either `populate_buy_trend()`, `populate_sell_trend()`, or to populate another indicator, otherwise performance may suffer. Sample: ```python -def advise_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: +def populate_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: """ Adds several different TA indicators to the given DataFrame """ From 787d6042de49e2e71ad4c27704460401560ac02e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 20:36:03 +0200 Subject: [PATCH 320/395] Switch from pair(str) to metadata(dict) --- docs/bot-optimization.md | 28 ++++++++++----- freqtrade/optimize/backtesting.py | 2 +- freqtrade/optimize/hyperopt.py | 4 +-- freqtrade/strategy/default_strategy.py | 12 +++---- freqtrade/strategy/interface.py | 42 +++++++++++------------ freqtrade/tests/strategy/test_strategy.py | 21 +++++------- user_data/strategies/test_strategy.py | 12 +++---- 7 files changed, 64 insertions(+), 57 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index ebd8cbe8c..0214ce5c5 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -39,7 +39,6 @@ A strategy file contains all the information needed to build a good strategy: - Sell strategy rules - Minimal ROI recommended - Stoploss recommended -- Hyperopt parameter The bot also include a sample strategy called `TestStrategy` you can update: `user_data/strategies/test_strategy.py`. You can test it with the parameter: `--strategy TestStrategy` @@ -61,17 +60,16 @@ file as reference.** ### Buy strategy -Edit the method `populate_buy_trend()` into your strategy file to -update your buy strategy. +Edit the method `populate_buy_trend()` into your strategy file to update your buy strategy. Sample from `user_data/strategies/test_strategy.py`: ```python -def populate_buy_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: +def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame populated with indicators - :param pair: Pair currently analyzed + :param metadata: Additional information, like the currently traded pair :return: DataFrame with buy column """ dataframe.loc[ @@ -93,11 +91,11 @@ Please note that the sell-signal is only used if `use_sell_signal` is set to tru Sample from `user_data/strategies/test_strategy.py`: ```python -def populate_sell_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: +def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame populated with indicators - :param pair: Pair currently analyzed + :param metadata: Additional information, like the currently traded pair :return: DataFrame with buy column """ dataframe.loc[ @@ -110,7 +108,7 @@ def populate_sell_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: return dataframe ``` -## Add more Indicator +## Add more Indicators As you have seen, buy and sell strategies need indicators. You can add more indicators by extending the list contained in the method `populate_indicators()` from your strategy file. @@ -119,9 +117,16 @@ You should only add the indicators used in either `populate_buy_trend()`, `popul Sample: ```python -def populate_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: +def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame + + Performance Note: For the best performance be frugal on the number of indicators + you are using. Let uncomment only the indicator you are using in your strategies + or your hyperopt configuration, otherwise you will waste your memory and CPU usage. + :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :param metadata: Additional information, like the currently traded pair + :return: a Dataframe with all mandatory indicators for the strategies """ dataframe['sar'] = ta.SAR(dataframe) dataframe['adx'] = ta.ADX(dataframe) @@ -152,6 +157,11 @@ def populate_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: return dataframe ``` +### Metadata dict + +The metadata-dict (available for `populate_buy_trend`, `populate_sell_trend`, `populate_indicators`) contains additional information. +Currently this is `pair`, which can be accessed using `metadata['pair']` - and will return a pair in the format `XRP/BTC`. + ### Want more indicator examples Look into the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py). diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e3c3974be..593af619c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -230,7 +230,7 @@ class Backtesting(object): pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run ticker_data = self.advise_sell( - self.advise_buy(pair_data, pair), pair)[headers].copy() + self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() # to avoid using data from future, we buy/sell with signal from previous candle ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f0d81e3ff..086cad5aa 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -75,7 +75,7 @@ class Hyperopt(Backtesting): return arg_dict @staticmethod - def populate_indicators(dataframe: DataFrame, pair: str) -> DataFrame: + def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe['adx'] = ta.ADX(dataframe) macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -228,7 +228,7 @@ class Hyperopt(Backtesting): """ Define the buy strategy parameters to be used by hyperopt """ - def populate_buy_trend(dataframe: DataFrame, pair: str) -> DataFrame: + def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: """ Buy strategy Hyperopt will build and use """ diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 9eaf093ae..f1646779b 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -28,7 +28,7 @@ class DefaultStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' - def populate_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame @@ -36,7 +36,7 @@ class DefaultStrategy(IStrategy): you are using. Let uncomment only the indicator you are using in your strategies or your hyperopt configuration, otherwise you will waste your memory and CPU usage. :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() - :param pair: Pair currently analyzed + :param metadata: Additional information, like the currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ @@ -199,11 +199,11 @@ class DefaultStrategy(IStrategy): return dataframe - def populate_buy_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame - :param pair: Pair currently analyzed + :param metadata: Additional information, like the currently traded pair :return: DataFrame with buy column """ dataframe.loc[ @@ -221,11 +221,11 @@ class DefaultStrategy(IStrategy): return dataframe - def populate_sell_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame - :param pair: Pair currently analyzed + :param metadata: Additional information, like the currently traded pair :return: DataFrame with buy column """ dataframe.loc[ diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 887c1d583..dfd624393 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -74,29 +74,29 @@ class IStrategy(ABC): self.config = config @abstractmethod - def populate_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Populate indicators that will be used in the Buy and Sell strategy :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() - :param pair: Pair currently analyzed + :param metadata: Additional information, like the currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ @abstractmethod - def populate_buy_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame - :param pair: Pair currently analyzed + :param metadata: Additional information, like the currently traded pair :return: DataFrame with buy column """ @abstractmethod - def populate_sell_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame - :param pair: Pair currently analyzed + :param metadata: Additional information, like the currently traded pair :return: DataFrame with sell column """ @@ -106,16 +106,16 @@ class IStrategy(ABC): """ return self.__class__.__name__ - def analyze_ticker(self, ticker_history: List[Dict], pair: str) -> DataFrame: + def analyze_ticker(self, ticker_history: List[Dict], metadata: dict) -> DataFrame: """ Parses the given ticker history and returns a populated DataFrame add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ dataframe = parse_ticker_dataframe(ticker_history) - dataframe = self.advise_indicators(dataframe, pair) - dataframe = self.advise_buy(dataframe, pair) - dataframe = self.advise_sell(dataframe, pair) + dataframe = self.advise_indicators(dataframe, metadata) + dataframe = self.advise_buy(dataframe, metadata) + dataframe = self.advise_sell(dataframe, metadata) return dataframe def get_signal(self, pair: str, interval: str, ticker_hist: List[Dict]) -> Tuple[bool, bool]: @@ -130,7 +130,7 @@ class IStrategy(ABC): return False, False try: - dataframe = self.analyze_ticker(ticker_hist, pair) + dataframe = self.analyze_ticker(ticker_hist, {'pair': pair}) except ValueError as error: logger.warning( 'Unable to analyze ticker for pair %s: %s', @@ -275,15 +275,15 @@ class IStrategy(ABC): """ Creates a dataframe and populates indicators for given ticker data """ - return {pair: self.advise_indicators(parse_ticker_dataframe(pair_data), pair) + return {pair: self.advise_indicators(parse_ticker_dataframe(pair_data), {'pair': pair}) for pair, pair_data in tickerdata.items()} - def advise_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: + def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Populate indicators that will be used in the Buy and Sell strategy This method should not be overridden. :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() - :param pair: The currently traded pair + :param metadata: Additional information, like the currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ if self._populate_fun_len == 2: @@ -291,14 +291,14 @@ class IStrategy(ABC): "the current function headers!", DeprecationWarning) return self.populate_indicators(dataframe) # type: ignore else: - return self.populate_indicators(dataframe, pair) + return self.populate_indicators(dataframe, metadata) - def advise_buy(self, dataframe: DataFrame, pair: str) -> DataFrame: + def advise_buy(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe This method should not be overridden. :param dataframe: DataFrame - :param pair: The currently traded pair + :param pair: Additional information, like the currently traded pair :return: DataFrame with buy column """ if self._buy_fun_len == 2: @@ -306,14 +306,14 @@ class IStrategy(ABC): "the current function headers!", DeprecationWarning) return self.populate_buy_trend(dataframe) # type: ignore else: - return self.populate_buy_trend(dataframe, pair) + return self.populate_buy_trend(dataframe, metadata) - def advise_sell(self, dataframe: DataFrame, pair: str) -> DataFrame: + def advise_sell(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe This method should not be overridden. :param dataframe: DataFrame - :param pair: The currently traded pair + :param pair: Additional information, like the currently traded pair :return: DataFrame with sell column """ if self._sell_fun_len == 2: @@ -321,4 +321,4 @@ class IStrategy(ABC): "the current function headers!", DeprecationWarning) return self.populate_sell_trend(dataframe) # type: ignore else: - return self.populate_sell_trend(dataframe, pair) + return self.populate_sell_trend(dataframe, metadata) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 02eea312f..1c8f80ca1 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -60,10 +60,7 @@ def test_search_strategy(): def test_load_strategy(result): resolver = StrategyResolver({'strategy': 'TestStrategy'}) pair = 'ETH/BTC' - assert len(resolver.strategy.populate_indicators.__annotations__) == 3 - assert 'dataframe' in resolver.strategy.populate_indicators.__annotations__ - assert 'pair' in resolver.strategy.populate_indicators.__annotations__ - assert 'adx' in resolver.strategy.advise_indicators(result, pair=pair) + assert 'adx' in resolver.strategy.advise_indicators(result, metadata=pair) def test_load_strategy_invalid_directory(result, caplog): @@ -92,7 +89,7 @@ def test_strategy(result): config = {'strategy': 'DefaultStrategy'} resolver = StrategyResolver(config) - pair = 'ETH/BTC' + metadata = {'pair': 'ETH/BTC'} assert resolver.strategy.minimal_roi[0] == 0.04 assert config["minimal_roi"]['0'] == 0.04 @@ -102,13 +99,13 @@ def test_strategy(result): assert resolver.strategy.ticker_interval == '5m' assert config['ticker_interval'] == '5m' - df_indicators = resolver.strategy.advise_indicators(result, pair=pair) + df_indicators = resolver.strategy.advise_indicators(result, metadata=metadata) assert 'adx' in df_indicators - dataframe = resolver.strategy.advise_buy(df_indicators, pair=pair) + dataframe = resolver.strategy.advise_buy(df_indicators, metadata=metadata) assert 'buy' in dataframe.columns - dataframe = resolver.strategy.advise_sell(df_indicators, pair='ETH/BTC') + dataframe = resolver.strategy.advise_sell(df_indicators, metadata=metadata) assert 'sell' in dataframe.columns @@ -196,21 +193,21 @@ def test_call_deprecated_function(result, monkeypatch): default_location = path.join(path.dirname(path.realpath(__file__))) resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', 'strategy_path': default_location}) - pair = 'ETH/BTC' + metadata = {'pair': 'ETH/BTC'} # Make sure we are using a legacy function assert resolver.strategy._populate_fun_len == 2 assert resolver.strategy._buy_fun_len == 2 assert resolver.strategy._sell_fun_len == 2 - indicator_df = resolver.strategy.advise_indicators(result, pair=pair) + indicator_df = resolver.strategy.advise_indicators(result, metadata=metadata) assert type(indicator_df) is DataFrame assert 'adx' in indicator_df.columns - buydf = resolver.strategy.advise_buy(result, pair=pair) + buydf = resolver.strategy.advise_buy(result, metadata=metadata) assert type(buydf) is DataFrame assert 'buy' in buydf.columns - selldf = resolver.strategy.advise_sell(result, pair=pair) + selldf = resolver.strategy.advise_sell(result, metadata=metadata) assert type(selldf) is DataFrame assert 'sell' in selldf diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 65e06558c..80c238d92 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -45,7 +45,7 @@ class TestStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' - def populate_indicators(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame @@ -53,7 +53,7 @@ class TestStrategy(IStrategy): you are using. Let uncomment only the indicator you are using in your strategies or your hyperopt configuration, otherwise you will waste your memory and CPU usage. :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() - :param pair: Pair currently analyzed + :param metadata: Additional information, like the currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ @@ -215,11 +215,11 @@ class TestStrategy(IStrategy): return dataframe - def populate_buy_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame populated with indicators - :param pair: Pair currently analyzed + :param metadata: Additional information, like the currently traded pair :return: DataFrame with buy column """ dataframe.loc[ @@ -232,11 +232,11 @@ class TestStrategy(IStrategy): return dataframe - def populate_sell_trend(self, dataframe: DataFrame, pair: str) -> DataFrame: + def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame populated with indicators - :param pair: Pair currently analyzed + :param metadata: Additional information, like the currently traded pair :return: DataFrame with buy column """ dataframe.loc[ From 2401fa15d2103af375b0095da82623cac59ae435 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 21:07:21 +0200 Subject: [PATCH 321/395] Change missed calls to advise_* functions --- freqtrade/tests/optimize/test_backtesting.py | 2 +- freqtrade/tests/optimize/test_hyperopt.py | 6 +++--- freqtrade/tests/strategy/legacy_strategy.py | 15 ++++----------- freqtrade/tests/strategy/test_default_strategy.py | 8 ++++---- freqtrade/tests/strategy/test_strategy.py | 6 +++--- scripts/plot_dataframe.py | 4 ++-- 6 files changed, 17 insertions(+), 24 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 91ea2eee1..e4177eab5 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -146,7 +146,7 @@ def _trend(signals, buy_value, sell_value): return signals -def _trend_alternate(dataframe=None, pair=None): +def _trend_alternate(dataframe=None, metadata=None): signals = dataframe low = signals['low'] n = len(low) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 9b7d301cc..f1e7ad1d7 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -247,7 +247,7 @@ def test_populate_indicators(init_hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = _HYPEROPT.tickerdata_to_dataframe(tickerlist) - dataframe = _HYPEROPT.populate_indicators(dataframes['UNITTEST/BTC'], 'UNITTEST/BTC') + dataframe = _HYPEROPT.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe @@ -259,7 +259,7 @@ def test_buy_strategy_generator(init_hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = _HYPEROPT.tickerdata_to_dataframe(tickerlist) - dataframe = _HYPEROPT.populate_indicators(dataframes['UNITTEST/BTC'], 'UNITTEST/BTC') + dataframe = _HYPEROPT.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) populate_buy_trend = _HYPEROPT.buy_strategy_generator( { @@ -274,7 +274,7 @@ def test_buy_strategy_generator(init_hyperopt) -> None: 'trigger': 'bb_lower' } ) - result = populate_buy_trend(dataframe, 'UNITTEST/BTC') + result = populate_buy_trend(dataframe, {'pair': 'UNITTEST/BTC'}) # Check if some indicators are generated. We will not test all of them assert 'buy' in result assert 1 in result['buy'] diff --git a/freqtrade/tests/strategy/legacy_strategy.py b/freqtrade/tests/strategy/legacy_strategy.py index cb97bd63b..2cd13b791 100644 --- a/freqtrade/tests/strategy/legacy_strategy.py +++ b/freqtrade/tests/strategy/legacy_strategy.py @@ -13,18 +13,11 @@ import numpy # noqa # This class is a sample. Feel free to customize it. class TestStrategyLegacy(IStrategy): """ - This is a test strategy to inspire you. - More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md + This is a test strategy using the legacy function headers, which will be + removed in a future update. + Please do not use this as a template, but refer to user_data/strategy/TestStrategy.py + for a uptodate version of this template. - You can: - - Rename the class name (Do not forget to update class_name) - - Add any methods you want to build your strategy - - Add any lib you need to build your strategy - - You must keep: - - the lib in the section "Do not remove these libs" - - the prototype for the methods: minimal_roi, stoploss, populate_indicators, populate_buy_trend, - populate_sell_trend, hyperopt_space, buy_strategy_generator """ # Minimal ROI designed for the strategy. diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index 2b10e9023..6acfc439f 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -25,11 +25,11 @@ def test_default_strategy_structure(): def test_default_strategy(result): strategy = DefaultStrategy({}) - pair = 'ETH/BTC' + metadata = {'pair': 'ETH/BTC'} assert type(strategy.minimal_roi) is dict assert type(strategy.stoploss) is float assert type(strategy.ticker_interval) is str - indicators = strategy.populate_indicators(result, pair) + indicators = strategy.populate_indicators(result, metadata) assert type(indicators) is DataFrame - assert type(strategy.populate_buy_trend(indicators, pair)) is DataFrame - assert type(strategy.populate_sell_trend(indicators, pair)) is DataFrame + assert type(strategy.populate_buy_trend(indicators, metadata)) is DataFrame + assert type(strategy.populate_sell_trend(indicators, metadata)) is DataFrame diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 1c8f80ca1..6bb17fc28 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -59,8 +59,8 @@ def test_search_strategy(): def test_load_strategy(result): resolver = StrategyResolver({'strategy': 'TestStrategy'}) - pair = 'ETH/BTC' - assert 'adx' in resolver.strategy.advise_indicators(result, metadata=pair) + metadata = {'pair': 'ETH/BTC'} + assert 'adx' in resolver.strategy.advise_indicators(result, metadata=metadata) def test_load_strategy_invalid_directory(result, caplog): @@ -74,7 +74,7 @@ def test_load_strategy_invalid_directory(result, caplog): 'Path "{}" does not exist'.format(extra_dir), ) in caplog.record_tuples - assert 'adx' in resolver.strategy.advise_indicators(result, 'ETH/BTC') + assert 'adx' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) def test_load_not_found_strategy(): diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 06e1cd1d8..fbb385a3c 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -159,8 +159,8 @@ def plot_analyzed_dataframe(args: Namespace) -> None: dataframes = strategy.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] - dataframe = strategy.advise_buy(dataframe, pair) - dataframe = strategy.advise_sell(dataframe, pair) + dataframe = strategy.advise_buy(dataframe, {'pair': pair}) + dataframe = strategy.advise_sell(dataframe, {'pair': pair}) if len(dataframe.index) > args.plot_limit: logger.warning('Ticker contained more than %s candles as defined ' From e242842805fc80e04abd919ff443fc2db6be0fc6 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 30 Jul 2018 11:37:29 +0300 Subject: [PATCH 322/395] remove more useless docstrings from tests --- freqtrade/tests/exchange/test_exchange.py | 2 - freqtrade/tests/optimize/test_backtesting.py | 45 ---- freqtrade/tests/optimize/test_hyperopt.py | 7 - freqtrade/tests/rpc/test_rpc.py | 3 - freqtrade/tests/strategy/test_interface.py | 3 - freqtrade/tests/test_arguments.py | 4 - freqtrade/tests/test_configuration.py | 3 - freqtrade/tests/test_freqtradebot.py | 204 +------------------ 8 files changed, 1 insertion(+), 270 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 814f56acc..4de17eb68 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -15,8 +15,6 @@ from freqtrade.tests.conftest import get_patched_exchange, log_has def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): - """Function to test ccxt exception handling """ - with pytest.raises(TemporaryError): api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 836c7c302..010419710 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -164,9 +164,6 @@ def _trend_alternate(dataframe=None): # Unit tests def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: - """ - Test setup_configuration() function - """ mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) @@ -205,9 +202,6 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None: - """ - Test setup_configuration() function - """ mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) @@ -276,10 +270,6 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: - """ - Test setup_configuration() function - """ - conf = deepcopy(default_conf) conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT @@ -298,9 +288,6 @@ def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog def test_start(mocker, fee, default_conf, caplog) -> None: - """ - Test start() function - """ start_mock = MagicMock() mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) patch_exchange(mocker) @@ -323,9 +310,6 @@ def test_start(mocker, fee, default_conf, caplog) -> None: def test_backtesting_init(mocker, default_conf) -> None: - """ - Test Backtesting._init() method - """ patch_exchange(mocker) get_fee = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5)) backtesting = Backtesting(default_conf) @@ -339,9 +323,6 @@ def test_backtesting_init(mocker, default_conf) -> None: def test_tickerdata_to_dataframe(default_conf, mocker) -> None: - """ - Test Backtesting.tickerdata_to_dataframe() method - """ patch_exchange(mocker) timerange = TimeRange(None, 'line', 0, -100) tick = optimize.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) @@ -358,9 +339,6 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: def test_get_timeframe(default_conf, mocker) -> None: - """ - Test Backtesting.get_timeframe() method - """ patch_exchange(mocker) backtesting = Backtesting(default_conf) @@ -377,9 +355,6 @@ def test_get_timeframe(default_conf, mocker) -> None: def test_generate_text_table(default_conf, mocker): - """ - Test Backtesting.generate_text_table() method - """ patch_exchange(mocker) backtesting = Backtesting(default_conf) @@ -408,9 +383,6 @@ def test_generate_text_table(default_conf, mocker): def test_generate_text_table_sell_reason(default_conf, mocker): - """ - Test Backtesting.generate_text_table_sell_reason() method - """ patch_exchange(mocker) backtesting = Backtesting(default_conf) @@ -437,10 +409,6 @@ def test_generate_text_table_sell_reason(default_conf, mocker): def test_backtesting_start(default_conf, mocker, caplog) -> None: - """ - Test Backtesting.start() method - """ - def get_timeframe(input1, input2): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) @@ -477,10 +445,6 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: - """ - Test Backtesting.start() method if no data is found - """ - def get_timeframe(input1, input2): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) @@ -510,9 +474,6 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: def test_backtest(default_conf, fee, mocker) -> None: - """ - Test Backtesting.backtest() method - """ mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) patch_exchange(mocker) backtesting = Backtesting(default_conf) @@ -560,9 +521,6 @@ def test_backtest(default_conf, fee, mocker) -> None: def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: - """ - Test Backtesting.backtest() method with 1 min ticker - """ mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) patch_exchange(mocker) backtesting = Backtesting(default_conf) @@ -583,9 +541,6 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: def test_processed(default_conf, mocker) -> None: - """ - Test Backtesting.backtest() method with offline data - """ patch_exchange(mocker) backtesting = Backtesting(default_conf) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index dd7bf7da0..8168adb6e 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -45,9 +45,6 @@ def create_trials(mocker) -> None: def test_start(mocker, default_conf, caplog) -> None: - """ - Test start() function - """ start_mock = MagicMock() mocker.patch( 'freqtrade.configuration.Configuration._load_config_file', @@ -204,10 +201,6 @@ def test_start_calls_optimizer(mocker, init_hyperopt, default_conf, caplog) -> N def test_format_results(init_hyperopt): - """ - Test Hyperopt.format_results() - """ - # Test with BTC as stake_currency trades = [ ('ETH/BTC', 2, 2, 123), diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 63624db85..85f5482d1 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -278,9 +278,6 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, def test_rpc_balance_handle(default_conf, mocker): - """ - Test rpc_balance() method - """ mock_balance = { 'BTC': { 'free': 10.0, diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 1099f4b5f..2c056870f 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -98,9 +98,6 @@ def test_get_signal_handles_exceptions(mocker, default_conf): def test_tickerdata_to_dataframe(default_conf) -> None: - """ - Test Analyze.tickerdata_to_dataframe() method - """ strategy = DefaultStrategy(default_conf) timerange = TimeRange(None, 'line', 0, -100) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index c7740ce47..79bd0254b 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -1,9 +1,5 @@ # pragma pylint: disable=missing-docstring, C0103 -""" -Unit test file for arguments.py -""" - import argparse import pytest diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index d4f9f46e1..114a33ffb 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -322,9 +322,6 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: def test_check_exchange(default_conf) -> None: - """ - Test the configuration validator with a missing attribute - """ conf = deepcopy(default_conf) configuration = Configuration(Namespace()) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index b1e08383b..1b1e202b4 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1,9 +1,6 @@ +# pragma pylint: disable=missing-docstring, C0103 # pragma pylint: disable=protected-access, too-many-lines, invalid-name, too-many-arguments -""" -Unit test file for freqtradebot.py -""" - import logging import re import time @@ -64,9 +61,6 @@ def patch_RPCManager(mocker) -> MagicMock: # Unit tests def test_freqtradebot(mocker, default_conf) -> None: - """ - Test __init__, _init_modules, update_state, and get_state methods - """ freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.state is State.RUNNING @@ -77,9 +71,6 @@ def test_freqtradebot(mocker, default_conf) -> None: def test_cleanup(mocker, default_conf, caplog) -> None: - """ - Test clean() method - """ mock_cleanup = MagicMock() mocker.patch('freqtrade.persistence.cleanup', mock_cleanup) @@ -90,9 +81,6 @@ def test_cleanup(mocker, default_conf, caplog) -> None: def test_worker_running(mocker, default_conf, caplog) -> None: - """ - Test worker() method. Test when we start the bot - """ mock_throttle = MagicMock() mocker.patch('freqtrade.freqtradebot.FreqtradeBot._throttle', mock_throttle) @@ -105,9 +93,6 @@ def test_worker_running(mocker, default_conf, caplog) -> None: def test_worker_stopped(mocker, default_conf, caplog) -> None: - """ - Test worker() method. Test when we stop the bot - """ mock_throttle = MagicMock() mocker.patch('freqtrade.freqtradebot.FreqtradeBot._throttle', mock_throttle) mock_sleep = mocker.patch('time.sleep', return_value=None) @@ -122,9 +107,6 @@ def test_worker_stopped(mocker, default_conf, caplog) -> None: def test_throttle(mocker, default_conf, caplog) -> None: - """ - Test _throttle() method - """ def func(): """ Test function to throttle @@ -147,9 +129,6 @@ def test_throttle(mocker, default_conf, caplog) -> None: def test_throttle_with_assets(mocker, default_conf) -> None: - """ - Test _throttle() method when the function passed can have parameters - """ def func(nb_assets=-1): """ Test function to throttle @@ -166,9 +145,6 @@ def test_throttle_with_assets(mocker, default_conf) -> None: def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: - """ - Test _gen_pair_whitelist() method - """ freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -193,17 +169,10 @@ def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: @pytest.mark.skip(reason="Test not implemented") def test_refresh_whitelist() -> None: - """ - Test _refresh_whitelist() method - """ pass def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: - """ - Test get_trade_stake_amount() method - """ - patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -222,17 +191,12 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, limit_buy_order, fee, mocker) -> None: - """ - Test get_trade_stake_amount() method - """ patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5) ) - - # test defined stake amount freqtrade = FreqtradeBot(default_conf) with pytest.raises(DependencyException, match=r'.*stake amount.*'): @@ -245,9 +209,6 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, fee, markets, mocker) -> None: - """ - Test get_trade_stake_amount() method - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -291,10 +252,6 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, def test_get_min_pair_stake_amount(mocker, default_conf) -> None: - """ - Test get_trade_stake_amount() method - """ - patch_RPCManager(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) freqtrade = FreqtradeBot(default_conf) @@ -430,9 +387,6 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: - """ - Test create_trade() method - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -468,9 +422,6 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocke def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: - """ - Test create_trade() method - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -491,9 +442,6 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: - """ - Test create_trade() method - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) @@ -505,7 +453,6 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, get_fee=fee, get_markets=markets ) - conf = deepcopy(default_conf) conf['stake_amount'] = 0.0005 freqtrade = FreqtradeBot(conf) @@ -518,9 +465,6 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: - """ - Test create_trade() method - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) @@ -544,9 +488,6 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: - """ - Test create_trade() method - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -570,9 +511,6 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: - """ - Test create_trade() method - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -598,9 +536,6 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, marke def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: - """ - Test create_trade() method - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -625,9 +560,6 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, def test_create_trade_no_signal(default_conf, fee, mocker) -> None: - """ - Test create_trade() method - """ conf = deepcopy(default_conf) conf['dry_run'] = True @@ -653,9 +585,6 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: def test_process_trade_creation(default_conf, ticker, limit_buy_order, markets, fee, mocker, caplog) -> None: - """ - Test the trade creation in _process() method - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -694,9 +623,6 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> None: - """ - Test _process() method when a RequestException happens - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -717,9 +643,6 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non def test_process_operational_exception(default_conf, ticker, markets, mocker) -> None: - """ - Test _process() method when an OperationalException happens - """ msg_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -742,9 +665,6 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) -> def test_process_trade_handling( default_conf, ticker, limit_buy_order, markets, fee, mocker) -> None: - """ - Test _process() - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -771,9 +691,6 @@ def test_process_trade_handling( def test_balance_fully_ask_side(mocker, default_conf) -> None: - """ - Test get_target_bid() method - """ default_conf['bid_strategy']['ask_last_balance'] = 0.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) @@ -781,9 +698,6 @@ def test_balance_fully_ask_side(mocker, default_conf) -> None: def test_balance_fully_last_side(mocker, default_conf) -> None: - """ - Test get_target_bid() method - """ default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) @@ -791,9 +705,6 @@ def test_balance_fully_last_side(mocker, default_conf) -> None: def test_balance_bigger_last_ask(mocker, default_conf) -> None: - """ - Test get_target_bid() method - """ default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) @@ -801,9 +712,6 @@ def test_balance_bigger_last_ask(mocker, default_conf) -> None: def test_process_maybe_execute_buy(mocker, default_conf) -> None: - """ - Test process_maybe_execute_buy() method - """ freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trade', MagicMock(return_value=True)) @@ -814,9 +722,6 @@ def test_process_maybe_execute_buy(mocker, default_conf) -> None: def test_process_maybe_execute_buy_exception(mocker, default_conf, caplog) -> None: - """ - Test exception on process_maybe_execute_buy() method - """ freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch( @@ -828,9 +733,6 @@ def test_process_maybe_execute_buy_exception(mocker, default_conf, caplog) -> No def test_process_maybe_execute_sell(mocker, default_conf, limit_buy_order, caplog) -> None: - """ - Test process_maybe_execute_sell() method - """ freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) @@ -864,9 +766,6 @@ def test_process_maybe_execute_sell(mocker, default_conf, limit_buy_order, caplo def test_process_maybe_execute_sell_exception(mocker, default_conf, limit_buy_order, caplog) -> None: - """ - Test the exceptions in process_maybe_execute_sell() - """ freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.get_order', return_value=limit_buy_order) @@ -893,9 +792,6 @@ def test_process_maybe_execute_sell_exception(mocker, default_conf, def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, markets, mocker) -> None: - """ - Test check_handle() method - """ patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -939,9 +835,6 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: - """ - Test check_handle() method - """ conf = deepcopy(default_conf) conf.update({'experimental': {'use_sell_signal': True}}) @@ -999,9 +892,6 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, def test_handle_trade_roi(default_conf, ticker, limit_buy_order, fee, mocker, markets, caplog) -> None: - """ - Test check_handle() method - """ caplog.set_level(logging.DEBUG) conf = deepcopy(default_conf) conf.update({'experimental': {'use_sell_signal': True}}) @@ -1038,9 +928,6 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, def test_handle_trade_experimental( default_conf, ticker, limit_buy_order, fee, mocker, markets, caplog) -> None: - """ - Test check_handle() method - """ caplog.set_level(logging.DEBUG) conf = deepcopy(default_conf) conf.update({'experimental': {'use_sell_signal': True}}) @@ -1074,9 +961,6 @@ def test_handle_trade_experimental( def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, fee, markets, mocker) -> None: - """ - Test check_handle() method - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1105,9 +989,6 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fee, mocker) -> None: - """ - Test check_handle_timedout() method - """ rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() patch_coinmarketcap(mocker) @@ -1146,9 +1027,6 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker) -> None: - """ - Test check_handle_timedout() method - """ rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker) cancel_order_mock = MagicMock() @@ -1186,9 +1064,6 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, mocker) -> None: - """ - Test check_handle_timedout() method - """ rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker) cancel_order_mock = MagicMock() @@ -1228,9 +1103,6 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) -> None: - """ - Test check_handle_timedout() method when get_order throw an exception - """ patch_RPCManager(mocker) cancel_order_mock = MagicMock() patch_coinmarketcap(mocker) @@ -1274,9 +1146,6 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - def test_handle_timedout_limit_buy(mocker, default_conf) -> None: - """ - Test handle_timedout_limit_buy() method - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) cancel_order_mock = MagicMock() @@ -1300,9 +1169,6 @@ def test_handle_timedout_limit_buy(mocker, default_conf) -> None: def test_handle_timedout_limit_sell(mocker, default_conf) -> None: - """ - Test handle_timedout_limit_sell() method - """ patch_RPCManager(mocker) cancel_order_mock = MagicMock() patch_coinmarketcap(mocker) @@ -1326,9 +1192,6 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None: def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: - """ - Test execute_sell() method with a ticker going UP - """ rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1376,9 +1239,6 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: - """ - Test execute_sell() method with a ticker going DOWN - """ rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1428,9 +1288,6 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: - """ - Test execute_sell() method with a ticker going DOWN and with a bot config empty - """ rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -1478,9 +1335,6 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: - """ - Test execute_sell() method with a ticker going DOWN and with a bot config empty - """ rpc_mock = patch_RPCManager(mocker) patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( @@ -1529,9 +1383,6 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, fee, markets, mocker) -> None: - """ - Test sell_profit_only feature when enabled - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1566,9 +1417,6 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, fee, markets, mocker) -> None: - """ - Test sell_profit_only feature when disabled - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1601,9 +1449,6 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, markets, mocker) -> None: - """ - Test sell_profit_only feature when enabled and we have a loss - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1637,9 +1482,6 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, markets, mocker) -> None: - """ - Test sell_profit_only feature when enabled and we have a loss - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1675,9 +1517,6 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: - """ - Test sell_profit_only feature when enabled and we have a loss - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1716,9 +1555,6 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, mocker) -> None: - """ - Test sell_profit_only feature when enabled and we have a loss - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1756,9 +1592,6 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets, caplog, mocker) -> None: - """ - Test sell_profit_only feature when enabled and we have a loss - """ buy_price = limit_buy_order['price'] patch_RPCManager(mocker) patch_coinmarketcap(mocker) @@ -1884,9 +1717,6 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, caplog, m def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: - """ - Test sell_profit_only feature when enabled and we have a loss - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch.multiple( @@ -1926,12 +1756,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, caplog, mocker): - """ - Test get_real_amount - fee in quote currency - """ - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -1954,10 +1779,6 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): - """ - Test get_real_amount - fee in quote currency - """ - mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) patch_RPCManager(mocker) @@ -1982,9 +1803,6 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, mocker): - """ - Test get_real_amount - fees in Stake currency - """ trades_for_order[0]['fee']['currency'] = 'ETH' patch_RPCManager(mocker) @@ -2007,10 +1825,6 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, mo def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mocker): - """ - Test get_real_amount - Fees in BNB - """ - trades_for_order[0]['fee']['currency'] = 'BNB' trades_for_order[0]['fee']['cost'] = 0.00094518 @@ -2034,10 +1848,6 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, mocker): - """ - Test get_real_amount with split trades (multiple trades for this order) - """ - patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) @@ -2061,9 +1871,6 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, caplog, mocker): - """ - Test get_real_amount with split trades (multiple trades for this order) - """ limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'} @@ -2091,9 +1898,6 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, mocker): - """ - Test get_real_amount with split trades (multiple trades for this order) - """ limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004} @@ -2117,9 +1921,6 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, mocker): - """ - Test get_real_amount - fees in Stake currency - """ # Remove "Currency" from fee dict trades_for_order[0]['fee'] = {'cost': 0.008} @@ -2142,9 +1943,6 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, def test_get_real_amount_open_trade(default_conf, mocker): - """ - Test get_real_amount condition trade.fee_open == 0 or order['status'] == 'open' - """ patch_RPCManager(mocker) patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) From df53e912f07ae203fcae9313efdc9521edae182a Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 30 Jul 2018 11:55:00 +0300 Subject: [PATCH 323/395] fix one more test that was missing mock and needed internet --- freqtrade/tests/test_freqtradebot.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1b1e202b4..fc1d2f2f4 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1651,10 +1651,8 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) -def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, caplog, mocker) -> None: - """ - Test sell_profit_only feature when enabled and we have a loss - """ +def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, + caplog, mocker, markets) -> None: buy_price = limit_buy_order['price'] patch_RPCManager(mocker) patch_coinmarketcap(mocker) @@ -1668,6 +1666,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, caplog, m }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, + get_markets=markets, ) conf = deepcopy(default_conf) From 1c20ef873de8b14ef63994e5a3bb5a88bd4e0ead Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 30 Jul 2018 11:57:11 +0300 Subject: [PATCH 324/395] remove parens --- freqtrade/tests/test_freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index fc1d2f2f4..f614f6b98 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -183,7 +183,7 @@ def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mock freqtrade = FreqtradeBot(default_conf) result = freqtrade._get_trade_stake_amount() - assert(result == default_conf['stake_amount']) + assert result == default_conf['stake_amount'] def test_get_trade_stake_amount_no_stake_amount(default_conf, From fb80964b695094b6fbef6b15504204ef2d72e8f8 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 30 Jul 2018 11:58:14 +0300 Subject: [PATCH 325/395] freqtradebot tests don't need to mock coinmarketcap anymore --- freqtrade/tests/rpc/test_rpc.py | 4 +- freqtrade/tests/rpc/test_rpc_telegram.py | 4 +- freqtrade/tests/test_freqtradebot.py | 50 +----------------------- 3 files changed, 5 insertions(+), 53 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 85f5482d1..70b7dcfd9 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -11,8 +11,8 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException from freqtrade.state import State -from freqtrade.tests.test_freqtradebot import (patch_coinmarketcap, - patch_get_signal) +from freqtrade.tests.test_freqtradebot import patch_get_signal +from freqtrade.tests.conftest import patch_coinmarketcap # Functions for recurrent object patching diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index ceb8a7808..14feef10b 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -21,8 +21,8 @@ from freqtrade.rpc.telegram import Telegram, authorized_only from freqtrade.state import State from freqtrade.tests.conftest import (get_patched_freqtradebot, log_has, patch_exchange) -from freqtrade.tests.test_freqtradebot import (patch_coinmarketcap, - patch_get_signal) +from freqtrade.tests.test_freqtradebot import patch_get_signal +from freqtrade.tests.conftest import patch_coinmarketcap class DummyCls(Telegram): diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index f614f6b98..9cb477489 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -18,7 +18,7 @@ from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType from freqtrade.state import State from freqtrade.strategy.interface import SellType, SellCheckTuple -from freqtrade.tests.conftest import log_has, patch_coinmarketcap, patch_exchange +from freqtrade.tests.conftest import log_has, patch_exchange # Functions for recurrent object patching @@ -32,7 +32,6 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) patch_exchange(mocker) - patch_coinmarketcap(mocker) return FreqtradeBot(config) @@ -210,7 +209,6 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -388,7 +386,6 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -423,7 +420,6 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocke def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -443,7 +439,6 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -466,7 +461,6 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -489,7 +483,6 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -512,7 +505,6 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -537,7 +529,6 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, marke def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -564,7 +555,6 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: conf['dry_run'] = True patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -586,7 +576,6 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: def test_process_trade_creation(default_conf, ticker, limit_buy_order, markets, fee, mocker, caplog) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -624,7 +613,6 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -644,7 +632,6 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non def test_process_operational_exception(default_conf, ticker, markets, mocker) -> None: msg_mock = patch_RPCManager(mocker) - patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -666,7 +653,6 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) -> def test_process_trade_handling( default_conf, ticker, limit_buy_order, markets, fee, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -806,8 +792,6 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, get_fee=fee, get_markets=markets ) - patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) - freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -839,7 +823,6 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, conf.update({'experimental': {'use_sell_signal': True}}) patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -897,7 +880,6 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, conf.update({'experimental': {'use_sell_signal': True}}) patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -933,7 +915,6 @@ def test_handle_trade_experimental( conf.update({'experimental': {'use_sell_signal': True}}) patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -962,7 +943,6 @@ def test_handle_trade_experimental( def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -991,7 +971,6 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fee, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1028,7 +1007,6 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker) -> None: rpc_mock = patch_RPCManager(mocker) - patch_coinmarketcap(mocker) cancel_order_mock = MagicMock() mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -1065,7 +1043,6 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, mocker) -> None: rpc_mock = patch_RPCManager(mocker) - patch_coinmarketcap(mocker) cancel_order_mock = MagicMock() mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -1105,7 +1082,6 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) -> None: patch_RPCManager(mocker) cancel_order_mock = MagicMock() - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', @@ -1147,7 +1123,6 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) - def test_handle_timedout_limit_buy(mocker, default_conf) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) cancel_order_mock = MagicMock() mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -1171,7 +1146,6 @@ def test_handle_timedout_limit_buy(mocker, default_conf) -> None: def test_handle_timedout_limit_sell(mocker, default_conf) -> None: patch_RPCManager(mocker) cancel_order_mock = MagicMock() - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1193,7 +1167,6 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None: def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1240,7 +1213,6 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1289,7 +1261,6 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) - patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1336,7 +1307,6 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) - patch_coinmarketcap(mocker, value={'price_usd': 12345.0}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1384,7 +1354,6 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1418,7 +1387,6 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1450,7 +1418,6 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1483,7 +1450,6 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1518,7 +1484,6 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1556,7 +1521,6 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1594,7 +1558,6 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets caplog, mocker) -> None: buy_price = limit_buy_order['price'] patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1655,7 +1618,6 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, caplog, mocker, markets) -> None: buy_price = limit_buy_order['price'] patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1717,7 +1679,6 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), @@ -1757,7 +1718,6 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, caplog, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -1781,7 +1741,6 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) amount = buy_order_fee['amount'] trade = Trade( @@ -1805,7 +1764,6 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, mo trades_for_order[0]['fee']['currency'] = 'ETH' patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) @@ -1828,7 +1786,6 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock trades_for_order[0]['fee']['cost'] = 0.00094518 patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) @@ -1848,7 +1805,6 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, mocker): patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order2) amount = float(sum(x['amount'] for x in trades_for_order2)) @@ -1874,7 +1830,6 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'} patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[trades_for_order]) @@ -1901,7 +1856,6 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order limit_buy_order['fee'] = {'cost': 0.004} patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) amount = float(sum(x['amount'] for x in trades_for_order)) @@ -1924,7 +1878,6 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, trades_for_order[0]['fee'] = {'cost': 0.008} patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) @@ -1943,7 +1896,6 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, def test_get_real_amount_open_trade(default_conf, mocker): patch_RPCManager(mocker) - patch_coinmarketcap(mocker) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock(return_value=True)) amount = 12345 trade = Trade( From affdeb8fd8914a36375fa2b0deeb3f21a74e1763 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 30 Jul 2018 12:06:16 +0300 Subject: [PATCH 326/395] rename func to throttled_func --- freqtrade/tests/test_freqtradebot.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 9cb477489..21b1de41d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -106,40 +106,34 @@ def test_worker_stopped(mocker, default_conf, caplog) -> None: def test_throttle(mocker, default_conf, caplog) -> None: - def func(): - """ - Test function to throttle - """ + def throttled_func(): return 42 caplog.set_level(logging.DEBUG) freqtrade = get_patched_freqtradebot(mocker, default_conf) start = time.time() - result = freqtrade._throttle(func, min_secs=0.1) + result = freqtrade._throttle(throttled_func, min_secs=0.1) end = time.time() assert result == 42 assert end - start > 0.1 - assert log_has('Throttling func for 0.10 seconds', caplog.record_tuples) + assert log_has('Throttling throttled_func for 0.10 seconds', caplog.record_tuples) - result = freqtrade._throttle(func, min_secs=-1) + result = freqtrade._throttle(throttled_func, min_secs=-1) assert result == 42 def test_throttle_with_assets(mocker, default_conf) -> None: - def func(nb_assets=-1): - """ - Test function to throttle - """ + def throttled_func(nb_assets=-1): return nb_assets freqtrade = get_patched_freqtradebot(mocker, default_conf) - result = freqtrade._throttle(func, min_secs=0.1, nb_assets=666) + result = freqtrade._throttle(throttled_func, min_secs=0.1, nb_assets=666) assert result == 666 - result = freqtrade._throttle(func, min_secs=0.1) + result = freqtrade._throttle(throttled_func, min_secs=0.1) assert result == -1 From 3083e5d2bea5a44d623e56ade8c372a375f95ab8 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 30 Jul 2018 13:26:54 +0300 Subject: [PATCH 327/395] use pytest fixture properly in test_hyperopt --- freqtrade/tests/optimize/test_hyperopt.py | 90 +++++++++-------------- 1 file changed, 35 insertions(+), 55 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 8168adb6e..7907cd206 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -12,29 +12,22 @@ from freqtrade.strategy.resolver import StrategyResolver from freqtrade.tests.conftest import log_has, patch_exchange from freqtrade.tests.optimize.test_backtesting import get_args -# Avoid to reinit the same object again and again -_HYPEROPT_INITIALIZED = False -_HYPEROPT = None - @pytest.fixture(scope='function') -def init_hyperopt(default_conf, mocker): - global _HYPEROPT_INITIALIZED, _HYPEROPT - if not _HYPEROPT_INITIALIZED: - patch_exchange(mocker) - _HYPEROPT = Hyperopt(default_conf) - _HYPEROPT_INITIALIZED = True +def hyperopt(default_conf, mocker): + patch_exchange(mocker) + return Hyperopt(default_conf) # Functions for recurrent object patching -def create_trials(mocker) -> None: +def create_trials(mocker, hyperopt) -> None: """ When creating trials, mock the hyperopt Trials so that *by default* - we don't create any pickle'd files in the filesystem - we might have a pickle'd file so make sure that we return false when looking for it """ - _HYPEROPT.trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') + hyperopt.trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=False) mocker.patch('freqtrade.optimize.hyperopt.os.path.getsize', return_value=1) @@ -73,8 +66,7 @@ def test_start(mocker, default_conf, caplog) -> None: assert start_mock.call_count == 1 -def test_loss_calculation_prefer_correct_trade_count(init_hyperopt) -> None: - hyperopt = _HYPEROPT +def test_loss_calculation_prefer_correct_trade_count(hyperopt) -> None: StrategyResolver({'strategy': 'DefaultStrategy'}) correct = hyperopt.calculate_loss(1, hyperopt.target_trades, 20) @@ -84,17 +76,13 @@ def test_loss_calculation_prefer_correct_trade_count(init_hyperopt) -> None: assert under > correct -def test_loss_calculation_prefer_shorter_trades(init_hyperopt) -> None: - hyperopt = _HYPEROPT - +def test_loss_calculation_prefer_shorter_trades(hyperopt) -> None: shorter = hyperopt.calculate_loss(1, 100, 20) longer = hyperopt.calculate_loss(1, 100, 30) assert shorter < longer -def test_loss_calculation_has_limited_profit(init_hyperopt) -> None: - hyperopt = _HYPEROPT - +def test_loss_calculation_has_limited_profit(hyperopt) -> None: correct = hyperopt.calculate_loss(hyperopt.expected_max_profit, hyperopt.target_trades, 20) over = hyperopt.calculate_loss(hyperopt.expected_max_profit * 2, hyperopt.target_trades, 20) under = hyperopt.calculate_loss(hyperopt.expected_max_profit / 2, hyperopt.target_trades, 20) @@ -102,8 +90,7 @@ def test_loss_calculation_has_limited_profit(init_hyperopt) -> None: assert under > correct -def test_log_results_if_loss_improves(init_hyperopt, capsys) -> None: - hyperopt = _HYPEROPT +def test_log_results_if_loss_improves(hyperopt, capsys) -> None: hyperopt.current_best_loss = 2 hyperopt.log_results( { @@ -117,8 +104,7 @@ def test_log_results_if_loss_improves(init_hyperopt, capsys) -> None: assert ' 1/2: foo. Loss 1.00000'in out -def test_no_log_if_loss_does_not_improve(init_hyperopt, caplog) -> None: - hyperopt = _HYPEROPT +def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None: hyperopt.current_best_loss = 2 hyperopt.log_results( { @@ -128,13 +114,10 @@ def test_no_log_if_loss_does_not_improve(init_hyperopt, caplog) -> None: assert caplog.record_tuples == [] -def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: - trials = create_trials(mocker) +def test_save_trials_saves_trials(mocker, hyperopt, caplog) -> None: + trials = create_trials(mocker, hyperopt) mock_dump = mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) - - hyperopt = _HYPEROPT - _HYPEROPT.trials = trials - + hyperopt.trials = trials hyperopt.save_trials() trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') @@ -145,11 +128,9 @@ def test_save_trials_saves_trials(mocker, init_hyperopt, caplog) -> None: mock_dump.assert_called_once() -def test_read_trials_returns_trials_file(mocker, init_hyperopt, caplog) -> None: - trials = create_trials(mocker) +def test_read_trials_returns_trials_file(mocker, hyperopt, caplog) -> None: + trials = create_trials(mocker, hyperopt) mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=trials) - - hyperopt = _HYPEROPT hyperopt_trial = hyperopt.read_trials() trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') assert log_has( @@ -160,7 +141,7 @@ def test_read_trials_returns_trials_file(mocker, init_hyperopt, caplog) -> None: mock_load.assert_called_once() -def test_roi_table_generation(init_hyperopt) -> None: +def test_roi_table_generation(hyperopt) -> None: params = { 'roi_t1': 5, 'roi_t2': 10, @@ -170,11 +151,10 @@ def test_roi_table_generation(init_hyperopt) -> None: 'roi_p3': 3, } - hyperopt = _HYPEROPT assert hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} -def test_start_calls_optimizer(mocker, init_hyperopt, default_conf, caplog) -> None: +def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.multiprocessing.cpu_count', MagicMock(return_value=1)) @@ -200,7 +180,7 @@ def test_start_calls_optimizer(mocker, init_hyperopt, default_conf, caplog) -> N assert dumper.called -def test_format_results(init_hyperopt): +def test_format_results(hyperopt): # Test with BTC as stake_currency trades = [ ('ETH/BTC', 2, 2, 123), @@ -210,7 +190,7 @@ def test_format_results(init_hyperopt): labels = ['currency', 'profit_percent', 'profit_abs', 'trade_duration'] df = pd.DataFrame.from_records(trades, columns=labels) - result = _HYPEROPT.format_results(df) + result = hyperopt.format_results(df) assert result.find(' 66.67%') assert result.find('Total profit 1.00000000 BTC') assert result.find('2.0000Σ %') @@ -222,25 +202,25 @@ def test_format_results(init_hyperopt): ('XPR/EUR', -1, -2, -246) ] df = pd.DataFrame.from_records(trades, columns=labels) - result = _HYPEROPT.format_results(df) + result = hyperopt.format_results(df) assert result.find('Total profit 1.00000000 EUR') -def test_has_space(init_hyperopt): - _HYPEROPT.config.update({'spaces': ['buy', 'roi']}) - assert _HYPEROPT.has_space('roi') - assert _HYPEROPT.has_space('buy') - assert not _HYPEROPT.has_space('stoploss') +def test_has_space(hyperopt): + hyperopt.config.update({'spaces': ['buy', 'roi']}) + assert hyperopt.has_space('roi') + assert hyperopt.has_space('buy') + assert not hyperopt.has_space('stoploss') - _HYPEROPT.config.update({'spaces': ['all']}) - assert _HYPEROPT.has_space('buy') + hyperopt.config.update({'spaces': ['all']}) + assert hyperopt.has_space('buy') -def test_populate_indicators(init_hyperopt) -> None: +def test_populate_indicators(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} - dataframes = _HYPEROPT.tickerdata_to_dataframe(tickerlist) - dataframe = _HYPEROPT.populate_indicators(dataframes['UNITTEST/BTC']) + dataframes = hyperopt.tickerdata_to_dataframe(tickerlist) + dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC']) # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe @@ -248,13 +228,13 @@ def test_populate_indicators(init_hyperopt) -> None: assert 'rsi' in dataframe -def test_buy_strategy_generator(init_hyperopt) -> None: +def test_buy_strategy_generator(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} - dataframes = _HYPEROPT.tickerdata_to_dataframe(tickerlist) - dataframe = _HYPEROPT.populate_indicators(dataframes['UNITTEST/BTC']) + dataframes = hyperopt.tickerdata_to_dataframe(tickerlist) + dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC']) - populate_buy_trend = _HYPEROPT.buy_strategy_generator( + populate_buy_trend = hyperopt.buy_strategy_generator( { 'adx-value': 20, 'fastd-value': 20, @@ -273,7 +253,7 @@ def test_buy_strategy_generator(init_hyperopt) -> None: assert 1 in result['buy'] -def test_generate_optimizer(mocker, init_hyperopt, default_conf) -> None: +def test_generate_optimizer(mocker, default_conf) -> None: conf = deepcopy(default_conf) conf.update({'config': 'config.json.example'}) conf.update({'timerange': None}) From 67d1693901a4a7d1c9c7b158460aa42b67495f59 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 30 Jul 2018 14:57:51 +0300 Subject: [PATCH 328/395] avoid validating default_conf hundreds of times --- freqtrade/tests/conftest.py | 3 --- freqtrade/tests/test_configuration.py | 7 ++++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 8d0809367..d18016e16 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -8,10 +8,8 @@ from unittest.mock import MagicMock import arrow import pytest -from jsonschema import validate from telegram import Chat, Message, Update -from freqtrade import constants from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot @@ -127,7 +125,6 @@ def default_conf(): "db_url": "sqlite://", "loglevel": logging.DEBUG, } - validate(configuration, constants.CONF_SCHEMA) return configuration diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 114a33ffb..595280225 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -7,8 +7,9 @@ import logging from unittest.mock import MagicMock import pytest -from jsonschema import ValidationError +from jsonschema import validate, ValidationError +from freqtrade import constants from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration, set_loggers @@ -395,3 +396,7 @@ def test_set_loggers() -> None: assert logging.getLogger('requests').level is logging.DEBUG assert logging.getLogger('ccxt.base.exchange').level is logging.DEBUG assert logging.getLogger('telegram').level is logging.INFO + + +def test_validate_default_conf(default_conf) -> None: + validate(default_conf, constants.CONF_SCHEMA) From 3ecc502d863eb9798a5111258657a8ef04e4ee64 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 30 Jul 2018 14:24:06 +0200 Subject: [PATCH 329/395] Update ccxt from 1.17.45 to 1.17.49 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3a00111ac..964da51e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.45 +ccxt==1.17.49 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 8b8d3f3b75b6e7ad9b8a2f83692a25605f86ecdd Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Mon, 30 Jul 2018 15:40:52 +0300 Subject: [PATCH 330/395] default_conf is function-scoped fixture, no need to deepcopy it --- freqtrade/tests/exchange/test_exchange.py | 11 +- freqtrade/tests/optimize/test_backtesting.py | 41 +++---- freqtrade/tests/optimize/test_hyperopt.py | 21 ++-- freqtrade/tests/rpc/test_rpc_manager.py | 28 ++--- freqtrade/tests/rpc/test_rpc_telegram.py | 34 ++--- freqtrade/tests/test_configuration.py | 41 +++---- freqtrade/tests/test_freqtradebot.py | 123 +++++++------------ freqtrade/tests/test_persistence.py | 27 ++-- 8 files changed, 128 insertions(+), 198 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 4de17eb68..eed7d6b7b 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1,7 +1,6 @@ # pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement # pragma pylint: disable=protected-access import logging -from copy import deepcopy from datetime import datetime from random import randint from unittest.mock import MagicMock, PropertyMock @@ -78,12 +77,11 @@ def test_validate_pairs_not_compatible(default_conf, mocker): api_mock.load_markets = MagicMock(return_value={ 'ETH/BTC': '', 'TKN/BTC': '', 'TRST/BTC': '', 'SWT/BTC': '', 'BCC/BTC': '' }) - conf = deepcopy(default_conf) - conf['stake_currency'] = 'ETH' + default_conf['stake_currency'] = 'ETH' mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) with pytest.raises(OperationalException, match=r'not compatible'): - Exchange(conf) + Exchange(default_conf) def test_validate_pairs_exception(default_conf, mocker, caplog): @@ -108,8 +106,7 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): def test_validate_pairs_stake_exception(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - conf = deepcopy(default_conf) - conf['stake_currency'] = 'ETH' + default_conf['stake_currency'] = 'ETH' api_mock = MagicMock() api_mock.name = MagicMock(return_value='binance') mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) @@ -119,7 +116,7 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): OperationalException, match=r'Pair ETH/BTC not compatible with stake_currency: ETH' ): - Exchange(conf) + Exchange(default_conf) def test_validate_timeframes(default_conf, mocker): diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 010419710..a523f4126 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -3,7 +3,6 @@ import json import math import random -from copy import deepcopy from typing import List from unittest.mock import MagicMock @@ -270,11 +269,10 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: - conf = deepcopy(default_conf) - conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT + default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT mocker.patch('freqtrade.configuration.open', mocker.mock_open( - read_data=json.dumps(conf) + read_data=json.dumps(default_conf) )) args = [ @@ -422,15 +420,14 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: get_timeframe=get_timeframe, ) - conf = deepcopy(default_conf) - conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - conf['ticker_interval'] = 1 - conf['live'] = False - conf['datadir'] = None - conf['export'] = None - conf['timerange'] = '-100' + default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] + default_conf['ticker_interval'] = 1 + default_conf['live'] = False + default_conf['datadir'] = None + default_conf['export'] = None + default_conf['timerange'] = '-100' - backtesting = Backtesting(conf) + backtesting = Backtesting(default_conf) backtesting.start() # check the logs, that will contain the backtest result exists = [ @@ -458,15 +455,14 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: get_timeframe=get_timeframe, ) - conf = deepcopy(default_conf) - conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - conf['ticker_interval'] = "1m" - conf['live'] = False - conf['datadir'] = None - conf['export'] = None - conf['timerange'] = '20180101-20180102' + default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] + default_conf['ticker_interval'] = "1m" + default_conf['live'] = False + default_conf['datadir'] = None + default_conf['export'] = None + default_conf['timerange'] = '20180101-20180102' - backtesting = Backtesting(conf) + backtesting = Backtesting(default_conf) backtesting.start() # check the logs, that will contain the backtest result @@ -680,15 +676,14 @@ def test_backtest_record(default_conf, fee, mocker): def test_backtest_start_live(default_conf, mocker, caplog): - conf = deepcopy(default_conf) - conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] + default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', new=lambda s, n, i: _load_pair_as_ticks(n, i)) patch_exchange(mocker) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock()) mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock()) mocker.patch('freqtrade.configuration.open', mocker.mock_open( - read_data=json.dumps(conf) + read_data=json.dumps(default_conf) )) args = MagicMock() diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 7907cd206..35f33a061 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,6 +1,5 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 import os -from copy import deepcopy from unittest.mock import MagicMock import pandas as pd @@ -164,13 +163,12 @@ def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: ) patch_exchange(mocker) - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'epochs': 1}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) + default_conf.update({'config': 'config.json.example'}) + default_conf.update({'epochs': 1}) + default_conf.update({'timerange': None}) + default_conf.update({'spaces': 'all'}) - hyperopt = Hyperopt(conf) + hyperopt = Hyperopt(default_conf) hyperopt.tickerdata_to_dataframe = MagicMock() hyperopt.start() @@ -254,10 +252,9 @@ def test_buy_strategy_generator(hyperopt) -> None: def test_generate_optimizer(mocker, default_conf) -> None: - conf = deepcopy(default_conf) - conf.update({'config': 'config.json.example'}) - conf.update({'timerange': None}) - conf.update({'spaces': 'all'}) + default_conf.update({'config': 'config.json.example'}) + default_conf.update({'timerange': None}) + default_conf.update({'spaces': 'all'}) trades = [ ('POWR/BTC', 0.023117, 0.000233, 100) @@ -297,6 +294,6 @@ def test_generate_optimizer(mocker, default_conf) -> None: 'params': optimizer_param } - hyperopt = Hyperopt(conf) + hyperopt = Hyperopt(default_conf) generate_optimizer_value = hyperopt.generate_optimizer(list(optimizer_param.values())) assert generate_optimizer_value == response_expected diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index c4f27787b..90c693830 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -1,7 +1,6 @@ # pragma pylint: disable=missing-docstring, C0103 import logging -from copy import deepcopy from unittest.mock import MagicMock from freqtrade.rpc import RPCMessageType, RPCManager @@ -9,18 +8,16 @@ from freqtrade.tests.conftest import log_has, get_patched_freqtradebot def test__init__(mocker, default_conf) -> None: - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False + default_conf['telegram']['enabled'] = False - rpc_manager = RPCManager(get_patched_freqtradebot(mocker, conf)) + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) assert rpc_manager.registered_modules == [] def test_init_telegram_disabled(mocker, default_conf, caplog) -> None: caplog.set_level(logging.DEBUG) - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False - rpc_manager = RPCManager(get_patched_freqtradebot(mocker, conf)) + default_conf['telegram']['enabled'] = False + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) assert not log_has('Enabling rpc.telegram ...', caplog.record_tuples) assert rpc_manager.registered_modules == [] @@ -40,10 +37,9 @@ def test_init_telegram_enabled(mocker, default_conf, caplog) -> None: def test_cleanup_telegram_disabled(mocker, default_conf, caplog) -> None: caplog.set_level(logging.DEBUG) telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.cleanup', MagicMock()) - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False + default_conf['telegram']['enabled'] = False - freqtradebot = get_patched_freqtradebot(mocker, conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc_manager = RPCManager(freqtradebot) rpc_manager.cleanup() @@ -70,10 +66,9 @@ def test_cleanup_telegram_enabled(mocker, default_conf, caplog) -> None: def test_send_msg_telegram_disabled(mocker, default_conf, caplog) -> None: telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False + default_conf['telegram']['enabled'] = False - freqtradebot = get_patched_freqtradebot(mocker, conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc_manager = RPCManager(freqtradebot) rpc_manager.send_msg({ 'type': RPCMessageType.STATUS_NOTIFICATION, @@ -101,10 +96,9 @@ def test_send_msg_telegram_enabled(mocker, default_conf, caplog) -> None: def test_init_webhook_disabled(mocker, default_conf, caplog) -> None: caplog.set_level(logging.DEBUG) - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False - conf['webhook'] = {'enabled': False} - rpc_manager = RPCManager(get_patched_freqtradebot(mocker, conf)) + default_conf['telegram']['enabled'] = False + default_conf['webhook'] = {'enabled': False} + rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) assert not log_has('Enabling rpc.webhook ...', caplog.record_tuples) assert rpc_manager.registered_modules == [] diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 14feef10b..4b2fe4cf5 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -3,7 +3,6 @@ # pragma pylint: disable=too-many-lines, too-many-arguments import re -from copy import deepcopy from datetime import datetime from random import randint from unittest.mock import MagicMock, ANY @@ -96,9 +95,8 @@ def test_authorized_only(default_conf, mocker, caplog) -> None: update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat) - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False - bot = FreqtradeBot(conf) + default_conf['telegram']['enabled'] = False + bot = FreqtradeBot(default_conf) patch_get_signal(bot, (True, False)) dummy = DummyCls(bot) dummy.dummy_handler(bot=MagicMock(), update=update) @@ -124,9 +122,8 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat) - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False - bot = FreqtradeBot(conf) + default_conf['telegram']['enabled'] = False + bot = FreqtradeBot(default_conf) patch_get_signal(bot, (True, False)) dummy = DummyCls(bot) dummy.dummy_handler(bot=MagicMock(), update=update) @@ -152,10 +149,9 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: update = Update(randint(1, 100)) update.message = Message(randint(1, 100), 0, datetime.utcnow(), Chat(0, 0)) - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False + default_conf['telegram']['enabled'] = False - bot = FreqtradeBot(conf) + bot = FreqtradeBot(default_conf) patch_get_signal(bot, (True, False)) dummy = DummyCls(bot) @@ -177,9 +173,8 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: update.message.chat.id = 123 - conf = deepcopy(default_conf) - conf['telegram']['enabled'] = False - conf['telegram']['chat_id'] = 123 + default_conf['telegram']['enabled'] = False + default_conf['telegram']['chat_id'] = 123 patch_coinmarketcap(mocker) @@ -214,7 +209,7 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(conf) + freqtradebot = FreqtradeBot(default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -294,9 +289,8 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) ) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - conf = deepcopy(default_conf) - conf['stake_amount'] = 15.0 - freqtradebot = FreqtradeBot(conf) + default_conf['stake_amount'] = 15.0 + freqtradebot = FreqtradeBot(default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -1181,9 +1175,8 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: def test__send_msg(default_conf, mocker) -> None: patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) - conf = deepcopy(default_conf) bot = MagicMock() - freqtradebot = get_patched_freqtradebot(mocker, conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) telegram._config['telegram']['enabled'] = True @@ -1194,10 +1187,9 @@ def test__send_msg(default_conf, mocker) -> None: def test__send_msg_network_error(default_conf, mocker, caplog) -> None: patch_coinmarketcap(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) - conf = deepcopy(default_conf) bot = MagicMock() bot.send_message = MagicMock(side_effect=NetworkError('Oh snap')) - freqtradebot = get_patched_freqtradebot(mocker, conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) telegram._config['telegram']['enabled'] = True diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 595280225..e48553bdf 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -2,7 +2,6 @@ import json from argparse import Namespace -from copy import deepcopy import logging from unittest.mock import MagicMock @@ -18,30 +17,27 @@ from freqtrade.tests.conftest import log_has def test_load_config_invalid_pair(default_conf) -> None: - conf = deepcopy(default_conf) - conf['exchange']['pair_whitelist'].append('ETH-BTC') + default_conf['exchange']['pair_whitelist'].append('ETH-BTC') with pytest.raises(ValidationError, match=r'.*does not match.*'): configuration = Configuration(Namespace()) - configuration._validate_config(conf) + configuration._validate_config(default_conf) def test_load_config_missing_attributes(default_conf) -> None: - conf = deepcopy(default_conf) - conf.pop('exchange') + default_conf.pop('exchange') with pytest.raises(ValidationError, match=r'.*\'exchange\' is a required property.*'): configuration = Configuration(Namespace()) - configuration._validate_config(conf) + configuration._validate_config(default_conf) def test_load_config_incorrect_stake_amount(default_conf) -> None: - conf = deepcopy(default_conf) - conf['stake_amount'] = 'fake' + default_conf['stake_amount'] = 'fake' with pytest.raises(ValidationError, match=r'.*\'fake\' does not match \'unlimited\'.*'): configuration = Configuration(Namespace()) - configuration._validate_config(conf) + configuration._validate_config(default_conf) def test_load_config_file(default_conf, mocker, caplog) -> None: @@ -58,10 +54,9 @@ def test_load_config_file(default_conf, mocker, caplog) -> None: def test_load_config_max_open_trades_zero(default_conf, mocker, caplog) -> None: - conf = deepcopy(default_conf) - conf['max_open_trades'] = 0 + default_conf['max_open_trades'] = 0 file_mock = mocker.patch('freqtrade.configuration.open', mocker.mock_open( - read_data=json.dumps(conf) + read_data=json.dumps(default_conf) )) Configuration(Namespace())._load_config_file('somefile') @@ -152,13 +147,12 @@ def test_load_config_with_params(default_conf, mocker) -> None: def test_load_custom_strategy(default_conf, mocker) -> None: - custom_conf = deepcopy(default_conf) - custom_conf.update({ + default_conf.update({ 'strategy': 'CustomStrategy', 'strategy_path': '/tmp/strategies', }) mocker.patch('freqtrade.configuration.open', mocker.mock_open( - read_data=json.dumps(custom_conf) + read_data=json.dumps(default_conf) )) args = Arguments([], '').get_parsed_arg() @@ -323,26 +317,25 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: def test_check_exchange(default_conf) -> None: - conf = deepcopy(default_conf) configuration = Configuration(Namespace()) # Test a valid exchange - conf.get('exchange').update({'name': 'BITTREX'}) - assert configuration.check_exchange(conf) + default_conf.get('exchange').update({'name': 'BITTREX'}) + assert configuration.check_exchange(default_conf) # Test a valid exchange - conf.get('exchange').update({'name': 'binance'}) - assert configuration.check_exchange(conf) + default_conf.get('exchange').update({'name': 'binance'}) + assert configuration.check_exchange(default_conf) # Test a invalid exchange - conf.get('exchange').update({'name': 'unknown_exchange'}) - configuration.config = conf + default_conf.get('exchange').update({'name': 'unknown_exchange'}) + configuration.config = default_conf with pytest.raises( OperationalException, match=r'.*Exchange "unknown_exchange" not supported.*' ): - configuration.check_exchange(conf) + configuration.check_exchange(default_conf) def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 21b1de41d..69f349107 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -63,9 +63,8 @@ def test_freqtradebot(mocker, default_conf) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.state is State.RUNNING - conf = deepcopy(default_conf) - conf.pop('initial_state') - freqtrade = FreqtradeBot(conf) + default_conf.pop('initial_state') + freqtrade = FreqtradeBot(default_conf) assert freqtrade.state is State.STOPPED @@ -442,14 +441,13 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, get_fee=fee, get_markets=markets ) - conf = deepcopy(default_conf) - conf['stake_amount'] = 0.0005 - freqtrade = FreqtradeBot(conf) + default_conf['stake_amount'] = 0.0005 + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.create_trade() rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2] - assert rate * amount >= conf['stake_amount'] + assert rate * amount >= default_conf['stake_amount'] def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order, @@ -465,9 +463,8 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord get_markets=markets ) - conf = deepcopy(default_conf) - conf['stake_amount'] = 0.000000005 - freqtrade = FreqtradeBot(conf) + default_conf['stake_amount'] = 0.000000005 + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) result = freqtrade.create_trade() @@ -486,11 +483,10 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, get_fee=fee, get_markets=markets ) - conf = deepcopy(default_conf) - conf['max_open_trades'] = 0 - conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT + default_conf['max_open_trades'] = 0 + default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT - freqtrade = FreqtradeBot(conf) + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) assert freqtrade.create_trade() is False @@ -508,10 +504,9 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, marke get_markets=markets ) - conf = deepcopy(default_conf) - conf['exchange']['pair_whitelist'] = ["ETH/BTC"] - conf['exchange']['pair_blacklist'] = ["ETH/BTC"] - freqtrade = FreqtradeBot(conf) + default_conf['exchange']['pair_whitelist'] = ["ETH/BTC"] + default_conf['exchange']['pair_blacklist'] = ["ETH/BTC"] + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.create_trade() @@ -531,11 +526,9 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, get_fee=fee, get_markets=markets ) - - conf = deepcopy(default_conf) - conf['exchange']['pair_whitelist'] = ["ETH/BTC"] - conf['exchange']['pair_blacklist'] = ["ETH/BTC"] - freqtrade = FreqtradeBot(conf) + default_conf['exchange']['pair_whitelist'] = ["ETH/BTC"] + default_conf['exchange']['pair_blacklist'] = ["ETH/BTC"] + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.create_trade() @@ -545,8 +538,7 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker, def test_create_trade_no_signal(default_conf, fee, mocker) -> None: - conf = deepcopy(default_conf) - conf['dry_run'] = True + default_conf['dry_run'] = True patch_RPCManager(mocker) mocker.patch.multiple( @@ -556,10 +548,8 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: get_balance=MagicMock(return_value=20), get_fee=fee, ) - - conf = deepcopy(default_conf) - conf['stake_amount'] = 10 - freqtrade = FreqtradeBot(conf) + default_conf['stake_amount'] = 10 + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade, value=(False, False)) Trade.query = MagicMock() @@ -813,8 +803,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: - conf = deepcopy(default_conf) - conf.update({'experimental': {'use_sell_signal': True}}) + default_conf.update({'experimental': {'use_sell_signal': True}}) patch_RPCManager(mocker) mocker.patch.multiple( @@ -826,7 +815,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, get_markets=markets ) - freqtrade = FreqtradeBot(conf) + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade, value=(True, True)) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False @@ -870,8 +859,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, def test_handle_trade_roi(default_conf, ticker, limit_buy_order, fee, mocker, markets, caplog) -> None: caplog.set_level(logging.DEBUG) - conf = deepcopy(default_conf) - conf.update({'experimental': {'use_sell_signal': True}}) + default_conf.update({'experimental': {'use_sell_signal': True}}) patch_RPCManager(mocker) mocker.patch.multiple( @@ -883,7 +871,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, get_markets=markets ) - freqtrade = FreqtradeBot(conf) + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade, value=(True, False)) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True @@ -905,9 +893,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, def test_handle_trade_experimental( default_conf, ticker, limit_buy_order, fee, mocker, markets, caplog) -> None: caplog.set_level(logging.DEBUG) - conf = deepcopy(default_conf) - conf.update({'experimental': {'use_sell_signal': True}}) - + default_conf.update({'experimental': {'use_sell_signal': True}}) patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -918,7 +904,7 @@ def test_handle_trade_experimental( get_markets=markets ) - freqtrade = FreqtradeBot(conf) + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() @@ -1360,12 +1346,11 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, get_fee=fee, get_markets=markets ) - conf = deepcopy(default_conf) - conf['experimental'] = { + default_conf['experimental'] = { 'use_sell_signal': True, 'sell_profit_only': True, } - freqtrade = FreqtradeBot(conf) + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False @@ -1393,12 +1378,11 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, get_fee=fee, get_markets=markets ) - conf = deepcopy(default_conf) - conf['experimental'] = { + default_conf['experimental'] = { 'use_sell_signal': True, 'sell_profit_only': False, } - freqtrade = FreqtradeBot(conf) + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() @@ -1424,12 +1408,11 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market get_fee=fee, get_markets=markets ) - conf = deepcopy(default_conf) - conf['experimental'] = { + default_conf['experimental'] = { 'use_sell_signal': True, 'sell_profit_only': True, } - freqtrade = FreqtradeBot(conf) + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.stop_loss_reached = \ lambda current_rate, trade, current_time, current_profit: SellCheckTuple( @@ -1456,14 +1439,12 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke get_fee=fee, get_markets=markets ) - - conf = deepcopy(default_conf) - conf['experimental'] = { + default_conf['experimental'] = { 'use_sell_signal': True, 'sell_profit_only': False, } - freqtrade = FreqtradeBot(conf) + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False @@ -1490,13 +1471,10 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m get_fee=fee, get_markets=markets ) - - conf = deepcopy(default_conf) - conf['experimental'] = { + default_conf['experimental'] = { 'ignore_roi_if_buy_signal': True } - - freqtrade = FreqtradeBot(conf) + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True @@ -1527,11 +1505,8 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, get_fee=fee, get_markets=markets, ) - - conf = deepcopy(default_conf) - conf['trailing_stop'] = True - print(limit_buy_order) - freqtrade = FreqtradeBot(conf) + default_conf['trailing_stop'] = True + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False @@ -1564,11 +1539,9 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets get_fee=fee, get_markets=markets, ) - - conf = deepcopy(default_conf) - conf['trailing_stop'] = True - conf['trailing_stop_positive'] = 0.01 - freqtrade = FreqtradeBot(conf) + default_conf['trailing_stop'] = True + default_conf['trailing_stop_positive'] = 0.01 + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() @@ -1625,11 +1598,10 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, get_markets=markets, ) - conf = deepcopy(default_conf) - conf['trailing_stop'] = True - conf['trailing_stop_positive'] = 0.01 - conf['trailing_stop_positive_offset'] = 0.011 - freqtrade = FreqtradeBot(conf) + default_conf['trailing_stop'] = True + default_conf['trailing_stop_positive'] = 0.01 + default_conf['trailing_stop_positive_offset'] = 0.011 + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() @@ -1685,13 +1657,10 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, get_fee=fee, get_markets=markets ) - - conf = deepcopy(default_conf) - conf['experimental'] = { + default_conf['experimental'] = { 'ignore_roi_if_buy_signal': False } - - freqtrade = FreqtradeBot(conf) + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 7baddf60a..26932136a 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,5 +1,4 @@ # pragma pylint: disable=missing-docstring, C0103 -from copy import deepcopy from unittest.mock import MagicMock import pytest @@ -23,46 +22,40 @@ def test_init_create_session(default_conf): def test_init_custom_db_url(default_conf, mocker): - conf = deepcopy(default_conf) - # Update path to a value other than default, but still in-memory - conf.update({'db_url': 'sqlite:///tmp/freqtrade2_test.sqlite'}) + default_conf.update({'db_url': 'sqlite:///tmp/freqtrade2_test.sqlite'}) create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) - init(conf) + init(default_conf) assert create_engine_mock.call_count == 1 assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tmp/freqtrade2_test.sqlite' def test_init_invalid_db_url(default_conf): - conf = deepcopy(default_conf) - # Update path to a value other than default, but still in-memory - conf.update({'db_url': 'unknown:///some.url'}) + default_conf.update({'db_url': 'unknown:///some.url'}) with pytest.raises(OperationalException, match=r'.*no valid database URL*'): - init(conf) + init(default_conf) def test_init_prod_db(default_conf, mocker): - conf = deepcopy(default_conf) - conf.update({'dry_run': False}) - conf.update({'db_url': constants.DEFAULT_DB_PROD_URL}) + default_conf.update({'dry_run': False}) + default_conf.update({'db_url': constants.DEFAULT_DB_PROD_URL}) create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) - init(conf) + init(default_conf) assert create_engine_mock.call_count == 1 assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tradesv3.sqlite' def test_init_dryrun_db(default_conf, mocker): - conf = deepcopy(default_conf) - conf.update({'dry_run': True}) - conf.update({'db_url': constants.DEFAULT_DB_DRYRUN_URL}) + default_conf.update({'dry_run': True}) + default_conf.update({'db_url': constants.DEFAULT_DB_DRYRUN_URL}) create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) - init(conf) + init(default_conf) assert create_engine_mock.call_count == 1 assert create_engine_mock.mock_calls[0][1][0] == 'sqlite://' From 012fe94333d3b2187ab06fb85e912b6f0b1064a2 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 30 Jul 2018 16:49:58 +0000 Subject: [PATCH 331/395] Recommitted as new branch with unit tests - GIT screwd me on the last PR --- freqtrade/exchange/__init__.py | 31 +++++++++++++++ freqtrade/tests/exchange/test_exchange.py | 48 ++++++++++++++++++++++- 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 972ff49ca..0f89eb660 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -4,6 +4,7 @@ import logging from random import randint from typing import List, Dict, Any, Optional from datetime import datetime +from math import floor, ceil import ccxt import arrow @@ -150,6 +151,28 @@ class Exchange(object): """ return endpoint in self._api.has and self._api.has[endpoint] + def symbol_amount_prec(self, pair, amount: float): + ''' + Returns the amount to buy or sell to a precision the Exchange accepts + Rounded down + ''' + if self._api.markets[pair]['precision']['amount']: + symbol_prec = self._api.markets[pair]['precision']['amount'] + big_amount = amount * pow(10, symbol_prec) + amount = floor(big_amount) / pow(10, symbol_prec) + return amount + + def symbol_price_prec(self, pair, price: float): + ''' + Returns the price buying or selling with to the precision the Exchange accepts + Rounds up + ''' + if self._api.markets[pair]['precision']['price']: + symbol_prec = self._api.markets[pair]['precision']['price'] + big_price = price * pow(10, symbol_prec) + price = ceil(big_price) / pow(10, symbol_prec) + return price + def buy(self, pair: str, rate: float, amount: float) -> Dict: if self._conf['dry_run']: order_id = f'dry_run_buy_{randint(0, 10**6)}' @@ -167,6 +190,10 @@ class Exchange(object): return {'id': order_id} try: + # Set the precision for amount and price(rate) as accepted by the exchange + amount = self.symbol_amount_prec(pair, amount) + rate = self.symbol_price_prec(pair, rate) + return self._api.create_limit_buy_order(pair, amount, rate) except ccxt.InsufficientFunds as e: raise DependencyException( @@ -200,6 +227,10 @@ class Exchange(object): return {'id': order_id} try: + # Set the precision for amount and price(rate) as accepted by the exchange + amount = self.symbol_amount_prec(pair, amount) + rate = self.symbol_price_prec(pair, rate) + return self._api.create_limit_sell_order(pair, amount, rate) except ccxt.InsufficientFunds as e: raise DependencyException( diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 814f56acc..7bfbb68ce 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -52,6 +52,52 @@ def test_init_exception(default_conf, mocker): Exchange(default_conf) +def test_symbol_amount_prec(default_conf, mocker): + ''' + Test rounds down to 4 Decimal places + ''' + api_mock = MagicMock() + api_mock.load_markets = MagicMock(return_value={ + 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' + }) + mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='binance')) + + markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'amount': 4}}}) + type(api_mock).markets = markets + + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + exchange = Exchange(default_conf) + + amount = 2.34559 + pair = 'ETH/BTC' + amount = exchange.symbol_amount_prec(pair, amount) + assert amount == 2.3455 + + +def test_symbol_price_prec(default_conf, mocker): + ''' + Test rounds up to 4 decimal places + ''' + api_mock = MagicMock() + api_mock.load_markets = MagicMock(return_value={ + 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' + }) + mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='binance')) + + markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'price': 4}}}) + type(api_mock).markets = markets + + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + exchange = Exchange(default_conf) + + price = 2.34559 + pair = 'ETH/BTC' + price = exchange.symbol_price_prec(pair, price) + assert price == 2.3456 + + def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() api_mock.load_markets = MagicMock(return_value={ @@ -173,7 +219,7 @@ def test_validate_timeframes_not_in_config(default_conf, mocker): Exchange(default_conf) -def test_exchange_has(default_conf, mocker): +def test_exchangehas(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) assert not exchange.exchange_has('ASDFASDF') api_mock = MagicMock() From fe27ca63b4ba7a9f1695d95aba77f2afcdc4e131 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Mon, 30 Jul 2018 17:08:33 +0000 Subject: [PATCH 332/395] Update test_exchange.py --- freqtrade/tests/exchange/test_exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 7bfbb68ce..4acd5c6b2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -219,7 +219,7 @@ def test_validate_timeframes_not_in_config(default_conf, mocker): Exchange(default_conf) -def test_exchangehas(default_conf, mocker): +def test_exchange_has(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) assert not exchange.exchange_has('ASDFASDF') api_mock = MagicMock() From b83487cc36fb2d7c5440fbe3a89dabdad794e881 Mon Sep 17 00:00:00 2001 From: Gert Date: Mon, 30 Jul 2018 13:00:08 -0700 Subject: [PATCH 333/395] added required changes --- freqtrade/strategy/resolver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 6ae779669..ea4f5c5e3 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -6,16 +6,16 @@ This module load custom strategies import importlib.util import inspect import logging +import os +import tempfile from base64 import urlsafe_b64decode from collections import OrderedDict +from pathlib import Path from typing import Dict, Optional, Type from freqtrade import constants from freqtrade.strategy import import_strategy from freqtrade.strategy.interface import IStrategy -import tempfile -import os -from pathlib import Path logger = logging.getLogger(__name__) @@ -83,7 +83,7 @@ class StrategyResolver(object): abs_paths.insert(0, extra_dir) if ":" in strategy_name: - logger.debug("loading base64 endocded strategy") + logger.info("loading base64 endocded strategy") strat = strategy_name.split(":") if len(strat) == 2: From be1298dbd219bca61b13457a00b1d28ffcb195fa Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 31 Jul 2018 14:19:16 +0200 Subject: [PATCH 334/395] Initializing CCXT with rate_limit parameter optional (default to false) --- config.json.example | 1 + freqtrade/exchange/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index e8473e919..af496685c 100644 --- a/config.json.example +++ b/config.json.example @@ -17,6 +17,7 @@ "name": "bittrex", "key": "your_exchange_key", "secret": "your_exchange_secret", + "ccxt_rate_limit": false, "pair_whitelist": [ "ETH/BTC", "LTC/BTC", diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 423e38246..f011161be 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -91,7 +91,7 @@ class Exchange(object): 'secret': exchange_config.get('secret'), 'password': exchange_config.get('password'), 'uid': exchange_config.get('uid', ''), - 'enableRateLimit': True, + 'enableRateLimit': exchange_config.get('ccxt_rate_limit', False), }) except (KeyError, AttributeError): raise OperationalException(f'Exchange {name} is not supported') From ab4343b7c0d461ea9b9cdc970e4f3008eed69a3a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 31 Jul 2018 14:25:06 +0200 Subject: [PATCH 335/395] Update ccxt from 1.17.49 to 1.17.56 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 964da51e3..bcef543b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.49 +ccxt==1.17.56 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 72480188b7b0f7fea3db50a6ac87253f6bc40b41 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 31 Jul 2018 14:25:07 +0200 Subject: [PATCH 336/395] Update pytest from 3.6.4 to 3.7.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bcef543b1..77501b7a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.0 TA-Lib==0.4.17 -pytest==3.6.4 +pytest==3.7.0 pytest-mock==1.10.0 pytest-cov==2.5.1 tabulate==0.8.2 From 74fa4ddca4498d3a2d487b880b5a982f4e8c2278 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 31 Jul 2018 16:54:02 +0200 Subject: [PATCH 337/395] CCXT rate limit config default to => true + adding config to config_full.json.example --- config.json.example | 2 +- config_full.json.example | 1 + freqtrade/exchange/__init__.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index af496685c..8bd3942e6 100644 --- a/config.json.example +++ b/config.json.example @@ -17,7 +17,7 @@ "name": "bittrex", "key": "your_exchange_key", "secret": "your_exchange_secret", - "ccxt_rate_limit": false, + "ccxt_rate_limit": true, "pair_whitelist": [ "ETH/BTC", "LTC/BTC", diff --git a/config_full.json.example b/config_full.json.example index b0714535f..cc3b3d630 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -26,6 +26,7 @@ "name": "bittrex", "key": "your_exchange_key", "secret": "your_exchange_secret", + "ccxt_rate_limit": true, "pair_whitelist": [ "ETH/BTC", "LTC/BTC", diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index f011161be..810957902 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -91,7 +91,7 @@ class Exchange(object): 'secret': exchange_config.get('secret'), 'password': exchange_config.get('password'), 'uid': exchange_config.get('uid', ''), - 'enableRateLimit': exchange_config.get('ccxt_rate_limit', False), + 'enableRateLimit': exchange_config.get('ccxt_rate_limit', True), }) except (KeyError, AttributeError): raise OperationalException(f'Exchange {name} is not supported') From e7d043974199ab041ccfd44111c73d330be62f0e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Jul 2018 23:00:50 +0200 Subject: [PATCH 338/395] Add new arguments --- freqtrade/arguments.py | 8 ++++++++ freqtrade/configuration.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 022a2c739..042eeedf1 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -142,6 +142,14 @@ class Arguments(object): action='store_true', dest='refresh_pairs', ) + parser.add_argument( + '--strategy-list', + help='Provide a commaseparated list of strategies to backtest ' + 'Please note that ticker-interval needs to be set either in config ' + 'or via command line', + nargs='+', + dest='strategy_list', + ) parser.add_argument( '--export', help='export backtest results, argument are: trades\ diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index dcc6e4332..aa452c79d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -187,6 +187,14 @@ class Configuration(object): config.update({'refresh_pairs': True}) logger.info('Parameter -r/--refresh-pairs-cached detected ...') + if 'strategy_list' in self.args and self.args.strategy_list: + config.update({'strategy_list': self.args.strategy_list}) + logger.info('using strategy list of %s Strategies', len(self.args.strategy_list)) + + if 'ticker_interval' in self.args and self.args.ticker_interval: + config.update({'ticker_interval': self.args.ticker_interval}) + logger.info('Overriding ticker interval with Command line argument') + # If --export is used we add it to the configuration if 'export' in self.args and self.args.export: config.update({'export': self.args.export}) From 56046b3cb39c16ba8a43e43cca88de2d5ecfa51c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Jul 2018 23:01:52 +0200 Subject: [PATCH 339/395] Add strategylist option to backtesting --- freqtrade/optimize/backtesting.py | 126 +++++++++++++++++------------- 1 file changed, 71 insertions(+), 55 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 593af619c..4146c25dd 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,6 +6,7 @@ This module contains the backtesting logic import logging import operator from argparse import Namespace +from copy import deepcopy from datetime import datetime, timedelta from typing import Any, Dict, List, NamedTuple, Optional, Tuple @@ -54,11 +55,6 @@ class Backtesting(object): """ def __init__(self, config: Dict[str, Any]) -> None: self.config = config - self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.ticker_interval = self.strategy.ticker_interval - self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.advise_buy = self.strategy.advise_buy - self.advise_sell = self.strategy.advise_sell # Reset keys for backtesting self.config['exchange']['key'] = '' @@ -279,6 +275,19 @@ class Backtesting(object): pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + strategylist: List[IStrategy] = [] + if self.config.get('strategy_list', None): + # Force one interval + self.ticker_interval = self.config.get('ticker_interval') + for strat in self.config.get('strategy_list'): + stratconf = deepcopy(self.config) + stratconf['strategy'] = strat + s = StrategyResolver(stratconf).strategy + strategylist.append(s) + + else: + # only one strategy + strategylist.append(StrategyResolver(self.config).strategy) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') @@ -308,61 +317,68 @@ class Backtesting(object): logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 - preprocessed = self.tickerdata_to_dataframe(data) + for strat in strategylist: + self.strategy = strat + self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe + self.populate_buy_trend = self.strategy.populate_buy_trend + self.populate_sell_trend = self.strategy.populate_sell_trend - # Print timeframe - min_date, max_date = self.get_timeframe(preprocessed) - logger.info( - 'Measuring data from %s up to %s (%s days)..', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days - ) + # need to reprocess data every time to populate signals + preprocessed = self.tickerdata_to_dataframe(data) - # Execute backtest and print results - results = self.backtest( - { - 'stake_amount': self.config.get('stake_amount'), - 'processed': preprocessed, - 'max_open_trades': max_open_trades, - 'position_stacking': self.config.get('position_stacking', False), - } - ) - - if self.config.get('export', False): - self._store_backtest_result(self.config.get('exportfilename'), results) - - logger.info( - '\n' + '=' * 49 + - ' BACKTESTING REPORT ' + - '=' * 50 + '\n' - '%s', - self._generate_text_table( - data, - results + # Print timeframe + min_date, max_date = self.get_timeframe(preprocessed) + logger.info( + 'Measuring data from %s up to %s (%s days)..', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days ) - ) - # logger.info( - # results[['sell_reason']].groupby('sell_reason').count() - # ) - logger.info( - '\n' + - ' SELL READON STATS '.center(119, '=') + - '\n%s \n', - self._generate_text_table_sell_reason(data, results) - - ) - - logger.info( - '\n' + - ' LEFT OPEN TRADES REPORT '.center(119, '=') + - '\n%s', - self._generate_text_table( - data, - results.loc[results.open_at_end] + # Execute backtest and print results + results = self.backtest( + { + 'stake_amount': self.config.get('stake_amount'), + 'processed': preprocessed, + 'max_open_trades': max_open_trades, + 'position_stacking': self.config.get('position_stacking', False), + } + ) + + if self.config.get('export', False): + self._store_backtest_result(self.config.get('exportfilename'), results) + + logger.info( + '\n' + '=' * 49 + + ' BACKTESTING REPORT ' + + '=' * 50 + '\n' + '%s', + self._generate_text_table( + data, + results + ) + ) + # logger.info( + # results[['sell_reason']].groupby('sell_reason').count() + # ) + + logger.info( + '\n' + + ' SELL READON STATS '.center(119, '=') + + '\n%s \n', + self._generate_text_table_sell_reason(data, results) + + ) + + logger.info( + '\n' + + ' LEFT OPEN TRADES REPORT '.center(119, '=') + + '\n%s', + self._generate_text_table( + data, + results.loc[results.open_at_end] + ) ) - ) def setup_configuration(args: Namespace) -> Dict[str, Any]: From 9a42aac0f24e82694e783d146c6069bb24663abe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 06:40:39 +0200 Subject: [PATCH 340/395] Add testcase for --strategylist --- freqtrade/configuration.py | 2 +- freqtrade/tests/test_arguments.py | 8 +++- freqtrade/tests/test_configuration.py | 55 +++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index aa452c79d..3da432b1d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -189,7 +189,7 @@ class Configuration(object): if 'strategy_list' in self.args and self.args.strategy_list: config.update({'strategy_list': self.args.strategy_list}) - logger.info('using strategy list of %s Strategies', len(self.args.strategy_list)) + logger.info('Using strategy list of %s Strategies', len(self.args.strategy_list)) if 'ticker_interval' in self.args and self.args.ticker_interval: config.update({'ticker_interval': self.args.ticker_interval}) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 79bd0254b..e09aeb1df 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -132,7 +132,11 @@ def test_parse_args_backtesting_custom() -> None: 'backtesting', '--live', '--ticker-interval', '1m', - '--refresh-pairs-cached'] + '--refresh-pairs-cached', + '--strategy-list', + 'DefaultStrategy', + 'TestStrategy' + ] call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.live is True @@ -141,6 +145,8 @@ def test_parse_args_backtesting_custom() -> None: assert call_args.func is not None assert call_args.ticker_interval == '1m' assert call_args.refresh_pairs is True + assert type(call_args.strategy_list) is list + assert len(call_args.strategy_list) == 2 def test_parse_args_hyperopt_custom() -> None: diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index e48553bdf..bf41aab83 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -292,6 +292,61 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non ) +def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> None: + """ + Test setup_configuration() function + """ + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + arglist = [ + '--config', 'config.json', + 'backtesting', + '--ticker-interval', '1m', + '--export', '/bar/foo', + '--strategy-list', + 'DefaultStrategy', + 'TestStrategy' + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + assert 'max_open_trades' in config + assert 'stake_currency' in config + assert 'stake_amount' in config + assert 'exchange' in config + assert 'pair_whitelist' in config['exchange'] + assert 'datadir' in config + assert log_has( + 'Using data folder: {} ...'.format(config['datadir']), + caplog.record_tuples + ) + assert 'ticker_interval' in config + assert log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples) + assert log_has( + 'Using ticker_interval: 1m ...', + caplog.record_tuples + ) + + assert 'strategy_list' in config + assert log_has('Using strategy list of 2 Strategies', caplog.record_tuples) + + assert 'position_stacking' not in config + + assert 'use_max_market_positions' not in config + + assert 'timerange' not in config + + assert 'export' in config + assert log_has( + 'Parameter --export detected: {} ...'.format(config['export']), + caplog.record_tuples + ) + + def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) From 65aaa3dffdea2ddae22795cdc202cccb1dbca56d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 06:54:33 +0200 Subject: [PATCH 341/395] Extract backtest strategy setting --- freqtrade/optimize/backtesting.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4146c25dd..14a66f2ac 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -65,6 +65,16 @@ class Backtesting(object): self.exchange = Exchange(self.config) self.fee = self.exchange.get_fee() + def set_strategy(self, strategy): + """ + Load strategy into backtesting + """ + self.strategy = strategy + self.ticker_interval = self.config.get('ticker_interval') + self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe + self.populate_buy_trend = strategy.populate_buy_trend + self.populate_sell_trend = strategy.populate_sell_trend + @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: """ @@ -288,6 +298,7 @@ class Backtesting(object): else: # only one strategy strategylist.append(StrategyResolver(self.config).strategy) + self.set_strategy(strategylist[0]) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') @@ -316,12 +327,11 @@ class Backtesting(object): else: logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 + all_results = {} for strat in strategylist: - self.strategy = strat - self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.populate_buy_trend = self.strategy.populate_buy_trend - self.populate_sell_trend = self.strategy.populate_sell_trend + logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) + self.set_strategy(strat) # need to reprocess data every time to populate signals preprocessed = self.tickerdata_to_dataframe(data) @@ -336,7 +346,7 @@ class Backtesting(object): ) # Execute backtest and print results - results = self.backtest( + all_results[self.strategy.get_strategy_name()] = self.backtest( { 'stake_amount': self.config.get('stake_amount'), 'processed': preprocessed, @@ -345,14 +355,16 @@ class Backtesting(object): } ) + for strategy, results in all_results.items(): + if self.config.get('export', False): self._store_backtest_result(self.config.get('exportfilename'), results) + logger.info("\nResult for strategy %s", strategy) logger.info( - '\n' + '=' * 49 + - ' BACKTESTING REPORT ' + - '=' * 50 + '\n' - '%s', + '\n' + + ' BACKTESTING REPORT '.center(119, '=') + + '\n%s', self._generate_text_table( data, results From 5f2e92ec5c7329ba3fe5b53ca4b5dd65332c7a2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 07:00:58 +0200 Subject: [PATCH 342/395] Refactor backtesting --- freqtrade/optimize/backtesting.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 14a66f2ac..a9121a3d0 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -53,6 +53,7 @@ class Backtesting(object): backtesting = Backtesting(config) backtesting.start() """ + def __init__(self, config: Dict[str, Any]) -> None: self.config = config @@ -62,10 +63,14 @@ class Backtesting(object): self.config['exchange']['password'] = '' self.config['exchange']['uid'] = '' self.config['dry_run'] = True + if not self.config.get('strategy_list'): + # In Single strategy mode, load strategy here to avoid problems with hyperopt + self._set_strategy(StrategyResolver(self.config).strategy) + self.exchange = Exchange(self.config) self.fee = self.exchange.get_fee() - def set_strategy(self, strategy): + def _set_strategy(self, strategy): """ Load strategy into backtesting """ @@ -297,8 +302,7 @@ class Backtesting(object): else: # only one strategy - strategylist.append(StrategyResolver(self.config).strategy) - self.set_strategy(strategylist[0]) + strategylist.append(self.strategy) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') @@ -331,7 +335,7 @@ class Backtesting(object): for strat in strategylist: logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) - self.set_strategy(strat) + self._set_strategy(strat) # need to reprocess data every time to populate signals preprocessed = self.tickerdata_to_dataframe(data) From 644f729aeabf42cd2ac7da45d43881b6bfdebe96 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 07:41:38 +0200 Subject: [PATCH 343/395] Refactor strategy loading to __init__ --- freqtrade/optimize/backtesting.py | 34 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a9121a3d0..ffd89635a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -63,9 +63,22 @@ class Backtesting(object): self.config['exchange']['password'] = '' self.config['exchange']['uid'] = '' self.config['dry_run'] = True - if not self.config.get('strategy_list'): - # In Single strategy mode, load strategy here to avoid problems with hyperopt - self._set_strategy(StrategyResolver(self.config).strategy) + self.strategylist: List[IStrategy] = [] + if self.config.get('strategy_list', None): + # Force one interval + self.ticker_interval = self.config.get('ticker_interval') + for strat in self.config.get('strategy_list'): + stratconf = deepcopy(self.config) + stratconf['strategy'] = strat + self.strategylist.append(StrategyResolver(stratconf).strategy) + + else: + # only one strategy + strat = StrategyResolver(self.config).strategy + + self.strategylist.append(StrategyResolver(self.config).strategy) + # Load one strategy + self._set_strategy(self.strategylist[0]) self.exchange = Exchange(self.config) self.fee = self.exchange.get_fee() @@ -290,19 +303,6 @@ class Backtesting(object): pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - strategylist: List[IStrategy] = [] - if self.config.get('strategy_list', None): - # Force one interval - self.ticker_interval = self.config.get('ticker_interval') - for strat in self.config.get('strategy_list'): - stratconf = deepcopy(self.config) - stratconf['strategy'] = strat - s = StrategyResolver(stratconf).strategy - strategylist.append(s) - - else: - # only one strategy - strategylist.append(self.strategy) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') @@ -333,7 +333,7 @@ class Backtesting(object): max_open_trades = 0 all_results = {} - for strat in strategylist: + for strat in self.strategylist: logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) self._set_strategy(strat) From bd3563df6738409d7b0e92e33d879d193a81b16b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 07:55:59 +0200 Subject: [PATCH 344/395] Add test for new functionality --- freqtrade/optimize/backtesting.py | 4 +- freqtrade/tests/optimize/test_backtesting.py | 63 +++++++++++++++++--- 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ffd89635a..0bd76b2c4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -66,8 +66,8 @@ class Backtesting(object): self.strategylist: List[IStrategy] = [] if self.config.get('strategy_list', None): # Force one interval - self.ticker_interval = self.config.get('ticker_interval') - for strat in self.config.get('strategy_list'): + self.ticker_interval = str(self.config.get('ticker_interval')) + for strat in list(self.config['strategy_list']): stratconf = deepcopy(self.config) stratconf['strategy'] = strat self.strategylist.append(StrategyResolver(stratconf).strategy) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 5d121d27c..d91781ffc 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -686,15 +686,6 @@ def test_backtest_start_live(default_conf, mocker, caplog): read_data=json.dumps(default_conf) )) - args = MagicMock() - args.ticker_interval = 1 - args.level = 10 - args.live = True - args.datadir = None - args.export = None - args.strategy = 'DefaultStrategy' - args.timerange = '-100' # needed due to MagicMock malleability - args = [ '--config', 'config.json', '--strategy', 'DefaultStrategy', @@ -725,3 +716,57 @@ def test_backtest_start_live(default_conf, mocker, caplog): for line in exists: assert log_has(line, caplog.record_tuples) + + +def test_backtest_start_multi_strat(default_conf, mocker, caplog): + conf = deepcopy(default_conf) + conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', + new=lambda s, n, i: _load_pair_as_ticks(n, i)) + patch_exchange(mocker) + backtestmock = MagicMock() + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) + gen_table_mock = MagicMock() + mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', gen_table_mock) + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(conf) + )) + + args = [ + '--config', 'config.json', + '--datadir', 'freqtrade/tests/testdata', + 'backtesting', + '--ticker-interval', '1m', + '--live', + '--timerange', '-100', + '--enable-position-stacking', + '--disable-max-market-positions', + '--strategy-list', + 'DefaultStrategy', + 'TestStrategy', + ] + args = get_args(args) + start(args) + # 2 backtests, 4 tables + assert backtestmock.call_count == 2 + assert gen_table_mock.call_count == 4 + + # check the logs, that will contain the backtest result + exists = [ + 'Parameter -i/--ticker-interval detected ...', + 'Using ticker_interval: 1m ...', + 'Parameter -l/--live detected ...', + 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', + 'Parameter --timerange detected: -100 ...', + 'Using data folder: freqtrade/tests/testdata ...', + 'Using stake_currency: BTC ...', + 'Using stake_amount: 0.001 ...', + 'Downloading data for all pairs in whitelist ...', + 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..', + 'Parameter --enable-position-stacking detected ...', + 'Running backtesting for Strategy DefaultStrategy', + 'Running backtesting for Strategy TestStrategy', + ] + + for line in exists: + assert log_has(line, caplog.record_tuples) From a57a2f4a750b52dcf5871e5aec329a34c5d60f32 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 21:55:47 +0200 Subject: [PATCH 345/395] Store backtest-result in different vars --- freqtrade/arguments.py | 4 +++- freqtrade/optimize/backtesting.py | 12 ++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 042eeedf1..501c1784f 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -146,7 +146,9 @@ class Arguments(object): '--strategy-list', help='Provide a commaseparated list of strategies to backtest ' 'Please note that ticker-interval needs to be set either in config ' - 'or via command line', + 'or via command line. When using this together with --export trades, ' + 'the strategy-name is injected into the filename ' + '(so backtest-data.json becomes backtest-data-DefaultStrategy.json', nargs='+', dest='strategy_list', ) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0bd76b2c4..69d48b027 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -8,6 +8,7 @@ import operator from argparse import Namespace from copy import deepcopy from datetime import datetime, timedelta +from pathlib import Path from typing import Any, Dict, List, NamedTuple, Optional, Tuple import arrow @@ -156,7 +157,8 @@ class Backtesting(object): tabular_data.append([reason.value, count]) return tabulate(tabular_data, headers=headers, tablefmt="pipe") - def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: + def _store_backtest_result(self, recordfilename: str, results: DataFrame, + strategyname: Optional[str] = None) -> None: records = [(t.pair, t.profit_percent, t.open_time.timestamp(), t.close_time.timestamp(), t.open_index - 1, t.trade_duration, @@ -164,6 +166,11 @@ class Backtesting(object): for index, t in results.iterrows()] if records: + if strategyname: + # Inject strategyname to filename + recname = Path(recordfilename) + recordfilename = str(Path.joinpath( + recname.parent, f'{recname.stem}-{strategyname}').with_suffix(recname.suffix)) logger.info('Dumping backtest results to %s', recordfilename) file_dump_json(recordfilename, records) @@ -362,7 +369,8 @@ class Backtesting(object): for strategy, results in all_results.items(): if self.config.get('export', False): - self._store_backtest_result(self.config.get('exportfilename'), results) + self._store_backtest_result(self.config['exportfilename'], results, + strategy if len(self.strategylist) > 1 else None) logger.info("\nResult for strategy %s", strategy) logger.info( From a8b55b8989387f083250b0b8bc7dbdefd05e4d3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 22:00:12 +0200 Subject: [PATCH 346/395] Add test for strategy-name injection --- freqtrade/tests/optimize/test_backtesting.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index d91781ffc..311fe7da4 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -654,6 +654,18 @@ def test_backtest_record(default_conf, fee, mocker): records = records[0] # Ensure records are of correct type assert len(records) == 4 + + # reset test to test with strategy name + names = [] + records = [] + backtesting._store_backtest_result("backtest-result.json", results, "DefStrat") + assert len(results) == 4 + # Assert file_dump_json was only called once + assert names == ['backtest-result-DefStrat.json'] + records = records[0] + # Ensure records are of correct type + assert len(records) == 4 + # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117) # Below follows just a typecheck of the schema/type of trade-records oix = None From 4ea6780153ae4ab1ddab61c4d4ea6eb636a5dc7a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 09:51:45 +0200 Subject: [PATCH 347/395] Update documentation with --strategy-list --- docs/backtesting.md | 19 ++++++++++++++++++- docs/bot-usage.md | 33 ++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 766875970..2d53303c5 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -151,7 +151,7 @@ cp freqtrade/tests/testdata/pairs.json user_data/data/binance Then run: ```bash -python scripts/download_backtest_data --exchange binance +python scripts/download_backtest_data.py --exchange binance ``` This will download ticker data for all the currency pairs you defined in `pairs.json`. @@ -238,6 +238,23 @@ On the other hand, if you set a too high `minimal_roi` like `"0": 0.55` profit. Hence, keep in mind that your performance is a mix of your strategies, your configuration, and the crypto-currency you have set up. +## Backtesting multiple strategies + +To backtest multiple strategies, a list of Strategies can be provided. + +This is limited to 1 ticker-interval per run, however, data is only loaded once from disk so if you have multiple +strategies you'd like to compare, this should give a nice runtime boost. + +All listed Strategies need to be in the same folder. + +``` bash +freqtrade backtesting --timerange 20180401-20180410 --ticker-interval 5m --strategy-list Strategy001 Strategy002 --export trades +``` + +This will save the results to `user_data/backtest_data/backtest-result-.json`, injecting the strategy-name into the target filename. +It will also output all results one after the other, so make sure to scroll up. + + ## Next step Great, your strategy is profitable. What if the bot can give your the diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 4e479adac..83a8ee833 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -1,13 +1,15 @@ # Bot usage -This page explains the difference parameters of the bot and how to run -it. + +This page explains the difference parameters of the bot and how to run it. ## Table of Contents + - [Bot commands](#bot-commands) - [Backtesting commands](#backtesting-commands) - [Hyperopt commands](#hyperopt-commands) ## Bot commands + ``` usage: freqtrade [-h] [-v] [--version] [-c PATH] [-d PATH] [-s NAME] [--strategy-path PATH] [--dynamic-whitelist [INT]] @@ -41,6 +43,7 @@ optional arguments: ``` ### How to use a different config file? + The bot allows you to select which config file you want to use. Per default, the bot will load the file `./config.json` @@ -49,6 +52,7 @@ python3 ./freqtrade/main.py -c path/far/far/away/config.json ``` ### How to use --strategy? + This parameter will allow you to load your custom strategy class. Per default without `--strategy` or `-s` the bot will load the `DefaultStrategy` included with the bot (`freqtrade/strategy/default_strategy.py`). @@ -60,6 +64,7 @@ To load a strategy, simply pass the class name (e.g.: `CustomStrategy`) in this **Example:** In `user_data/strategies` you have a file `my_awesome_strategy.py` which has a strategy class called `AwesomeStrategy` to load it: + ```bash python3 ./freqtrade/main.py --strategy AwesomeStrategy ``` @@ -70,6 +75,7 @@ message the reason (File not found, or errors in your code). Learn more about strategy file in [optimize your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md). ### How to use --strategy-path? + This parameter allows you to add an additional strategy lookup path, which gets checked before the default locations (The passed path must be a folder!): ```bash @@ -77,21 +83,25 @@ python3 ./freqtrade/main.py --strategy AwesomeStrategy --strategy-path /some/fol ``` #### How to install a strategy? + This is very simple. Copy paste your strategy file into the folder `user_data/strategies` or use `--strategy-path`. And voila, the bot is ready to use it. ### How to use --dynamic-whitelist? + Per default `--dynamic-whitelist` will retrieve the 20 currencies based on BaseVolume. This value can be changed when you run the script. **By Default** Get the 20 currencies based on BaseVolume. + ```bash python3 ./freqtrade/main.py --dynamic-whitelist ``` **Customize the number of currencies to retrieve** Get the 30 currencies based on BaseVolume. + ```bash python3 ./freqtrade/main.py --dynamic-whitelist 30 ``` @@ -102,6 +112,7 @@ negative value (e.g -2), `--dynamic-whitelist` will use the default value (20). ### How to use --db-url? + When you run the bot in Dry-run mode, per default no transactions are stored in a database. If you want to store your bot actions in a DB using `--db-url`. This can also be used to specify a custom database @@ -111,14 +122,14 @@ in production mode. Example command: python3 ./freqtrade/main.py -c config.json --db-url sqlite:///tradesv3.dry_run.sqlite ``` - ## Backtesting commands Backtesting also uses the config specified via `-c/--config`. ``` -usage: main.py backtesting [-h] [-i TICKER_INTERVAL] [--eps] [--dmmp] +usage: freqtrade backtesting [-h] [-i TICKER_INTERVAL] [--eps] [--dmmp] [--timerange TIMERANGE] [-l] [-r] + [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] [--export EXPORT] [--export-filename PATH] optional arguments: @@ -139,6 +150,13 @@ optional arguments: refresh the pairs files in tests/testdata with the latest data from the exchange. Use it if you want to run your backtesting with up-to-date data. + --strategy-list STRATEGY_LIST [STRATEGY_LIST ...] + Provide a commaseparated list of strategies to + backtest Please note that ticker-interval needs to be + set either in config or via command line. When using + this together with --export trades, the strategy-name + is injected into the filename (so backtest-data.json + becomes backtest-data-DefaultStrategy.json --export EXPORT export backtest results, argument are: trades Example --export=trades --export-filename PATH @@ -151,6 +169,7 @@ optional arguments: ``` ### How to use --refresh-pairs-cached parameter? + The first time your run Backtesting, it will take the pairs you have set in your config file and download data from Bittrex. @@ -162,7 +181,6 @@ to come back to the previous version.** To test your strategy with latest data, we recommend continuing using the parameter `-l` or `--live`. - ## Hyperopt commands To optimize your strategy, you can use hyperopt parameter hyperoptimization @@ -194,10 +212,11 @@ optional arguments: ``` ## A parameter missing in the configuration? + All parameters for `main.py`, `backtesting`, `hyperopt` are referenced in [misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc.py#L84) ## Next step -The optimal strategy of the bot will change with time depending of the -market trends. The next step is to + +The optimal strategy of the bot will change with time depending of the market trends. The next step is to [optimize your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md). From 5125076f5d9a809559da3f0717cf553d1fbd6c96 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 10:05:16 +0200 Subject: [PATCH 348/395] Fix typo --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 69d48b027..6e242ac1a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -388,7 +388,7 @@ class Backtesting(object): logger.info( '\n' + - ' SELL READON STATS '.center(119, '=') + + ' SELL REASON STATS '.center(119, '=') + '\n%s \n', self._generate_text_table_sell_reason(data, results) From 028589abd273f23cf708fc77430775c6661bdbfe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 13:07:11 +0200 Subject: [PATCH 349/395] Add strategy summary table --- freqtrade/optimize/backtesting.py | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6e242ac1a..6f571ae27 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -157,6 +157,30 @@ class Backtesting(object): tabular_data.append([reason.value, count]) return tabulate(tabular_data, headers=headers, tablefmt="pipe") + def _generate_text_table_strategy(self, all_results: dict) -> str: + """ + Generate summary table per strategy + """ + stake_currency = str(self.config.get('stake_currency')) + + floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f') + tabular_data = [] + headers = ['Strategy', 'buy count', 'avg profit %', 'cum profit %', + 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] + for strategy, results in all_results.items(): + tabular_data.append([ + strategy, + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_percent.sum() * 100.0, + results.profit_abs.sum(), + str(timedelta( + minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', + len(results[results.profit_abs > 0]), + len(results[results.profit_abs < 0]) + ]) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") + def _store_backtest_result(self, recordfilename: str, results: DataFrame, strategyname: Optional[str] = None) -> None: @@ -404,6 +428,15 @@ class Backtesting(object): ) ) + if len(all_results) > 1: + # Print Strategy summary table + logger.info( + '\n' + + ' Strategy Summary'.center(119, '=') + + '\n%s\n\nFor more details, please look at the detail tables above', + self._generate_text_table_strategy(all_results) + ) + def setup_configuration(args: Namespace) -> Dict[str, Any]: """ From 765d1c769c9552840d8fd6679dd3788b5e5ddd61 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 13:07:30 +0200 Subject: [PATCH 350/395] Add test for stratgy summary table --- freqtrade/tests/optimize/test_backtesting.py | 48 ++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 311fe7da4..02f16be85 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -406,6 +406,50 @@ def test_generate_text_table_sell_reason(default_conf, mocker): data={'ETH/BTC': {}}, results=results) == result_str +def test_generate_text_table_strategyn(default_conf, mocker): + """ + Test Backtesting.generate_text_table_sell_reason() method + """ + patch_exchange(mocker) + backtesting = Backtesting(default_conf) + results = {} + results['ETH/BTC'] = pd.DataFrame( + { + 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], + 'profit_percent': [0.1, 0.2, 0.3], + 'profit_abs': [0.2, 0.4, 0.5], + 'trade_duration': [10, 30, 10], + 'profit': [2, 0, 0], + 'loss': [0, 0, 1], + 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] + } + ) + results['LTC/BTC'] = pd.DataFrame( + { + 'pair': ['LTC/BTC', 'LTC/BTC', 'LTC/BTC'], + 'profit_percent': [0.4, 0.2, 0.3], + 'profit_abs': [0.4, 0.4, 0.5], + 'trade_duration': [15, 30, 15], + 'profit': [4, 1, 0], + 'loss': [0, 0, 1], + 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] + } + ) + + result_str = ( + '| Strategy | buy count | avg profit % | cum profit % ' + '| total profit BTC | avg duration | profit | loss |\n' + '|:-----------|------------:|---------------:|---------------:' + '|-------------------:|:---------------|---------:|-------:|\n' + '| ETH/BTC | 3 | 20.00 | 60.00 ' + '| 1.10000000 | 0:17:00 | 3 | 0 |\n' + '| LTC/BTC | 3 | 30.00 | 90.00 ' + '| 1.30000000 | 0:20:00 | 3 | 0 |' + ) + print(backtesting._generate_text_table_strategy(all_results=results)) + assert backtesting._generate_text_table_strategy(all_results=results) == result_str + + def test_backtesting_start(default_conf, mocker, caplog) -> None: def get_timeframe(input1, input2): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) @@ -740,6 +784,9 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) gen_table_mock = MagicMock() mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', gen_table_mock) + gen_strattable_mock = MagicMock() + mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table_strategy', + gen_strattable_mock) mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(conf) )) @@ -762,6 +809,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): # 2 backtests, 4 tables assert backtestmock.call_count == 2 assert gen_table_mock.call_count == 4 + assert gen_strattable_mock.call_count == 1 # check the logs, that will contain the backtest result exists = [ From c648e2acfcad9f3b0af714d3a7105d23d7ffe64a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 13:13:03 +0200 Subject: [PATCH 351/395] Adjust documentation to strategy table --- docs/backtesting.md | 10 +++++++++- freqtrade/optimize/backtesting.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 2d53303c5..cc8ecd6c7 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -252,8 +252,16 @@ freqtrade backtesting --timerange 20180401-20180410 --ticker-interval 5m --strat ``` This will save the results to `user_data/backtest_data/backtest-result-.json`, injecting the strategy-name into the target filename. -It will also output all results one after the other, so make sure to scroll up. +There will be an additional table comparing win/losses of the different strategies (identical to the "Total" row in the first table). +Detailed output for all strategies one after the other will be available, so make sure to scroll up. +``` +=================================================== Strategy Summary ==================================================== +| Strategy | buy count | avg profit % | cum profit % | total profit ETH | avg duration | profit | loss | +|:-----------|------------:|---------------:|---------------:|-------------------:|:----------------|---------:|-------:| +| Strategy1 | 19 | -0.76 | -14.39 | -0.01440287 | 15:48:00 | 15 | 4 | +| Strategy2 | 6 | -2.73 | -16.40 | -0.01641299 | 1 day, 14:12:00 | 3 | 3 | +``` ## Next step diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6f571ae27..067e7bdca 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -432,7 +432,7 @@ class Backtesting(object): # Print Strategy summary table logger.info( '\n' + - ' Strategy Summary'.center(119, '=') + + ' Strategy Summary '.center(119, '=') + '\n%s\n\nFor more details, please look at the detail tables above', self._generate_text_table_strategy(all_results) ) From 76fbb89a03b84bcb35cfcb585af40d8fcc3e4674 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 19:41:39 +0200 Subject: [PATCH 352/395] use print for backtest results to avoid odd newline-handling --- freqtrade/optimize/backtesting.py | 47 ++++++++----------------------- 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 067e7bdca..53071efaf 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -396,46 +396,21 @@ class Backtesting(object): self._store_backtest_result(self.config['exportfilename'], results, strategy if len(self.strategylist) > 1 else None) - logger.info("\nResult for strategy %s", strategy) - logger.info( - '\n' + - ' BACKTESTING REPORT '.center(119, '=') + - '\n%s', - self._generate_text_table( - data, - results - ) - ) - # logger.info( - # results[['sell_reason']].groupby('sell_reason').count() - # ) + print(f"Result for strategy {strategy}") + print(' BACKTESTING REPORT '.center(119, '=')) + print(self._generate_text_table(data, results)) - logger.info( - '\n' + - ' SELL REASON STATS '.center(119, '=') + - '\n%s \n', - self._generate_text_table_sell_reason(data, results) - - ) - - logger.info( - '\n' + - ' LEFT OPEN TRADES REPORT '.center(119, '=') + - '\n%s', - self._generate_text_table( - data, - results.loc[results.open_at_end] - ) - ) + print(' SELL REASON STATS '.center(119, '=')) + print(self._generate_text_table_sell_reason(data, results)) + print(' LEFT OPEN TRADES REPORT '.center(119, '=')) + print(self._generate_text_table(data, results.loc[results.open_at_end])) + print() if len(all_results) > 1: # Print Strategy summary table - logger.info( - '\n' + - ' Strategy Summary '.center(119, '=') + - '\n%s\n\nFor more details, please look at the detail tables above', - self._generate_text_table_strategy(all_results) - ) + print(' Strategy Summary '.center(119, '=')) + print(self._generate_text_table_strategy(all_results)) + print('\nFor more details, please look at the detail tables above') def setup_configuration(args: Namespace) -> Dict[str, Any]: From 40ee86b3579b35cd69e20766d3e7f1b869d36d86 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 21:08:03 +0200 Subject: [PATCH 353/395] Adapt after rebase --- freqtrade/optimize/backtesting.py | 4 ++-- freqtrade/tests/optimize/test_backtesting.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 53071efaf..3fd96221b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -91,8 +91,8 @@ class Backtesting(object): self.strategy = strategy self.ticker_interval = self.config.get('ticker_interval') self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe - self.populate_buy_trend = strategy.populate_buy_trend - self.populate_sell_trend = strategy.populate_sell_trend + self.advise_buy = strategy.advise_buy + self.advise_sell = strategy.advise_sell @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 02f16be85..0099a3e32 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -775,8 +775,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): def test_backtest_start_multi_strat(default_conf, mocker, caplog): - conf = deepcopy(default_conf) - conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] + default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', new=lambda s, n, i: _load_pair_as_ticks(n, i)) patch_exchange(mocker) @@ -788,7 +787,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table_strategy', gen_strattable_mock) mocker.patch('freqtrade.configuration.open', mocker.mock_open( - read_data=json.dumps(conf) + read_data=json.dumps(default_conf) )) args = [ From 5b8ee214f9e4287aa36366af0eff48c4da6c0782 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Aug 2018 07:28:12 +0200 Subject: [PATCH 354/395] Adapt to pair_to_strat methology --- freqtrade/tests/strategy/test_strategy.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 8135995a7..0cbd9f22c 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -67,9 +67,8 @@ def test_load_strategy(result): def test_load_strategy_byte64(result): with open("freqtrade/tests/strategy/test_strategy.py", "r") as file: encoded_string = urlsafe_b64encode(file.read().encode("utf-8")).decode("utf-8") - resolver = StrategyResolver({'strategy': 'TestStrategy:{}'.format(encoded_string)}) - assert hasattr(resolver.strategy, 'populate_indicators') - assert 'adx' in resolver.strategy.populate_indicators(result) + resolver = StrategyResolver({'strategy': 'TestStrategy:{}'.format(encoded_string)}) + assert 'adx' in resolver.strategy.advise_indicators(result, 'ETH/BTC') def test_load_strategy_invalid_directory(result, caplog): From 36f91fcdf564ad700534e06e46526b8b0beffb31 Mon Sep 17 00:00:00 2001 From: creslin Date: Wed, 1 Aug 2018 06:03:34 +0000 Subject: [PATCH 355/395] XBT missing as a market symbol for BTC in constants --- freqtrade/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 87e354455..b30add71b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -36,7 +36,7 @@ SUPPORTED_FIAT = [ "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD", - "BTC", "ETH", "XRP", "LTC", "BCH", "USDT" + "BTC", "XBT", "ETH", "XRP", "LTC", "BCH", "USDT" ] # Required json-schema for user specified config @@ -45,7 +45,7 @@ CONF_SCHEMA = { 'properties': { 'max_open_trades': {'type': 'integer', 'minimum': 0}, 'ticker_interval': {'type': 'string', 'enum': list(TICKER_INTERVAL_MINUTES.keys())}, - 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT', 'EUR', 'USD']}, + 'stake_currency': {'type': 'string', 'enum': ['BTC', 'XBT', 'ETH', 'USDT', 'EUR', 'USD']}, 'stake_amount': { "type": ["number", "string"], "minimum": 0.0005, From f7f75b4b04b772033f3c1bb42e4c79d8101a0bae Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 1 Aug 2018 14:26:05 +0200 Subject: [PATCH 356/395] Update ccxt from 1.17.56 to 1.17.60 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 77501b7a9..5ff5d3694 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.56 +ccxt==1.17.60 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From f619cd1d2aae971098d47f03480c396e027e631a Mon Sep 17 00:00:00 2001 From: creslin Date: Thu, 2 Aug 2018 08:45:28 +0000 Subject: [PATCH 357/395] renamed/refactored get_ticker_history to get_candle_history as it does not fetch any ticker data only candles and is causing confusion when developer are talking about candles /tickers incorreclty. OHLCV < candles and Tickers are two seperate datafeeds from the exchange --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/tests/exchange/test_exchange.py | 16 ++++++++-------- freqtrade/tests/test_freqtradebot.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 46fbb3a38..706435017 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -330,7 +330,7 @@ class FreqtradeBot(object): # Pick pair based on buy signals for _pair in whitelist: - thistory = self.exchange.get_ticker_history(_pair, interval) + thistory = self.exchange.get_candle_history(_pair, interval) (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) if buy and not sell: @@ -497,7 +497,7 @@ class FreqtradeBot(object): (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - ticker = self.exchange.get_ticker_history(trade.pair, self.strategy.ticker_interval) + ticker = self.exchange.get_candle_history(trade.pair, self.strategy.ticker_interval) (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, ticker) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index d327b97c7..6918e9da1 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -524,7 +524,7 @@ def make_fetch_ohlcv_mock(data): return fetch_ohlcv_mock -def test_get_ticker_history(default_conf, mocker): +def test_get_candle_history(default_conf, mocker): api_mock = MagicMock() tick = [ [ @@ -541,7 +541,7 @@ def test_get_ticker_history(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) # retrieve original ticker - ticks = exchange.get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert ticks[0][0] == 1511686200000 assert ticks[0][1] == 1 assert ticks[0][2] == 2 @@ -563,7 +563,7 @@ def test_get_ticker_history(default_conf, mocker): api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(new_tick)) exchange = get_patched_exchange(mocker, default_conf, api_mock) - ticks = exchange.get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert ticks[0][0] == 1511686210000 assert ticks[0][1] == 6 assert ticks[0][2] == 7 @@ -572,16 +572,16 @@ def test_get_ticker_history(default_conf, mocker): assert ticks[0][5] == 10 ccxt_exceptionhandlers(mocker, default_conf, api_mock, - "get_ticker_history", "fetch_ohlcv", + "get_candle_history", "fetch_ohlcv", pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_ticker_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + exchange.get_candle_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) -def test_get_ticker_history_sort(default_conf, mocker): +def test_get_candle_history_sort(default_conf, mocker): api_mock = MagicMock() # GDAX use-case (real data from GDAX) @@ -604,7 +604,7 @@ def test_get_ticker_history_sort(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) # Test the ticker history sort - ticks = exchange.get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert ticks[0][0] == 1527830400000 assert ticks[0][1] == 0.07649 assert ticks[0][2] == 0.07651 @@ -637,7 +637,7 @@ def test_get_ticker_history_sort(default_conf, mocker): api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) exchange = get_patched_exchange(mocker, default_conf, api_mock) # Test the ticker history sort - ticks = exchange.get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert ticks[0][0] == 1527827700000 assert ticks[0][1] == 0.07659999 assert ticks[0][2] == 0.0766 diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 69f349107..df73fff3c 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -43,7 +43,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: :return: None """ freqtrade.strategy.get_signal = lambda e, s, t: value - freqtrade.exchange.get_ticker_history = lambda p, i: None + freqtrade.exchange.get_candle_history = lambda p, i: None def patch_RPCManager(mocker) -> MagicMock: From a741f1144a43ec7116718cb5e8128b1ee41b8c77 Mon Sep 17 00:00:00 2001 From: creslin Date: Thu, 2 Aug 2018 08:58:04 +0000 Subject: [PATCH 358/395] missing __init__.py --- freqtrade/exchange/__init__.py | 2 +- freqtrade/exchange/exchange_helpers.py | 2 +- freqtrade/optimize/__init__.py | 2 +- freqtrade/optimize/backtesting.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 8 ++++---- freqtrade/tests/optimize/test_optimize.py | 16 ++++++++-------- freqtrade/tests/strategy/test_interface.py | 2 +- scripts/plot_dataframe.py | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 423e38246..0be89aaa5 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -330,7 +330,7 @@ class Exchange(object): return self._cached_ticker[pair] @retrier - def get_ticker_history(self, pair: str, tick_interval: str, + def get_candle_history(self, pair: str, tick_interval: str, since_ms: Optional[int] = None) -> List[Dict]: try: # last item should be in the time interval [now - tick_interval, now] diff --git a/freqtrade/exchange/exchange_helpers.py b/freqtrade/exchange/exchange_helpers.py index 254c16309..46f04328c 100644 --- a/freqtrade/exchange/exchange_helpers.py +++ b/freqtrade/exchange/exchange_helpers.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) def parse_ticker_dataframe(ticker: list) -> DataFrame: """ Analyses the trend for the given ticker history - :param ticker: See exchange.get_ticker_history + :param ticker: See exchange.get_candle_history :return: DataFrame """ cols = ['date', 'open', 'high', 'low', 'close', 'volume'] diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index e806ff2b8..8d5350fe5 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -219,7 +219,7 @@ def download_backtesting_testdata(datadir: str, logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') - new_data = exchange.get_ticker_history(pair=pair, tick_interval=tick_interval, + new_data = exchange.get_candle_history(pair=pair, tick_interval=tick_interval, since_ms=since_ms) data.extend(new_data) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 593af619c..fff658b6f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -283,7 +283,7 @@ class Backtesting(object): if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') for pair in pairs: - data[pair] = self.exchange.get_ticker_history(pair, self.ticker_interval) + data[pair] = self.exchange.get_candle_history(pair, self.ticker_interval) else: logger.info('Using local backtesting data (using whitelist in given config) ...') diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 5d121d27c..fc7b1f043 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -110,7 +110,7 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals return pairdata -# use for mock freqtrade.exchange.get_ticker_history' +# use for mock freqtrade.exchange.get_candle_history' def _load_pair_as_ticks(pair, tickfreq): ticks = optimize.load_data(None, ticker_interval=tickfreq, pairs=[pair]) ticks = trim_dictlist(ticks, -201) @@ -411,7 +411,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history') + mocker.patch('freqtrade.exchange.Exchange.get_candle_history') patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', @@ -446,7 +446,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history') + mocker.patch('freqtrade.exchange.Exchange.get_candle_history') patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', @@ -677,7 +677,7 @@ def test_backtest_record(default_conf, fee, mocker): def test_backtest_start_live(default_conf, mocker, caplog): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', new=lambda s, n, i: _load_pair_as_ticks(n, i)) patch_exchange(mocker) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock()) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index eef79bef3..13f65fbf5 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -53,7 +53,7 @@ def _clean_test_file(file: str) -> None: def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json') _backup_file(file, copy_file=True) optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m') @@ -63,7 +63,7 @@ def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json') _backup_file(file, copy_file=True) @@ -74,7 +74,7 @@ def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') _backup_file(file, copy_file=True) optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) @@ -87,7 +87,7 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog, default_co """ Test load_data() with 1 min ticker """ - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') @@ -118,7 +118,7 @@ def test_testdata_path() -> None: def test_download_pairs(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') @@ -261,7 +261,7 @@ def test_load_cached_data_for_updating(mocker) -> None: def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata', side_effect=BaseException('File Error')) exchange = get_patched_exchange(mocker, default_conf) @@ -279,7 +279,7 @@ def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) # Download a 1 min ticker file @@ -304,7 +304,7 @@ def test_download_backtesting_testdata2(mocker, default_conf) -> None: [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=tick) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='1m') download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='3m') diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 2c056870f..ec4ab0fd4 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -88,7 +88,7 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog): def test_get_signal_handles_exceptions(mocker, default_conf): - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=MagicMock()) exchange = get_patched_exchange(mocker, default_conf) mocker.patch.object( _STRATEGY, 'analyze_ticker', diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index fbb385a3c..f2f2e0c7f 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -138,7 +138,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: tickers = {} if args.live: logger.info('Downloading pair.') - tickers[pair] = exchange.get_ticker_history(pair, tick_interval) + tickers[pair] = exchange.get_candle_history(pair, tick_interval) else: tickers = optimize.load_data( datadir=_CONF.get("datadir"), From 1f97d0d78b79b6f4ac889a5ba2c8a2004c5b1111 Mon Sep 17 00:00:00 2001 From: creslin Date: Thu, 2 Aug 2018 09:15:02 +0000 Subject: [PATCH 359/395] fix --- freqtrade/tests/test_freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index df73fff3c..89adae6ab 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -544,7 +544,7 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), - get_ticker_history=MagicMock(return_value=20), + get_candle_history=MagicMock(return_value=20), get_balance=MagicMock(return_value=20), get_fee=fee, ) From e282d57a918513209ebdd41e9359e1e782de7dea Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Thu, 2 Aug 2018 12:57:47 +0300 Subject: [PATCH 360/395] fix broken test --- freqtrade/tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index f492384aa..32a5229c0 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -776,7 +776,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): def test_backtest_start_multi_strat(default_conf, mocker, caplog): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', new=lambda s, n, i: _load_pair_as_ticks(n, i)) patch_exchange(mocker) backtestmock = MagicMock() From 7f4472ad7789b846b37f7107b99baca586f25842 Mon Sep 17 00:00:00 2001 From: creslin Date: Thu, 2 Aug 2018 10:10:44 +0000 Subject: [PATCH 361/395] As requested in issue #1111 A python script to return - all exchanges supported by CCXT - all markets on a exchange Invoked as `python get_market_pairs.py` it will list exchanges Invoked as `python get_market_pairs binance` it will list all markets on binance --- scripts/get_market_pairs.py | 93 +++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 scripts/get_market_pairs.py diff --git a/scripts/get_market_pairs.py b/scripts/get_market_pairs.py new file mode 100644 index 000000000..6ee6464d3 --- /dev/null +++ b/scripts/get_market_pairs.py @@ -0,0 +1,93 @@ +import os +import sys + +root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.append(root + '/python') + +import ccxt # noqa: E402 + + +def style(s, style): + return style + s + '\033[0m' + + +def green(s): + return style(s, '\033[92m') + + +def blue(s): + return style(s, '\033[94m') + + +def yellow(s): + return style(s, '\033[93m') + + +def red(s): + return style(s, '\033[91m') + + +def pink(s): + return style(s, '\033[95m') + + +def bold(s): + return style(s, '\033[1m') + + +def underline(s): + return style(s, '\033[4m') + + +def dump(*args): + print(' '.join([str(arg) for arg in args])) + + +def print_supported_exchanges(): + dump('Supported exchanges:', green(', '.join(ccxt.exchanges))) + + +try: + + id = sys.argv[1] # get exchange id from command line arguments + + + # check if the exchange is supported by ccxt + exchange_found = id in ccxt.exchanges + + if exchange_found: + dump('Instantiating', green(id), 'exchange') + + # instantiate the exchange by id + exchange = getattr(ccxt, id)({ + # 'proxy':'https://cors-anywhere.herokuapp.com/', + }) + + # load all markets from the exchange + markets = exchange.load_markets() + + # output a list of all market symbols + dump(green(id), 'has', len(exchange.symbols), 'symbols:', exchange.symbols) + + tuples = list(ccxt.Exchange.keysort(markets).items()) + + # debug + for (k, v) in tuples: + print(v) + + # output a table of all markets + dump(pink('{:<15} {:<15} {:<15} {:<15}'.format('id', 'symbol', 'base', 'quote'))) + + for (k, v) in tuples: + dump('{:<15} {:<15} {:<15} {:<15}'.format(v['id'], v['symbol'], v['base'], v['quote'])) + + else: + + dump('Exchange ' + red(id) + ' not found') + print_supported_exchanges() + +except Exception as e: + dump('[' + type(e).__name__ + ']', str(e)) + dump("Usage: python " + sys.argv[0], green('id')) + print_supported_exchanges() + From 0fc4a7910d01f79491d97b1a37d52cb4cd24c72e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Aug 2018 20:15:18 +0200 Subject: [PATCH 362/395] Add note to readme for binance users --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index da691230f..7b6b4996b 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ hesitate to read the source code and understand the mechanism of this bot. ## Exchange marketplaces supported - [X] [Bittrex](https://bittrex.com/) -- [X] [Binance](https://www.binance.com/) +- [X] [Binance](https://www.binance.com/) ([*Note for binance users](#a-note-on-binance)) - [ ] [113 others to tests](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ## Features @@ -152,6 +152,13 @@ The project is currently setup in two main branches: - `develop` - This branch has often new features, but might also cause breaking changes. - `master` - This branch contains the latest stable release. The bot 'should' be stable on this branch, and is generally well tested. +- `feat/*` - This are feature branches, which are beeing worked on heavily. Please don't use these unless you want to test a specific feature. + + +## A note on Binance + +For Binance, please add `"BNB/"` to your blacklist to avoid issues. +Accounts having BNB accounts use this to pay for fees - if your first trade happens to be on `BNB`, further trades will consume this position and make the initial BNB order unsellable as the expected amount is not there anymore. ## Support From 00b81e3f0df781c2e718b94b23dff3e467a7e4dd Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Aug 2018 11:45:28 +0200 Subject: [PATCH 363/395] fix readme.md spelling --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b6b4996b..02b870209 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ The project is currently setup in two main branches: - `develop` - This branch has often new features, but might also cause breaking changes. - `master` - This branch contains the latest stable release. The bot 'should' be stable on this branch, and is generally well tested. -- `feat/*` - This are feature branches, which are beeing worked on heavily. Please don't use these unless you want to test a specific feature. +- `feat/*` - These are feature branches, which are beeing worked on heavily. Please don't use these unless you want to test a specific feature. ## A note on Binance From 145008421f9fd62e964366873239b0d83635b874 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 2 Aug 2018 14:26:07 +0200 Subject: [PATCH 364/395] Update ccxt from 1.17.60 to 1.17.63 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5ff5d3694..ff6457a8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.60 +ccxt==1.17.63 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 85c73ea8507d012353bb9744b4371d978bf07af6 Mon Sep 17 00:00:00 2001 From: Gert Date: Thu, 2 Aug 2018 16:39:13 -0700 Subject: [PATCH 365/395] added index --- freqtrade/persistence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 8fb01d074..c21b902bc 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -157,8 +157,8 @@ class Trade(_DECL_BASE): id = Column(Integer, primary_key=True) exchange = Column(String, nullable=False) - pair = Column(String, nullable=False) - is_open = Column(Boolean, nullable=False, default=True) + pair = Column(String, nullable=False,index=True) + is_open = Column(Boolean, nullable=False, default=True, index=True) fee_open = Column(Float, nullable=False, default=0.0) fee_close = Column(Float, nullable=False, default=0.0) open_rate = Column(Float) From 2cfa3b7607874879584484c7c99d47c969517fb5 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Thu, 2 Aug 2018 17:08:14 -0700 Subject: [PATCH 366/395] updated dockerfile and requirements --- Dockerfile | 7 ++++++- requirements.txt | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 309763d2a..10cd14bfe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.6.6-slim-stretch # Install TA-lib -RUN apt-get update && apt-get -y install curl build-essential && apt-get clean +RUN apt-get update && apt-get -y install curl build-essential git && apt-get clean RUN curl -L http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz | \ tar xzvf - && \ cd ta-lib && \ @@ -13,6 +13,11 @@ ENV LD_LIBRARY_PATH /usr/local/lib RUN mkdir /freqtrade WORKDIR /freqtrade +# Update PIP +RUN python -m pip install --upgrade pip +RUN pip install future +RUN pip install numpy + # Install dependencies COPY requirements.txt /freqtrade/ RUN pip install numpy \ diff --git a/requirements.txt b/requirements.txt index ff6457a8c..183d79cdf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,3 +23,13 @@ scikit-optimize==0.5.2 # Required for plotting data #plotly==3.0.0 + +# Required for plotting data +plotly==3.0.0 + +# find first, C search in arrays +py_find_1st==1.1.1 + +#Load ticker files 30% faster +ujson==1.35 +git+git://github.com/berlinguyinca/technical.git@master From 3037d85529fc0504506a902a46fb27ca2ae20091 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 3 Aug 2018 14:26:06 +0200 Subject: [PATCH 367/395] Update ccxt from 1.17.63 to 1.17.66 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ff6457a8c..0c523ddec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.63 +ccxt==1.17.66 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From b963b95ee9909d2c03f5ef244c49ac6bc6d78130 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 3 Aug 2018 14:26:07 +0200 Subject: [PATCH 368/395] Update pytest from 3.7.0 to 3.7.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0c523ddec..8670b4074 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.0 TA-Lib==0.4.17 -pytest==3.7.0 +pytest==3.7.1 pytest-mock==1.10.0 pytest-cov==2.5.1 tabulate==0.8.2 From 721341e4128a422a33ef6a059db4e7e97a164a9a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 4 Aug 2018 14:26:05 +0200 Subject: [PATCH 369/395] Update ccxt from 1.17.66 to 1.17.73 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8670b4074..221bdf968 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.66 +ccxt==1.17.73 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From ea506b05c67c4da1b66e328bdf5d8b79bf33ed4a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 4 Aug 2018 20:22:16 +0200 Subject: [PATCH 370/395] Add test for failing database migration --- freqtrade/tests/test_persistence.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 26932136a..e52500071 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -404,6 +404,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): Test Database migration (starting with new pairformat) """ amount = 103.223 + # Always create all columns apart from the last! create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( id INTEGER NOT NULL, exchange VARCHAR NOT NULL, @@ -418,14 +419,21 @@ def test_migrate_new(mocker, default_conf, fee, caplog): open_date DATETIME NOT NULL, close_date DATETIME, open_order_id VARCHAR, + stop_loss FLOAT, + initial_stop_loss FLOAT, + max_rate FLOAT, + sell_reason VARCHAR, + strategy VARCHAR, PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, - open_rate, stake_amount, amount, open_date) + open_rate, stake_amount, amount, open_date, + stop_loss, initial_stop_loss, max_rate) VALUES ('binance', 'ETC/BTC', 1, {fee}, 0.00258580, {stake}, {amount}, - '2019-11-28 12:44:24.000000') + '2019-11-28 12:44:24.000000', + 0.0, 0.0, 0.0) """.format(fee=fee.return_value, stake=default_conf.get("stake_amount"), amount=amount From d73d0a5253016b65fadb97578a5eb5b1c80180c8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 4 Aug 2018 20:22:45 +0200 Subject: [PATCH 371/395] Fix database migration --- freqtrade/persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 8fb01d074..6eaa5008a 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -82,7 +82,7 @@ def check_migrate(engine) -> None: logger.info(f'trying {table_back_name}') # Check for latest column - if not has_column(cols, 'max_rate'): + if not has_column(cols, 'ticker_interval'): fee_open = get_column_def(cols, 'fee_open', 'fee') fee_close = get_column_def(cols, 'fee_close', 'fee') open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') From be9436b2a6d7e50bd61c90cc51a832e0cd13ceb8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 5 Aug 2018 14:26:07 +0200 Subject: [PATCH 372/395] Update ccxt from 1.17.73 to 1.17.78 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 221bdf968..3c2b10847 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.73 +ccxt==1.17.78 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From ba4de4137e033319ed34ce8ffca155f56b100480 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 5 Aug 2018 14:26:08 +0200 Subject: [PATCH 373/395] Update pandas from 0.23.3 to 0.23.4 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3c2b10847..edeb07527 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ cachetools==2.1.0 requests==2.19.1 urllib3==1.22 wrapt==1.10.11 -pandas==0.23.3 +pandas==0.23.4 scikit-learn==0.19.2 scipy==1.1.0 jsonschema==2.6.0 From 0b825e96aac2bf5e93606309ccc12a209cdf6582 Mon Sep 17 00:00:00 2001 From: Axel Cherubin Date: Sun, 5 Aug 2018 16:08:49 -0400 Subject: [PATCH 374/395] fix talib bug on bollinger bands and other indicators when working on small assets, rise talib prescision and add test associated --- Dockerfile | 1 + freqtrade/tests/test_talib.py | 15 +++++++++++++++ install_ta-lib.sh | 8 ++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 freqtrade/tests/test_talib.py diff --git a/Dockerfile b/Dockerfile index 309763d2a..e959b9296 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ RUN apt-get update && apt-get -y install curl build-essential && apt-get clean RUN curl -L http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz | \ tar xzvf - && \ cd ta-lib && \ + sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && \ ./configure && make && make install && \ cd .. && rm -rf ta-lib ENV LD_LIBRARY_PATH /usr/local/lib diff --git a/freqtrade/tests/test_talib.py b/freqtrade/tests/test_talib.py new file mode 100644 index 000000000..f5e51c553 --- /dev/null +++ b/freqtrade/tests/test_talib.py @@ -0,0 +1,15 @@ + + +import talib.abstract as ta +import pandas as pd + +def test_talib_bollingerbands_near_zero_values(): + inputs = pd.DataFrame([ + {'close': 0.00000010}, + {'close': 0.00000011}, + {'close': 0.00000012}, + {'close': 0.00000013}, + {'close': 0.00000014} + ]) + bollinger = ta.BBANDS(inputs, matype=0, timeperiod=2) + assert (bollinger['upperband'][3] != bollinger['middleband'][3]) \ No newline at end of file diff --git a/install_ta-lib.sh b/install_ta-lib.sh index 21e69cbba..d5d7cf03e 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -1,7 +1,11 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib && ./configure && make && sudo make install && cd .. + cd ta-lib && \ + sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && \ + ./configure && make && sudo make install && cd .. else echo "TA-lib already installed, skipping download and build." - cd ta-lib && sudo make install && cd .. + cd ta-lib && \ + sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && \ + sudo make install && cd .. fi From a5554604e0e3a9a01582d5221f13e754766d7e87 Mon Sep 17 00:00:00 2001 From: Axel Cherubin Date: Sun, 5 Aug 2018 16:59:18 -0400 Subject: [PATCH 375/395] add sed command in doc, fix travis error --- docs/installation.md | 1 + install_ta-lib.sh | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 7a7719fc0..4de05c121 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -267,6 +267,7 @@ Official webpage: https://mrjbq7.github.io/ta-lib/install.html wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar xvzf ta-lib-0.4.0-src.tar.gz cd ta-lib +sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h ./configure --prefix=/usr make make install diff --git a/install_ta-lib.sh b/install_ta-lib.sh index d5d7cf03e..1639bd3a2 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -1,11 +1,7 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib && \ - sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && \ - ./configure && make && sudo make install && cd .. + cd ta-lib && sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd .. else echo "TA-lib already installed, skipping download and build." - cd ta-lib && \ - sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && \ - sudo make install && cd .. + cd ta-lib && sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && sudo make install && cd .. fi From 848ecb91bbb537e834cc38221d2360fd4a0118a0 Mon Sep 17 00:00:00 2001 From: Axel Cherubin Date: Sun, 5 Aug 2018 17:28:53 -0400 Subject: [PATCH 376/395] remove unnecessary seb command --- install_ta-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_ta-lib.sh b/install_ta-lib.sh index 1639bd3a2..18e7b8bbb 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -3,5 +3,5 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then cd ta-lib && sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd .. else echo "TA-lib already installed, skipping download and build." - cd ta-lib && sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && sudo make install && cd .. + cd ta-lib && sudo make install && cd .. fi From 65f7b75c343693ed560a15addaac6413865fd865 Mon Sep 17 00:00:00 2001 From: Axel Cherubin Date: Sun, 5 Aug 2018 17:52:06 -0400 Subject: [PATCH 377/395] fix flake8 issue --- freqtrade/tests/test_talib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_talib.py b/freqtrade/tests/test_talib.py index f5e51c553..093c3023c 100644 --- a/freqtrade/tests/test_talib.py +++ b/freqtrade/tests/test_talib.py @@ -3,6 +3,7 @@ import talib.abstract as ta import pandas as pd + def test_talib_bollingerbands_near_zero_values(): inputs = pd.DataFrame([ {'close': 0.00000010}, @@ -12,4 +13,4 @@ def test_talib_bollingerbands_near_zero_values(): {'close': 0.00000014} ]) bollinger = ta.BBANDS(inputs, matype=0, timeperiod=2) - assert (bollinger['upperband'][3] != bollinger['middleband'][3]) \ No newline at end of file + assert (bollinger['upperband'][3] != bollinger['middleband'][3]) From bc62f626c529ed7478f7876bf52b7c6dd2fb42a3 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 6 Aug 2018 14:26:06 +0200 Subject: [PATCH 378/395] Update ccxt from 1.17.78 to 1.17.81 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index edeb07527..f3135f9bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.78 +ccxt==1.17.81 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 131d268721f7a9499961db619d337261ee7b4f62 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Aug 2018 19:15:30 +0200 Subject: [PATCH 379/395] Fix failing tests when metadata in `analyze_ticker` is actually used --- freqtrade/tests/test_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index ce144e118..dc030d630 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -14,7 +14,7 @@ def load_dataframe_pair(pairs, strategy): assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] - dataframe = strategy.analyze_ticker(dataframe, pairs[0]) + dataframe = strategy.analyze_ticker(dataframe, {'pair': pairs[0]}) return dataframe From 3d94720be98953f658f0c96d867d055a0c6d5f91 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 7 Aug 2018 14:26:07 +0200 Subject: [PATCH 380/395] Update ccxt from 1.17.81 to 1.17.84 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f3135f9bb..2db78bd2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.81 +ccxt==1.17.84 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 4d03fc213f51acbe5a23a5f7e13e94c5ad02b428 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 8 Aug 2018 14:26:07 +0200 Subject: [PATCH 381/395] Update ccxt from 1.17.84 to 1.17.86 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2db78bd2c..82c739a70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.84 +ccxt==1.17.86 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 1bcd4333fc3ffef27e978f33c3d8b47e554414f4 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 9 Aug 2018 14:26:06 +0200 Subject: [PATCH 382/395] Update ccxt from 1.17.86 to 1.17.94 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 82c739a70..91ecf71c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.86 +ccxt==1.17.94 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 5bec389e853ec6ab9c6fd48a0b2866af4e1fd069 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 11 Aug 2018 14:26:06 +0200 Subject: [PATCH 383/395] Update ccxt from 1.17.94 to 1.17.106 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 91ecf71c9..d3ff4e6d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.94 +ccxt==1.17.106 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 5f8ec82319f63630db3f58f15b0ab6d6c3c17284 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Aug 2018 09:18:30 +0200 Subject: [PATCH 384/395] Revert "updated dockerfile and requirements" This reverts commit 2cfa3b7607874879584484c7c99d47c969517fb5. --- Dockerfile | 7 +------ requirements.txt | 10 ---------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 10cd14bfe..309763d2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.6.6-slim-stretch # Install TA-lib -RUN apt-get update && apt-get -y install curl build-essential git && apt-get clean +RUN apt-get update && apt-get -y install curl build-essential && apt-get clean RUN curl -L http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz | \ tar xzvf - && \ cd ta-lib && \ @@ -13,11 +13,6 @@ ENV LD_LIBRARY_PATH /usr/local/lib RUN mkdir /freqtrade WORKDIR /freqtrade -# Update PIP -RUN python -m pip install --upgrade pip -RUN pip install future -RUN pip install numpy - # Install dependencies COPY requirements.txt /freqtrade/ RUN pip install numpy \ diff --git a/requirements.txt b/requirements.txt index 183d79cdf..ff6457a8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,13 +23,3 @@ scikit-optimize==0.5.2 # Required for plotting data #plotly==3.0.0 - -# Required for plotting data -plotly==3.0.0 - -# find first, C search in arrays -py_find_1st==1.1.1 - -#Load ticker files 30% faster -ujson==1.35 -git+git://github.com/berlinguyinca/technical.git@master From ffa47151ee50ece9b00dece77e6fe3f0e6edfabf Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Aug 2018 09:30:12 +0200 Subject: [PATCH 385/395] Flake8 fix --- freqtrade/persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index c21b902bc..a169bc042 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -157,7 +157,7 @@ class Trade(_DECL_BASE): id = Column(Integer, primary_key=True) exchange = Column(String, nullable=False) - pair = Column(String, nullable=False,index=True) + pair = Column(String, nullable=False, index=True) is_open = Column(Boolean, nullable=False, default=True, index=True) fee_open = Column(Float, nullable=False, default=0.0) fee_close = Column(Float, nullable=False, default=0.0) From 7f6f5791ea0d56bd7ff6d029e40dd0287a335cc6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Aug 2018 10:25:19 +0200 Subject: [PATCH 386/395] update plotly dependency --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d3ff4e6d7..c13cd4fa8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,4 @@ coinmarketcap==5.0.3 scikit-optimize==0.5.2 # Required for plotting data -#plotly==3.0.0 +#plotly==3.1.1 From 2e7837976da309dfcdc7d85186a5268766e21ae2 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 12 Aug 2018 14:26:06 +0200 Subject: [PATCH 387/395] Update ccxt from 1.17.106 to 1.17.113 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d3ff4e6d7..c1bd768c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.106 +ccxt==1.17.113 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From eca8682528d525885149693a1b6a1946ff08838a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 13 Aug 2018 14:26:06 +0200 Subject: [PATCH 388/395] Update ccxt from 1.17.113 to 1.17.118 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c1bd768c5..d373f8c73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.113 +ccxt==1.17.118 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 04878da66b47d522004aa45f1b6175f746651d36 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 14 Aug 2018 14:27:07 +0200 Subject: [PATCH 389/395] Update ccxt from 1.17.118 to 1.17.122 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d373f8c73..67fda790f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.118 +ccxt==1.17.122 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 2bc7a668a3282d346dae13a900ad62e195c07e07 Mon Sep 17 00:00:00 2001 From: Nullart2 Date: Wed, 15 Aug 2018 10:39:32 +0800 Subject: [PATCH 390/395] informative startup --- freqtrade/exchange/__init__.py | 4 ++-- freqtrade/freqtradebot.py | 34 ++++++++++++++++++++++++++++++++++ freqtrade/rpc/rpc.py | 2 ++ freqtrade/rpc/telegram.py | 6 ++++++ freqtrade/tests/test_talib.py | 2 +- 5 files changed, 45 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index a6ec70636..cd75a7229 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -116,8 +116,8 @@ class Exchange(object): api.urls['api'] = api.urls['test'] logger.info("Enabled Sandbox API on %s", name) else: - logger.warning(self, "No Sandbox URL in CCXT, exiting. " - "Please check your config.json") + logger.warning(self._api.name, "No Sandbox URL in CCXT, exiting. " + "Please check your config.json") raise OperationalException(f'Exchange {name} does not provide a sandbox api') def validate_pairs(self, pairs: List[str]) -> None: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 706435017..ba081d960 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -94,6 +94,8 @@ class FreqtradeBot(object): 'status': f'{state.name.lower()}' }) logger.info('Changing state to: %s', state.name) + if state == State.RUNNING: + self._startup_messages() if state == State.STOPPED: time.sleep(1) @@ -110,6 +112,38 @@ class FreqtradeBot(object): nb_assets=nb_assets) return state + def _startup_messages(self) -> None: + if self.config.get('dry_run', False): + self.rpc.send_msg({ + 'type': RPCMessageType.WARNING_NOTIFICATION, + 'status': 'Dry run is enabled. All trades are simulated.' + }) + stake_currency = self.config['stake_currency'] + stake_amount = self.config['stake_amount'] + minimal_roi = self.config['minimal_roi'] + ticker_interval = self.config['ticker_interval'] + exchange_name = self.config['exchange']['name'] + strategy_name = self.config.get('strategy', '') + self.rpc.send_msg({ + 'type': RPCMessageType.CUSTOM_NOTIFICATION, + 'status': f'*Exchange:* `{exchange_name}`\n' + f'*Stake per trade:* `{stake_amount} {stake_currency}`\n' + f'*Minimum ROI:* `{minimal_roi}`\n' + f'*Ticker Interval:* `{ticker_interval}`\n' + f'*Strategy:* `{strategy_name}`' + }) + if self.config.get('dynamic_whitelist', False): + top_pairs = 'top ' + str(self.config.get('dynamic_whitelist', False)) + specific_pairs = '' + else: + top_pairs = 'whitelisted' + specific_pairs = '\n' + ', '.join(self.config['exchange'].get('pair_whitelist', '')) + self.rpc.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, + 'status': f'Searching for {top_pairs} {stake_currency} pairs to buy and sell...\ + {specific_pairs}' + }) + def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: """ Throttles the given callable that it diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f58fbae9a..80bac0dd4 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -24,6 +24,8 @@ logger = logging.getLogger(__name__) class RPCMessageType(Enum): STATUS_NOTIFICATION = 'status' + WARNING_NOTIFICATION = 'warning' + CUSTOM_NOTIFICATION = 'custom' BUY_NOTIFICATION = 'buy' SELL_NOTIFICATION = 'sell' diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 3b5ce3f74..64708ef74 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -154,6 +154,12 @@ class Telegram(RPC): elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION: message = '*Status:* `{status}`'.format(**msg) + elif msg['type'] == RPCMessageType.WARNING_NOTIFICATION: + message = '*Warning:* `{status}`'.format(**msg) + + elif msg['type'] == RPCMessageType.CUSTOM_NOTIFICATION: + message = '{status}'.format(**msg) + else: raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) diff --git a/freqtrade/tests/test_talib.py b/freqtrade/tests/test_talib.py index 093c3023c..0fefbccb3 100644 --- a/freqtrade/tests/test_talib.py +++ b/freqtrade/tests/test_talib.py @@ -13,4 +13,4 @@ def test_talib_bollingerbands_near_zero_values(): {'close': 0.00000014} ]) bollinger = ta.BBANDS(inputs, matype=0, timeperiod=2) - assert (bollinger['upperband'][3] != bollinger['middleband'][3]) + assert (bollinger['upperband'][3] == bollinger['middleband'][3]) From 48e218d6c08fb5380b12eab07e0e7613ffdd1259 Mon Sep 17 00:00:00 2001 From: Nullart2 Date: Wed, 15 Aug 2018 11:01:59 +0800 Subject: [PATCH 391/395] test_talib fix --- freqtrade/tests/test_talib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_talib.py b/freqtrade/tests/test_talib.py index 0fefbccb3..093c3023c 100644 --- a/freqtrade/tests/test_talib.py +++ b/freqtrade/tests/test_talib.py @@ -13,4 +13,4 @@ def test_talib_bollingerbands_near_zero_values(): {'close': 0.00000014} ]) bollinger = ta.BBANDS(inputs, matype=0, timeperiod=2) - assert (bollinger['upperband'][3] == bollinger['middleband'][3]) + assert (bollinger['upperband'][3] != bollinger['middleband'][3]) From b34aa461811c341d926bf9737d2cc799f52612eb Mon Sep 17 00:00:00 2001 From: Nullart2 Date: Wed, 15 Aug 2018 12:05:56 +0800 Subject: [PATCH 392/395] additional tests --- freqtrade/freqtradebot.py | 2 +- freqtrade/tests/rpc/test_rpc_telegram.py | 32 ++++++++++++++++++++++++ freqtrade/tests/test_freqtradebot.py | 6 +++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ba081d960..2ff999fdb 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -133,7 +133,7 @@ class FreqtradeBot(object): f'*Strategy:* `{strategy_name}`' }) if self.config.get('dynamic_whitelist', False): - top_pairs = 'top ' + str(self.config.get('dynamic_whitelist', False)) + top_pairs = 'top ' + str(self.config.get('dynamic_whitelist', 20)) specific_pairs = '' else: top_pairs = 'whitelisted' diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 4b2fe4cf5..4d2b9cda2 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1095,6 +1095,38 @@ def test_send_msg_status_notification(default_conf, mocker) -> None: assert msg_mock.call_args[0][0] == '*Status:* `running`' +def test_warning_notification(default_conf, mocker) -> None: + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + telegram.send_msg({ + 'type': RPCMessageType.WARNING_NOTIFICATION, + 'status': 'message' + }) + assert msg_mock.call_args[0][0] == '*Warning:* `message`' + + +def test_custom_notification(default_conf, mocker) -> None: + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + telegram.send_msg({ + 'type': RPCMessageType.CUSTOM_NOTIFICATION, + 'status': '*Custom:* `Hello World`' + }) + assert msg_mock.call_args[0][0] == '*Custom:* `Hello World`' + + def test_send_msg_unknown_type(default_conf, mocker) -> None: msg_mock = MagicMock() mocker.patch.multiple( diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 89adae6ab..fa6bc7c2a 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1876,3 +1876,9 @@ def test_get_real_amount_open_trade(default_conf, mocker): freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) assert freqtrade.get_real_amount(trade, order) == amount + + +def test_startup_messages(default_conf, mocker): + default_conf['dynamic_whitelist'] = 20 + freqtrade = get_patched_freqtradebot(mocker, default_conf) + assert freqtrade.state is State.RUNNING From 1edbc494ee91b5303090fe6d46b92ed093b11ed4 Mon Sep 17 00:00:00 2001 From: Nullart2 Date: Wed, 15 Aug 2018 12:37:30 +0800 Subject: [PATCH 393/395] refactor --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2ff999fdb..a2090d267 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -140,8 +140,8 @@ class FreqtradeBot(object): specific_pairs = '\n' + ', '.join(self.config['exchange'].get('pair_whitelist', '')) self.rpc.send_msg({ 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'Searching for {top_pairs} {stake_currency} pairs to buy and sell...\ - {specific_pairs}' + 'status': f'Searching for {top_pairs} {stake_currency} pairs to buy and sell...' + f'{specific_pairs}' }) def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: From dd7f540e5add2f75a023d98c1f63f6aa19eb01bb Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Wed, 15 Aug 2018 08:25:04 +0300 Subject: [PATCH 394/395] Push develop as 0.17.2 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index ac00264f0..46825f548 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ FreqTrade bot """ -__version__ = '0.17.1' +__version__ = '0.17.2' class DependencyException(BaseException): From be373e7563154bedb1e8abe44cee4b18780ebd95 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 15 Aug 2018 14:27:06 +0200 Subject: [PATCH 395/395] Update ccxt from 1.17.122 to 1.17.126 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f44085505..c2b90fff7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.122 +ccxt==1.17.126 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1