From d1306a21778d9ad15928bbd4449152d3f890b213 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Aug 2018 19:15:30 +0200 Subject: [PATCH 01/10] Fix failing tests when metadata in `analyze_ticker` is actually used --- freqtrade/tests/test_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index ce144e118..dc030d630 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -14,7 +14,7 @@ def load_dataframe_pair(pairs, strategy): assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] - dataframe = strategy.analyze_ticker(dataframe, pairs[0]) + dataframe = strategy.analyze_ticker(dataframe, {'pair': pairs[0]}) return dataframe From 98730939d4a3b0607774b8668d771505c6b1c287 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 13:02:41 +0200 Subject: [PATCH 02/10] Refactor to use a plain dict * check config-setting first - avoids any call to "candle_seen" eventually --- freqtrade/strategy/interface.py | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 5e0837767..0f03cddb3 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -19,22 +19,6 @@ from freqtrade.persistence import Trade logger = logging.getLogger(__name__) -class CandleAnalyzed: - ''' - Maintains dictionary of the last candle date a pair was processed with - This allows analyze_ticker to test if analysed the candle row in dataframe prior. - To not keep testing the same candle data, which is wasteful in CPU and time - ''' - def __init__(self, last_seen={}): - self.last_seen = last_seen - - def get_last_seen(self, pair): - return self.last_seen.get(pair) - - def set_last_seen(self, pair, candle_date): - self.last_seen[pair] = candle_date - - class SignalType(Enum): """ Enum to distinguish between buy and sell signals @@ -86,9 +70,11 @@ class IStrategy(ABC): # associated ticker interval ticker_interval: str + # Dict to determine if analysis is necessary + candle_seen: Dict[str, datetime] = {} + def __init__(self, config: dict) -> None: self.config = config - self.candleSeen = CandleAnalyzed() @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -133,23 +119,23 @@ class IStrategy(ABC): # Test if seen this pair and last candle before. dataframe = parse_ticker_dataframe(ticker_history) - pair = metadata.get('pair') - last_seen = self.candleSeen.get_last_seen(pair) + pair = str(metadata.get('pair')) - if last_seen != dataframe.iloc[-1]['date'] or self.config.get('ta_on_candle') is False: + if (not self.config.get('ta_on_candle') or + self.candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. - logging.info("TA Analysis Launched") + logging.debug("TA Analysis Launched") dataframe = self.advise_indicators(dataframe, metadata) dataframe = self.advise_buy(dataframe, metadata) dataframe = self.advise_sell(dataframe, metadata) - self.candleSeen.set_last_seen(pair=pair, candle_date=dataframe.iloc[-1]['date']) + self.candle_seen[pair] = dataframe.iloc[-1]['date'] else: dataframe.loc['buy'] = 0 dataframe.loc['sell'] = 0 # Other Defs in strategy that want to be called every loop here # twitter_sell = self.watch_twitter_feed(dataframe, metadata) - logging.info("Loop Analysis Launched") + logging.debug("Loop Analysis Launched") return dataframe From 029d61b8c5a3fdcfc0d352dc04bbd1d3ad46f79c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 13:12:12 +0200 Subject: [PATCH 03/10] Add ta_on_candle descripton to support strategy --- docs/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4c43975d7..64e75c51e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -22,8 +22,8 @@ The table below will list all configuration parameters. | `stake_amount` | 0.05 | Yes | Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to 'unlimited' to allow the bot to use all avaliable balance. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes | `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. -| `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. -| `ta_on_candle` | false | No | if set to true indicators are processed each new candle. If false each bot loop, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. +| `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. +| `ta_on_candle` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). From c4e43039f20accf285d201bd6e93177d768697d1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 19:24:00 +0200 Subject: [PATCH 04/10] Allow control from strategy --- freqtrade/strategy/interface.py | 6 +++++- freqtrade/strategy/resolver.py | 9 +++++++++ user_data/strategies/test_strategy.py | 3 +++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 0f03cddb3..494f65547 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -70,6 +70,9 @@ class IStrategy(ABC): # associated ticker interval ticker_interval: str + # run "populate_indicators" only for new candle + ta_on_candle: bool = False + # Dict to determine if analysis is necessary candle_seen: Dict[str, datetime] = {} @@ -121,7 +124,8 @@ class IStrategy(ABC): pair = str(metadata.get('pair')) - if (not self.config.get('ta_on_candle') or + # always run if ta_on_candle is set to true + if (not self.ta_on_candle or self.candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. logging.debug("TA Analysis Launched") diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 7aeec300e..1a15ba63f 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -65,6 +65,15 @@ class StrategyResolver(object): else: config['ticker_interval'] = self.strategy.ticker_interval + if 'ta_on_candle' in config: + self.strategy.ta_on_candle = config['ta_on_candle'] + logger.info( + "Override ta_on_candle \'ta_on_candle\' with value in config file: %s.", + config['ta_on_candle'] + ) + else: + config['ta_on_candle'] = self.strategy.ta_on_candle + # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 80c238d92..7c3892b77 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -45,6 +45,9 @@ class TestStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' + # run "populate_indicators" only for new candle + ta_on_candle = False + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame From e36067afd35c00799965b3f0aeb14a1ebc7576f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 19:53:47 +0200 Subject: [PATCH 05/10] refactor candle_seen to private --- freqtrade/strategy/interface.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 494f65547..f8965d440 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -74,10 +74,11 @@ class IStrategy(ABC): ta_on_candle: bool = False # Dict to determine if analysis is necessary - candle_seen: Dict[str, datetime] = {} + _candle_seen: Dict[str, datetime] = {} def __init__(self, config: dict) -> None: self.config = config + self._candle_seen = {} @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -126,16 +127,16 @@ class IStrategy(ABC): # always run if ta_on_candle is set to true if (not self.ta_on_candle or - self.candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): + self._candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. logging.debug("TA Analysis Launched") dataframe = self.advise_indicators(dataframe, metadata) dataframe = self.advise_buy(dataframe, metadata) dataframe = self.advise_sell(dataframe, metadata) - self.candle_seen[pair] = dataframe.iloc[-1]['date'] + self._candle_seen[pair] = dataframe.iloc[-1]['date'] else: - dataframe.loc['buy'] = 0 - dataframe.loc['sell'] = 0 + dataframe['buy'] = 0 + dataframe['sell'] = 0 # Other Defs in strategy that want to be called every loop here # twitter_sell = self.watch_twitter_feed(dataframe, metadata) From 4ece5d6d7ac9f20c3f868dc5569e12a9e709bdcc Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:02:24 +0200 Subject: [PATCH 06/10] Add tests for ta_on_candle --- freqtrade/tests/strategy/test_interface.py | 57 ++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index ec4ab0fd4..45f650938 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -105,3 +105,60 @@ def test_tickerdata_to_dataframe(default_conf) -> None: tickerlist = {'UNITTEST/BTC': tick} data = strategy.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed + + +def test_analyze_ticker_default(ticker_history, mocker) -> None: + + ind_mock = MagicMock(side_effect=lambda x, meta: x) + buy_mock = MagicMock(side_effect=lambda x, meta: x) + sell_mock = MagicMock(side_effect=lambda x, meta: x) + mocker.patch.multiple( + 'freqtrade.strategy.interface.IStrategy', + advise_indicators=ind_mock, + advise_buy=buy_mock, + advise_sell=sell_mock, + + ) + strategy = DefaultStrategy({}) + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + assert ind_mock.call_count == 1 + assert buy_mock.call_count == 1 + assert buy_mock.call_count == 1 + + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + # No analysis happens as ta_on_candle is true + assert ind_mock.call_count == 2 + assert buy_mock.call_count == 2 + assert buy_mock.call_count == 2 + + +def test_analyze_ticker_only_once(ticker_history, mocker) -> None: + + ind_mock = MagicMock(side_effect=lambda x, meta: x) + buy_mock = MagicMock(side_effect=lambda x, meta: x) + sell_mock = MagicMock(side_effect=lambda x, meta: x) + mocker.patch.multiple( + 'freqtrade.strategy.interface.IStrategy', + advise_indicators=ind_mock, + advise_buy=buy_mock, + advise_sell=sell_mock, + + ) + strategy = DefaultStrategy({}) + strategy.ta_on_candle = True + + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + assert ind_mock.call_count == 1 + assert buy_mock.call_count == 1 + assert buy_mock.call_count == 1 + + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + # No analysis happens as ta_on_candle is true + assert ind_mock.call_count == 1 + assert buy_mock.call_count == 1 + assert buy_mock.call_count == 1 + # only skipped analyze adds buy and sell columns, otherwise it's all mocked + assert 'buy' in ret + assert 'sell' in ret + assert ret['buy'].sum() == 0 + assert ret['sell'].sum() == 0 From df960241bd7e627c1aeec356c62e4b6f7322c882 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:07:01 +0200 Subject: [PATCH 07/10] Add log-message for skipped candle and tests --- freqtrade/tests/strategy/test_interface.py | 23 ++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 45f650938..75deecda2 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -107,8 +107,8 @@ def test_tickerdata_to_dataframe(default_conf) -> None: assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed -def test_analyze_ticker_default(ticker_history, mocker) -> None: - +def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: + caplog.set_level(logging.DEBUG) ind_mock = MagicMock(side_effect=lambda x, meta: x) buy_mock = MagicMock(side_effect=lambda x, meta: x) sell_mock = MagicMock(side_effect=lambda x, meta: x) @@ -125,15 +125,23 @@ def test_analyze_ticker_default(ticker_history, mocker) -> None: assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 + assert log_has('TA Analysis Launched', caplog.record_tuples) + assert not log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) + caplog.clear() + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) # No analysis happens as ta_on_candle is true assert ind_mock.call_count == 2 assert buy_mock.call_count == 2 assert buy_mock.call_count == 2 + assert log_has('TA Analysis Launched', caplog.record_tuples) + assert not log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) -def test_analyze_ticker_only_once(ticker_history, mocker) -> None: - +def test_analyze_ticker_skip_analyze(ticker_history, mocker, caplog) -> None: + caplog.set_level(logging.DEBUG) ind_mock = MagicMock(side_effect=lambda x, meta: x) buy_mock = MagicMock(side_effect=lambda x, meta: x) sell_mock = MagicMock(side_effect=lambda x, meta: x) @@ -151,6 +159,10 @@ def test_analyze_ticker_only_once(ticker_history, mocker) -> None: assert ind_mock.call_count == 1 assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 + assert log_has('TA Analysis Launched', caplog.record_tuples) + assert not log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) + caplog.clear() ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) # No analysis happens as ta_on_candle is true @@ -162,3 +174,6 @@ def test_analyze_ticker_only_once(ticker_history, mocker) -> None: assert 'sell' in ret assert ret['buy'].sum() == 0 assert ret['sell'].sum() == 0 + assert not log_has('TA Analysis Launched', caplog.record_tuples) + assert log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) From 3b2f161573712c4823e591fe41d6e9e0ce758aea Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:12:45 +0200 Subject: [PATCH 08/10] Add test for ta_on_candle override --- freqtrade/strategy/interface.py | 1 + freqtrade/tests/strategy/test_strategy.py | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index f8965d440..3957139d2 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -135,6 +135,7 @@ class IStrategy(ABC): dataframe = self.advise_sell(dataframe, metadata) self._candle_seen[pair] = dataframe.iloc[-1]['date'] else: + logging.debug("Skippinig TA Analysis for already analyzed candle") dataframe['buy'] = 0 dataframe['sell'] = 0 diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 0cbd9f22c..d45715a69 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -130,7 +130,7 @@ def test_strategy_override_minimal_roi(caplog): assert resolver.strategy.minimal_roi[0] == 0.5 assert ('freqtrade.strategy.resolver', logging.INFO, - 'Override strategy \'minimal_roi\' with value in config file.' + "Override strategy 'minimal_roi' with value in config file." ) in caplog.record_tuples @@ -145,7 +145,7 @@ def test_strategy_override_stoploss(caplog): assert resolver.strategy.stoploss == -0.5 assert ('freqtrade.strategy.resolver', logging.INFO, - 'Override strategy \'stoploss\' with value in config file: -0.5.' + "Override strategy 'stoploss' with value in config file: -0.5." ) in caplog.record_tuples @@ -161,10 +161,25 @@ def test_strategy_override_ticker_interval(caplog): assert resolver.strategy.ticker_interval == 60 assert ('freqtrade.strategy.resolver', logging.INFO, - 'Override strategy \'ticker_interval\' with value in config file: 60.' + "Override strategy 'ticker_interval' with value in config file: 60." ) in caplog.record_tuples +def test_strategy_override_ta_on_candle(caplog): + caplog.set_level(logging.INFO) + + config = { + 'strategy': 'DefaultStrategy', + 'ta_on_candle': True + } + resolver = StrategyResolver(config) + + assert resolver.strategy.ta_on_candle == True + assert ('freqtrade.strategy.resolver', + logging.INFO, + "Override ta_on_candle 'ta_on_candle' with value in config file: True." + ) in caplog.record_tuples + def test_deprecate_populate_indicators(result): default_location = path.join(path.dirname(path.realpath(__file__))) resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', From b008649d790e74f81b114ef82c36b05da4045648 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:13:07 +0200 Subject: [PATCH 09/10] Remove unnecessary quote escaping --- freqtrade/strategy/resolver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 1a15ba63f..75fb99d69 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -44,14 +44,14 @@ class StrategyResolver(object): # Check if we need to override configuration if 'minimal_roi' in config: self.strategy.minimal_roi = config['minimal_roi'] - logger.info("Override strategy \'minimal_roi\' with value in config file.") + logger.info("Override strategy 'minimal_roi' with value in config file.") else: config['minimal_roi'] = self.strategy.minimal_roi if 'stoploss' in config: self.strategy.stoploss = config['stoploss'] logger.info( - "Override strategy \'stoploss\' with value in config file: %s.", config['stoploss'] + "Override strategy 'stoploss' with value in config file: %s.", config['stoploss'] ) else: config['stoploss'] = self.strategy.stoploss @@ -59,7 +59,7 @@ class StrategyResolver(object): if 'ticker_interval' in config: self.strategy.ticker_interval = config['ticker_interval'] logger.info( - "Override strategy \'ticker_interval\' with value in config file: %s.", + "Override strategy 'ticker_interval' with value in config file: %s.", config['ticker_interval'] ) else: @@ -68,7 +68,7 @@ class StrategyResolver(object): if 'ta_on_candle' in config: self.strategy.ta_on_candle = config['ta_on_candle'] logger.info( - "Override ta_on_candle \'ta_on_candle\' with value in config file: %s.", + "Override ta_on_candle 'ta_on_candle' with value in config file: %s.", config['ta_on_candle'] ) else: From 56768f1a617c4fca5c648ffcbe95b3e53b7d7f9c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:17:55 +0200 Subject: [PATCH 10/10] Flake8 in tests ... --- freqtrade/tests/strategy/test_interface.py | 4 ++-- freqtrade/tests/strategy/test_strategy.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 75deecda2..e96dfb024 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -120,7 +120,7 @@ def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: ) strategy = DefaultStrategy({}) - ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) assert ind_mock.call_count == 1 assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 @@ -130,7 +130,7 @@ def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: caplog.record_tuples) caplog.clear() - ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) # No analysis happens as ta_on_candle is true assert ind_mock.call_count == 2 assert buy_mock.call_count == 2 diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index d45715a69..14b1ef1bd 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -174,12 +174,13 @@ def test_strategy_override_ta_on_candle(caplog): } resolver = StrategyResolver(config) - assert resolver.strategy.ta_on_candle == True + assert resolver.strategy.ta_on_candle assert ('freqtrade.strategy.resolver', logging.INFO, "Override ta_on_candle 'ta_on_candle' with value in config file: True." ) in caplog.record_tuples + def test_deprecate_populate_indicators(result): default_location = path.join(path.dirname(path.realpath(__file__))) resolver = StrategyResolver({'strategy': 'TestStrategyLegacy',