From c38d94df2d226ede89ab71f88dac5169fcd5b628 Mon Sep 17 00:00:00 2001 From: creslin Date: Fri, 3 Aug 2018 07:33:34 +0000 Subject: [PATCH 01/19] Resubmitting - because GIT. This is the last cut that was in #1117 before i closed that PR This PR allows a user to set the flag "ta_on_candle" in their config.json This will change the behaviour of the the bot to only process indicators when there is a new candle to be processed for that pair. The test is made up of "last dataframe row date + pair" is different to last_seen OR ta_on_candle is not True --- docs/configuration.md | 1 + freqtrade/constants.py | 1 + freqtrade/strategy/interface.py | 45 ++++++++++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index ff5ce118c..6f03117ba 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -23,6 +23,7 @@ The table below will list all configuration parameters. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes | `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. +| `ta_on_candle` | false | No | if set to true indicators are processed each new candle. If false each bot loop, this will mean the same candle is processed many times creating system load and buy/sell signals. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 87e354455..189b78617 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -53,6 +53,7 @@ CONF_SCHEMA = { }, 'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT}, 'dry_run': {'type': 'boolean'}, + 'ta_on_candle': {'type': 'boolean'}, 'minimal_roi': { 'type': 'object', 'patternProperties': { diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index dfd624393..e68d484de 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -19,6 +19,24 @@ from freqtrade.persistence import Trade logger = logging.getLogger(__name__) +class CandleAnalyzed: + ''' + Maintains candle_row, the last df ['date'], set by analyze_ticker. + This allows analyze_ticker to test if analysed the candle row in dataframe prior. + To not keep testing the same candle data, which is wasteful in CPU and time + ''' + def __init__(self, candle_row=0): + self.candle_row = candle_row + + def get_candle_row(self): + return self._candle_row + + def set_candle_row(self, row): + self._candle_row = row + + candle_row = property(get_candle_row, set_candle_row) + + class SignalType(Enum): """ Enum to distinguish between buy and sell signals @@ -72,6 +90,7 @@ class IStrategy(ABC): def __init__(self, config: dict) -> None: self.config = config + self.r = CandleAnalyzed() @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -112,10 +131,30 @@ class IStrategy(ABC): add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ + + # Test if seen this pair and last candle before. dataframe = parse_ticker_dataframe(ticker_history) - dataframe = self.advise_indicators(dataframe, metadata) - dataframe = self.advise_buy(dataframe, metadata) - dataframe = self.advise_sell(dataframe, metadata) + + last_seen = metadata['pair'] + str(dataframe.iloc[-1]['date']) + last_candle_processed = self.r.get_candle_row() + + if last_candle_processed != last_seen or self.config.get('ta_on_candle') is False: + # Defs that only make change on new candle data. + logging.info("TA Analysis Launched") + dataframe = self.advise_indicators(dataframe, metadata) + dataframe = self.advise_buy(dataframe, metadata) + dataframe = self.advise_sell(dataframe, metadata) + + last_seen = metadata['pair'] + str(dataframe.iloc[-1]['date']) + self.r.set_candle_row(last_seen) + else: + dataframe.loc['buy'] = 0 + dataframe.loc['sell'] = 0 + + # Other Defs in strategy that want to be called every loop here + # twitter_sell = self.watch_twitter_feed(dataframe, metadata) + logging.info("Loop Analysis Launched") + return dataframe def get_signal(self, pair: str, interval: str, ticker_hist: List[Dict]) -> Tuple[bool, bool]: From 6b3e8dcc3396753e649e9ecc7e9035810fced686 Mon Sep 17 00:00:00 2001 From: creslin Date: Fri, 3 Aug 2018 08:33:37 +0000 Subject: [PATCH 02/19] holds a dict of each pair last seen. to correctly manage the last seen of a pair. --- freqtrade/strategy/interface.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index e68d484de..b44d6997a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -21,20 +21,20 @@ logger = logging.getLogger(__name__) class CandleAnalyzed: ''' - Maintains candle_row, the last df ['date'], set by analyze_ticker. + Maintains dictionary of the last candle date a pair was processed with This allows analyze_ticker to test if analysed the candle row in dataframe prior. To not keep testing the same candle data, which is wasteful in CPU and time ''' - def __init__(self, candle_row=0): - self.candle_row = candle_row + def __init__(self, last_seen = {}): + self.last_seen = last_seen - def get_candle_row(self): - return self._candle_row + def get_last_seen(self, pair): + return self.last_seen.get(pair) - def set_candle_row(self, row): - self._candle_row = row + def set_last_seen(self, pair, candle_date): + self.last_seen[pair] = candle_date - candle_row = property(get_candle_row, set_candle_row) + candle_row = property(get_last_seen, set_last_seen) class SignalType(Enum): @@ -90,7 +90,7 @@ class IStrategy(ABC): def __init__(self, config: dict) -> None: self.config = config - self.r = CandleAnalyzed() + self.candleSeen = CandleAnalyzed() @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -135,18 +135,16 @@ class IStrategy(ABC): # Test if seen this pair and last candle before. dataframe = parse_ticker_dataframe(ticker_history) - last_seen = metadata['pair'] + str(dataframe.iloc[-1]['date']) - last_candle_processed = self.r.get_candle_row() + pair = metadata['pair'] + last_candle_seen = self.candleSeen.get_last_seen(pair) - if last_candle_processed != last_seen or self.config.get('ta_on_candle') is False: + if last_candle_seen != dataframe.iloc[-1]['date'] or self.config.get('ta_on_candle') is False: # Defs that only make change on new candle data. logging.info("TA Analysis Launched") dataframe = self.advise_indicators(dataframe, metadata) dataframe = self.advise_buy(dataframe, metadata) dataframe = self.advise_sell(dataframe, metadata) - - last_seen = metadata['pair'] + str(dataframe.iloc[-1]['date']) - self.r.set_candle_row(last_seen) + self.candleSeen.set_last_seen(pair=pair, candle_date=dataframe.iloc[-1]['date']) else: dataframe.loc['buy'] = 0 dataframe.loc['sell'] = 0 From d2a728cebd381bb7e5c1d7de218ab5ae4f31d418 Mon Sep 17 00:00:00 2001 From: creslin Date: Fri, 3 Aug 2018 08:38:13 +0000 Subject: [PATCH 03/19] flake 8 --- freqtrade/strategy/interface.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index b44d6997a..6035f50bf 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -25,7 +25,7 @@ class CandleAnalyzed: This allows analyze_ticker to test if analysed the candle row in dataframe prior. To not keep testing the same candle data, which is wasteful in CPU and time ''' - def __init__(self, last_seen = {}): + def __init__(self, last_seen={}): self.last_seen = last_seen def get_last_seen(self, pair): @@ -136,9 +136,9 @@ class IStrategy(ABC): dataframe = parse_ticker_dataframe(ticker_history) pair = metadata['pair'] - last_candle_seen = self.candleSeen.get_last_seen(pair) + last_seen = self.candleSeen.get_last_seen(pair) - if last_candle_seen != dataframe.iloc[-1]['date'] or self.config.get('ta_on_candle') is False: + if last_seen != dataframe.iloc[-1]['date'] or self.config.get('ta_on_candle') is False: # Defs that only make change on new candle data. logging.info("TA Analysis Launched") dataframe = self.advise_indicators(dataframe, metadata) From 1fef384bbae17b18e7d4842d47581a1cef7dae4f Mon Sep 17 00:00:00 2001 From: creslin Date: Fri, 3 Aug 2018 08:40:16 +0000 Subject: [PATCH 04/19] flake 8 --- freqtrade/strategy/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 6035f50bf..a17fdb625 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -135,7 +135,7 @@ class IStrategy(ABC): # Test if seen this pair and last candle before. dataframe = parse_ticker_dataframe(ticker_history) - pair = metadata['pair'] + pair = metadata.get('pair') last_seen = self.candleSeen.get_last_seen(pair) if last_seen != dataframe.iloc[-1]['date'] or self.config.get('ta_on_candle') is False: From 71b0e15182cf4bd45f816df0d3c1196f066cc2d9 Mon Sep 17 00:00:00 2001 From: creslin Date: Fri, 3 Aug 2018 08:45:24 +0000 Subject: [PATCH 05/19] updated configuration.md --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6f03117ba..4c43975d7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -23,7 +23,7 @@ The table below will list all configuration parameters. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes | `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. -| `ta_on_candle` | false | No | if set to true indicators are processed each new candle. If false each bot loop, this will mean the same candle is processed many times creating system load and buy/sell signals. +| `ta_on_candle` | false | No | if set to true indicators are processed each new candle. If false each bot loop, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). From 10ab6c7ffa54442c2d90e0b95653adb0be776bdc Mon Sep 17 00:00:00 2001 From: creslin Date: Fri, 3 Aug 2018 09:14:16 +0000 Subject: [PATCH 06/19] Removed unneeded property code --- freqtrade/strategy/interface.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index a17fdb625..5e0837767 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -34,8 +34,6 @@ class CandleAnalyzed: def set_last_seen(self, pair, candle_date): self.last_seen[pair] = candle_date - candle_row = property(get_last_seen, set_last_seen) - class SignalType(Enum): """ From d1306a21778d9ad15928bbd4449152d3f890b213 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Aug 2018 19:15:30 +0200 Subject: [PATCH 07/19] Fix failing tests when metadata in `analyze_ticker` is actually used --- freqtrade/tests/test_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index ce144e118..dc030d630 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -14,7 +14,7 @@ def load_dataframe_pair(pairs, strategy): assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] - dataframe = strategy.analyze_ticker(dataframe, pairs[0]) + dataframe = strategy.analyze_ticker(dataframe, {'pair': pairs[0]}) return dataframe From 98730939d4a3b0607774b8668d771505c6b1c287 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 13:02:41 +0200 Subject: [PATCH 08/19] Refactor to use a plain dict * check config-setting first - avoids any call to "candle_seen" eventually --- freqtrade/strategy/interface.py | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 5e0837767..0f03cddb3 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -19,22 +19,6 @@ from freqtrade.persistence import Trade logger = logging.getLogger(__name__) -class CandleAnalyzed: - ''' - Maintains dictionary of the last candle date a pair was processed with - This allows analyze_ticker to test if analysed the candle row in dataframe prior. - To not keep testing the same candle data, which is wasteful in CPU and time - ''' - def __init__(self, last_seen={}): - self.last_seen = last_seen - - def get_last_seen(self, pair): - return self.last_seen.get(pair) - - def set_last_seen(self, pair, candle_date): - self.last_seen[pair] = candle_date - - class SignalType(Enum): """ Enum to distinguish between buy and sell signals @@ -86,9 +70,11 @@ class IStrategy(ABC): # associated ticker interval ticker_interval: str + # Dict to determine if analysis is necessary + candle_seen: Dict[str, datetime] = {} + def __init__(self, config: dict) -> None: self.config = config - self.candleSeen = CandleAnalyzed() @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -133,23 +119,23 @@ class IStrategy(ABC): # Test if seen this pair and last candle before. dataframe = parse_ticker_dataframe(ticker_history) - pair = metadata.get('pair') - last_seen = self.candleSeen.get_last_seen(pair) + pair = str(metadata.get('pair')) - if last_seen != dataframe.iloc[-1]['date'] or self.config.get('ta_on_candle') is False: + if (not self.config.get('ta_on_candle') or + self.candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. - logging.info("TA Analysis Launched") + logging.debug("TA Analysis Launched") dataframe = self.advise_indicators(dataframe, metadata) dataframe = self.advise_buy(dataframe, metadata) dataframe = self.advise_sell(dataframe, metadata) - self.candleSeen.set_last_seen(pair=pair, candle_date=dataframe.iloc[-1]['date']) + self.candle_seen[pair] = dataframe.iloc[-1]['date'] else: dataframe.loc['buy'] = 0 dataframe.loc['sell'] = 0 # Other Defs in strategy that want to be called every loop here # twitter_sell = self.watch_twitter_feed(dataframe, metadata) - logging.info("Loop Analysis Launched") + logging.debug("Loop Analysis Launched") return dataframe From 029d61b8c5a3fdcfc0d352dc04bbd1d3ad46f79c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 13:12:12 +0200 Subject: [PATCH 09/19] Add ta_on_candle descripton to support strategy --- docs/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4c43975d7..64e75c51e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -22,8 +22,8 @@ The table below will list all configuration parameters. | `stake_amount` | 0.05 | Yes | Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to 'unlimited' to allow the bot to use all avaliable balance. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes | `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. -| `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. -| `ta_on_candle` | false | No | if set to true indicators are processed each new candle. If false each bot loop, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. +| `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. +| `ta_on_candle` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). From c4e43039f20accf285d201bd6e93177d768697d1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 19:24:00 +0200 Subject: [PATCH 10/19] Allow control from strategy --- freqtrade/strategy/interface.py | 6 +++++- freqtrade/strategy/resolver.py | 9 +++++++++ user_data/strategies/test_strategy.py | 3 +++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 0f03cddb3..494f65547 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -70,6 +70,9 @@ class IStrategy(ABC): # associated ticker interval ticker_interval: str + # run "populate_indicators" only for new candle + ta_on_candle: bool = False + # Dict to determine if analysis is necessary candle_seen: Dict[str, datetime] = {} @@ -121,7 +124,8 @@ class IStrategy(ABC): pair = str(metadata.get('pair')) - if (not self.config.get('ta_on_candle') or + # always run if ta_on_candle is set to true + if (not self.ta_on_candle or self.candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. logging.debug("TA Analysis Launched") diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 7aeec300e..1a15ba63f 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -65,6 +65,15 @@ class StrategyResolver(object): else: config['ticker_interval'] = self.strategy.ticker_interval + if 'ta_on_candle' in config: + self.strategy.ta_on_candle = config['ta_on_candle'] + logger.info( + "Override ta_on_candle \'ta_on_candle\' with value in config file: %s.", + config['ta_on_candle'] + ) + else: + config['ta_on_candle'] = self.strategy.ta_on_candle + # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 80c238d92..7c3892b77 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -45,6 +45,9 @@ class TestStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' + # run "populate_indicators" only for new candle + ta_on_candle = False + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame From e36067afd35c00799965b3f0aeb14a1ebc7576f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 19:53:47 +0200 Subject: [PATCH 11/19] refactor candle_seen to private --- freqtrade/strategy/interface.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 494f65547..f8965d440 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -74,10 +74,11 @@ class IStrategy(ABC): ta_on_candle: bool = False # Dict to determine if analysis is necessary - candle_seen: Dict[str, datetime] = {} + _candle_seen: Dict[str, datetime] = {} def __init__(self, config: dict) -> None: self.config = config + self._candle_seen = {} @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -126,16 +127,16 @@ class IStrategy(ABC): # always run if ta_on_candle is set to true if (not self.ta_on_candle or - self.candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): + self._candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. logging.debug("TA Analysis Launched") dataframe = self.advise_indicators(dataframe, metadata) dataframe = self.advise_buy(dataframe, metadata) dataframe = self.advise_sell(dataframe, metadata) - self.candle_seen[pair] = dataframe.iloc[-1]['date'] + self._candle_seen[pair] = dataframe.iloc[-1]['date'] else: - dataframe.loc['buy'] = 0 - dataframe.loc['sell'] = 0 + dataframe['buy'] = 0 + dataframe['sell'] = 0 # Other Defs in strategy that want to be called every loop here # twitter_sell = self.watch_twitter_feed(dataframe, metadata) From 4ece5d6d7ac9f20c3f868dc5569e12a9e709bdcc Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:02:24 +0200 Subject: [PATCH 12/19] Add tests for ta_on_candle --- freqtrade/tests/strategy/test_interface.py | 57 ++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index ec4ab0fd4..45f650938 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -105,3 +105,60 @@ def test_tickerdata_to_dataframe(default_conf) -> None: tickerlist = {'UNITTEST/BTC': tick} data = strategy.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed + + +def test_analyze_ticker_default(ticker_history, mocker) -> None: + + ind_mock = MagicMock(side_effect=lambda x, meta: x) + buy_mock = MagicMock(side_effect=lambda x, meta: x) + sell_mock = MagicMock(side_effect=lambda x, meta: x) + mocker.patch.multiple( + 'freqtrade.strategy.interface.IStrategy', + advise_indicators=ind_mock, + advise_buy=buy_mock, + advise_sell=sell_mock, + + ) + strategy = DefaultStrategy({}) + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + assert ind_mock.call_count == 1 + assert buy_mock.call_count == 1 + assert buy_mock.call_count == 1 + + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + # No analysis happens as ta_on_candle is true + assert ind_mock.call_count == 2 + assert buy_mock.call_count == 2 + assert buy_mock.call_count == 2 + + +def test_analyze_ticker_only_once(ticker_history, mocker) -> None: + + ind_mock = MagicMock(side_effect=lambda x, meta: x) + buy_mock = MagicMock(side_effect=lambda x, meta: x) + sell_mock = MagicMock(side_effect=lambda x, meta: x) + mocker.patch.multiple( + 'freqtrade.strategy.interface.IStrategy', + advise_indicators=ind_mock, + advise_buy=buy_mock, + advise_sell=sell_mock, + + ) + strategy = DefaultStrategy({}) + strategy.ta_on_candle = True + + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + assert ind_mock.call_count == 1 + assert buy_mock.call_count == 1 + assert buy_mock.call_count == 1 + + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + # No analysis happens as ta_on_candle is true + assert ind_mock.call_count == 1 + assert buy_mock.call_count == 1 + assert buy_mock.call_count == 1 + # only skipped analyze adds buy and sell columns, otherwise it's all mocked + assert 'buy' in ret + assert 'sell' in ret + assert ret['buy'].sum() == 0 + assert ret['sell'].sum() == 0 From df960241bd7e627c1aeec356c62e4b6f7322c882 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:07:01 +0200 Subject: [PATCH 13/19] Add log-message for skipped candle and tests --- freqtrade/tests/strategy/test_interface.py | 23 ++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 45f650938..75deecda2 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -107,8 +107,8 @@ def test_tickerdata_to_dataframe(default_conf) -> None: assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed -def test_analyze_ticker_default(ticker_history, mocker) -> None: - +def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: + caplog.set_level(logging.DEBUG) ind_mock = MagicMock(side_effect=lambda x, meta: x) buy_mock = MagicMock(side_effect=lambda x, meta: x) sell_mock = MagicMock(side_effect=lambda x, meta: x) @@ -125,15 +125,23 @@ def test_analyze_ticker_default(ticker_history, mocker) -> None: assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 + assert log_has('TA Analysis Launched', caplog.record_tuples) + assert not log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) + caplog.clear() + ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) # No analysis happens as ta_on_candle is true assert ind_mock.call_count == 2 assert buy_mock.call_count == 2 assert buy_mock.call_count == 2 + assert log_has('TA Analysis Launched', caplog.record_tuples) + assert not log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) -def test_analyze_ticker_only_once(ticker_history, mocker) -> None: - +def test_analyze_ticker_skip_analyze(ticker_history, mocker, caplog) -> None: + caplog.set_level(logging.DEBUG) ind_mock = MagicMock(side_effect=lambda x, meta: x) buy_mock = MagicMock(side_effect=lambda x, meta: x) sell_mock = MagicMock(side_effect=lambda x, meta: x) @@ -151,6 +159,10 @@ def test_analyze_ticker_only_once(ticker_history, mocker) -> None: assert ind_mock.call_count == 1 assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 + assert log_has('TA Analysis Launched', caplog.record_tuples) + assert not log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) + caplog.clear() ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) # No analysis happens as ta_on_candle is true @@ -162,3 +174,6 @@ def test_analyze_ticker_only_once(ticker_history, mocker) -> None: assert 'sell' in ret assert ret['buy'].sum() == 0 assert ret['sell'].sum() == 0 + assert not log_has('TA Analysis Launched', caplog.record_tuples) + assert log_has('Skippinig TA Analysis for already analyzed candle', + caplog.record_tuples) From 3b2f161573712c4823e591fe41d6e9e0ce758aea Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:12:45 +0200 Subject: [PATCH 14/19] Add test for ta_on_candle override --- freqtrade/strategy/interface.py | 1 + freqtrade/tests/strategy/test_strategy.py | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index f8965d440..3957139d2 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -135,6 +135,7 @@ class IStrategy(ABC): dataframe = self.advise_sell(dataframe, metadata) self._candle_seen[pair] = dataframe.iloc[-1]['date'] else: + logging.debug("Skippinig TA Analysis for already analyzed candle") dataframe['buy'] = 0 dataframe['sell'] = 0 diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 0cbd9f22c..d45715a69 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -130,7 +130,7 @@ def test_strategy_override_minimal_roi(caplog): assert resolver.strategy.minimal_roi[0] == 0.5 assert ('freqtrade.strategy.resolver', logging.INFO, - 'Override strategy \'minimal_roi\' with value in config file.' + "Override strategy 'minimal_roi' with value in config file." ) in caplog.record_tuples @@ -145,7 +145,7 @@ def test_strategy_override_stoploss(caplog): assert resolver.strategy.stoploss == -0.5 assert ('freqtrade.strategy.resolver', logging.INFO, - 'Override strategy \'stoploss\' with value in config file: -0.5.' + "Override strategy 'stoploss' with value in config file: -0.5." ) in caplog.record_tuples @@ -161,10 +161,25 @@ def test_strategy_override_ticker_interval(caplog): assert resolver.strategy.ticker_interval == 60 assert ('freqtrade.strategy.resolver', logging.INFO, - 'Override strategy \'ticker_interval\' with value in config file: 60.' + "Override strategy 'ticker_interval' with value in config file: 60." ) in caplog.record_tuples +def test_strategy_override_ta_on_candle(caplog): + caplog.set_level(logging.INFO) + + config = { + 'strategy': 'DefaultStrategy', + 'ta_on_candle': True + } + resolver = StrategyResolver(config) + + assert resolver.strategy.ta_on_candle == True + assert ('freqtrade.strategy.resolver', + logging.INFO, + "Override ta_on_candle 'ta_on_candle' with value in config file: True." + ) in caplog.record_tuples + def test_deprecate_populate_indicators(result): default_location = path.join(path.dirname(path.realpath(__file__))) resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', From b008649d790e74f81b114ef82c36b05da4045648 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:13:07 +0200 Subject: [PATCH 15/19] Remove unnecessary quote escaping --- freqtrade/strategy/resolver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 1a15ba63f..75fb99d69 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -44,14 +44,14 @@ class StrategyResolver(object): # Check if we need to override configuration if 'minimal_roi' in config: self.strategy.minimal_roi = config['minimal_roi'] - logger.info("Override strategy \'minimal_roi\' with value in config file.") + logger.info("Override strategy 'minimal_roi' with value in config file.") else: config['minimal_roi'] = self.strategy.minimal_roi if 'stoploss' in config: self.strategy.stoploss = config['stoploss'] logger.info( - "Override strategy \'stoploss\' with value in config file: %s.", config['stoploss'] + "Override strategy 'stoploss' with value in config file: %s.", config['stoploss'] ) else: config['stoploss'] = self.strategy.stoploss @@ -59,7 +59,7 @@ class StrategyResolver(object): if 'ticker_interval' in config: self.strategy.ticker_interval = config['ticker_interval'] logger.info( - "Override strategy \'ticker_interval\' with value in config file: %s.", + "Override strategy 'ticker_interval' with value in config file: %s.", config['ticker_interval'] ) else: @@ -68,7 +68,7 @@ class StrategyResolver(object): if 'ta_on_candle' in config: self.strategy.ta_on_candle = config['ta_on_candle'] logger.info( - "Override ta_on_candle \'ta_on_candle\' with value in config file: %s.", + "Override ta_on_candle 'ta_on_candle' with value in config file: %s.", config['ta_on_candle'] ) else: From 56768f1a617c4fca5c648ffcbe95b3e53b7d7f9c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Aug 2018 20:17:55 +0200 Subject: [PATCH 16/19] Flake8 in tests ... --- freqtrade/tests/strategy/test_interface.py | 4 ++-- freqtrade/tests/strategy/test_strategy.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 75deecda2..e96dfb024 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -120,7 +120,7 @@ def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: ) strategy = DefaultStrategy({}) - ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) assert ind_mock.call_count == 1 assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 @@ -130,7 +130,7 @@ def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: caplog.record_tuples) caplog.clear() - ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) # No analysis happens as ta_on_candle is true assert ind_mock.call_count == 2 assert buy_mock.call_count == 2 diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index d45715a69..14b1ef1bd 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -174,12 +174,13 @@ def test_strategy_override_ta_on_candle(caplog): } resolver = StrategyResolver(config) - assert resolver.strategy.ta_on_candle == True + assert resolver.strategy.ta_on_candle assert ('freqtrade.strategy.resolver', logging.INFO, "Override ta_on_candle 'ta_on_candle' with value in config file: True." ) in caplog.record_tuples + def test_deprecate_populate_indicators(result): default_location = path.join(path.dirname(path.realpath(__file__))) resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', From cb46aeb73cc277a2883c75ba3feadeb7ad2c132b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Sep 2018 19:50:45 +0200 Subject: [PATCH 17/19] rename variable to be more expressive --- freqtrade/strategy/interface.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 3957139d2..a9838a5cb 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -74,11 +74,11 @@ class IStrategy(ABC): ta_on_candle: bool = False # Dict to determine if analysis is necessary - _candle_seen: Dict[str, datetime] = {} + _last_candle_seen_per_pair: Dict[str, datetime] = {} def __init__(self, config: dict) -> None: self.config = config - self._candle_seen = {} + self._last_candle_seen_per_pair = {} @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -119,7 +119,6 @@ class IStrategy(ABC): add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ - # Test if seen this pair and last candle before. dataframe = parse_ticker_dataframe(ticker_history) @@ -127,13 +126,13 @@ class IStrategy(ABC): # always run if ta_on_candle is set to true if (not self.ta_on_candle or - self._candle_seen.get(pair, None) != dataframe.iloc[-1]['date']): + self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. logging.debug("TA Analysis Launched") dataframe = self.advise_indicators(dataframe, metadata) dataframe = self.advise_buy(dataframe, metadata) dataframe = self.advise_sell(dataframe, metadata) - self._candle_seen[pair] = dataframe.iloc[-1]['date'] + self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date'] else: logging.debug("Skippinig TA Analysis for already analyzed candle") dataframe['buy'] = 0 From d35d3bb38cf9de50b0d6d2a234297d6c8b892cee Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Sep 2018 19:52:40 +0200 Subject: [PATCH 18/19] rename ta_on_candle to process_only_new_candles be more expressive --- docs/configuration.md | 2 +- freqtrade/constants.py | 2 +- freqtrade/strategy/interface.py | 6 +++--- freqtrade/strategy/resolver.py | 10 +++++----- freqtrade/tests/strategy/test_interface.py | 6 +++--- freqtrade/tests/strategy/test_strategy.py | 9 +++++---- 6 files changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 64e75c51e..1d5fc0922 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -23,7 +23,7 @@ The table below will list all configuration parameters. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes | `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. -| `ta_on_candle` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. +| `process_only_new_candles` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 189b78617..92b090432 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -53,7 +53,7 @@ CONF_SCHEMA = { }, 'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT}, 'dry_run': {'type': 'boolean'}, - 'ta_on_candle': {'type': 'boolean'}, + 'process_only_new_candles': {'type': 'boolean'}, 'minimal_roi': { 'type': 'object', 'patternProperties': { diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index a9838a5cb..05ede129e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -71,7 +71,7 @@ class IStrategy(ABC): ticker_interval: str # run "populate_indicators" only for new candle - ta_on_candle: bool = False + process_only_new_candles: bool = False # Dict to determine if analysis is necessary _last_candle_seen_per_pair: Dict[str, datetime] = {} @@ -124,8 +124,8 @@ class IStrategy(ABC): pair = str(metadata.get('pair')) - # always run if ta_on_candle is set to true - if (not self.ta_on_candle or + # always run if process_only_new_candles is set to true + if (not self.process_only_new_candles or self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. logging.debug("TA Analysis Launched") diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 75fb99d69..35aee8d20 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -65,14 +65,14 @@ class StrategyResolver(object): else: config['ticker_interval'] = self.strategy.ticker_interval - if 'ta_on_candle' in config: - self.strategy.ta_on_candle = config['ta_on_candle'] + if 'process_only_new_candles' in config: + self.strategy.process_only_new_candles = config['process_only_new_candles'] logger.info( - "Override ta_on_candle 'ta_on_candle' with value in config file: %s.", - config['ta_on_candle'] + "Override process_only_new_candles 'process_only_new_candles' " + "with value in config file: %s.", config['process_only_new_candles'] ) else: - config['ta_on_candle'] = self.strategy.ta_on_candle + config['process_only_new_candles'] = self.strategy.process_only_new_candles # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index e96dfb024..5afffd87f 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -131,7 +131,7 @@ def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: caplog.clear() strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) - # No analysis happens as ta_on_candle is true + # No analysis happens as process_only_new_candles is true assert ind_mock.call_count == 2 assert buy_mock.call_count == 2 assert buy_mock.call_count == 2 @@ -153,7 +153,7 @@ def test_analyze_ticker_skip_analyze(ticker_history, mocker, caplog) -> None: ) strategy = DefaultStrategy({}) - strategy.ta_on_candle = True + strategy.process_only_new_candles = True ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) assert ind_mock.call_count == 1 @@ -165,7 +165,7 @@ def test_analyze_ticker_skip_analyze(ticker_history, mocker, caplog) -> None: caplog.clear() ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) - # No analysis happens as ta_on_candle is true + # No analysis happens as process_only_new_candles is true assert ind_mock.call_count == 1 assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 14b1ef1bd..8a6e7e617 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -165,19 +165,20 @@ def test_strategy_override_ticker_interval(caplog): ) in caplog.record_tuples -def test_strategy_override_ta_on_candle(caplog): +def test_strategy_override_process_only_new_candles(caplog): caplog.set_level(logging.INFO) config = { 'strategy': 'DefaultStrategy', - 'ta_on_candle': True + 'process_only_new_candles': True } resolver = StrategyResolver(config) - assert resolver.strategy.ta_on_candle + assert resolver.strategy.process_only_new_candles assert ('freqtrade.strategy.resolver', logging.INFO, - "Override ta_on_candle 'ta_on_candle' with value in config file: True." + "Override process_only_new_candles 'process_only_new_candles' " + "with value in config file: True." ) in caplog.record_tuples From 2ec5a536aadf63006e734b8989fbefa904a95945 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 Sep 2018 19:53:49 +0200 Subject: [PATCH 19/19] Fix comment location --- freqtrade/strategy/interface.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 05ede129e..6d865f4bc 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -119,11 +119,12 @@ class IStrategy(ABC): add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ - # Test if seen this pair and last candle before. + dataframe = parse_ticker_dataframe(ticker_history) pair = str(metadata.get('pair')) + # Test if seen this pair and last candle before. # always run if process_only_new_candles is set to true if (not self.process_only_new_candles or self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']):