From fb6ba621582f64ae2c77483c5d1212954f6b3d75 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Nov 2021 19:52:29 +0100 Subject: [PATCH 1/5] Add default to "is_new_pair" --- freqtrade/exchange/binance.py | 2 +- freqtrade/exchange/exchange.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 8dced3894..06d64999d 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -93,7 +93,7 @@ class Binance(Exchange): raise OperationalException(e) from e async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool + since_ms: int, is_new_pair: bool = False ) -> List: """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index f9553699b..b22556f60 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1223,8 +1223,8 @@ class Exchange: drop_incomplete=self._ohlcv_partial_candle) async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool - ) -> List: + since_ms: int, is_new_pair: bool = False + ) -> Tuple[str, str, List]: """ Download historic ohlcv :param is_new_pair: used by binance subclass to allow "fast" new pair downloading From 9fa64c264768c4156c7cfc84b5a86badd7ee62ef Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Nov 2021 20:00:02 +0100 Subject: [PATCH 2/5] Allow multiple calls to get more candles in live-run --- freqtrade/exchange/binance.py | 10 ++++---- freqtrade/exchange/exchange.py | 42 +++++++++++++++++++++++---------- tests/exchange/test_binance.py | 9 ++++--- tests/exchange/test_exchange.py | 8 ++++--- 4 files changed, 47 insertions(+), 22 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 06d64999d..4ba30b626 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -1,6 +1,6 @@ """ Binance exchange subclass """ import logging -from typing import Dict, List +from typing import Dict, List, Tuple import arrow import ccxt @@ -93,8 +93,9 @@ class Binance(Exchange): raise OperationalException(e) from e async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool = False - ) -> List: + since_ms: int, is_new_pair: bool = False, + raise_: bool = False + ) -> Tuple[str, str, List]: """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date Does not work for other exchanges, which don't return the earliest data when called with "0" @@ -107,4 +108,5 @@ class Binance(Exchange): logger.info(f"Candle-data for {pair} available starting with " f"{arrow.get(since_ms // 1000).isoformat()}.") return await super()._async_get_historic_ohlcv( - pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair) + pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair, + raise_=raise_) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b22556f60..01f0864c4 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -7,7 +7,7 @@ import http import inspect import logging from copy import deepcopy -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from math import ceil from typing import Any, Dict, List, Optional, Tuple @@ -1205,9 +1205,11 @@ class Exchange: :param since_ms: Timestamp in milliseconds to get history from :return: List with candle (OHLCV) data """ - return asyncio.get_event_loop().run_until_complete( + pair, timeframe, data = asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair)) + logger.info(f"Downloaded data for {pair} with length {len(data)}.") + return data def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, since_ms: int) -> DataFrame: @@ -1223,7 +1225,8 @@ class Exchange: drop_incomplete=self._ohlcv_partial_candle) async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool = False + since_ms: int, is_new_pair: bool = False, + raise_: bool = False ) -> Tuple[str, str, List]: """ Download historic ohlcv @@ -1248,15 +1251,17 @@ class Exchange: for res in results: if isinstance(res, Exception): logger.warning("Async code raised an exception: %s", res.__class__.__name__) + if raise_: + raise continue - # Deconstruct tuple if it's not an exception - p, _, new_data = res - if p == pair: - data.extend(new_data) + else: + # Deconstruct tuple if it's not an exception + p, _, new_data = res + if p == pair: + data.extend(new_data) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) - logger.info(f"Downloaded data for {pair} with length {len(data)}.") - return data + return pair, timeframe, data def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, since_ms: Optional[int] = None, cache: bool = True @@ -1276,10 +1281,23 @@ class Exchange: cached_pairs = [] # Gather coroutines to run for pair, timeframe in set(pair_list): - if (((pair, timeframe) not in self._klines) + if ((pair, timeframe) not in self._klines or self._now_is_time_to_refresh(pair, timeframe)): - input_coroutines.append(self._async_get_candle_history(pair, timeframe, - since_ms=since_ms)) + call_count = self._ft_has.get('ohlcv_candle_call_count', 1) + if not since_ms and call_count > 1: + # Multiple calls for one pair - to get more history + one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) + move_to = one_call * call_count + now = timeframe_to_next_date(timeframe) + since_ms = int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000) + + if since_ms: + input_coroutines.append(self._async_get_historic_ohlcv( + pair, timeframe, since_ms=since_ms, raise_=True)) + else: + # One call ... "regular" refresh + input_coroutines.append(self._async_get_candle_history( + pair, timeframe, since_ms=since_ms)) else: logger.debug( "Using cached candle (OHLCV) data for pair %s, timeframe %s ...", diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index dd85c3abe..d88ae9b1d 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -126,13 +126,16 @@ async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog): exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) pair = 'ETH/BTC' - res = await exchange._async_get_historic_ohlcv(pair, "5m", - 1500000000000, is_new_pair=False) + respair, restf, res = await exchange._async_get_historic_ohlcv( + pair, "5m", 1500000000000, is_new_pair=False) + assert respair == pair + assert restf == '5m' # Call with very old timestamp - causes tons of requests assert exchange._api_async.fetch_ohlcv.call_count > 400 # assert res == ohlcv exchange._api_async.fetch_ohlcv.reset_mock() - res = await exchange._async_get_historic_ohlcv(pair, "5m", 1500000000000, is_new_pair=True) + _, _, res = await exchange._async_get_historic_ohlcv( + pair, "5m", 1500000000000, is_new_pair=True) # Called twice - one "init" call - and one to get the actual data. assert exchange._api_async.fetch_ohlcv.call_count == 2 diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index e3369182d..34e2b04ab 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1506,6 +1506,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): assert exchange._async_get_candle_history.call_count == 2 # Returns twice the above OHLCV data assert len(ret) == 2 + assert log_has_re(r'Downloaded data for .* with length .*\.', caplog) caplog.clear() @@ -1587,12 +1588,13 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) pair = 'ETH/USDT' - res = await exchange._async_get_historic_ohlcv(pair, "5m", - 1500000000000, is_new_pair=False) + respair, restf, res = await exchange._async_get_historic_ohlcv( + pair, "5m", 1500000000000, is_new_pair=False) + assert respair == pair + assert restf == '5m' # Call with very old timestamp - causes tons of requests assert exchange._api_async.fetch_ohlcv.call_count > 200 assert res[0] == ohlcv[0] - assert log_has_re(r'Downloaded data for .* with length .*\.', caplog) def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: From a08dd17bc19c81156ace7735bad0cbb2fb98e2ea Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Nov 2021 13:10:40 +0100 Subject: [PATCH 3/5] Use startup_candle-count to determine call count --- freqtrade/exchange/exchange.py | 28 ++++++++++++++++++++-------- tests/exchange/test_exchange.py | 24 ++++++++++++++++++++---- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 01f0864c4..4e9a6c71d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -155,8 +155,8 @@ class Exchange: self.validate_pairs(config['exchange']['pair_whitelist']) self.validate_ordertypes(config.get('order_types', {})) self.validate_order_time_in_force(config.get('order_time_in_force', {})) - self.validate_required_startup_candles(config.get('startup_candle_count', 0), - config.get('timeframe', '')) + self.required_candle_call_count = self.validate_required_startup_candles( + config.get('startup_candle_count', 0), config.get('timeframe', '')) # Converts the interval provided in minutes in config to seconds self.markets_refresh_interval: int = exchange_config.get( @@ -477,10 +477,23 @@ class Exchange: Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default. """ candle_limit = self.ohlcv_candle_limit(timeframe) - if startup_candles + 5 > candle_limit: + # Require one more candle - to account for the still open candle. + candle_count = startup_candles + 1 + # Allow 5 calls to the exchange per pair + required_candle_call_count = int( + (candle_count / candle_limit) + (0 if candle_count % candle_limit == 0 else 1)) + + if required_candle_call_count > 5: + # Only allow 5 calls per pair to somewhat limit the impact raise OperationalException( - f"This strategy requires {startup_candles} candles to start. " - f"{self.name} only provides {candle_limit - 5} for {timeframe}.") + f"This strategy requires {startup_candles} candles to start, which is more than 5x " + f"the amount of candles {self.name} provides for {timeframe}.") + + if required_candle_call_count > 1: + logger.warning(f"Using {required_candle_call_count} calls to get OHLCV. " + f"This can result in slower operations for the bot. Please check " + f"if you really need {startup_candles} candles for your strategy") + return required_candle_call_count def exchange_has(self, endpoint: str) -> bool: """ @@ -1283,11 +1296,10 @@ class Exchange: for pair, timeframe in set(pair_list): if ((pair, timeframe) not in self._klines or self._now_is_time_to_refresh(pair, timeframe)): - call_count = self._ft_has.get('ohlcv_candle_call_count', 1) - if not since_ms and call_count > 1: + if not since_ms and self.required_candle_call_count > 1: # Multiple calls for one pair - to get more history one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) - move_to = one_call * call_count + move_to = one_call * self.required_candle_call_count now = timeframe_to_next_date(timeframe) since_ms = int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 34e2b04ab..ff78321c5 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -941,12 +941,26 @@ def test_validate_required_startup_candles(default_conf, mocker, caplog): default_conf['startup_candle_count'] = 20 ex = Exchange(default_conf) assert ex - default_conf['startup_candle_count'] = 600 + # assumption is that the exchange provides 500 candles per call.s + assert ex.validate_required_startup_candles(200, '5m') == 1 + assert ex.validate_required_startup_candles(499, '5m') == 1 + assert ex.validate_required_startup_candles(600, '5m') == 2 + assert ex.validate_required_startup_candles(501, '5m') == 2 + assert ex.validate_required_startup_candles(499, '5m') == 1 + assert ex.validate_required_startup_candles(1000, '5m') == 3 + assert ex.validate_required_startup_candles(2499, '5m') == 5 + assert log_has_re(r'Using 5 calls to get OHLCV. This.*', caplog) - with pytest.raises(OperationalException, match=r'This strategy requires 600.*'): + with pytest.raises(OperationalException, match=r'This strategy requires 2500.*'): + ex.validate_required_startup_candles(2500, '5m') + + # Ensure the same also happens on init + default_conf['startup_candle_count'] = 6000 + with pytest.raises(OperationalException, match=r'This strategy requires 6000.*'): Exchange(default_conf) + def test_exchange_has(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) assert not exchange.exchange_has('ASDFASDF') @@ -1632,12 +1646,14 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: assert exchange._api_async.fetch_ohlcv.call_count == 2 exchange._api_async.fetch_ohlcv.reset_mock() + exchange.required_candle_call_count = 2 res = exchange.refresh_latest_ohlcv(pairs) assert len(res) == len(pairs) assert log_has(f'Refreshing candle (OHLCV) data for {len(pairs)} pairs', caplog) assert exchange._klines - assert exchange._api_async.fetch_ohlcv.call_count == 2 + assert exchange._api_async.fetch_ohlcv.call_count == 4 + exchange._api_async.fetch_ohlcv.reset_mock() for pair in pairs: assert isinstance(exchange.klines(pair), DataFrame) assert len(exchange.klines(pair)) > 0 @@ -1653,7 +1669,7 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m')]) assert len(res) == len(pairs) - assert exchange._api_async.fetch_ohlcv.call_count == 2 + assert exchange._api_async.fetch_ohlcv.call_count == 0 assert log_has(f"Using cached candle (OHLCV) data for pair {pairs[0][0]}, " f"timeframe {pairs[0][1]} ...", caplog) From de4bc7204d99c77afd19c446bc0811181911e616 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Nov 2021 13:14:29 +0100 Subject: [PATCH 4/5] Update documentation to clarify new behaviour --- docs/strategy-customization.md | 9 +++++++-- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_exchange.py | 1 - 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 62e7509b3..d445234b5 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -134,7 +134,7 @@ Additional technical libraries can be installed as necessary, or custom indicato ### Strategy startup period -Most indicators have an instable startup period, in which they are either not available, or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be. +Most indicators have an instable startup period, in which they are either not available (NaN), or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be. To account for this, the strategy can be assigned the `startup_candle_count` attribute. This should be set to the maximum number of candles that the strategy requires to calculate stable indicators. @@ -146,8 +146,13 @@ In this example strategy, this should be set to 100 (`startup_candle_count = 100 By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt. +!!! Warning "Using x calls to get OHLCV" + If you receive a warning like `WARNING - Using 3 calls to get OHLCV. This can result in slower operations for the bot. Please check if you really need 1500 candles for your strategy` - you should consider if you really need this much lookback. + This will make Freqtrade take longer to refresh candles - and should be avoided if possible. + This is capped to 5 total calls to avoid overloading the exchange, or make freqtrade too slow. + !!! Warning - `startup_candle_count` should be below `ohlcv_candle_limit` (which is 500 for most exchanges) - since only this amount of candles will be available during Dry-Run/Live Trade operations. + `startup_candle_count` should be below `ohlcv_candle_limit * 5` (which is 500 * 5 for most exchanges) - since only this amount of candles will be available during Dry-Run/Live Trade operations. #### Example diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 4e9a6c71d..19ad4e4b6 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -471,7 +471,7 @@ class Exchange: raise OperationalException( f'Time in force policies are not supported for {self.name} yet.') - def validate_required_startup_candles(self, startup_candles: int, timeframe: str) -> None: + def validate_required_startup_candles(self, startup_candles: int, timeframe: str) -> int: """ Checks if required startup_candles is more than ohlcv_candle_limit(). Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default. diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ff78321c5..8a8569dc4 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -960,7 +960,6 @@ def test_validate_required_startup_candles(default_conf, mocker, caplog): Exchange(default_conf) - def test_exchange_has(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) assert not exchange.exchange_has('ASDFASDF') From 84261237a0cb90051e493c9e0ceef8c7e37370f1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 8 Nov 2021 08:09:33 +0100 Subject: [PATCH 5/5] Improve doc wording --- docs/strategy-customization.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index d445234b5..f3658dd5a 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -147,8 +147,9 @@ In this example strategy, this should be set to 100 (`startup_candle_count = 100 By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt. !!! Warning "Using x calls to get OHLCV" - If you receive a warning like `WARNING - Using 3 calls to get OHLCV. This can result in slower operations for the bot. Please check if you really need 1500 candles for your strategy` - you should consider if you really need this much lookback. - This will make Freqtrade take longer to refresh candles - and should be avoided if possible. + If you receive a warning like `WARNING - Using 3 calls to get OHLCV. This can result in slower operations for the bot. Please check if you really need 1500 candles for your strategy` - you should consider if you really need this much historic data for your signals. + Having this will cause Freqtrade to make multiple calls for the same pair, which will obviously be slower than one network request. + As a consequence, Freqtrade will take longer to refresh candles - and should therefore be avoided if possible. This is capped to 5 total calls to avoid overloading the exchange, or make freqtrade too slow. !!! Warning