diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index ae07e36e9..350c730a4 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -478,7 +478,9 @@ class Exchange(object): # Because some exchange sort Tickers ASC and other DESC. # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) # when GDAX returns a list of tickers DESC (newest first, oldest last) - data = sorted(data, key=lambda x: x[0]) + # Only sort if necessary to save computing time + if data and data[0][0] > data[-1][0]: + data = sorted(data, key=lambda x: x[0]) # keeping last candle time as last refreshed time of the pair if data: @@ -500,51 +502,6 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(f'Could not fetch ticker data. Msg: {e}') - @retrier - def get_candle_history(self, pair: str, tick_interval: str, - since_ms: Optional[int] = None) -> List[Dict]: - try: - # last item should be in the time interval [now - tick_interval, now] - till_time_ms = arrow.utcnow().shift( - minutes=-constants.TICKER_INTERVAL_MINUTES[tick_interval] - ).timestamp * 1000 - # it looks as if some exchanges return cached data - # and they update it one in several minute, so 10 mins interval - # is necessary to skeep downloading of an empty array when all - # chached data was already downloaded - till_time_ms = min(till_time_ms, arrow.utcnow().shift(minutes=-10).timestamp * 1000) - - data: List[Dict[Any, Any]] = [] - while not since_ms or since_ms < till_time_ms: - data_part = self._api.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) - - # Because some exchange sort Tickers ASC and other DESC. - # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) - # when GDAX returns a list of tickers DESC (newest first, oldest last) - data_part = sorted(data_part, key=lambda x: x[0]) - - if not data_part: - break - - logger.debug('Downloaded data for %s time range [%s, %s]', - pair, - arrow.get(data_part[0][0] / 1000).format(), - arrow.get(data_part[-1][0] / 1000).format()) - - data.extend(data_part) - since_ms = data[-1][0] + 1 - - return data - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching historical candlestick data.' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(f'Could not fetch ticker data. Msg: {e}') - @retrier def cancel_order(self, order_id: str, pair: str) -> None: if self._conf['dry_run']: diff --git a/freqtrade/exchange/exchange_helpers.py b/freqtrade/exchange/exchange_helpers.py index 8f4b03daf..84e68d4bb 100644 --- a/freqtrade/exchange/exchange_helpers.py +++ b/freqtrade/exchange/exchange_helpers.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) def parse_ticker_dataframe(ticker: list) -> DataFrame: """ Analyses the trend for the given ticker history - :param ticker: See exchange.get_candle_history + :param ticker: ticker list, as returned by exchange.async_get_candle_history :return: DataFrame """ cols = ['date', 'open', 'high', 'low', 'close', 'volume'] diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 207f14efe..dbb8d4ec2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -875,65 +875,10 @@ def make_fetch_ohlcv_mock(data): return fetch_ohlcv_mock -def test_get_candle_history(default_conf, mocker): - api_mock = MagicMock() - tick = [ - [ - 1511686200000, # unix timestamp ms - 1, # open - 2, # high - 3, # low - 4, # close - 5, # volume (in quote currency) - ] - ] - type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - - # retrieve original ticker - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) - assert ticks[0][0] == 1511686200000 - assert ticks[0][1] == 1 - assert ticks[0][2] == 2 - assert ticks[0][3] == 3 - assert ticks[0][4] == 4 - assert ticks[0][5] == 5 - - # change ticker and ensure tick changes - new_tick = [ - [ - 1511686210000, # unix timestamp ms - 6, # open - 7, # high - 8, # low - 9, # close - 10, # volume (in quote currency) - ] - ] - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(new_tick)) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) - assert ticks[0][0] == 1511686210000 - assert ticks[0][1] == 6 - assert ticks[0][2] == 7 - assert ticks[0][3] == 8 - assert ticks[0][4] == 9 - assert ticks[0][5] == 10 - - ccxt_exceptionhandlers(mocker, default_conf, api_mock, - "get_candle_history", "fetch_ohlcv", - pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) - - with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_candle_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) - - -def test_get_candle_history_sort(default_conf, mocker): - api_mock = MagicMock() +@pytest.mark.asyncio +async def test___async_get_candle_history_sort(default_conf, mocker): + def sort_data(data, key): + return sorted(data, key=key) # GDAX use-case (real data from GDAX) # This ticker history is ordered DESC (newest first, oldest last) @@ -949,13 +894,15 @@ def test_get_candle_history_sort(default_conf, mocker): [1527830700000, 0.07652, 0.07652, 0.07651, 0.07652, 10.04822687], [1527830400000, 0.07649, 0.07651, 0.07649, 0.07651, 2.5734867] ] - type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - - exchange = get_patched_exchange(mocker, default_conf, api_mock) - + exchange = get_patched_exchange(mocker, default_conf) + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data)) # Test the ticker history sort - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) + res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) + assert res[0] == 'ETH/BTC' + ticks = res[1] + + assert sort_mock.call_count == 1 assert ticks[0][0] == 1527830400000 assert ticks[0][1] == 0.07649 assert ticks[0][2] == 0.07651 @@ -984,11 +931,15 @@ def test_get_candle_history_sort(default_conf, mocker): [1527830100000, 0.076695, 0.07671, 0.07624171, 0.07671, 1.80689244], [1527830400000, 0.07671, 0.07674399, 0.07629216, 0.07655213, 2.31452783] ] - type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + # Reset sort mock + sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data)) # Test the ticker history sort - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) + res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) + assert res[0] == 'ETH/BTC' + ticks = res[1] + # Sorted not called again - data is already in order + assert sort_mock.call_count == 0 assert ticks[0][0] == 1527827700000 assert ticks[0][1] == 0.07659999 assert ticks[0][2] == 0.0766