diff --git a/.travis.yml b/.travis.yml index 88121945f..981eedcf8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,12 +13,12 @@ addons: install: - ./install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH -- pip install --upgrade flake8 coveralls pytest-random-order mypy +- pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy - pip install -r requirements.txt - pip install -e . jobs: include: - - script: + - script: - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - coveralls - script: diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index e7f94c499..f663420e0 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,12 +1,15 @@ # pragma pylint: disable=W0603 """ Cryptocurrency Exchanges support """ import logging +import inspect from random import randint -from typing import List, Dict, Any, Optional +from typing import List, Dict, Tuple, Any, Optional from datetime import datetime from math import floor, ceil +import asyncio import ccxt +import ccxt.async_support as ccxt_async import arrow from freqtrade import constants, OperationalException, DependencyException, TemporaryError @@ -23,6 +26,24 @@ _EXCHANGE_URLS = { } +def retrier_async(f): + async def wrapper(*args, **kwargs): + count = kwargs.pop('count', API_RETRY_COUNT) + try: + return await f(*args, **kwargs) + except (TemporaryError, DependencyException) as ex: + logger.warning('%s() returned exception: "%s"', f.__name__, ex) + if count > 0: + count -= 1 + kwargs.update({'count': count}) + logger.warning('retrying %s() still for %s times', f.__name__, count) + return await wrapper(*args, **kwargs) + else: + logger.warning('Giving up retrying: %s()', f.__name__) + raise ex + return wrapper + + def retrier(f): def wrapper(*args, **kwargs): count = kwargs.pop('count', API_RETRY_COUNT) @@ -45,8 +66,8 @@ class Exchange(object): # Current selected exchange _api: ccxt.Exchange = None + _api_async: ccxt_async.Exchange = None _conf: Dict = {} - _cached_ticker: Dict[str, Any] = {} # Holds all open sell orders for dry_run _dry_run_open_orders: Dict[str, Any] = {} @@ -60,11 +81,20 @@ class Exchange(object): """ self._conf.update(config) + self._cached_ticker: Dict[str, Any] = {} + + # Holds last candle refreshed time of each pair + self._pairs_last_refresh_time: Dict[str, int] = {} + + # Holds candles + self.klines: Dict[str, Any] = {} + if config['dry_run']: logger.info('Instance is running with dry_run enabled') exchange_config = config['exchange'] self._api = self._init_ccxt(exchange_config) + self._api_async = self._init_ccxt(exchange_config, ccxt_async) logger.info('Using Exchange "%s"', self.name) @@ -75,7 +105,15 @@ class Exchange(object): # Check if timeframe is available self.validate_timeframes(config['ticker_interval']) - def _init_ccxt(self, exchange_config: dict) -> ccxt.Exchange: + def __del__(self): + """ + Destructor - clean up async stuff + """ + logger.debug("Exchange object destroyed, closing async loop") + if self._api_async and inspect.iscoroutinefunction(self._api_async.close): + asyncio.get_event_loop().run_until_complete(self._api_async.close()) + + def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid ccxt instance. @@ -83,15 +121,15 @@ class Exchange(object): # Find matching class for the given exchange name name = exchange_config['name'] - if name not in ccxt.exchanges: + if name not in ccxt_module.exchanges: raise OperationalException(f'Exchange {name} is not supported') try: - api = getattr(ccxt, name.lower())({ + api = getattr(ccxt_module, name.lower())({ 'apiKey': exchange_config.get('key'), 'secret': exchange_config.get('secret'), 'password': exchange_config.get('password'), 'uid': exchange_config.get('uid', ''), - 'enableRateLimit': exchange_config.get('ccxt_rate_limit', True), + 'enableRateLimit': exchange_config.get('ccxt_rate_limit', True) }) except (KeyError, AttributeError): raise OperationalException(f'Exchange {name} is not supported') @@ -120,6 +158,15 @@ class Exchange(object): "Please check your config.json") raise OperationalException(f'Exchange {name} does not provide a sandbox api') + def _load_async_markets(self) -> None: + try: + if self._api_async: + asyncio.get_event_loop().run_until_complete(self._api_async.load_markets()) + + except ccxt.BaseError as e: + logger.warning('Could not load async markets. Reason: %s', e) + return + def validate_pairs(self, pairs: List[str]) -> None: """ Checks if all given pairs are tradable on the current exchange. @@ -130,6 +177,7 @@ class Exchange(object): try: markets = self._api.load_markets() + self._load_async_markets() except ccxt.BaseError as e: logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e) return @@ -329,6 +377,102 @@ class Exchange(object): logger.info("returning cached ticker-data for %s", pair) return self._cached_ticker[pair] + def get_history(self, pair: str, tick_interval: str, + since_ms: int) -> List: + """ + Gets candle history using asyncio and returns the list of candles. + Handles all async doing. + """ + return asyncio.get_event_loop().run_until_complete( + self._async_get_history(pair=pair, tick_interval=tick_interval, + since_ms=since_ms)) + + async def _async_get_history(self, pair: str, + tick_interval: str, + since_ms: int) -> List: + # Assume exchange returns 500 candles + _LIMIT = 500 + + one_call = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 * _LIMIT * 1000 + logger.debug("one_call: %s", one_call) + input_coroutines = [self._async_get_candle_history( + pair, tick_interval, since) for since in + range(since_ms, arrow.utcnow().timestamp * 1000, one_call)] + tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) + + # Combine tickers + data: List = [] + for tick in tickers: + if tick[0] == pair: + data.extend(tick[1]) + # Sort data again after extending the result - above calls return in "async order" order + data = sorted(data, key=lambda x: x[0]) + logger.info("downloaded %s with length %s.", pair, len(data)) + return data + + def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> None: + """ + Refresh tickers asyncronously and return the result. + """ + logger.debug("Refreshing klines for %d pairs", len(pair_list)) + asyncio.get_event_loop().run_until_complete( + self.async_get_candles_history(pair_list, ticker_interval)) + + async def async_get_candles_history(self, pairs: List[str], + tick_interval: str) -> List[Tuple[str, List]]: + """Download ohlcv history for pair-list asyncronously """ + input_coroutines = [self._async_get_candle_history( + symbol, tick_interval) for symbol in pairs] + tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) + return tickers + + @retrier_async + async def _async_get_candle_history(self, pair: str, tick_interval: str, + since_ms: Optional[int] = None) -> Tuple[str, List]: + try: + # fetch ohlcv asynchronously + logger.debug("fetching %s since %s ...", pair, since_ms) + + # Calculating ticker interval in second + interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 + + # If (last update time) + (interval in second) is greater or equal than now + # that means we don't have to hit the API as there is no new candle + # so we fetch it from local cache + if (not since_ms and + self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= + arrow.utcnow().timestamp): + data = self.klines[pair] + logger.debug("Using cached klines data for %s ...", pair) + else: + data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, + since=since_ms) + + # Because some exchange sort Tickers ASC and other DESC. + # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) + # when GDAX returns a list of tickers DESC (newest first, oldest last) + data = sorted(data, key=lambda x: x[0]) + + # keeping last candle time as last refreshed time of the pair + if data: + self._pairs_last_refresh_time[pair] = data[-1][0] // 1000 + + # keeping candles in cache + self.klines[pair] = data + + logger.debug("done fetching %s ...", pair) + return pair, data + + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching historical candlestick data.' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(f'Could not fetch ticker data. Msg: {e}') + @retrier def get_candle_history(self, pair: str, tick_interval: str, since_ms: Optional[int] = None) -> List[Dict]: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3d3737372..5c943ba3d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -11,6 +11,7 @@ from typing import Any, Callable, Dict, List, Optional import arrow import requests + from cachetools import TTLCache, cached from freqtrade import (DependencyException, OperationalException, @@ -181,6 +182,9 @@ class FreqtradeBot(object): final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list self.config['exchange']['pair_whitelist'] = final_list + # Refreshing candles + self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval) + # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -358,7 +362,7 @@ class FreqtradeBot(object): amount_reserve_percent += self.strategy.stoploss # it should not be more than 50% amount_reserve_percent = max(amount_reserve_percent, 0.5) - return min(min_stake_amounts)/amount_reserve_percent + return min(min_stake_amounts) / amount_reserve_percent def create_trade(self) -> bool: """ @@ -387,11 +391,10 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') - # Pick pair based on buy signals + # running get_signal on historical data fetched + # to find buy signals for _pair in whitelist: - thistory = self.exchange.get_candle_history(_pair, interval) - (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) - + (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) if buy and not sell: bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ get('check_depth_of_market', {}) @@ -402,6 +405,7 @@ class FreqtradeBot(object): else: return False return self.execute_buy(_pair, stake_amount) + return False def _check_depth_of_market_buy(self, pair: str, conf: Dict) -> bool: @@ -581,7 +585,7 @@ class FreqtradeBot(object): (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - ticker = self.exchange.get_candle_history(trade.pair, self.strategy.ticker_interval) + ticker = self.exchange.klines.get(trade.pair) (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, ticker) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 0478dbda6..74c842427 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -205,19 +205,18 @@ def download_backtesting_testdata(datadir: str, timerange: Optional[TimeRange] = None) -> None: """ - Download the latest ticker intervals from the exchange for the pairs passed in parameters + Download the latest ticker intervals from the exchange for the pair passed in parameters The data is downloaded starting from the last correct ticker interval data that - esists in a cache. If timerange starts earlier than the data in the cache, + exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data - :param pairs: list of pairs to download + :param pair: pair to download :param tick_interval: ticker interval :param timerange: range of time to download :return: None """ - path = make_testdata_path(datadir) filepair = pair.replace("/", "_") filename = os.path.join(path, f'{filepair}-{tick_interval}.json') @@ -233,8 +232,11 @@ def download_backtesting_testdata(datadir: str, logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') - new_data = exchange.get_candle_history(pair=pair, tick_interval=tick_interval, - since_ms=since_ms) + # Default since_ms to 30 days if nothing is given + new_data = exchange.get_history(pair=pair, tick_interval=tick_interval, + since_ms=since_ms if since_ms + else + int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9e68318f7..d0b70afc7 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -330,15 +330,15 @@ class Backtesting(object): Run a backtesting end-to-end :return: None """ - data = {} + data: Dict[str, Any] = {} pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') - for pair in pairs: - data[pair] = self.exchange.get_candle_history(pair, self.ticker_interval) + self.exchange.refresh_tickers(pairs, self.ticker_interval) + data = self.exchange.klines else: logger.info('Using local backtesting data (using whitelist in given config) ...') diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index c86b85e62..6afa4161b 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -6,7 +6,7 @@ import logging from abc import ABC, abstractmethod from datetime import datetime from enum import Enum -from typing import Dict, List, NamedTuple, Tuple +from typing import Dict, List, NamedTuple, Optional, Tuple import warnings import arrow @@ -145,7 +145,8 @@ class IStrategy(ABC): return dataframe - def get_signal(self, pair: str, interval: str, ticker_hist: List[Dict]) -> Tuple[bool, bool]: + def get_signal(self, pair: str, interval: str, + ticker_hist: Optional[List[Dict]]) -> Tuple[bool, bool]: """ Calculates current signal based several technical analysis indicators :param pair: pair in format ANT/BTC diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 701eeb776..de720b3d9 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -3,8 +3,9 @@ import logging from datetime import datetime from random import randint -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import Mock, MagicMock, PropertyMock +import arrow import ccxt import pytest @@ -13,6 +14,14 @@ from freqtrade.exchange import API_RETRY_COUNT, Exchange from freqtrade.tests.conftest import get_patched_exchange, log_has +# Source: https://stackoverflow.com/questions/29881236/how-to-mock-asyncio-coroutines +def get_mock_coro(return_value): + async def mock_coro(*args, **kwargs): + return return_value + + return Mock(wraps=mock_coro) + + def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): with pytest.raises(TemporaryError): api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) @@ -27,12 +36,32 @@ def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, * assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 +async def async_ccxt_exception(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): + with pytest.raises(TemporaryError): + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + await getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1 + + with pytest.raises(OperationalException): + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + await getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 + + def test_init(default_conf, mocker, caplog): caplog.set_level(logging.INFO) get_patched_exchange(mocker, default_conf) assert log_has('Instance is running with dry_run enabled', caplog.record_tuples) +def test_destroy(default_conf, mocker, caplog): + caplog.set_level(logging.DEBUG) + get_patched_exchange(mocker, default_conf) + assert log_has('Exchange object destroyed, closing async loop', caplog.record_tuples) + + def test_init_exception(default_conf, mocker): default_conf['exchange']['name'] = 'wrong_exchange_name' @@ -64,6 +93,7 @@ def test_symbol_amount_prec(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) exchange = Exchange(default_conf) amount = 2.34559 @@ -87,6 +117,7 @@ def test_symbol_price_prec(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) exchange = Exchange(default_conf) price = 2.34559 @@ -108,6 +139,7 @@ def test_set_sandbox(default_conf, mocker): type(api_mock).urls = url_mock mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) exchange = Exchange(default_conf) liveurl = exchange._api.urls['api'] @@ -129,6 +161,7 @@ def test_set_sandbox_exception(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'does not provide a sandbox api'): exchange = Exchange(default_conf) @@ -136,6 +169,20 @@ def test_set_sandbox_exception(default_conf, mocker): exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname') +def test__load_async_markets(default_conf, mocker, caplog): + exchange = get_patched_exchange(mocker, default_conf) + exchange._api_async.load_markets = get_mock_coro(None) + exchange._load_async_markets() + assert exchange._api_async.load_markets.call_count == 1 + caplog.set_level(logging.DEBUG) + + exchange._api_async.load_markets = Mock(side_effect=ccxt.BaseError("deadbeef")) + exchange._load_async_markets() + + assert log_has('Could not load async markets. Reason: deadbeef', + caplog.record_tuples) + + def test_validate_pairs(default_conf, mocker): api_mock = MagicMock() api_mock.load_markets = MagicMock(return_value={ @@ -146,6 +193,7 @@ def test_validate_pairs(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) Exchange(default_conf) @@ -154,6 +202,7 @@ def test_validate_pairs_not_available(default_conf, mocker): api_mock.load_markets = MagicMock(return_value={}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'not available'): Exchange(default_conf) @@ -167,6 +216,7 @@ def test_validate_pairs_not_compatible(default_conf, mocker): default_conf['stake_currency'] = 'ETH' mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'not compatible'): Exchange(default_conf) @@ -179,6 +229,7 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): api_mock.load_markets = MagicMock(return_value={}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available at Binance'): Exchange(default_conf) @@ -198,6 +249,7 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): api_mock.name = MagicMock(return_value='binance') mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises( OperationalException, @@ -515,6 +567,160 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=True) +def test_get_history(default_conf, mocker, caplog): + exchange = get_patched_exchange(mocker, default_conf) + tick = [ + [ + arrow.utcnow().timestamp * 1000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + pair = 'ETH/BTC' + + async def mock_candle_hist(pair, tick_interval, since_ms): + return pair, tick + + exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) + # one_call calculation * 1.8 should do 2 calls + since = 5 * 60 * 500 * 1.8 + print(f"since = {since}") + ret = exchange.get_history(pair, "5m", int((arrow.utcnow().timestamp - since) * 1000)) + + assert exchange._async_get_candle_history.call_count == 2 + # Returns twice the above tick + assert len(ret) == 2 + + +def test_refresh_tickers(mocker, default_conf, caplog) -> None: + tick = [ + [ + 1511686200000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + caplog.set_level(logging.DEBUG) + exchange = get_patched_exchange(mocker, default_conf) + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + + pairs = ['IOTA/ETH', 'XRP/ETH'] + # empty dicts + assert not exchange.klines + exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m') + + assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples) + assert exchange.klines + for pair in pairs: + assert exchange.klines[pair] + + +@pytest.mark.asyncio +async def test__async_get_candle_history(default_conf, mocker, caplog): + tick = [ + [ + arrow.utcnow().timestamp * 1000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + caplog.set_level(logging.DEBUG) + exchange = get_patched_exchange(mocker, default_conf) + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + + exchange = Exchange(default_conf) + pair = 'ETH/BTC' + res = await exchange._async_get_candle_history(pair, "5m") + assert type(res) is tuple + assert len(res) == 2 + assert res[0] == pair + assert res[1] == tick + assert exchange._api_async.fetch_ohlcv.call_count == 1 + assert not log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples) + # test caching + res = await exchange._async_get_candle_history(pair, "5m") + assert exchange._api_async.fetch_ohlcv.call_count == 1 + assert log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples) + + # exchange = Exchange(default_conf) + await async_ccxt_exception(mocker, default_conf, MagicMock(), + "_async_get_candle_history", "fetch_ohlcv", + pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + + api_mock = MagicMock() + with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): + api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + await exchange._async_get_candle_history(pair, "5m", + (arrow.utcnow().timestamp - 2000) * 1000) + + +@pytest.mark.asyncio +async def test__async_get_candle_history_empty(default_conf, mocker, caplog): + """ Test empty exchange result """ + tick = [] + + caplog.set_level(logging.DEBUG) + exchange = get_patched_exchange(mocker, default_conf) + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro([]) + + exchange = Exchange(default_conf) + pair = 'ETH/BTC' + res = await exchange._async_get_candle_history(pair, "5m") + assert type(res) is tuple + assert len(res) == 2 + assert res[0] == pair + assert res[1] == tick + assert exchange._api_async.fetch_ohlcv.call_count == 1 + + +@pytest.mark.asyncio +async def test_async_get_candles_history(default_conf, mocker): + tick = [ + [ + 1511686200000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + async def mock_get_candle_hist(pair, tick_interval, since_ms=None): + return (pair, tick) + + exchange = get_patched_exchange(mocker, default_conf) + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + + exchange._async_get_candle_history = Mock(wraps=mock_get_candle_hist) + + pairs = ['ETH/BTC', 'XRP/BTC'] + res = await exchange.async_get_candles_history(pairs, "5m") + assert type(res) is list + assert len(res) == 2 + assert type(res[0]) is tuple + assert res[0][0] == pairs[0] + assert res[0][1] == tick + assert res[1][0] == pairs[1] + assert res[1][1] == tick + assert exchange._async_get_candle_history.call_count == 2 + + def test_get_order_book(default_conf, mocker, order_book_l2): default_conf['exchange']['name'] = 'binance' api_mock = MagicMock() diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 32a5229c0..a17867b3a 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -110,7 +110,7 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals return pairdata -# use for mock freqtrade.exchange.get_candle_history' +# use for mock ccxt.fetch_ohlvc' def _load_pair_as_ticks(pair, tickfreq): ticks = optimize.load_data(None, ticker_interval=tickfreq, pairs=[pair]) ticks = trim_dictlist(ticks, -201) @@ -455,7 +455,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) - mocker.patch('freqtrade.exchange.Exchange.get_candle_history') + mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', @@ -490,7 +490,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) - mocker.patch('freqtrade.exchange.Exchange.get_candle_history') + mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', @@ -733,9 +733,14 @@ def test_backtest_record(default_conf, fee, mocker): def test_backtest_start_live(default_conf, mocker, caplog): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', - new=lambda s, n, i: _load_pair_as_ticks(n, i)) - patch_exchange(mocker) + + async def load_pairs(pair, timeframe, since): + return _load_pair_as_ticks(pair, timeframe) + + api_mock = MagicMock() + api_mock.fetch_ohlcv = load_pairs + + patch_exchange(mocker, api_mock) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock()) mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock()) mocker.patch('freqtrade.configuration.open', mocker.mock_open( @@ -776,9 +781,13 @@ def test_backtest_start_live(default_conf, mocker, caplog): def test_backtest_start_multi_strat(default_conf, mocker, caplog): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', - new=lambda s, n, i: _load_pair_as_ticks(n, i)) - patch_exchange(mocker) + + async def load_pairs(pair, timeframe, since): + return _load_pair_as_ticks(pair, timeframe) + api_mock = MagicMock() + api_mock.fetch_ohlcv = load_pairs + + patch_exchange(mocker, api_mock) backtestmock = MagicMock() mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) gen_table_mock = MagicMock() diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 13f65fbf5..77fa3e3b1 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -53,7 +53,7 @@ def _clean_test_file(file: str) -> None: def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json') _backup_file(file, copy_file=True) optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m') @@ -63,7 +63,7 @@ def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json') _backup_file(file, copy_file=True) @@ -74,7 +74,7 @@ def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') _backup_file(file, copy_file=True) optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) @@ -87,7 +87,7 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog, default_co """ Test load_data() with 1 min ticker """ - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') @@ -118,7 +118,7 @@ def test_testdata_path() -> None: def test_download_pairs(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') @@ -261,7 +261,7 @@ def test_load_cached_data_for_updating(mocker) -> None: def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata', side_effect=BaseException('File Error')) exchange = get_patched_exchange(mocker, default_conf) @@ -279,7 +279,7 @@ def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) # Download a 1 min ticker file @@ -304,7 +304,7 @@ def test_download_backtesting_testdata2(mocker, default_conf) -> None: [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=tick) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='1m') download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='3m') diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 29a6dc1de..fedd355af 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -89,7 +89,6 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog): def test_get_signal_handles_exceptions(mocker, default_conf): - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=MagicMock()) exchange = get_patched_exchange(mocker, default_conf) mocker.patch.object( _STRATEGY, 'analyze_ticker', diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 49c9c4431..5e982f11a 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -43,7 +43,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: :return: None """ freqtrade.strategy.get_signal = lambda e, s, t: value - freqtrade.exchange.get_candle_history = lambda p, i: None + freqtrade.exchange.refresh_tickers = lambda p, i: None def patch_RPCManager(mocker) -> MagicMock: @@ -553,7 +553,6 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), - get_candle_history=MagicMock(return_value=20), get_balance=MagicMock(return_value=20), get_fee=fee, ) diff --git a/requirements.txt b/requirements.txt index e9da93b1d..feeff061f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ numpy==1.15.1 TA-Lib==0.4.17 pytest==3.8.0 pytest-mock==1.10.0 +pytest-asyncio==0.9.0 pytest-cov==2.6.0 tabulate==0.8.2 coinmarketcap==5.0.3 diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 686098f94..27c4c1e1c 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""This script generate json data from bittrex""" +"""This script generate json data""" import json import sys from pathlib import Path @@ -52,9 +52,10 @@ exchange = Exchange({'key': '', 'stake_currency': '', 'dry_run': True, 'exchange': { - 'name': args.exchange, - 'pair_whitelist': [] - } + 'name': args.exchange, + 'pair_whitelist': [], + 'ccxt_rate_limit': False + } }) pairs_not_available = [] diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index f2f2e0c7f..0f0a3d4cb 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -138,7 +138,8 @@ def plot_analyzed_dataframe(args: Namespace) -> None: tickers = {} if args.live: logger.info('Downloading pair.') - tickers[pair] = exchange.get_candle_history(pair, tick_interval) + exchange.refresh_tickers([pair], tick_interval) + tickers[pair] = exchange.klines[pair] else: tickers = optimize.load_data( datadir=_CONF.get("datadir"),