diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 449a8270d..de1310751 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -54,7 +54,7 @@ class Exchange(object): _cached_ticker: Dict[str, Any] = {} # Holds last candle refreshed time of each pair - _pairs_last_refreshed_time = {} + _pairs_last_refresh_time: Dict[str, int] = {} # Holds candles _cached_klines: Dict[str, Any] = {} @@ -132,6 +132,15 @@ class Exchange(object): "Please check your config.json") raise OperationalException(f'Exchange {name} does not provide a sandbox api') + def _load_async_markets(self) -> None: + try: + if self._api_async: + asyncio.get_event_loop().run_until_complete(self._api_async.load_markets()) + + except ccxt.BaseError as e: + logger.warning('Could not load async markets. Reason: %s', e) + return + def validate_pairs(self, pairs: List[str]) -> None: """ Checks if all given pairs are tradable on the current exchange. @@ -142,6 +151,7 @@ class Exchange(object): try: markets = self._api.load_markets() + self._load_async_markets() except ccxt.BaseError as e: logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e) return @@ -341,11 +351,43 @@ class Exchange(object): logger.info("returning cached ticker-data for %s", pair) return self._cached_ticker[pair] - async def async_get_candles_history(self, pairs, tick_interval) -> List[Tuple[str, List]]: + def get_history(self, pair: str, tick_interval: str, + since_ms: int) -> List: + """ + Gets candle history using asyncio and returns the list of candles. + Handles all async doing. + """ + return asyncio.get_event_loop().run_until_complete( + self._async_get_history(pair=pair, tick_interval=tick_interval, + since_ms=since_ms)) + + async def _async_get_history(self, pair: str, + tick_interval: str, + since_ms: int) -> List: + # Assume exchange returns 500 candles + _LIMIT = 500 + + one_call = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 * _LIMIT * 1000 + logger.debug("one_call: %s", one_call) + input_coroutines = [self.async_get_candle_history( + pair, tick_interval, since) for since in + range(since_ms, int(time.time() * 1000), one_call)] + tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) + + # Combine tickers + data: List = [] + for tick in tickers: + if tick[0] == pair: + data.extend(tick[1]) + logger.info("downloaded %s with length %s.", pair, len(data)) + return data + + async def async_get_candles_history(self, pairs: List[str], + tick_interval: str) -> List[Tuple[str, List]]: # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ? # loop = asyncio.new_event_loop() # asyncio.set_event_loop(loop) - await self._api_async.load_markets() + # await self._api_async.load_markets() input_coroutines = [self.async_get_candle_history( symbol, tick_interval) for symbol in pairs] tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) @@ -356,21 +398,30 @@ class Exchange(object): since_ms: Optional[int] = None) -> Tuple[str, List]: try: # fetch ohlcv asynchronously - logger.debug("fetching %s ...", pair) + logger.debug("fetching %s since %s ...", pair, since_ms) # Calculating ticker interval in second - interval_in_seconds = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 + interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 - # If (last update time) + (interval in second) + (1 second) is greater than now + # If (last update time) + (interval in second) is greater or equal than now # that means we don't have to hit the API as there is no new candle # so we fetch it from local cache - if self._pairs_last_refreshed_time.get(pair, 0) + interval_in_seconds + 1 > round(time.time()): + if (not since_ms and + self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= + int(time.time())): data = self._cached_klines[pair] - else: - data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) + logger.debug("Using cached klines data for %s ...", pair) + else: + data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, + since=since_ms) + + # Because some exchange sort Tickers ASC and other DESC. + # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) + # when GDAX returns a list of tickers DESC (newest first, oldest last) + data = sorted(data, key=lambda x: x[0]) # keeping last candle time as last refreshed time of the pair - self._pairs_last_refreshed_time[pair] = data[-1][0] / 1000 + self._pairs_last_refresh_time[pair] = data[-1][0] // 1000 # keeping candles in cache self._cached_klines[pair] = data diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 8d5350fe5..4332f84a4 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -191,19 +191,18 @@ def download_backtesting_testdata(datadir: str, timerange: Optional[TimeRange] = None) -> None: """ - Download the latest ticker intervals from the exchange for the pairs passed in parameters + Download the latest ticker intervals from the exchange for the pair passed in parameters The data is downloaded starting from the last correct ticker interval data that - esists in a cache. If timerange starts earlier than the data in the cache, + exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data - :param pairs: list of pairs to download + :param pair: pair to download :param tick_interval: ticker interval :param timerange: range of time to download :return: None """ - path = make_testdata_path(datadir) filepair = pair.replace("/", "_") filename = os.path.join(path, f'{filepair}-{tick_interval}.json') @@ -219,8 +218,11 @@ def download_backtesting_testdata(datadir: str, logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') - new_data = exchange.get_candle_history(pair=pair, tick_interval=tick_interval, - since_ms=since_ms) + # Default since_ms to 30 days if nothing is given + new_data = exchange.get_history(pair=pair, tick_interval=tick_interval, + since_ms=since_ms if since_ms + else + int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 3a7573f70..8fa7a6fec 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -13,6 +13,10 @@ from freqtrade.exchange import API_RETRY_COUNT, Exchange from freqtrade.tests.conftest import get_patched_exchange, log_has +async def async_load_markets(): + return {} + + def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): with pytest.raises(TemporaryError): api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError) @@ -78,6 +82,7 @@ def test_symbol_amount_prec(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) exchange = Exchange(default_conf) amount = 2.34559 @@ -101,6 +106,7 @@ def test_symbol_price_prec(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) exchange = Exchange(default_conf) price = 2.34559 @@ -122,6 +128,7 @@ def test_set_sandbox(default_conf, mocker): type(api_mock).urls = url_mock mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) exchange = Exchange(default_conf) liveurl = exchange._api.urls['api'] @@ -143,6 +150,7 @@ def test_set_sandbox_exception(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'does not provide a sandbox api'): exchange = Exchange(default_conf) @@ -160,6 +168,7 @@ def test_validate_pairs(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) Exchange(default_conf) @@ -168,6 +177,7 @@ def test_validate_pairs_not_available(default_conf, mocker): api_mock.load_markets = MagicMock(return_value={}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'not available'): Exchange(default_conf) @@ -181,6 +191,7 @@ def test_validate_pairs_not_compatible(default_conf, mocker): default_conf['stake_currency'] = 'ETH' mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'not compatible'): Exchange(default_conf) @@ -193,6 +204,7 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): api_mock.load_markets = MagicMock(return_value={}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available at Binance'): Exchange(default_conf) @@ -212,6 +224,7 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog): api_mock.name = MagicMock(return_value='binance') mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) with pytest.raises( OperationalException, @@ -588,7 +601,8 @@ async def test_async_get_candles_history(default_conf, mocker): # Monkey-patch async function exchange._api_async.fetch_ohlcv = async_fetch_ohlcv - exchange = Exchange(default_conf) + exchange._api_async.load_markets = async_load_markets + pairs = ['ETH/BTC', 'XRP/BTC'] res = await exchange.async_get_candles_history(pairs, "5m") assert type(res) is list diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 13f65fbf5..77fa3e3b1 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -53,7 +53,7 @@ def _clean_test_file(file: str) -> None: def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json') _backup_file(file, copy_file=True) optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m') @@ -63,7 +63,7 @@ def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json') _backup_file(file, copy_file=True) @@ -74,7 +74,7 @@ def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') _backup_file(file, copy_file=True) optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) @@ -87,7 +87,7 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog, default_co """ Test load_data() with 1 min ticker """ - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') @@ -118,7 +118,7 @@ def test_testdata_path() -> None: def test_download_pairs(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') @@ -261,7 +261,7 @@ def test_load_cached_data_for_updating(mocker) -> None: def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata', side_effect=BaseException('File Error')) exchange = get_patched_exchange(mocker, default_conf) @@ -279,7 +279,7 @@ def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) # Download a 1 min ticker file @@ -304,7 +304,7 @@ def test_download_backtesting_testdata2(mocker, default_conf) -> None: [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) - mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=tick) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='1m') download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='3m') diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 686098f94..27c4c1e1c 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""This script generate json data from bittrex""" +"""This script generate json data""" import json import sys from pathlib import Path @@ -52,9 +52,10 @@ exchange = Exchange({'key': '', 'stake_currency': '', 'dry_run': True, 'exchange': { - 'name': args.exchange, - 'pair_whitelist': [] - } + 'name': args.exchange, + 'pair_whitelist': [], + 'ccxt_rate_limit': False + } }) pairs_not_available = []