Merge pull request #1101 from mishaker/ccxt-async
use ccxt async for ticker_history download
This commit is contained in:
Matthias
GitHub
commit
179bcf3907
@ -13,7 +13,7 @@ addons:
|
||||
install:
|
||||
- ./install_ta-lib.sh
|
||||
- export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
|
||||
- pip install --upgrade flake8 coveralls pytest-random-order mypy
|
||||
- pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy
|
||||
- pip install -r requirements.txt
|
||||
- pip install -e .
|
||||
jobs:
|
||||
|
||||
@ -1,12 +1,15 @@
|
||||
# pragma pylint: disable=W0603
|
||||
""" Cryptocurrency Exchanges support """
|
||||
import logging
|
||||
import inspect
|
||||
from random import randint
|
||||
from typing import List, Dict, Any, Optional
|
||||
from typing import List, Dict, Tuple, Any, Optional
|
||||
from datetime import datetime
|
||||
from math import floor, ceil
|
||||
|
||||
import asyncio
|
||||
import ccxt
|
||||
import ccxt.async_support as ccxt_async
|
||||
import arrow
|
||||
|
||||
from freqtrade import constants, OperationalException, DependencyException, TemporaryError
|
||||
@ -23,6 +26,24 @@ _EXCHANGE_URLS = {
|
||||
}
|
||||
|
||||
|
||||
def retrier_async(f):
|
||||
async def wrapper(*args, **kwargs):
|
||||
count = kwargs.pop('count', API_RETRY_COUNT)
|
||||
try:
|
||||
return await f(*args, **kwargs)
|
||||
except (TemporaryError, DependencyException) as ex:
|
||||
logger.warning('%s() returned exception: "%s"', f.__name__, ex)
|
||||
if count > 0:
|
||||
count -= 1
|
||||
kwargs.update({'count': count})
|
||||
logger.warning('retrying %s() still for %s times', f.__name__, count)
|
||||
return await wrapper(*args, **kwargs)
|
||||
else:
|
||||
logger.warning('Giving up retrying: %s()', f.__name__)
|
||||
raise ex
|
||||
return wrapper
|
||||
|
||||
|
||||
def retrier(f):
|
||||
def wrapper(*args, **kwargs):
|
||||
count = kwargs.pop('count', API_RETRY_COUNT)
|
||||
@ -45,8 +66,8 @@ class Exchange(object):
|
||||
|
||||
# Current selected exchange
|
||||
_api: ccxt.Exchange = None
|
||||
_api_async: ccxt_async.Exchange = None
|
||||
_conf: Dict = {}
|
||||
_cached_ticker: Dict[str, Any] = {}
|
||||
|
||||
# Holds all open sell orders for dry_run
|
||||
_dry_run_open_orders: Dict[str, Any] = {}
|
||||
@ -60,11 +81,20 @@ class Exchange(object):
|
||||
"""
|
||||
self._conf.update(config)
|
||||
|
||||
self._cached_ticker: Dict[str, Any] = {}
|
||||
|
||||
# Holds last candle refreshed time of each pair
|
||||
self._pairs_last_refresh_time: Dict[str, int] = {}
|
||||
|
||||
# Holds candles
|
||||
self.klines: Dict[str, Any] = {}
|
||||
|
||||
if config['dry_run']:
|
||||
logger.info('Instance is running with dry_run enabled')
|
||||
|
||||
exchange_config = config['exchange']
|
||||
self._api = self._init_ccxt(exchange_config)
|
||||
self._api_async = self._init_ccxt(exchange_config, ccxt_async)
|
||||
|
||||
logger.info('Using Exchange "%s"', self.name)
|
||||
|
||||
@ -75,7 +105,15 @@ class Exchange(object):
|
||||
# Check if timeframe is available
|
||||
self.validate_timeframes(config['ticker_interval'])
|
||||
|
||||
def _init_ccxt(self, exchange_config: dict) -> ccxt.Exchange:
|
||||
def __del__(self):
|
||||
"""
|
||||
Destructor - clean up async stuff
|
||||
"""
|
||||
logger.debug("Exchange object destroyed, closing async loop")
|
||||
if self._api_async and inspect.iscoroutinefunction(self._api_async.close):
|
||||
asyncio.get_event_loop().run_until_complete(self._api_async.close())
|
||||
|
||||
def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt) -> ccxt.Exchange:
|
||||
"""
|
||||
Initialize ccxt with given config and return valid
|
||||
ccxt instance.
|
||||
@ -83,15 +121,15 @@ class Exchange(object):
|
||||
# Find matching class for the given exchange name
|
||||
name = exchange_config['name']
|
||||
|
||||
if name not in ccxt.exchanges:
|
||||
if name not in ccxt_module.exchanges:
|
||||
raise OperationalException(f'Exchange {name} is not supported')
|
||||
try:
|
||||
api = getattr(ccxt, name.lower())({
|
||||
api = getattr(ccxt_module, name.lower())({
|
||||
'apiKey': exchange_config.get('key'),
|
||||
'secret': exchange_config.get('secret'),
|
||||
'password': exchange_config.get('password'),
|
||||
'uid': exchange_config.get('uid', ''),
|
||||
'enableRateLimit': exchange_config.get('ccxt_rate_limit', True),
|
||||
'enableRateLimit': exchange_config.get('ccxt_rate_limit', True)
|
||||
})
|
||||
except (KeyError, AttributeError):
|
||||
raise OperationalException(f'Exchange {name} is not supported')
|
||||
@ -120,6 +158,15 @@ class Exchange(object):
|
||||
"Please check your config.json")
|
||||
raise OperationalException(f'Exchange {name} does not provide a sandbox api')
|
||||
|
||||
def _load_async_markets(self) -> None:
|
||||
try:
|
||||
if self._api_async:
|
||||
asyncio.get_event_loop().run_until_complete(self._api_async.load_markets())
|
||||
|
||||
except ccxt.BaseError as e:
|
||||
logger.warning('Could not load async markets. Reason: %s', e)
|
||||
return
|
||||
|
||||
def validate_pairs(self, pairs: List[str]) -> None:
|
||||
"""
|
||||
Checks if all given pairs are tradable on the current exchange.
|
||||
@ -130,6 +177,7 @@ class Exchange(object):
|
||||
|
||||
try:
|
||||
markets = self._api.load_markets()
|
||||
self._load_async_markets()
|
||||
except ccxt.BaseError as e:
|
||||
logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e)
|
||||
return
|
||||
@ -329,6 +377,102 @@ class Exchange(object):
|
||||
logger.info("returning cached ticker-data for %s", pair)
|
||||
return self._cached_ticker[pair]
|
||||
|
||||
def get_history(self, pair: str, tick_interval: str,
|
||||
since_ms: int) -> List:
|
||||
"""
|
||||
Gets candle history using asyncio and returns the list of candles.
|
||||
Handles all async doing.
|
||||
"""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self._async_get_history(pair=pair, tick_interval=tick_interval,
|
||||
since_ms=since_ms))
|
||||
|
||||
async def _async_get_history(self, pair: str,
|
||||
tick_interval: str,
|
||||
since_ms: int) -> List:
|
||||
# Assume exchange returns 500 candles
|
||||
_LIMIT = 500
|
||||
|
||||
one_call = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 * _LIMIT * 1000
|
||||
logger.debug("one_call: %s", one_call)
|
||||
input_coroutines = [self._async_get_candle_history(
|
||||
pair, tick_interval, since) for since in
|
||||
range(since_ms, arrow.utcnow().timestamp * 1000, one_call)]
|
||||
tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
|
||||
|
||||
# Combine tickers
|
||||
data: List = []
|
||||
for tick in tickers:
|
||||
if tick[0] == pair:
|
||||
data.extend(tick[1])
|
||||
# Sort data again after extending the result - above calls return in "async order" order
|
||||
data = sorted(data, key=lambda x: x[0])
|
||||
logger.info("downloaded %s with length %s.", pair, len(data))
|
||||
return data
|
||||
|
||||
def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> None:
|
||||
"""
|
||||
Refresh tickers asyncronously and return the result.
|
||||
"""
|
||||
logger.debug("Refreshing klines for %d pairs", len(pair_list))
|
||||
asyncio.get_event_loop().run_until_complete(
|
||||
self.async_get_candles_history(pair_list, ticker_interval))
|
||||
|
||||
async def async_get_candles_history(self, pairs: List[str],
|
||||
tick_interval: str) -> List[Tuple[str, List]]:
|
||||
"""Download ohlcv history for pair-list asyncronously """
|
||||
input_coroutines = [self._async_get_candle_history(
|
||||
symbol, tick_interval) for symbol in pairs]
|
||||
tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
|
||||
return tickers
|
||||
|
||||
@retrier_async
|
||||
async def _async_get_candle_history(self, pair: str, tick_interval: str,
|
||||
since_ms: Optional[int] = None) -> Tuple[str, List]:
|
||||
try:
|
||||
# fetch ohlcv asynchronously
|
||||
logger.debug("fetching %s since %s ...", pair, since_ms)
|
||||
|
||||
# Calculating ticker interval in second
|
||||
interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60
|
||||
|
||||
# If (last update time) + (interval in second) is greater or equal than now
|
||||
# that means we don't have to hit the API as there is no new candle
|
||||
# so we fetch it from local cache
|
||||
if (not since_ms and
|
||||
self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >=
|
||||
arrow.utcnow().timestamp):
|
||||
data = self.klines[pair]
|
||||
logger.debug("Using cached klines data for %s ...", pair)
|
||||
else:
|
||||
data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval,
|
||||
since=since_ms)
|
||||
|
||||
# Because some exchange sort Tickers ASC and other DESC.
|
||||
# Ex: Bittrex returns a list of tickers ASC (oldest first, newest last)
|
||||
# when GDAX returns a list of tickers DESC (newest first, oldest last)
|
||||
data = sorted(data, key=lambda x: x[0])
|
||||
|
||||
# keeping last candle time as last refreshed time of the pair
|
||||
if data:
|
||||
self._pairs_last_refresh_time[pair] = data[-1][0] // 1000
|
||||
|
||||
# keeping candles in cache
|
||||
self.klines[pair] = data
|
||||
|
||||
logger.debug("done fetching %s ...", pair)
|
||||
return pair, data
|
||||
|
||||
except ccxt.NotSupported as e:
|
||||
raise OperationalException(
|
||||
f'Exchange {self._api.name} does not support fetching historical candlestick data.'
|
||||
f'Message: {e}')
|
||||
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
|
||||
raise TemporaryError(
|
||||
f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}')
|
||||
except ccxt.BaseError as e:
|
||||
raise OperationalException(f'Could not fetch ticker data. Msg: {e}')
|
||||
|
||||
@retrier
|
||||
def get_candle_history(self, pair: str, tick_interval: str,
|
||||
since_ms: Optional[int] = None) -> List[Dict]:
|
||||
|
||||
@ -11,6 +11,7 @@ from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
import arrow
|
||||
import requests
|
||||
|
||||
from cachetools import TTLCache, cached
|
||||
|
||||
from freqtrade import (DependencyException, OperationalException,
|
||||
@ -181,6 +182,9 @@ class FreqtradeBot(object):
|
||||
final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list
|
||||
self.config['exchange']['pair_whitelist'] = final_list
|
||||
|
||||
# Refreshing candles
|
||||
self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval)
|
||||
|
||||
# Query trades from persistence layer
|
||||
trades = Trade.query.filter(Trade.is_open.is_(True)).all()
|
||||
|
||||
@ -358,7 +362,7 @@ class FreqtradeBot(object):
|
||||
amount_reserve_percent += self.strategy.stoploss
|
||||
# it should not be more than 50%
|
||||
amount_reserve_percent = max(amount_reserve_percent, 0.5)
|
||||
return min(min_stake_amounts)/amount_reserve_percent
|
||||
return min(min_stake_amounts) / amount_reserve_percent
|
||||
|
||||
def create_trade(self) -> bool:
|
||||
"""
|
||||
@ -387,11 +391,10 @@ class FreqtradeBot(object):
|
||||
if not whitelist:
|
||||
raise DependencyException('No currency pairs in whitelist')
|
||||
|
||||
# Pick pair based on buy signals
|
||||
# running get_signal on historical data fetched
|
||||
# to find buy signals
|
||||
for _pair in whitelist:
|
||||
thistory = self.exchange.get_candle_history(_pair, interval)
|
||||
(buy, sell) = self.strategy.get_signal(_pair, interval, thistory)
|
||||
|
||||
(buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair))
|
||||
if buy and not sell:
|
||||
bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\
|
||||
get('check_depth_of_market', {})
|
||||
@ -402,6 +405,7 @@ class FreqtradeBot(object):
|
||||
else:
|
||||
return False
|
||||
return self.execute_buy(_pair, stake_amount)
|
||||
|
||||
return False
|
||||
|
||||
def _check_depth_of_market_buy(self, pair: str, conf: Dict) -> bool:
|
||||
@ -581,7 +585,7 @@ class FreqtradeBot(object):
|
||||
(buy, sell) = (False, False)
|
||||
experimental = self.config.get('experimental', {})
|
||||
if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'):
|
||||
ticker = self.exchange.get_candle_history(trade.pair, self.strategy.ticker_interval)
|
||||
ticker = self.exchange.klines.get(trade.pair)
|
||||
(buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval,
|
||||
ticker)
|
||||
|
||||
|
||||
@ -205,19 +205,18 @@ def download_backtesting_testdata(datadir: str,
|
||||
timerange: Optional[TimeRange] = None) -> None:
|
||||
|
||||
"""
|
||||
Download the latest ticker intervals from the exchange for the pairs passed in parameters
|
||||
Download the latest ticker intervals from the exchange for the pair passed in parameters
|
||||
The data is downloaded starting from the last correct ticker interval data that
|
||||
esists in a cache. If timerange starts earlier than the data in the cache,
|
||||
exists in a cache. If timerange starts earlier than the data in the cache,
|
||||
the full data will be redownloaded
|
||||
|
||||
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
|
||||
:param pairs: list of pairs to download
|
||||
:param pair: pair to download
|
||||
:param tick_interval: ticker interval
|
||||
:param timerange: range of time to download
|
||||
:return: None
|
||||
|
||||
"""
|
||||
|
||||
path = make_testdata_path(datadir)
|
||||
filepair = pair.replace("/", "_")
|
||||
filename = os.path.join(path, f'{filepair}-{tick_interval}.json')
|
||||
@ -233,8 +232,11 @@ def download_backtesting_testdata(datadir: str,
|
||||
logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None')
|
||||
logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None')
|
||||
|
||||
new_data = exchange.get_candle_history(pair=pair, tick_interval=tick_interval,
|
||||
since_ms=since_ms)
|
||||
# Default since_ms to 30 days if nothing is given
|
||||
new_data = exchange.get_history(pair=pair, tick_interval=tick_interval,
|
||||
since_ms=since_ms if since_ms
|
||||
else
|
||||
int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000)
|
||||
data.extend(new_data)
|
||||
|
||||
logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))
|
||||
|
||||
@ -330,15 +330,15 @@ class Backtesting(object):
|
||||
Run a backtesting end-to-end
|
||||
:return: None
|
||||
"""
|
||||
data = {}
|
||||
data: Dict[str, Any] = {}
|
||||
pairs = self.config['exchange']['pair_whitelist']
|
||||
logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
|
||||
logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
|
||||
|
||||
if self.config.get('live'):
|
||||
logger.info('Downloading data for all pairs in whitelist ...')
|
||||
for pair in pairs:
|
||||
data[pair] = self.exchange.get_candle_history(pair, self.ticker_interval)
|
||||
self.exchange.refresh_tickers(pairs, self.ticker_interval)
|
||||
data = self.exchange.klines
|
||||
else:
|
||||
logger.info('Using local backtesting data (using whitelist in given config) ...')
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@ import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Dict, List, NamedTuple, Tuple
|
||||
from typing import Dict, List, NamedTuple, Optional, Tuple
|
||||
import warnings
|
||||
|
||||
import arrow
|
||||
@ -145,7 +145,8 @@ class IStrategy(ABC):
|
||||
|
||||
return dataframe
|
||||
|
||||
def get_signal(self, pair: str, interval: str, ticker_hist: List[Dict]) -> Tuple[bool, bool]:
|
||||
def get_signal(self, pair: str, interval: str,
|
||||
ticker_hist: Optional[List[Dict]]) -> Tuple[bool, bool]:
|
||||
"""
|
||||
Calculates current signal based several technical analysis indicators
|
||||
:param pair: pair in format ANT/BTC
|
||||
|
||||
@ -3,8 +3,9 @@
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from random import randint
|
||||
from unittest.mock import MagicMock, PropertyMock
|
||||
from unittest.mock import Mock, MagicMock, PropertyMock
|
||||
|
||||
import arrow
|
||||
import ccxt
|
||||
import pytest
|
||||
|
||||
@ -13,6 +14,14 @@ from freqtrade.exchange import API_RETRY_COUNT, Exchange
|
||||
from freqtrade.tests.conftest import get_patched_exchange, log_has
|
||||
|
||||
|
||||
# Source: https://stackoverflow.com/questions/29881236/how-to-mock-asyncio-coroutines
|
||||
def get_mock_coro(return_value):
|
||||
async def mock_coro(*args, **kwargs):
|
||||
return return_value
|
||||
|
||||
return Mock(wraps=mock_coro)
|
||||
|
||||
|
||||
def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs):
|
||||
with pytest.raises(TemporaryError):
|
||||
api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError)
|
||||
@ -27,12 +36,32 @@ def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, *
|
||||
assert api_mock.__dict__[mock_ccxt_fun].call_count == 1
|
||||
|
||||
|
||||
async def async_ccxt_exception(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs):
|
||||
with pytest.raises(TemporaryError):
|
||||
api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError)
|
||||
exchange = get_patched_exchange(mocker, default_conf, api_mock)
|
||||
await getattr(exchange, fun)(**kwargs)
|
||||
assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1
|
||||
|
||||
with pytest.raises(OperationalException):
|
||||
api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError)
|
||||
exchange = get_patched_exchange(mocker, default_conf, api_mock)
|
||||
await getattr(exchange, fun)(**kwargs)
|
||||
assert api_mock.__dict__[mock_ccxt_fun].call_count == 1
|
||||
|
||||
|
||||
def test_init(default_conf, mocker, caplog):
|
||||
caplog.set_level(logging.INFO)
|
||||
get_patched_exchange(mocker, default_conf)
|
||||
assert log_has('Instance is running with dry_run enabled', caplog.record_tuples)
|
||||
|
||||
|
||||
def test_destroy(default_conf, mocker, caplog):
|
||||
caplog.set_level(logging.DEBUG)
|
||||
get_patched_exchange(mocker, default_conf)
|
||||
assert log_has('Exchange object destroyed, closing async loop', caplog.record_tuples)
|
||||
|
||||
|
||||
def test_init_exception(default_conf, mocker):
|
||||
default_conf['exchange']['name'] = 'wrong_exchange_name'
|
||||
|
||||
@ -64,6 +93,7 @@ def test_symbol_amount_prec(default_conf, mocker):
|
||||
|
||||
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
|
||||
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
|
||||
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
|
||||
exchange = Exchange(default_conf)
|
||||
|
||||
amount = 2.34559
|
||||
@ -87,6 +117,7 @@ def test_symbol_price_prec(default_conf, mocker):
|
||||
|
||||
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
|
||||
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
|
||||
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
|
||||
exchange = Exchange(default_conf)
|
||||
|
||||
price = 2.34559
|
||||
@ -108,6 +139,7 @@ def test_set_sandbox(default_conf, mocker):
|
||||
type(api_mock).urls = url_mock
|
||||
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
|
||||
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
|
||||
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
|
||||
|
||||
exchange = Exchange(default_conf)
|
||||
liveurl = exchange._api.urls['api']
|
||||
@ -129,6 +161,7 @@ def test_set_sandbox_exception(default_conf, mocker):
|
||||
|
||||
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
|
||||
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
|
||||
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
|
||||
|
||||
with pytest.raises(OperationalException, match=r'does not provide a sandbox api'):
|
||||
exchange = Exchange(default_conf)
|
||||
@ -136,6 +169,20 @@ def test_set_sandbox_exception(default_conf, mocker):
|
||||
exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname')
|
||||
|
||||
|
||||
def test__load_async_markets(default_conf, mocker, caplog):
|
||||
exchange = get_patched_exchange(mocker, default_conf)
|
||||
exchange._api_async.load_markets = get_mock_coro(None)
|
||||
exchange._load_async_markets()
|
||||
assert exchange._api_async.load_markets.call_count == 1
|
||||
caplog.set_level(logging.DEBUG)
|
||||
|
||||
exchange._api_async.load_markets = Mock(side_effect=ccxt.BaseError("deadbeef"))
|
||||
exchange._load_async_markets()
|
||||
|
||||
assert log_has('Could not load async markets. Reason: deadbeef',
|
||||
caplog.record_tuples)
|
||||
|
||||
|
||||
def test_validate_pairs(default_conf, mocker):
|
||||
api_mock = MagicMock()
|
||||
api_mock.load_markets = MagicMock(return_value={
|
||||
@ -146,6 +193,7 @@ def test_validate_pairs(default_conf, mocker):
|
||||
|
||||
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
|
||||
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
|
||||
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
|
||||
Exchange(default_conf)
|
||||
|
||||
|
||||
@ -154,6 +202,7 @@ def test_validate_pairs_not_available(default_conf, mocker):
|
||||
api_mock.load_markets = MagicMock(return_value={})
|
||||
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
|
||||
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
|
||||
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
|
||||
|
||||
with pytest.raises(OperationalException, match=r'not available'):
|
||||
Exchange(default_conf)
|
||||
@ -167,6 +216,7 @@ def test_validate_pairs_not_compatible(default_conf, mocker):
|
||||
default_conf['stake_currency'] = 'ETH'
|
||||
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
|
||||
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
|
||||
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
|
||||
with pytest.raises(OperationalException, match=r'not compatible'):
|
||||
Exchange(default_conf)
|
||||
|
||||
@ -179,6 +229,7 @@ def test_validate_pairs_exception(default_conf, mocker, caplog):
|
||||
api_mock.load_markets = MagicMock(return_value={})
|
||||
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
|
||||
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
|
||||
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
|
||||
|
||||
with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available at Binance'):
|
||||
Exchange(default_conf)
|
||||
@ -198,6 +249,7 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog):
|
||||
api_mock.name = MagicMock(return_value='binance')
|
||||
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
|
||||
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
|
||||
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
|
||||
|
||||
with pytest.raises(
|
||||
OperationalException,
|
||||
@ -515,6 +567,160 @@ def test_get_ticker(default_conf, mocker):
|
||||
exchange.get_ticker(pair='ETH/BTC', refresh=True)
|
||||
|
||||
|
||||
def test_get_history(default_conf, mocker, caplog):
|
||||
exchange = get_patched_exchange(mocker, default_conf)
|
||||
tick = [
|
||||
[
|
||||
arrow.utcnow().timestamp * 1000, # unix timestamp ms
|
||||
1, # open
|
||||
2, # high
|
||||
3, # low
|
||||
4, # close
|
||||
5, # volume (in quote currency)
|
||||
]
|
||||
]
|
||||
pair = 'ETH/BTC'
|
||||
|
||||
async def mock_candle_hist(pair, tick_interval, since_ms):
|
||||
return pair, tick
|
||||
|
||||
exchange._async_get_candle_history = Mock(wraps=mock_candle_hist)
|
||||
# one_call calculation * 1.8 should do 2 calls
|
||||
since = 5 * 60 * 500 * 1.8
|
||||
print(f"since = {since}")
|
||||
ret = exchange.get_history(pair, "5m", int((arrow.utcnow().timestamp - since) * 1000))
|
||||
|
||||
assert exchange._async_get_candle_history.call_count == 2
|
||||
# Returns twice the above tick
|
||||
assert len(ret) == 2
|
||||
|
||||
|
||||
def test_refresh_tickers(mocker, default_conf, caplog) -> None:
|
||||
tick = [
|
||||
[
|
||||
1511686200000, # unix timestamp ms
|
||||
1, # open
|
||||
2, # high
|
||||
3, # low
|
||||
4, # close
|
||||
5, # volume (in quote currency)
|
||||
]
|
||||
]
|
||||
|
||||
caplog.set_level(logging.DEBUG)
|
||||
exchange = get_patched_exchange(mocker, default_conf)
|
||||
exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
|
||||
|
||||
pairs = ['IOTA/ETH', 'XRP/ETH']
|
||||
# empty dicts
|
||||
assert not exchange.klines
|
||||
exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m')
|
||||
|
||||
assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples)
|
||||
assert exchange.klines
|
||||
for pair in pairs:
|
||||
assert exchange.klines[pair]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test__async_get_candle_history(default_conf, mocker, caplog):
|
||||
tick = [
|
||||
[
|
||||
arrow.utcnow().timestamp * 1000, # unix timestamp ms
|
||||
1, # open
|
||||
2, # high
|
||||
3, # low
|
||||
4, # close
|
||||
5, # volume (in quote currency)
|
||||
]
|
||||
]
|
||||
|
||||
caplog.set_level(logging.DEBUG)
|
||||
exchange = get_patched_exchange(mocker, default_conf)
|
||||
# Monkey-patch async function
|
||||
exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
|
||||
|
||||
exchange = Exchange(default_conf)
|
||||
pair = 'ETH/BTC'
|
||||
res = await exchange._async_get_candle_history(pair, "5m")
|
||||
assert type(res) is tuple
|
||||
assert len(res) == 2
|
||||
assert res[0] == pair
|
||||
assert res[1] == tick
|
||||
assert exchange._api_async.fetch_ohlcv.call_count == 1
|
||||
assert not log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples)
|
||||
# test caching
|
||||
res = await exchange._async_get_candle_history(pair, "5m")
|
||||
assert exchange._api_async.fetch_ohlcv.call_count == 1
|
||||
assert log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples)
|
||||
|
||||
# exchange = Exchange(default_conf)
|
||||
await async_ccxt_exception(mocker, default_conf, MagicMock(),
|
||||
"_async_get_candle_history", "fetch_ohlcv",
|
||||
pair='ABCD/BTC', tick_interval=default_conf['ticker_interval'])
|
||||
|
||||
api_mock = MagicMock()
|
||||
with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'):
|
||||
api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError)
|
||||
exchange = get_patched_exchange(mocker, default_conf, api_mock)
|
||||
await exchange._async_get_candle_history(pair, "5m",
|
||||
(arrow.utcnow().timestamp - 2000) * 1000)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test__async_get_candle_history_empty(default_conf, mocker, caplog):
|
||||
""" Test empty exchange result """
|
||||
tick = []
|
||||
|
||||
caplog.set_level(logging.DEBUG)
|
||||
exchange = get_patched_exchange(mocker, default_conf)
|
||||
# Monkey-patch async function
|
||||
exchange._api_async.fetch_ohlcv = get_mock_coro([])
|
||||
|
||||
exchange = Exchange(default_conf)
|
||||
pair = 'ETH/BTC'
|
||||
res = await exchange._async_get_candle_history(pair, "5m")
|
||||
assert type(res) is tuple
|
||||
assert len(res) == 2
|
||||
assert res[0] == pair
|
||||
assert res[1] == tick
|
||||
assert exchange._api_async.fetch_ohlcv.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_candles_history(default_conf, mocker):
|
||||
tick = [
|
||||
[
|
||||
1511686200000, # unix timestamp ms
|
||||
1, # open
|
||||
2, # high
|
||||
3, # low
|
||||
4, # close
|
||||
5, # volume (in quote currency)
|
||||
]
|
||||
]
|
||||
|
||||
async def mock_get_candle_hist(pair, tick_interval, since_ms=None):
|
||||
return (pair, tick)
|
||||
|
||||
exchange = get_patched_exchange(mocker, default_conf)
|
||||
# Monkey-patch async function
|
||||
exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
|
||||
|
||||
exchange._async_get_candle_history = Mock(wraps=mock_get_candle_hist)
|
||||
|
||||
pairs = ['ETH/BTC', 'XRP/BTC']
|
||||
res = await exchange.async_get_candles_history(pairs, "5m")
|
||||
assert type(res) is list
|
||||
assert len(res) == 2
|
||||
assert type(res[0]) is tuple
|
||||
assert res[0][0] == pairs[0]
|
||||
assert res[0][1] == tick
|
||||
assert res[1][0] == pairs[1]
|
||||
assert res[1][1] == tick
|
||||
assert exchange._async_get_candle_history.call_count == 2
|
||||
|
||||
|
||||
def test_get_order_book(default_conf, mocker, order_book_l2):
|
||||
default_conf['exchange']['name'] = 'binance'
|
||||
api_mock = MagicMock()
|
||||
|
||||
@ -110,7 +110,7 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals
|
||||
return pairdata
|
||||
|
||||
|
||||
# use for mock freqtrade.exchange.get_candle_history'
|
||||
# use for mock ccxt.fetch_ohlvc'
|
||||
def _load_pair_as_ticks(pair, tickfreq):
|
||||
ticks = optimize.load_data(None, ticker_interval=tickfreq, pairs=[pair])
|
||||
ticks = trim_dictlist(ticks, -201)
|
||||
@ -455,7 +455,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None:
|
||||
return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
|
||||
|
||||
mocker.patch('freqtrade.optimize.load_data', mocked_load_data)
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_candle_history')
|
||||
mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock())
|
||||
patch_exchange(mocker)
|
||||
mocker.patch.multiple(
|
||||
'freqtrade.optimize.backtesting.Backtesting',
|
||||
@ -490,7 +490,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None:
|
||||
return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
|
||||
|
||||
mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={}))
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_candle_history')
|
||||
mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock())
|
||||
patch_exchange(mocker)
|
||||
mocker.patch.multiple(
|
||||
'freqtrade.optimize.backtesting.Backtesting',
|
||||
@ -733,9 +733,14 @@ def test_backtest_record(default_conf, fee, mocker):
|
||||
|
||||
def test_backtest_start_live(default_conf, mocker, caplog):
|
||||
default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_candle_history',
|
||||
new=lambda s, n, i: _load_pair_as_ticks(n, i))
|
||||
patch_exchange(mocker)
|
||||
|
||||
async def load_pairs(pair, timeframe, since):
|
||||
return _load_pair_as_ticks(pair, timeframe)
|
||||
|
||||
api_mock = MagicMock()
|
||||
api_mock.fetch_ohlcv = load_pairs
|
||||
|
||||
patch_exchange(mocker, api_mock)
|
||||
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
|
||||
mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock())
|
||||
mocker.patch('freqtrade.configuration.open', mocker.mock_open(
|
||||
@ -776,9 +781,13 @@ def test_backtest_start_live(default_conf, mocker, caplog):
|
||||
|
||||
def test_backtest_start_multi_strat(default_conf, mocker, caplog):
|
||||
default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_candle_history',
|
||||
new=lambda s, n, i: _load_pair_as_ticks(n, i))
|
||||
patch_exchange(mocker)
|
||||
|
||||
async def load_pairs(pair, timeframe, since):
|
||||
return _load_pair_as_ticks(pair, timeframe)
|
||||
api_mock = MagicMock()
|
||||
api_mock.fetch_ohlcv = load_pairs
|
||||
|
||||
patch_exchange(mocker, api_mock)
|
||||
backtestmock = MagicMock()
|
||||
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
|
||||
gen_table_mock = MagicMock()
|
||||
|
||||
@ -53,7 +53,7 @@ def _clean_test_file(file: str) -> None:
|
||||
|
||||
|
||||
def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None:
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
|
||||
file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json')
|
||||
_backup_file(file, copy_file=True)
|
||||
optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m')
|
||||
@ -63,7 +63,7 @@ def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) ->
|
||||
|
||||
|
||||
def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None:
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
|
||||
|
||||
file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json')
|
||||
_backup_file(file, copy_file=True)
|
||||
@ -74,7 +74,7 @@ def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) ->
|
||||
|
||||
|
||||
def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None:
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
|
||||
file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json')
|
||||
_backup_file(file, copy_file=True)
|
||||
optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC'])
|
||||
@ -87,7 +87,7 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog, default_co
|
||||
"""
|
||||
Test load_data() with 1 min ticker
|
||||
"""
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
|
||||
exchange = get_patched_exchange(mocker, default_conf)
|
||||
file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
|
||||
|
||||
@ -118,7 +118,7 @@ def test_testdata_path() -> None:
|
||||
|
||||
|
||||
def test_download_pairs(ticker_history, mocker, default_conf) -> None:
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
|
||||
exchange = get_patched_exchange(mocker, default_conf)
|
||||
file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
|
||||
file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json')
|
||||
@ -261,7 +261,7 @@ def test_load_cached_data_for_updating(mocker) -> None:
|
||||
|
||||
|
||||
def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) -> None:
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
|
||||
mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata',
|
||||
side_effect=BaseException('File Error'))
|
||||
exchange = get_patched_exchange(mocker, default_conf)
|
||||
@ -279,7 +279,7 @@ def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf)
|
||||
|
||||
|
||||
def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None:
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
|
||||
exchange = get_patched_exchange(mocker, default_conf)
|
||||
|
||||
# Download a 1 min ticker file
|
||||
@ -304,7 +304,7 @@ def test_download_backtesting_testdata2(mocker, default_conf) -> None:
|
||||
[1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199]
|
||||
]
|
||||
json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None)
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=tick)
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick)
|
||||
exchange = get_patched_exchange(mocker, default_conf)
|
||||
download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='1m')
|
||||
download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='3m')
|
||||
|
||||
@ -89,7 +89,6 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog):
|
||||
|
||||
|
||||
def test_get_signal_handles_exceptions(mocker, default_conf):
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=MagicMock())
|
||||
exchange = get_patched_exchange(mocker, default_conf)
|
||||
mocker.patch.object(
|
||||
_STRATEGY, 'analyze_ticker',
|
||||
|
||||
@ -43,7 +43,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None:
|
||||
:return: None
|
||||
"""
|
||||
freqtrade.strategy.get_signal = lambda e, s, t: value
|
||||
freqtrade.exchange.get_candle_history = lambda p, i: None
|
||||
freqtrade.exchange.refresh_tickers = lambda p, i: None
|
||||
|
||||
|
||||
def patch_RPCManager(mocker) -> MagicMock:
|
||||
@ -553,7 +553,6 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None:
|
||||
mocker.patch.multiple(
|
||||
'freqtrade.exchange.Exchange',
|
||||
validate_pairs=MagicMock(),
|
||||
get_candle_history=MagicMock(return_value=20),
|
||||
get_balance=MagicMock(return_value=20),
|
||||
get_fee=fee,
|
||||
)
|
||||
|
||||
@ -14,6 +14,7 @@ numpy==1.15.1
|
||||
TA-Lib==0.4.17
|
||||
pytest==3.8.0
|
||||
pytest-mock==1.10.0
|
||||
pytest-asyncio==0.9.0
|
||||
pytest-cov==2.6.0
|
||||
tabulate==0.8.2
|
||||
coinmarketcap==5.0.3
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""This script generate json data from bittrex"""
|
||||
"""This script generate json data"""
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@ -53,7 +53,8 @@ exchange = Exchange({'key': '',
|
||||
'dry_run': True,
|
||||
'exchange': {
|
||||
'name': args.exchange,
|
||||
'pair_whitelist': []
|
||||
'pair_whitelist': [],
|
||||
'ccxt_rate_limit': False
|
||||
}
|
||||
})
|
||||
pairs_not_available = []
|
||||
|
||||
@ -138,7 +138,8 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
|
||||
tickers = {}
|
||||
if args.live:
|
||||
logger.info('Downloading pair.')
|
||||
tickers[pair] = exchange.get_candle_history(pair, tick_interval)
|
||||
exchange.refresh_tickers([pair], tick_interval)
|
||||
tickers[pair] = exchange.klines[pair]
|
||||
else:
|
||||
tickers = optimize.load_data(
|
||||
datadir=_CONF.get("datadir"),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user