Merge pull request #1101 from mishaker/ccxt-async

use ccxt async for ticker_history download
This commit is contained in:
Matthias 2018-09-09 08:39:57 +02:00 committed by GitHub
commit 179bcf3907
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 418 additions and 51 deletions

View File

@ -13,7 +13,7 @@ addons:
install:
- ./install_ta-lib.sh
- export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
- pip install --upgrade flake8 coveralls pytest-random-order mypy
- pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy
- pip install -r requirements.txt
- pip install -e .
jobs:

View File

@ -1,12 +1,15 @@
# pragma pylint: disable=W0603
""" Cryptocurrency Exchanges support """
import logging
import inspect
from random import randint
from typing import List, Dict, Any, Optional
from typing import List, Dict, Tuple, Any, Optional
from datetime import datetime
from math import floor, ceil
import asyncio
import ccxt
import ccxt.async_support as ccxt_async
import arrow
from freqtrade import constants, OperationalException, DependencyException, TemporaryError
@ -23,6 +26,24 @@ _EXCHANGE_URLS = {
}
def retrier_async(f):
async def wrapper(*args, **kwargs):
count = kwargs.pop('count', API_RETRY_COUNT)
try:
return await f(*args, **kwargs)
except (TemporaryError, DependencyException) as ex:
logger.warning('%s() returned exception: "%s"', f.__name__, ex)
if count > 0:
count -= 1
kwargs.update({'count': count})
logger.warning('retrying %s() still for %s times', f.__name__, count)
return await wrapper(*args, **kwargs)
else:
logger.warning('Giving up retrying: %s()', f.__name__)
raise ex
return wrapper
def retrier(f):
def wrapper(*args, **kwargs):
count = kwargs.pop('count', API_RETRY_COUNT)
@ -45,8 +66,8 @@ class Exchange(object):
# Current selected exchange
_api: ccxt.Exchange = None
_api_async: ccxt_async.Exchange = None
_conf: Dict = {}
_cached_ticker: Dict[str, Any] = {}
# Holds all open sell orders for dry_run
_dry_run_open_orders: Dict[str, Any] = {}
@ -60,11 +81,20 @@ class Exchange(object):
"""
self._conf.update(config)
self._cached_ticker: Dict[str, Any] = {}
# Holds last candle refreshed time of each pair
self._pairs_last_refresh_time: Dict[str, int] = {}
# Holds candles
self.klines: Dict[str, Any] = {}
if config['dry_run']:
logger.info('Instance is running with dry_run enabled')
exchange_config = config['exchange']
self._api = self._init_ccxt(exchange_config)
self._api_async = self._init_ccxt(exchange_config, ccxt_async)
logger.info('Using Exchange "%s"', self.name)
@ -75,7 +105,15 @@ class Exchange(object):
# Check if timeframe is available
self.validate_timeframes(config['ticker_interval'])
def _init_ccxt(self, exchange_config: dict) -> ccxt.Exchange:
def __del__(self):
"""
Destructor - clean up async stuff
"""
logger.debug("Exchange object destroyed, closing async loop")
if self._api_async and inspect.iscoroutinefunction(self._api_async.close):
asyncio.get_event_loop().run_until_complete(self._api_async.close())
def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt) -> ccxt.Exchange:
"""
Initialize ccxt with given config and return valid
ccxt instance.
@ -83,15 +121,15 @@ class Exchange(object):
# Find matching class for the given exchange name
name = exchange_config['name']
if name not in ccxt.exchanges:
if name not in ccxt_module.exchanges:
raise OperationalException(f'Exchange {name} is not supported')
try:
api = getattr(ccxt, name.lower())({
api = getattr(ccxt_module, name.lower())({
'apiKey': exchange_config.get('key'),
'secret': exchange_config.get('secret'),
'password': exchange_config.get('password'),
'uid': exchange_config.get('uid', ''),
'enableRateLimit': exchange_config.get('ccxt_rate_limit', True),
'enableRateLimit': exchange_config.get('ccxt_rate_limit', True)
})
except (KeyError, AttributeError):
raise OperationalException(f'Exchange {name} is not supported')
@ -120,6 +158,15 @@ class Exchange(object):
"Please check your config.json")
raise OperationalException(f'Exchange {name} does not provide a sandbox api')
def _load_async_markets(self) -> None:
try:
if self._api_async:
asyncio.get_event_loop().run_until_complete(self._api_async.load_markets())
except ccxt.BaseError as e:
logger.warning('Could not load async markets. Reason: %s', e)
return
def validate_pairs(self, pairs: List[str]) -> None:
"""
Checks if all given pairs are tradable on the current exchange.
@ -130,6 +177,7 @@ class Exchange(object):
try:
markets = self._api.load_markets()
self._load_async_markets()
except ccxt.BaseError as e:
logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e)
return
@ -329,6 +377,102 @@ class Exchange(object):
logger.info("returning cached ticker-data for %s", pair)
return self._cached_ticker[pair]
def get_history(self, pair: str, tick_interval: str,
since_ms: int) -> List:
"""
Gets candle history using asyncio and returns the list of candles.
Handles all async doing.
"""
return asyncio.get_event_loop().run_until_complete(
self._async_get_history(pair=pair, tick_interval=tick_interval,
since_ms=since_ms))
async def _async_get_history(self, pair: str,
tick_interval: str,
since_ms: int) -> List:
# Assume exchange returns 500 candles
_LIMIT = 500
one_call = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 * _LIMIT * 1000
logger.debug("one_call: %s", one_call)
input_coroutines = [self._async_get_candle_history(
pair, tick_interval, since) for since in
range(since_ms, arrow.utcnow().timestamp * 1000, one_call)]
tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
# Combine tickers
data: List = []
for tick in tickers:
if tick[0] == pair:
data.extend(tick[1])
# Sort data again after extending the result - above calls return in "async order" order
data = sorted(data, key=lambda x: x[0])
logger.info("downloaded %s with length %s.", pair, len(data))
return data
def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> None:
"""
Refresh tickers asyncronously and return the result.
"""
logger.debug("Refreshing klines for %d pairs", len(pair_list))
asyncio.get_event_loop().run_until_complete(
self.async_get_candles_history(pair_list, ticker_interval))
async def async_get_candles_history(self, pairs: List[str],
tick_interval: str) -> List[Tuple[str, List]]:
"""Download ohlcv history for pair-list asyncronously """
input_coroutines = [self._async_get_candle_history(
symbol, tick_interval) for symbol in pairs]
tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
return tickers
@retrier_async
async def _async_get_candle_history(self, pair: str, tick_interval: str,
since_ms: Optional[int] = None) -> Tuple[str, List]:
try:
# fetch ohlcv asynchronously
logger.debug("fetching %s since %s ...", pair, since_ms)
# Calculating ticker interval in second
interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60
# If (last update time) + (interval in second) is greater or equal than now
# that means we don't have to hit the API as there is no new candle
# so we fetch it from local cache
if (not since_ms and
self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >=
arrow.utcnow().timestamp):
data = self.klines[pair]
logger.debug("Using cached klines data for %s ...", pair)
else:
data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval,
since=since_ms)
# Because some exchange sort Tickers ASC and other DESC.
# Ex: Bittrex returns a list of tickers ASC (oldest first, newest last)
# when GDAX returns a list of tickers DESC (newest first, oldest last)
data = sorted(data, key=lambda x: x[0])
# keeping last candle time as last refreshed time of the pair
if data:
self._pairs_last_refresh_time[pair] = data[-1][0] // 1000
# keeping candles in cache
self.klines[pair] = data
logger.debug("done fetching %s ...", pair)
return pair, data
except ccxt.NotSupported as e:
raise OperationalException(
f'Exchange {self._api.name} does not support fetching historical candlestick data.'
f'Message: {e}')
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(
f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}')
except ccxt.BaseError as e:
raise OperationalException(f'Could not fetch ticker data. Msg: {e}')
@retrier
def get_candle_history(self, pair: str, tick_interval: str,
since_ms: Optional[int] = None) -> List[Dict]:

View File

@ -11,6 +11,7 @@ from typing import Any, Callable, Dict, List, Optional
import arrow
import requests
from cachetools import TTLCache, cached
from freqtrade import (DependencyException, OperationalException,
@ -181,6 +182,9 @@ class FreqtradeBot(object):
final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list
self.config['exchange']['pair_whitelist'] = final_list
# Refreshing candles
self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval)
# Query trades from persistence layer
trades = Trade.query.filter(Trade.is_open.is_(True)).all()
@ -358,7 +362,7 @@ class FreqtradeBot(object):
amount_reserve_percent += self.strategy.stoploss
# it should not be more than 50%
amount_reserve_percent = max(amount_reserve_percent, 0.5)
return min(min_stake_amounts)/amount_reserve_percent
return min(min_stake_amounts) / amount_reserve_percent
def create_trade(self) -> bool:
"""
@ -387,11 +391,10 @@ class FreqtradeBot(object):
if not whitelist:
raise DependencyException('No currency pairs in whitelist')
# Pick pair based on buy signals
# running get_signal on historical data fetched
# to find buy signals
for _pair in whitelist:
thistory = self.exchange.get_candle_history(_pair, interval)
(buy, sell) = self.strategy.get_signal(_pair, interval, thistory)
(buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair))
if buy and not sell:
bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\
get('check_depth_of_market', {})
@ -402,6 +405,7 @@ class FreqtradeBot(object):
else:
return False
return self.execute_buy(_pair, stake_amount)
return False
def _check_depth_of_market_buy(self, pair: str, conf: Dict) -> bool:
@ -581,7 +585,7 @@ class FreqtradeBot(object):
(buy, sell) = (False, False)
experimental = self.config.get('experimental', {})
if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'):
ticker = self.exchange.get_candle_history(trade.pair, self.strategy.ticker_interval)
ticker = self.exchange.klines.get(trade.pair)
(buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval,
ticker)

View File

@ -205,19 +205,18 @@ def download_backtesting_testdata(datadir: str,
timerange: Optional[TimeRange] = None) -> None:
"""
Download the latest ticker intervals from the exchange for the pairs passed in parameters
Download the latest ticker intervals from the exchange for the pair passed in parameters
The data is downloaded starting from the last correct ticker interval data that
esists in a cache. If timerange starts earlier than the data in the cache,
exists in a cache. If timerange starts earlier than the data in the cache,
the full data will be redownloaded
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
:param pairs: list of pairs to download
:param pair: pair to download
:param tick_interval: ticker interval
:param timerange: range of time to download
:return: None
"""
path = make_testdata_path(datadir)
filepair = pair.replace("/", "_")
filename = os.path.join(path, f'{filepair}-{tick_interval}.json')
@ -233,8 +232,11 @@ def download_backtesting_testdata(datadir: str,
logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None')
logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None')
new_data = exchange.get_candle_history(pair=pair, tick_interval=tick_interval,
since_ms=since_ms)
# Default since_ms to 30 days if nothing is given
new_data = exchange.get_history(pair=pair, tick_interval=tick_interval,
since_ms=since_ms if since_ms
else
int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000)
data.extend(new_data)
logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))

View File

@ -330,15 +330,15 @@ class Backtesting(object):
Run a backtesting end-to-end
:return: None
"""
data = {}
data: Dict[str, Any] = {}
pairs = self.config['exchange']['pair_whitelist']
logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
if self.config.get('live'):
logger.info('Downloading data for all pairs in whitelist ...')
for pair in pairs:
data[pair] = self.exchange.get_candle_history(pair, self.ticker_interval)
self.exchange.refresh_tickers(pairs, self.ticker_interval)
data = self.exchange.klines
else:
logger.info('Using local backtesting data (using whitelist in given config) ...')

View File

@ -6,7 +6,7 @@ import logging
from abc import ABC, abstractmethod
from datetime import datetime
from enum import Enum
from typing import Dict, List, NamedTuple, Tuple
from typing import Dict, List, NamedTuple, Optional, Tuple
import warnings
import arrow
@ -145,7 +145,8 @@ class IStrategy(ABC):
return dataframe
def get_signal(self, pair: str, interval: str, ticker_hist: List[Dict]) -> Tuple[bool, bool]:
def get_signal(self, pair: str, interval: str,
ticker_hist: Optional[List[Dict]]) -> Tuple[bool, bool]:
"""
Calculates current signal based several technical analysis indicators
:param pair: pair in format ANT/BTC

View File

@ -3,8 +3,9 @@
import logging
from datetime import datetime
from random import randint
from unittest.mock import MagicMock, PropertyMock
from unittest.mock import Mock, MagicMock, PropertyMock
import arrow
import ccxt
import pytest
@ -13,6 +14,14 @@ from freqtrade.exchange import API_RETRY_COUNT, Exchange
from freqtrade.tests.conftest import get_patched_exchange, log_has
# Source: https://stackoverflow.com/questions/29881236/how-to-mock-asyncio-coroutines
def get_mock_coro(return_value):
async def mock_coro(*args, **kwargs):
return return_value
return Mock(wraps=mock_coro)
def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs):
with pytest.raises(TemporaryError):
api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError)
@ -27,12 +36,32 @@ def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, *
assert api_mock.__dict__[mock_ccxt_fun].call_count == 1
async def async_ccxt_exception(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs):
with pytest.raises(TemporaryError):
api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError)
exchange = get_patched_exchange(mocker, default_conf, api_mock)
await getattr(exchange, fun)(**kwargs)
assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1
with pytest.raises(OperationalException):
api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError)
exchange = get_patched_exchange(mocker, default_conf, api_mock)
await getattr(exchange, fun)(**kwargs)
assert api_mock.__dict__[mock_ccxt_fun].call_count == 1
def test_init(default_conf, mocker, caplog):
caplog.set_level(logging.INFO)
get_patched_exchange(mocker, default_conf)
assert log_has('Instance is running with dry_run enabled', caplog.record_tuples)
def test_destroy(default_conf, mocker, caplog):
caplog.set_level(logging.DEBUG)
get_patched_exchange(mocker, default_conf)
assert log_has('Exchange object destroyed, closing async loop', caplog.record_tuples)
def test_init_exception(default_conf, mocker):
default_conf['exchange']['name'] = 'wrong_exchange_name'
@ -64,6 +93,7 @@ def test_symbol_amount_prec(default_conf, mocker):
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
exchange = Exchange(default_conf)
amount = 2.34559
@ -87,6 +117,7 @@ def test_symbol_price_prec(default_conf, mocker):
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
exchange = Exchange(default_conf)
price = 2.34559
@ -108,6 +139,7 @@ def test_set_sandbox(default_conf, mocker):
type(api_mock).urls = url_mock
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
exchange = Exchange(default_conf)
liveurl = exchange._api.urls['api']
@ -129,6 +161,7 @@ def test_set_sandbox_exception(default_conf, mocker):
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
with pytest.raises(OperationalException, match=r'does not provide a sandbox api'):
exchange = Exchange(default_conf)
@ -136,6 +169,20 @@ def test_set_sandbox_exception(default_conf, mocker):
exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname')
def test__load_async_markets(default_conf, mocker, caplog):
exchange = get_patched_exchange(mocker, default_conf)
exchange._api_async.load_markets = get_mock_coro(None)
exchange._load_async_markets()
assert exchange._api_async.load_markets.call_count == 1
caplog.set_level(logging.DEBUG)
exchange._api_async.load_markets = Mock(side_effect=ccxt.BaseError("deadbeef"))
exchange._load_async_markets()
assert log_has('Could not load async markets. Reason: deadbeef',
caplog.record_tuples)
def test_validate_pairs(default_conf, mocker):
api_mock = MagicMock()
api_mock.load_markets = MagicMock(return_value={
@ -146,6 +193,7 @@ def test_validate_pairs(default_conf, mocker):
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
Exchange(default_conf)
@ -154,6 +202,7 @@ def test_validate_pairs_not_available(default_conf, mocker):
api_mock.load_markets = MagicMock(return_value={})
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
with pytest.raises(OperationalException, match=r'not available'):
Exchange(default_conf)
@ -167,6 +216,7 @@ def test_validate_pairs_not_compatible(default_conf, mocker):
default_conf['stake_currency'] = 'ETH'
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
with pytest.raises(OperationalException, match=r'not compatible'):
Exchange(default_conf)
@ -179,6 +229,7 @@ def test_validate_pairs_exception(default_conf, mocker, caplog):
api_mock.load_markets = MagicMock(return_value={})
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available at Binance'):
Exchange(default_conf)
@ -198,6 +249,7 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog):
api_mock.name = MagicMock(return_value='binance')
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
with pytest.raises(
OperationalException,
@ -515,6 +567,160 @@ def test_get_ticker(default_conf, mocker):
exchange.get_ticker(pair='ETH/BTC', refresh=True)
def test_get_history(default_conf, mocker, caplog):
exchange = get_patched_exchange(mocker, default_conf)
tick = [
[
arrow.utcnow().timestamp * 1000, # unix timestamp ms
1, # open
2, # high
3, # low
4, # close
5, # volume (in quote currency)
]
]
pair = 'ETH/BTC'
async def mock_candle_hist(pair, tick_interval, since_ms):
return pair, tick
exchange._async_get_candle_history = Mock(wraps=mock_candle_hist)
# one_call calculation * 1.8 should do 2 calls
since = 5 * 60 * 500 * 1.8
print(f"since = {since}")
ret = exchange.get_history(pair, "5m", int((arrow.utcnow().timestamp - since) * 1000))
assert exchange._async_get_candle_history.call_count == 2
# Returns twice the above tick
assert len(ret) == 2
def test_refresh_tickers(mocker, default_conf, caplog) -> None:
tick = [
[
1511686200000, # unix timestamp ms
1, # open
2, # high
3, # low
4, # close
5, # volume (in quote currency)
]
]
caplog.set_level(logging.DEBUG)
exchange = get_patched_exchange(mocker, default_conf)
exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
pairs = ['IOTA/ETH', 'XRP/ETH']
# empty dicts
assert not exchange.klines
exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m')
assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples)
assert exchange.klines
for pair in pairs:
assert exchange.klines[pair]
@pytest.mark.asyncio
async def test__async_get_candle_history(default_conf, mocker, caplog):
tick = [
[
arrow.utcnow().timestamp * 1000, # unix timestamp ms
1, # open
2, # high
3, # low
4, # close
5, # volume (in quote currency)
]
]
caplog.set_level(logging.DEBUG)
exchange = get_patched_exchange(mocker, default_conf)
# Monkey-patch async function
exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
exchange = Exchange(default_conf)
pair = 'ETH/BTC'
res = await exchange._async_get_candle_history(pair, "5m")
assert type(res) is tuple
assert len(res) == 2
assert res[0] == pair
assert res[1] == tick
assert exchange._api_async.fetch_ohlcv.call_count == 1
assert not log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples)
# test caching
res = await exchange._async_get_candle_history(pair, "5m")
assert exchange._api_async.fetch_ohlcv.call_count == 1
assert log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples)
# exchange = Exchange(default_conf)
await async_ccxt_exception(mocker, default_conf, MagicMock(),
"_async_get_candle_history", "fetch_ohlcv",
pair='ABCD/BTC', tick_interval=default_conf['ticker_interval'])
api_mock = MagicMock()
with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'):
api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError)
exchange = get_patched_exchange(mocker, default_conf, api_mock)
await exchange._async_get_candle_history(pair, "5m",
(arrow.utcnow().timestamp - 2000) * 1000)
@pytest.mark.asyncio
async def test__async_get_candle_history_empty(default_conf, mocker, caplog):
""" Test empty exchange result """
tick = []
caplog.set_level(logging.DEBUG)
exchange = get_patched_exchange(mocker, default_conf)
# Monkey-patch async function
exchange._api_async.fetch_ohlcv = get_mock_coro([])
exchange = Exchange(default_conf)
pair = 'ETH/BTC'
res = await exchange._async_get_candle_history(pair, "5m")
assert type(res) is tuple
assert len(res) == 2
assert res[0] == pair
assert res[1] == tick
assert exchange._api_async.fetch_ohlcv.call_count == 1
@pytest.mark.asyncio
async def test_async_get_candles_history(default_conf, mocker):
tick = [
[
1511686200000, # unix timestamp ms
1, # open
2, # high
3, # low
4, # close
5, # volume (in quote currency)
]
]
async def mock_get_candle_hist(pair, tick_interval, since_ms=None):
return (pair, tick)
exchange = get_patched_exchange(mocker, default_conf)
# Monkey-patch async function
exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
exchange._async_get_candle_history = Mock(wraps=mock_get_candle_hist)
pairs = ['ETH/BTC', 'XRP/BTC']
res = await exchange.async_get_candles_history(pairs, "5m")
assert type(res) is list
assert len(res) == 2
assert type(res[0]) is tuple
assert res[0][0] == pairs[0]
assert res[0][1] == tick
assert res[1][0] == pairs[1]
assert res[1][1] == tick
assert exchange._async_get_candle_history.call_count == 2
def test_get_order_book(default_conf, mocker, order_book_l2):
default_conf['exchange']['name'] = 'binance'
api_mock = MagicMock()

View File

@ -110,7 +110,7 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals
return pairdata
# use for mock freqtrade.exchange.get_candle_history'
# use for mock ccxt.fetch_ohlvc'
def _load_pair_as_ticks(pair, tickfreq):
ticks = optimize.load_data(None, ticker_interval=tickfreq, pairs=[pair])
ticks = trim_dictlist(ticks, -201)
@ -455,7 +455,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None:
return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
mocker.patch('freqtrade.optimize.load_data', mocked_load_data)
mocker.patch('freqtrade.exchange.Exchange.get_candle_history')
mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock())
patch_exchange(mocker)
mocker.patch.multiple(
'freqtrade.optimize.backtesting.Backtesting',
@ -490,7 +490,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None:
return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={}))
mocker.patch('freqtrade.exchange.Exchange.get_candle_history')
mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock())
patch_exchange(mocker)
mocker.patch.multiple(
'freqtrade.optimize.backtesting.Backtesting',
@ -733,9 +733,14 @@ def test_backtest_record(default_conf, fee, mocker):
def test_backtest_start_live(default_conf, mocker, caplog):
default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
mocker.patch('freqtrade.exchange.Exchange.get_candle_history',
new=lambda s, n, i: _load_pair_as_ticks(n, i))
patch_exchange(mocker)
async def load_pairs(pair, timeframe, since):
return _load_pair_as_ticks(pair, timeframe)
api_mock = MagicMock()
api_mock.fetch_ohlcv = load_pairs
patch_exchange(mocker, api_mock)
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock())
mocker.patch('freqtrade.configuration.open', mocker.mock_open(
@ -776,9 +781,13 @@ def test_backtest_start_live(default_conf, mocker, caplog):
def test_backtest_start_multi_strat(default_conf, mocker, caplog):
default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
mocker.patch('freqtrade.exchange.Exchange.get_candle_history',
new=lambda s, n, i: _load_pair_as_ticks(n, i))
patch_exchange(mocker)
async def load_pairs(pair, timeframe, since):
return _load_pair_as_ticks(pair, timeframe)
api_mock = MagicMock()
api_mock.fetch_ohlcv = load_pairs
patch_exchange(mocker, api_mock)
backtestmock = MagicMock()
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
gen_table_mock = MagicMock()

View File

@ -53,7 +53,7 @@ def _clean_test_file(file: str) -> None:
def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json')
_backup_file(file, copy_file=True)
optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m')
@ -63,7 +63,7 @@ def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) ->
def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json')
_backup_file(file, copy_file=True)
@ -74,7 +74,7 @@ def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) ->
def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json')
_backup_file(file, copy_file=True)
optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC'])
@ -87,7 +87,7 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog, default_co
"""
Test load_data() with 1 min ticker
"""
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
exchange = get_patched_exchange(mocker, default_conf)
file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
@ -118,7 +118,7 @@ def test_testdata_path() -> None:
def test_download_pairs(ticker_history, mocker, default_conf) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
exchange = get_patched_exchange(mocker, default_conf)
file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json')
@ -261,7 +261,7 @@ def test_load_cached_data_for_updating(mocker) -> None:
def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata',
side_effect=BaseException('File Error'))
exchange = get_patched_exchange(mocker, default_conf)
@ -279,7 +279,7 @@ def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf)
def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
exchange = get_patched_exchange(mocker, default_conf)
# Download a 1 min ticker file
@ -304,7 +304,7 @@ def test_download_backtesting_testdata2(mocker, default_conf) -> None:
[1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199]
]
json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None)
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=tick)
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick)
exchange = get_patched_exchange(mocker, default_conf)
download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='1m')
download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='3m')

View File

@ -89,7 +89,6 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog):
def test_get_signal_handles_exceptions(mocker, default_conf):
mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=MagicMock())
exchange = get_patched_exchange(mocker, default_conf)
mocker.patch.object(
_STRATEGY, 'analyze_ticker',

View File

@ -43,7 +43,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None:
:return: None
"""
freqtrade.strategy.get_signal = lambda e, s, t: value
freqtrade.exchange.get_candle_history = lambda p, i: None
freqtrade.exchange.refresh_tickers = lambda p, i: None
def patch_RPCManager(mocker) -> MagicMock:
@ -553,7 +553,6 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None:
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
validate_pairs=MagicMock(),
get_candle_history=MagicMock(return_value=20),
get_balance=MagicMock(return_value=20),
get_fee=fee,
)

View File

@ -14,6 +14,7 @@ numpy==1.15.1
TA-Lib==0.4.17
pytest==3.8.0
pytest-mock==1.10.0
pytest-asyncio==0.9.0
pytest-cov==2.6.0
tabulate==0.8.2
coinmarketcap==5.0.3

View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
"""This script generate json data from bittrex"""
"""This script generate json data"""
import json
import sys
from pathlib import Path
@ -52,9 +52,10 @@ exchange = Exchange({'key': '',
'stake_currency': '',
'dry_run': True,
'exchange': {
'name': args.exchange,
'pair_whitelist': []
}
'name': args.exchange,
'pair_whitelist': [],
'ccxt_rate_limit': False
}
})
pairs_not_available = []

View File

@ -138,7 +138,8 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
tickers = {}
if args.live:
logger.info('Downloading pair.')
tickers[pair] = exchange.get_candle_history(pair, tick_interval)
exchange.refresh_tickers([pair], tick_interval)
tickers[pair] = exchange.klines[pair]
else:
tickers = optimize.load_data(
datadir=_CONF.get("datadir"),