adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3381 from freqtrade/fix/orderbook_keyerror

Browse Source 
Fix orderbook keyerror
This commit is contained in:
hroff-1902 2020-05-27 11:52:29 +03:00 committed by GitHub
commit a0556689ea
7 changed files with 92 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
freqtrade/data/dataprovider.py
View File

@ -110,12 +110,13 @@ class DataProvider:
def orderbook(self, pair: str, maximum: int) -> Dict[str, List]:
"""
fetch latest orderbook data
Fetch latest l2 orderbook data
Warning: Does a network request - so use with common sense.
:param pair: pair to get the data for
:param maximum: Maximum number of orderbook entries to query
:return: dict including bids/asks with a total of `maximum` entries.
"""
return self._exchange.get_order_book(pair, maximum)
return self._exchange.fetch_l2_order_book(pair, maximum)
@property
def runmode(self) -> RunMode:

8
freqtrade/exceptions.py
View File

@ -21,6 +21,14 @@ class DependencyException(FreqtradeException):
"""
class PricingError(DependencyException):
"""
Subclass of DependencyException.
Indicates that the price could not be determined.
Implicitly a buy / sell operation.
"""
class InvalidOrderException(FreqtradeException):
"""
This is returned when the order is not valid. Example:

4
freqtrade/exchange/binance.py
View File

@ -20,7 +20,7 @@ class Binance(Exchange):
"trades_pagination_arg": "fromId",
}
def get_order_book(self, pair: str, limit: int = 100) -> dict:
def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
"""
get order book level 2 from exchange
@ -30,7 +30,7 @@ class Binance(Exchange):
# get next-higher step in the limit_range list
limit = min(list(filter(lambda x: limit <= x, limit_range)))
return super().get_order_book(pair, limit)
return super().fetch_l2_order_book(pair, limit)
def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool:
"""

2
freqtrade/exchange/exchange.py
View File

@ -998,7 +998,7 @@ class Exchange:
raise OperationalException(e) from e
@retrier
def get_order_book(self, pair: str, limit: int = 100) -> dict:
def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
"""
get order book level 2 from exchange

35
freqtrade/freqtradebot.py
View File

@ -18,7 +18,7 @@ from freqtrade.configuration import validate_config_consistency
from freqtrade.data.converter import order_book_to_dataframe
from freqtrade.data.dataprovider import DataProvider
from freqtrade.edge import Edge
from freqtrade.exceptions import DependencyException, InvalidOrderException
from freqtrade.exceptions import DependencyException, InvalidOrderException, PricingError
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
from freqtrade.misc import safe_value_fallback
from freqtrade.pairlist.pairlistmanager import PairListManager
@ -260,12 +260,19 @@ class FreqtradeBot:
f"Getting price from order book {bid_strategy['price_side'].capitalize()} side."
)
order_book_top = bid_strategy.get('order_book_top', 1)
order_book = self.exchange.get_order_book(pair, order_book_top)
order_book = self.exchange.fetch_l2_order_book(pair, order_book_top)
logger.debug('order_book %s', order_book)
# top 1 = index 0
order_book_rate = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0]
logger.info(f'...top {order_book_top} order book buy rate {order_book_rate:.8f}')
used_rate = order_book_rate
try:
rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0]
except (IndexError, KeyError) as e:
logger.warning(
"Buy Price from orderbook could not be determined."
f"Orderbook: {order_book}"
)
raise PricingError from e
logger.info(f'...top {order_book_top} order book buy rate {rate_from_l2:.8f}')
used_rate = rate_from_l2
else:
logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price")
ticker = self.exchange.fetch_ticker(pair)
@ -446,7 +453,7 @@ class FreqtradeBot:
"""
conf_bids_to_ask_delta = conf.get('bids_to_ask_delta', 0)
logger.info(f"Checking depth of market for {pair} ...")
order_book = self.exchange.get_order_book(pair, 1000)
order_book = self.exchange.fetch_l2_order_book(pair, 1000)
order_book_data_frame = order_book_to_dataframe(order_book['bids'], order_book['asks'])
order_book_bids = order_book_data_frame['b_size'].sum()
order_book_asks = order_book_data_frame['a_size'].sum()
@ -635,7 +642,7 @@ class FreqtradeBot:
"""
Helper generator to query orderbook in loop (used for early sell-order placing)
"""
order_book = self.exchange.get_order_book(pair, order_book_max)
order_book = self.exchange.fetch_l2_order_book(pair, order_book_max)
for i in range(order_book_min, order_book_max + 1):
yield order_book[side][i - 1][0]
@ -662,8 +669,11 @@ class FreqtradeBot:
logger.info(
f"Getting price from order book {ask_strategy['price_side'].capitalize()} side."
)
try:
rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s"))
except (IndexError, KeyError) as e:
logger.warning("Sell Price at location from orderbook could not be determined.")
raise PricingError from e
else:
rate = self.exchange.fetch_ticker(pair)[ask_strategy['price_side']]
self._sell_rate_cache[pair] = rate
@ -690,16 +700,23 @@ class FreqtradeBot:
self.dataprovider.ohlcv(trade.pair, self.strategy.ticker_interval))
if config_ask_strategy.get('use_order_book', False):
logger.debug(f'Using order book for selling {trade.pair}...')
# logger.debug('Order book %s',orderBook)
order_book_min = config_ask_strategy.get('order_book_min', 1)
order_book_max = config_ask_strategy.get('order_book_max', 1)
logger.info(f'Using order book between {order_book_min} and {order_book_max} '
f'for selling {trade.pair}...')
order_book = self._order_book_gen(trade.pair, f"{config_ask_strategy['price_side']}s",
order_book_min=order_book_min,
order_book_max=order_book_max)
for i in range(order_book_min, order_book_max + 1):
try:
sell_rate = next(order_book)
except (IndexError, KeyError) as e:
logger.warning(
f"Sell Price at location {i} from orderbook could not be determined."
)
raise PricingError from e
logger.debug(f" order book {config_ask_strategy['price_side']} top {i}: "
f"{sell_rate:0.8f}")

12
tests/exchange/test_exchange.py
View File

@ -1413,13 +1413,13 @@ def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog):
@pytest.mark.parametrize("exchange_name", EXCHANGES)
def test_get_order_book(default_conf, mocker, order_book_l2, exchange_name):
def test_fetch_l2_order_book(default_conf, mocker, order_book_l2, exchange_name):
default_conf['exchange']['name'] = exchange_name
api_mock = MagicMock()
api_mock.fetch_l2_order_book = order_book_l2
exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
order_book = exchange.get_order_book(pair='ETH/BTC', limit=10)
order_book = exchange.fetch_l2_order_book(pair='ETH/BTC', limit=10)
assert 'bids' in order_book
assert 'asks' in order_book
assert len(order_book['bids']) == 10
@ -1427,20 +1427,20 @@ def test_get_order_book(default_conf, mocker, order_book_l2, exchange_name):
@pytest.mark.parametrize("exchange_name", EXCHANGES)
def test_get_order_book_exception(default_conf, mocker, exchange_name):
def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name):
api_mock = MagicMock()
with pytest.raises(OperationalException):
api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.NotSupported("Not supported"))
exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
exchange.get_order_book(pair='ETH/BTC', limit=50)
exchange.fetch_l2_order_book(pair='ETH/BTC', limit=50)
with pytest.raises(TemporaryError):
api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.NetworkError("DeadBeef"))
exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
exchange.get_order_book(pair='ETH/BTC', limit=50)
exchange.fetch_l2_order_book(pair='ETH/BTC', limit=50)
with pytest.raises(OperationalException):
api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.BaseError("DeadBeef"))
exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
exchange.get_order_book(pair='ETH/BTC', limit=50)
exchange.fetch_l2_order_book(pair='ETH/BTC', limit=50)
def make_fetch_ohlcv_mock(data):

66
tests/test_freqtradebot.py
View File

@ -11,18 +11,21 @@ import arrow
import pytest
import requests
from freqtrade.constants import MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT, CANCEL_REASON
from freqtrade.constants import (CANCEL_REASON, MATH_CLOSE_PREC,
UNLIMITED_STAKE_AMOUNT)
from freqtrade.exceptions import (DependencyException, InvalidOrderException,
OperationalException, TemporaryError)
OperationalException, PricingError,
TemporaryError)
from freqtrade.freqtradebot import FreqtradeBot
from freqtrade.persistence import Trade
from freqtrade.rpc import RPCMessageType
from freqtrade.state import RunMode, State
from freqtrade.strategy.interface import SellCheckTuple, SellType
from freqtrade.worker import Worker
from tests.conftest import (get_patched_freqtradebot, get_patched_worker,
log_has, log_has_re, patch_edge, patch_exchange,
patch_get_signal, patch_wallet, patch_whitelist, create_mock_trades)
from tests.conftest import (create_mock_trades, get_patched_freqtradebot,
get_patched_worker, log_has, log_has_re,
patch_edge, patch_exchange, patch_get_signal,
patch_wallet, patch_whitelist)
def patch_RPCManager(mocker) -> MagicMock:
@ -3695,7 +3698,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee,
default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1
patch_RPCManager(mocker)
patch_exchange(mocker)
mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2)
mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2)
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
fetch_ticker=ticker,
@ -3732,7 +3735,7 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_o
default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100
patch_RPCManager(mocker)
patch_exchange(mocker)
mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2)
mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2)
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
fetch_ticker=ticker,
@ -3757,7 +3760,7 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None:
ticker_mock = MagicMock(return_value={'ask': 0.045, 'last': 0.046})
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
get_order_book=order_book_l2,
fetch_l2_order_book=order_book_l2,
fetch_ticker=ticker_mock,
)
@ -3772,29 +3775,26 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None:
assert ticker_mock.call_count == 0
def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2) -> None:
"""
test if function get_buy_rate will return the ask rate (since its value is lower)
instead of the order book rate (even if enabled)
"""
def test_order_book_bid_strategy_exception(mocker, default_conf, caplog) -> None:
patch_exchange(mocker)
ticker_mock = MagicMock(return_value={'ask': 0.042, 'last': 0.046})
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
get_order_book=order_book_l2,
fetch_l2_order_book=MagicMock(return_value={'bids': [[]], 'asks': [[]]}),
fetch_ticker=ticker_mock,
)
default_conf['exchange']['name'] = 'binance'
default_conf['bid_strategy']['use_order_book'] = True
default_conf['bid_strategy']['order_book_top'] = 2
default_conf['bid_strategy']['order_book_top'] = 1
default_conf['bid_strategy']['ask_last_balance'] = 0
default_conf['telegram']['enabled'] = False
freqtrade = FreqtradeBot(default_conf)
# orderbook shall be used even if tickers would be lower.
assert freqtrade.get_buy_rate('ETH/BTC', True) != 0.042
assert ticker_mock.call_count == 0
with pytest.raises(PricingError):
freqtrade.get_buy_rate('ETH/BTC', refresh=True)
assert log_has_re(r'Buy Price from orderbook could not be determined.', caplog)
def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None:
@ -3804,7 +3804,7 @@ def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None:
patch_exchange(mocker)
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
get_order_book=order_book_l2
fetch_l2_order_book=order_book_l2
)
default_conf['telegram']['enabled'] = False
default_conf['exchange']['name'] = 'binance'
@ -3818,11 +3818,11 @@ def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None:
def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order,
fee, mocker, order_book_l2) -> None:
fee, mocker, order_book_l2, caplog) -> None:
"""
test order book ask strategy
"""
mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2)
mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2)
default_conf['exchange']['name'] = 'binance'
default_conf['ask_strategy']['use_order_book'] = True
default_conf['ask_strategy']['order_book_min'] = 1
@ -3856,6 +3856,13 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order
patch_get_signal(freqtrade, value=(False, True))
assert freqtrade.handle_trade(trade) is True
assert trade.close_rate_requested == order_book_l2.return_value['asks'][0][0]
mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book',
return_value={'bids': [[]], 'asks': [[]]})
with pytest.raises(PricingError):
freqtrade.handle_trade(trade)
assert log_has('Sell Price at location 1 from orderbook could not be determined.', caplog)
@pytest.mark.parametrize('side,ask,bid,expected', [
@ -3896,9 +3903,8 @@ def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, o
default_conf['ask_strategy']['use_order_book'] = True
default_conf['ask_strategy']['order_book_min'] = 1
default_conf['ask_strategy']['order_book_max'] = 2
# TODO: min/max is irrelevant for this test until refactoring
pair = "ETH/BTC"
mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2)
mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2)
ft = get_patched_freqtradebot(mocker, default_conf)
rate = ft.get_sell_rate(pair, True)
assert not log_has("Using cached sell rate for ETH/BTC.", caplog)
@ -3909,6 +3915,22 @@ def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, o
assert log_has("Using cached sell rate for ETH/BTC.", caplog)
def test_get_sell_rate_orderbook_exception(default_conf, mocker, caplog):
# Test orderbook mode
default_conf['ask_strategy']['price_side'] = 'ask'
default_conf['ask_strategy']['use_order_book'] = True
default_conf['ask_strategy']['order_book_min'] = 1
default_conf['ask_strategy']['order_book_max'] = 2
pair = "ETH/BTC"
# Test What happens if the exchange returns an empty orderbook.
mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book',
return_value={'bids': [[]], 'asks': [[]]})
ft = get_patched_freqtradebot(mocker, default_conf)
with pytest.raises(PricingError):
ft.get_sell_rate(pair, True)
assert log_has("Sell Price at location from orderbook could not be determined.", caplog)
def test_startup_state(default_conf, mocker):
default_conf['pairlist'] = {'method': 'VolumePairList',
'config': {'number_assets': 20}