From 4a9bf78770624357d91de4675a10c541fc488a74 Mon Sep 17 00:00:00 2001 From: Nullart2 Date: Sun, 5 Aug 2018 12:41:06 +0800 Subject: [PATCH] Order Book with tests --- config.json.example | 16 +++- config_full.json.example | 16 +++- docs/configuration.md | 7 ++ freqtrade/constants.py | 25 ++++- freqtrade/exchange/__init__.py | 23 +++++ freqtrade/exchange/exchange_helpers.py | 24 +++++ freqtrade/freqtradebot.py | 111 ++++++++++++++++++++-- freqtrade/tests/conftest.py | 16 ++++ freqtrade/tests/exchange/test_exchange.py | 10 ++ freqtrade/tests/test_freqtradebot.py | 39 +++++++- 10 files changed, 271 insertions(+), 16 deletions(-) diff --git a/config.json.example b/config.json.example index 8bd3942e6..c3dc6b5b6 100644 --- a/config.json.example +++ b/config.json.example @@ -37,7 +37,21 @@ "experimental": { "use_sell_signal": false, "sell_profit_only": false, - "ignore_roi_if_buy_signal": false + "ignore_roi_if_buy_signal": false, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + }, + "bid_strategy": { + "use_order_book": false, + "order_book_top": 2, + "percent_from_top": 0 + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 + } }, "telegram": { "enabled": true, diff --git a/config_full.json.example b/config_full.json.example index cc3b3d630..5a364a93b 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -46,7 +46,21 @@ "experimental": { "use_sell_signal": false, "sell_profit_only": false, - "ignore_roi_if_buy_signal": false + "ignore_roi_if_buy_signal": false, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + }, + "bid_strategy": { + "use_order_book": false, + "order_book_top": 2, + "percent_from_top": 0 + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 + }s }, "telegram": { "enabled": true, diff --git a/docs/configuration.md b/docs/configuration.md index ff5ce118c..5ffe24556 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,6 +39,13 @@ The table below will list all configuration parameters. | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` +| `experimental.check_depth_of_market` | false | No | Does not sell if the % difference of buy orders and sell orders is met in Order Book. +| `experimental.bids_to_ask_delta` | 0 | No | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. +| `experimental.bid_strategy.use_order_book` | false | No | Allows buying of pair using the rates in Order Book Bids. +| `experimental.bid_strategy.order_book_top` | 0 | No | Bot will use the top N rate in Order Book Bids. Ie. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. +| `experimental.ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. +| `experimental.ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. +| `experimental.ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `telegram.enabled` | true | Yes | Enable or not the usage of Telegram. | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`. | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 87e354455..b7431af3c 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -89,7 +89,30 @@ CONF_SCHEMA = { 'properties': { 'use_sell_signal': {'type': 'boolean'}, 'sell_profit_only': {'type': 'boolean'}, - "ignore_roi_if_buy_signal_true": {'type': 'boolean'} + 'ignore_roi_if_buy_signal_true': {'type': 'boolean'}, + 'check_depth_of_market': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'bids_to_ask_delta': {'type': 'number', 'minimum': 0}, + } + }, + 'bid_strategy': { + 'type': 'object', + 'properties': { + 'percent_from_top': {'type': 'number', 'minimum': 0}, + 'use_order_book': {'type': 'boolean'}, + 'order_book_top': {'type': 'number', 'maximum': 20, 'minimum': 1} + } + }, + 'ask_strategy': { + 'type': 'object', + 'properties': { + 'use_order_book': {'type': 'boolean'}, + 'order_book_min': {'type': 'number', 'minimum': 1}, + 'order_book_max': {'type': 'number', 'minimum': 1, 'maximum': 50} + } + } } }, 'telegram': { diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 810957902..18b95b604 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -409,6 +409,29 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(e) + @retrier + def get_order_book(self, pair: str, limit: int = 100) -> dict: + try: + # 20180619: bittrex doesnt support limits -.- + # 20180619: binance support limits but only on specific range + if self._api.name == 'Binance': + limit_range = [5, 10, 20, 50, 100, 500, 1000] + for limitx in limit_range: + if limit <= limitx: + limit = limitx + break + + return self._api.fetch_l2_order_book(pair, limit) + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching order book.' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order book due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + @retrier def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: if self._conf['dry_run']: diff --git a/freqtrade/exchange/exchange_helpers.py b/freqtrade/exchange/exchange_helpers.py index 254c16309..6574f5a53 100644 --- a/freqtrade/exchange/exchange_helpers.py +++ b/freqtrade/exchange/exchange_helpers.py @@ -2,6 +2,7 @@ Functions to analyze ticker data with indicators and produce buy and sell signals """ import logging +import pandas as pd from pandas import DataFrame, to_datetime logger = logging.getLogger(__name__) @@ -31,3 +32,26 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame: }) frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle return frame + + +def order_book_to_dataframe(data: list) -> DataFrame: + """ + Gets order book list, returns dataframe with below format per suggested by creslin + ------------------------------------------------------------------- + b_sum b_size bids asks a_size a_sum + ------------------------------------------------------------------- + """ + cols = ['bids', 'b_size'] + bids_frame = DataFrame(data['bids'], columns=cols) + # add cumulative sum column + bids_frame['b_sum'] = bids_frame['b_size'].cumsum() + cols2 = ['asks', 'a_size'] + asks_frame = DataFrame(data['asks'], columns=cols2) + # add cumulative sum column + asks_frame['a_sum'] = asks_frame['a_size'].cumsum() + + frame = pd.concat([bids_frame['b_sum'], bids_frame['b_size'], bids_frame['bids'], + asks_frame['asks'], asks_frame['a_size'], asks_frame['a_sum']], axis=1, + keys=['b_sum', 'b_size', 'bids', 'asks', 'a_size', 'a_sum']) + # logger.info('order book %s', frame ) + return frame diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 46fbb3a38..0c4670971 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -21,6 +21,7 @@ from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from freqtrade.exchange.exchange_helpers import order_book_to_dataframe logger = logging.getLogger(__name__) @@ -233,16 +234,47 @@ class FreqtradeBot(object): return final_list - def get_target_bid(self, ticker: Dict[str, float]) -> float: + def get_target_bid(self, pair: str, ticker: Dict[str, float]) -> float: """ Calculates bid target between current ask price and last price :param ticker: Ticker to use for getting Ask and Last Price :return: float: Price """ if ticker['ask'] < ticker['last']: - return ticker['ask'] - balance = self.config['bid_strategy']['ask_last_balance'] - return ticker['ask'] + balance * (ticker['last'] - ticker['ask']) + ticker_rate = ticker['ask'] + else: + balance = self.config['bid_strategy']['ask_last_balance'] + ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask']) + + used_rate = ticker_rate + experimental_bid_strategy = self.config.get('experimental', {}).get('bid_strategy', {}) + if 'use_order_book' in experimental_bid_strategy and\ + experimental_bid_strategy.get('use_order_book', False): + logger.info('Getting price from order book') + order_book_top = experimental_bid_strategy.get('order_book_top', 1) + order_book = self.exchange.get_order_book(pair, order_book_top) + # top 1 = index 0 + order_book_rate = order_book['bids'][order_book_top - 1][0] + # if ticker has lower rate, then use ticker ( usefull if down trending ) + logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate) + if ticker_rate < order_book_rate: + logger.info('...using ticker rate instead %0.8f', ticker_rate) + used_rate = ticker_rate + used_rate = order_book_rate + else: + logger.info('Using Last Ask / Last Price') + used_rate = ticker_rate + percent_from_top = self.config.get('bid_strategy', {}).get('percent_from_top', 0) + if percent_from_top > 0: + used_rate = used_rate - (used_rate * percent_from_top) + used_rate = self._trunc_num(used_rate, 8) + logger.info('...percent_from_top enabled, new buy rate %0.8f', used_rate) + + return used_rate + + def _trunc_num(self, f, n): + import math + return math.floor(f * 10 ** n) / 10 ** n def _get_trade_stake_amount(self) -> Optional[float]: """ @@ -334,9 +366,37 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) if buy and not sell: + experimental_check_depth_of_market = self.config.get('experimental', {}).\ + get('check_depth_of_market', {}) + if (experimental_check_depth_of_market.get('enabled', False)) and\ + (experimental_check_depth_of_market.get('bids_to_ask_delta', 0) > 0): + if self._check_depth_of_market_buy(_pair): + return self.execute_buy(_pair, stake_amount) + else: + return False return self.execute_buy(_pair, stake_amount) return False + def _check_depth_of_market_buy(self, pair: str, ) -> bool: + """ + Checks depth of market before executing a buy + """ + experimental_check_depth_of_market = self.config.get('experimental', {}).\ + get('check_depth_of_market', {}) + conf_bids_to_ask_delta = experimental_check_depth_of_market.get('bids_to_ask_delta', 0) + logger.info('checking depth of market for %s', pair) + order_book = self.exchange.get_order_book(pair, 1000) + order_book_data_frame = order_book_to_dataframe(order_book) + order_book_bids = order_book_data_frame['b_size'].sum() + order_book_asks = order_book_data_frame['a_size'].sum() + bids_ask_delta = order_book_bids / order_book_asks + logger.info('bids: %s, asks: %s, delta: %s', order_book_bids, + order_book_asks, + order_book_bids / order_book_asks) + if bids_ask_delta >= conf_bids_to_ask_delta: + return True + return False + def execute_buy(self, pair: str, stake_amount: float) -> bool: """ Executes a limit buy for the given pair @@ -349,7 +409,7 @@ class FreqtradeBot(object): fiat_currency = self.config.get('fiat_display_currency', None) # Calculate amount - buy_limit = self.get_target_bid(self.exchange.get_ticker(pair)) + buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair)) min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit) if min_stake_amount is not None and min_stake_amount > stake_amount: @@ -492,7 +552,7 @@ class FreqtradeBot(object): raise ValueError(f'attempt to handle closed trade: {trade}') logger.debug('Handling %s ...', trade) - current_rate = self.exchange.get_ticker(trade.pair)['bid'] + sell_rate = self.exchange.get_ticker(trade.pair)['bid'] (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) @@ -501,13 +561,44 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, ticker) - should_sell = self.strategy.should_sell(trade, current_rate, datetime.utcnow(), buy, sell) - if should_sell.sell_flag: - self.execute_sell(trade, current_rate, should_sell.sell_type) - return True + experimental_ask_strategy = self.config.get('experimental', {}).get('ask_strategy', {}) + if 'use_order_book' in experimental_ask_strategy and\ + experimental_ask_strategy.get('use_order_book', False): + logger.info('Using order book for selling...') + # logger.debug('Order book %s',orderBook) + order_book_min = experimental_ask_strategy.get('order_book_min', 1) + order_book_max = experimental_ask_strategy.get('order_book_max', 1) + + order_book = self.exchange.get_order_book(trade.pair, order_book_max) + + for i in range(order_book_min, order_book_max + 1): + order_book_rate = order_book['asks'][i - 1][0] + + # if orderbook has higher rate (high profit), + # use orderbook, otherwise just use bids rate + logger.info(' order book asks top %s: %0.8f', i, order_book_rate) + if sell_rate < order_book_rate: + sell_rate = order_book_rate + + if self.check_sell(trade, sell_rate, buy, sell): + return True + break + else: + logger.info('checking sell') + if self.check_sell(trade, sell_rate, buy, sell): + return True + logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False + def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: + should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + if should_sell.sell_flag: + self.execute_sell(trade, sell_rate, should_sell.sell_type) + logger.info('excuted sell') + return True + return False + def check_handle_timedout(self) -> None: """ Check if any orders are timed out and cancel if neccessary diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index d18016e16..d7f7e96d9 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -116,6 +116,22 @@ def default_conf(): "NEO/BTC" ] }, + "experimental": { + "check_depth_of_market": { + "enabled": False, + "bids_to_ask_delta": 1 + }, + "bid_strategy": { + "percent_from_top": 0, + "use_order_book": False, + "order_book_top": 1 + }, + "ask_strategy": { + "use_order_book": False, + "order_book_min": 1, + "order_book_max": 1 + } + }, "telegram": { "enabled": True, "token": "token", diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index d327b97c7..d0917bda2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -515,6 +515,16 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=True) +def test_get_order_book(default_conf, mocker): + default_conf['exchange']['name'] = 'binance' + exchange = Exchange(default_conf) + order_book = exchange.get_order_book(pair='ETH/BTC', limit=50) + assert 'bids' in order_book + assert 'asks' in order_book + assert len(order_book['bids']) == 50 + assert len(order_book['asks']) == 50 + + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 69f349107..4718ab0fa 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -664,21 +664,21 @@ def test_balance_fully_ask_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 0.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 20 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 20 def test_balance_fully_last_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 10 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 10 def test_balance_bigger_last_ask(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_target_bid({'ask': 5, 'last': 10}) == 5 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 5, 'last': 10}) == 5 def test_process_maybe_execute_buy(mocker, default_conf) -> None: @@ -1876,3 +1876,36 @@ def test_get_real_amount_open_trade(default_conf, mocker): freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) assert freqtrade.get_real_amount(trade, order) == amount + + +def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, markets, mocker): + default_conf['experimental']['check_depth_of_market']['enabled'] = True + default_conf['experimental']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 + patch_RPCManager(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets + ) + + # Save state of current whitelist + whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade is not None + assert trade.stake_amount == 0.001 + assert trade.is_open + assert trade.open_date is not None + assert trade.exchange == 'bittrex' + + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) + + assert trade.open_rate == 0.00001099 + assert whitelist == default_conf['exchange']['pair_whitelist']