From e8ef36fb6ed8d984711531313487692eba9669c3 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Wed, 13 Feb 2019 02:55:55 +0300 Subject: [PATCH 1/7] execute_buy: do not use ticker if use_order_book:true is set in config This PR corresponds to: https://github.com/freqtrade/freqtrade/issues/1377#issue-386200394 in understanfing that pair Ticker is mostly statistics, but on the other side, create_trade/execute_buy. It resolves problem with some exchanges (BitMex) where ticker structure returned by ccxt does not contain bid and ask values. 1. On exchanges like Bitmex, set use_order_book: true for buys. FT won't request ticker and will use data from order book only. 2. On exchanges where order book is not available, set use_order_book: false, ticker data (including ask/last balance logic) will be used. 3. On other exchanges, either approach may be used in the config. Performance: current implementation fetches ticker every time even if order book data will be later used. With this change it's eliminated. Comparison of order book rate and ticker rate is removed (in order to split fetching order book and ticker completely in execute_buy), so some tests that touch this code may require adjustments. --- freqtrade/freqtradebot.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5413c57f9..4671b6053 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -204,19 +204,11 @@ class FreqtradeBot(object): self.state = State.STOPPED return state_changed - def get_target_bid(self, pair: str, ticker: Dict[str, float]) -> float: + def get_target_bid(self, pair: str) -> float: """ Calculates bid target between current ask price and last price - :param ticker: Ticker to use for getting Ask and Last Price :return: float: Price """ - if ticker['ask'] < ticker['last']: - ticker_rate = ticker['ask'] - else: - balance = self.config['bid_strategy']['ask_last_balance'] - ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask']) - - used_rate = ticker_rate config_bid_strategy = self.config.get('bid_strategy', {}) if 'use_order_book' in config_bid_strategy and\ config_bid_strategy.get('use_order_book', False): @@ -226,15 +218,16 @@ class FreqtradeBot(object): logger.debug('order_book %s', order_book) # top 1 = index 0 order_book_rate = order_book['bids'][order_book_top - 1][0] - # if ticker has lower rate, then use ticker ( usefull if down trending ) logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate) - if ticker_rate < order_book_rate: - logger.info('...using ticker rate instead %0.8f', ticker_rate) - used_rate = ticker_rate - else: - used_rate = order_book_rate + used_rate = order_book_rate else: logger.info('Using Last Ask / Last Price') + ticker = self.exchange.get_ticker(pair) + if ticker['ask'] < ticker['last']: + ticker_rate = ticker['ask'] + else: + balance = self.config['bid_strategy']['ask_last_balance'] + ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask']) used_rate = ticker_rate return used_rate @@ -380,7 +373,7 @@ class FreqtradeBot(object): buy_limit_requested = price else: # Calculate amount - buy_limit_requested = self.get_target_bid(pair, self.exchange.get_ticker(pair)) + buy_limit_requested = self.get_target_bid(pair) min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit_requested) if min_stake_amount is not None and min_stake_amount > stake_amount: From f551fb5ff7c135b039047cbae970e002e6c709f3 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Wed, 13 Feb 2019 03:14:24 +0300 Subject: [PATCH 2/7] adjusted for passing tests Don't like this test_ticker parameter, but it's needed for tests to pass prepared ticker. Any ideas? --- freqtrade/freqtradebot.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4671b6053..0183b5dc6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -204,9 +204,10 @@ class FreqtradeBot(object): self.state = State.STOPPED return state_changed - def get_target_bid(self, pair: str) -> float: + def get_target_bid(self, pair: str, test_ticker: Dict[str, float]) -> float: """ Calculates bid target between current ask price and last price + :param test_ticker: Ticker to use for getting Ask and Last Price; left for tests :return: float: Price """ config_bid_strategy = self.config.get('bid_strategy', {}) @@ -222,7 +223,10 @@ class FreqtradeBot(object): used_rate = order_book_rate else: logger.info('Using Last Ask / Last Price') - ticker = self.exchange.get_ticker(pair) + if test_ticker is not None: + ticker = test_ticker + else: + ticker = self.exchange.get_ticker(pair) if ticker['ask'] < ticker['last']: ticker_rate = ticker['ask'] else: From 91629807f7f64c00a9480d27c35f5087677473b0 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Wed, 13 Feb 2019 03:17:54 +0300 Subject: [PATCH 3/7] shame on me --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0183b5dc6..1118deb9d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -204,7 +204,7 @@ class FreqtradeBot(object): self.state = State.STOPPED return state_changed - def get_target_bid(self, pair: str, test_ticker: Dict[str, float]) -> float: + def get_target_bid(self, pair: str, test_ticker: Dict[str, float] = None) -> float: """ Calculates bid target between current ask price and last price :param test_ticker: Ticker to use for getting Ask and Last Price; left for tests From e8daadfb7e5d943bde8fa2208883211e2558e5b3 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Wed, 13 Feb 2019 03:54:57 +0300 Subject: [PATCH 4/7] same approach for the sell side (at handle_trade) --- freqtrade/freqtradebot.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1118deb9d..e5c1bf8ba 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -577,7 +577,6 @@ class FreqtradeBot(object): raise ValueError(f'Attempt to handle closed trade: {trade}') logger.debug('Handling %s ...', trade) - sell_rate = self.exchange.get_ticker(trade.pair)['bid'] (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) @@ -597,18 +596,15 @@ class FreqtradeBot(object): for i in range(order_book_min, order_book_max + 1): order_book_rate = order_book['asks'][i - 1][0] - - # if orderbook has higher rate (high profit), - # use orderbook, otherwise just use bids rate logger.info(' order book asks top %s: %0.8f', i, order_book_rate) - if sell_rate < order_book_rate: - sell_rate = order_book_rate + sell_rate = order_book_rate if self.check_sell(trade, sell_rate, buy, sell): return True else: logger.debug('checking sell') + sell_rate = self.exchange.get_ticker(trade.pair)['bid'] if self.check_sell(trade, sell_rate, buy, sell): return True From 69a24c12722a6f76e7ca401e7ec851da8ee3c6f7 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Wed, 13 Feb 2019 12:23:22 +0300 Subject: [PATCH 5/7] no need for test_ticker parameter just for making current tests happy, tests should be reimplemented --- freqtrade/freqtradebot.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e5c1bf8ba..dc54fce78 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -204,10 +204,9 @@ class FreqtradeBot(object): self.state = State.STOPPED return state_changed - def get_target_bid(self, pair: str, test_ticker: Dict[str, float] = None) -> float: + def get_target_bid(self, pair: str) -> float: """ Calculates bid target between current ask price and last price - :param test_ticker: Ticker to use for getting Ask and Last Price; left for tests :return: float: Price """ config_bid_strategy = self.config.get('bid_strategy', {}) @@ -223,10 +222,7 @@ class FreqtradeBot(object): used_rate = order_book_rate else: logger.info('Using Last Ask / Last Price') - if test_ticker is not None: - ticker = test_ticker - else: - ticker = self.exchange.get_ticker(pair) + ticker = self.exchange.get_ticker(pair) if ticker['ask'] < ticker['last']: ticker_rate = ticker['ask'] else: From f852be1a9be8c39666dc0195ced391b7098c256c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 14 Feb 2019 07:27:13 +0100 Subject: [PATCH 6/7] Fix tests for get_ticker fix --- freqtrade/tests/test_freqtradebot.py | 50 +++++++++++----------------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1a0e1e217..9083255ba 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -841,22 +841,27 @@ def test_process_informative_pairs_added(default_conf, ticker, markets, mocker) def test_balance_fully_ask_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 0.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(return_value={'ask': 20, 'last': 10})) - assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 20 + assert freqtrade.get_target_bid('ETH/BTC') == 20 def test_balance_fully_last_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(return_value={'ask': 20, 'last': 10})) - assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 10 + assert freqtrade.get_target_bid('ETH/BTC') == 10 def test_balance_bigger_last_ask(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - - assert freqtrade.get_target_bid('ETH/BTC', {'ask': 5, 'last': 10}) == 5 + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(return_value={'ask': 5, 'last': 10})) + assert freqtrade.get_target_bid('ETH/BTC') == 5 def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> None: @@ -2813,10 +2818,13 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, markets) instead of the ask rate """ patch_exchange(mocker) + ticker_mock = MagicMock(return_value={'ask': 0.045, 'last': 0.046}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_markets=markets, - get_order_book=order_book_l2 + get_order_book=order_book_l2, + get_ticker=ticker_mock, + ) default_conf['exchange']['name'] = 'binance' default_conf['bid_strategy']['use_order_book'] = True @@ -2825,7 +2833,8 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, markets) default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.045, 'last': 0.046}) == 0.043935 + assert freqtrade.get_target_bid('ETH/BTC') == 0.043935 + assert ticker_mock.call_count == 0 def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) -> None: @@ -2834,10 +2843,13 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) instead of the order book rate (even if enabled) """ patch_exchange(mocker) + ticker_mock = MagicMock(return_value={'ask': 0.042, 'last': 0.046}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_markets=markets, - get_order_book=order_book_l2 + get_order_book=order_book_l2, + get_ticker=ticker_mock, + ) default_conf['exchange']['name'] = 'binance' default_conf['bid_strategy']['use_order_book'] = True @@ -2846,29 +2858,7 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.042, 'last': 0.046}) == 0.042 - - -def test_order_book_bid_strategy3(default_conf, mocker, order_book_l2, markets) -> None: - """ - test if function get_target_bid will return ask rate instead - of the order book rate - """ - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - get_markets=markets, - get_order_book=order_book_l2 - ) - default_conf['exchange']['name'] = 'binance' - default_conf['bid_strategy']['use_order_book'] = True - default_conf['bid_strategy']['order_book_top'] = 1 - default_conf['bid_strategy']['ask_last_balance'] = 0 - default_conf['telegram']['enabled'] = False - - freqtrade = FreqtradeBot(default_conf) - - assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.03, 'last': 0.029}) == 0.03 + assert freqtrade.get_target_bid('ETH/BTC', ) == 0.042 def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2, markets) -> None: From 98bd7136244a1ef1a06ef15eec4e4117519ea33e Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 14 Feb 2019 19:15:16 +0100 Subject: [PATCH 7/7] iUpdate orderbook_bid_test --- freqtrade/tests/test_freqtradebot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 9083255ba..a0ac6ee99 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -2858,7 +2858,9 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.get_target_bid('ETH/BTC', ) == 0.042 + # ordrebook shall be used even if tickers would be lower. + assert freqtrade.get_target_bid('ETH/BTC', ) != 0.042 + assert ticker_mock.call_count == 0 def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2, markets) -> None: