From e8ef36fb6ed8d984711531313487692eba9669c3 Mon Sep 17 00:00:00 2001
From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
Date: Wed, 13 Feb 2019 02:55:55 +0300
Subject: [PATCH 01/13] execute_buy: do not use ticker if use_order_book:true
 is set in config

This PR corresponds to:
https://github.com/freqtrade/freqtrade/issues/1377#issue-386200394
in understanfing that pair Ticker is mostly statistics, but on the other side, create_trade/execute_buy.

It resolves problem with some exchanges (BitMex) where ticker structure returned by ccxt does not contain bid and ask values.

1. On exchanges like Bitmex, set use_order_book: true for buys. FT won't request ticker and will use data from order book only.
2. On exchanges where order book is not available, set use_order_book: false, ticker data (including ask/last balance logic) will be used.
3. On other exchanges, either approach may be used in the config.

Performance: current implementation fetches ticker every time even if order book data will be later used. With this change it's eliminated.

Comparison of order book rate and ticker rate is removed (in order to split fetching order book and ticker completely in execute_buy), so some tests that touch this code may require adjustments.
---
 freqtrade/freqtradebot.py | 25 +++++++++----------------
 1 file changed, 9 insertions(+), 16 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 5413c57f9..4671b6053 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -204,19 +204,11 @@ class FreqtradeBot(object):
             self.state = State.STOPPED
         return state_changed
 
-    def get_target_bid(self, pair: str, ticker: Dict[str, float]) -> float:
+    def get_target_bid(self, pair: str) -> float:
         """
         Calculates bid target between current ask price and last price
-        :param ticker: Ticker to use for getting Ask and Last Price
         :return: float: Price
         """
-        if ticker['ask'] < ticker['last']:
-            ticker_rate = ticker['ask']
-        else:
-            balance = self.config['bid_strategy']['ask_last_balance']
-            ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask'])
-
-        used_rate = ticker_rate
         config_bid_strategy = self.config.get('bid_strategy', {})
         if 'use_order_book' in config_bid_strategy and\
                 config_bid_strategy.get('use_order_book', False):
@@ -226,15 +218,16 @@ class FreqtradeBot(object):
             logger.debug('order_book %s', order_book)
             # top 1 = index 0
             order_book_rate = order_book['bids'][order_book_top - 1][0]
-            # if ticker has lower rate, then use ticker ( usefull if down trending )
             logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate)
-            if ticker_rate < order_book_rate:
-                logger.info('...using ticker rate instead %0.8f', ticker_rate)
-                used_rate = ticker_rate
-            else:
-                used_rate = order_book_rate
+            used_rate = order_book_rate
         else:
             logger.info('Using Last Ask / Last Price')
+            ticker = self.exchange.get_ticker(pair)
+            if ticker['ask'] < ticker['last']:
+                ticker_rate = ticker['ask']
+            else:
+                balance = self.config['bid_strategy']['ask_last_balance']
+                ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask'])
             used_rate = ticker_rate
 
         return used_rate
@@ -380,7 +373,7 @@ class FreqtradeBot(object):
             buy_limit_requested = price
         else:
             # Calculate amount
-            buy_limit_requested = self.get_target_bid(pair, self.exchange.get_ticker(pair))
+            buy_limit_requested = self.get_target_bid(pair)
 
         min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit_requested)
         if min_stake_amount is not None and min_stake_amount > stake_amount:

From f551fb5ff7c135b039047cbae970e002e6c709f3 Mon Sep 17 00:00:00 2001
From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
Date: Wed, 13 Feb 2019 03:14:24 +0300
Subject: [PATCH 02/13] adjusted for passing tests

Don't like this test_ticker parameter, but it's needed for tests to pass prepared ticker.
Any ideas?
---
 freqtrade/freqtradebot.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 4671b6053..0183b5dc6 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -204,9 +204,10 @@ class FreqtradeBot(object):
             self.state = State.STOPPED
         return state_changed
 
-    def get_target_bid(self, pair: str) -> float:
+    def get_target_bid(self, pair: str, test_ticker: Dict[str, float]) -> float:
         """
         Calculates bid target between current ask price and last price
+        :param test_ticker: Ticker to use for getting Ask and Last Price; left for tests
         :return: float: Price
         """
         config_bid_strategy = self.config.get('bid_strategy', {})
@@ -222,7 +223,10 @@ class FreqtradeBot(object):
             used_rate = order_book_rate
         else:
             logger.info('Using Last Ask / Last Price')
-            ticker = self.exchange.get_ticker(pair)
+            if test_ticker is not None:
+                ticker = test_ticker
+            else:
+                ticker = self.exchange.get_ticker(pair)
             if ticker['ask'] < ticker['last']:
                 ticker_rate = ticker['ask']
             else:

From 91629807f7f64c00a9480d27c35f5087677473b0 Mon Sep 17 00:00:00 2001
From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
Date: Wed, 13 Feb 2019 03:17:54 +0300
Subject: [PATCH 03/13] shame on me

---
 freqtrade/freqtradebot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 0183b5dc6..1118deb9d 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -204,7 +204,7 @@ class FreqtradeBot(object):
             self.state = State.STOPPED
         return state_changed
 
-    def get_target_bid(self, pair: str, test_ticker: Dict[str, float]) -> float:
+    def get_target_bid(self, pair: str, test_ticker: Dict[str, float] = None) -> float:
         """
         Calculates bid target between current ask price and last price
         :param test_ticker: Ticker to use for getting Ask and Last Price; left for tests

From e8daadfb7e5d943bde8fa2208883211e2558e5b3 Mon Sep 17 00:00:00 2001
From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
Date: Wed, 13 Feb 2019 03:54:57 +0300
Subject: [PATCH 04/13] same approach for the sell side (at handle_trade)

---
 freqtrade/freqtradebot.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 1118deb9d..e5c1bf8ba 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -577,7 +577,6 @@ class FreqtradeBot(object):
             raise ValueError(f'Attempt to handle closed trade: {trade}')
 
         logger.debug('Handling %s ...', trade)
-        sell_rate = self.exchange.get_ticker(trade.pair)['bid']
 
         (buy, sell) = (False, False)
         experimental = self.config.get('experimental', {})
@@ -597,18 +596,15 @@ class FreqtradeBot(object):
 
             for i in range(order_book_min, order_book_max + 1):
                 order_book_rate = order_book['asks'][i - 1][0]
-
-                # if orderbook has higher rate (high profit),
-                # use orderbook, otherwise just use bids rate
                 logger.info('  order book asks top %s: %0.8f', i, order_book_rate)
-                if sell_rate < order_book_rate:
-                    sell_rate = order_book_rate
+                sell_rate = order_book_rate
 
                 if self.check_sell(trade, sell_rate, buy, sell):
                     return True
 
         else:
             logger.debug('checking sell')
+            sell_rate = self.exchange.get_ticker(trade.pair)['bid']
             if self.check_sell(trade, sell_rate, buy, sell):
                 return True
 

From 69a24c12722a6f76e7ca401e7ec851da8ee3c6f7 Mon Sep 17 00:00:00 2001
From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
Date: Wed, 13 Feb 2019 12:23:22 +0300
Subject: [PATCH 05/13] no need for test_ticker parameter just for making
 current tests happy, tests should be reimplemented

---
 freqtrade/freqtradebot.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index e5c1bf8ba..dc54fce78 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -204,10 +204,9 @@ class FreqtradeBot(object):
             self.state = State.STOPPED
         return state_changed
 
-    def get_target_bid(self, pair: str, test_ticker: Dict[str, float] = None) -> float:
+    def get_target_bid(self, pair: str) -> float:
         """
         Calculates bid target between current ask price and last price
-        :param test_ticker: Ticker to use for getting Ask and Last Price; left for tests
         :return: float: Price
         """
         config_bid_strategy = self.config.get('bid_strategy', {})
@@ -223,10 +222,7 @@ class FreqtradeBot(object):
             used_rate = order_book_rate
         else:
             logger.info('Using Last Ask / Last Price')
-            if test_ticker is not None:
-                ticker = test_ticker
-            else:
-                ticker = self.exchange.get_ticker(pair)
+            ticker = self.exchange.get_ticker(pair)
             if ticker['ask'] < ticker['last']:
                 ticker_rate = ticker['ask']
             else:

From f852be1a9be8c39666dc0195ced391b7098c256c Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 14 Feb 2019 07:27:13 +0100
Subject: [PATCH 06/13] Fix tests for get_ticker fix

---
 freqtrade/tests/test_freqtradebot.py | 50 +++++++++++-----------------
 1 file changed, 20 insertions(+), 30 deletions(-)

diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py
index 1a0e1e217..9083255ba 100644
--- a/freqtrade/tests/test_freqtradebot.py
+++ b/freqtrade/tests/test_freqtradebot.py
@@ -841,22 +841,27 @@ def test_process_informative_pairs_added(default_conf, ticker, markets, mocker)
 def test_balance_fully_ask_side(mocker, default_conf) -> None:
     default_conf['bid_strategy']['ask_last_balance'] = 0.0
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    mocker.patch('freqtrade.exchange.Exchange.get_ticker',
+                 MagicMock(return_value={'ask': 20, 'last': 10}))
 
-    assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 20
+    assert freqtrade.get_target_bid('ETH/BTC') == 20
 
 
 def test_balance_fully_last_side(mocker, default_conf) -> None:
     default_conf['bid_strategy']['ask_last_balance'] = 1.0
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    mocker.patch('freqtrade.exchange.Exchange.get_ticker',
+                 MagicMock(return_value={'ask': 20, 'last': 10}))
 
-    assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 10
+    assert freqtrade.get_target_bid('ETH/BTC') == 10
 
 
 def test_balance_bigger_last_ask(mocker, default_conf) -> None:
     default_conf['bid_strategy']['ask_last_balance'] = 1.0
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
-
-    assert freqtrade.get_target_bid('ETH/BTC', {'ask': 5, 'last': 10}) == 5
+    mocker.patch('freqtrade.exchange.Exchange.get_ticker',
+                 MagicMock(return_value={'ask': 5, 'last': 10}))
+    assert freqtrade.get_target_bid('ETH/BTC') == 5
 
 
 def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> None:
@@ -2813,10 +2818,13 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, markets)
     instead of the ask rate
     """
     patch_exchange(mocker)
+    ticker_mock = MagicMock(return_value={'ask': 0.045, 'last': 0.046})
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         get_markets=markets,
-        get_order_book=order_book_l2
+        get_order_book=order_book_l2,
+        get_ticker=ticker_mock,
+
     )
     default_conf['exchange']['name'] = 'binance'
     default_conf['bid_strategy']['use_order_book'] = True
@@ -2825,7 +2833,8 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, markets)
     default_conf['telegram']['enabled'] = False
 
     freqtrade = FreqtradeBot(default_conf)
-    assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.045, 'last': 0.046}) == 0.043935
+    assert freqtrade.get_target_bid('ETH/BTC') == 0.043935
+    assert ticker_mock.call_count == 0
 
 
 def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) -> None:
@@ -2834,10 +2843,13 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets)
     instead of the order book rate (even if enabled)
     """
     patch_exchange(mocker)
+    ticker_mock = MagicMock(return_value={'ask': 0.042, 'last': 0.046})
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         get_markets=markets,
-        get_order_book=order_book_l2
+        get_order_book=order_book_l2,
+        get_ticker=ticker_mock,
+
     )
     default_conf['exchange']['name'] = 'binance'
     default_conf['bid_strategy']['use_order_book'] = True
@@ -2846,29 +2858,7 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets)
     default_conf['telegram']['enabled'] = False
 
     freqtrade = FreqtradeBot(default_conf)
-    assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.042, 'last': 0.046}) == 0.042
-
-
-def test_order_book_bid_strategy3(default_conf, mocker, order_book_l2, markets) -> None:
-    """
-    test if function get_target_bid will return ask rate instead
-    of the order book rate
-    """
-    patch_exchange(mocker)
-    mocker.patch.multiple(
-        'freqtrade.exchange.Exchange',
-        get_markets=markets,
-        get_order_book=order_book_l2
-    )
-    default_conf['exchange']['name'] = 'binance'
-    default_conf['bid_strategy']['use_order_book'] = True
-    default_conf['bid_strategy']['order_book_top'] = 1
-    default_conf['bid_strategy']['ask_last_balance'] = 0
-    default_conf['telegram']['enabled'] = False
-
-    freqtrade = FreqtradeBot(default_conf)
-
-    assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.03, 'last': 0.029}) == 0.03
+    assert freqtrade.get_target_bid('ETH/BTC', ) == 0.042
 
 
 def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2, markets) -> None:

From 98bd7136244a1ef1a06ef15eec4e4117519ea33e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 14 Feb 2019 19:15:16 +0100
Subject: [PATCH 07/13] iUpdate orderbook_bid_test

---
 freqtrade/tests/test_freqtradebot.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py
index 9083255ba..a0ac6ee99 100644
--- a/freqtrade/tests/test_freqtradebot.py
+++ b/freqtrade/tests/test_freqtradebot.py
@@ -2858,7 +2858,9 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets)
     default_conf['telegram']['enabled'] = False
 
     freqtrade = FreqtradeBot(default_conf)
-    assert freqtrade.get_target_bid('ETH/BTC', ) == 0.042
+    # ordrebook shall be used even if tickers would be lower.
+    assert freqtrade.get_target_bid('ETH/BTC', ) != 0.042
+    assert ticker_mock.call_count == 0
 
 
 def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2, markets) -> None:

From c2bc316e2fb4940e3944dca767ae9586993c8d7c Mon Sep 17 00:00:00 2001
From: pyup-bot <github-bot@pyup.io>
Date: Sat, 16 Feb 2019 13:32:04 +0100
Subject: [PATCH 08/13] Update ccxt from 1.18.234 to 1.18.243

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 9d3bc6d0d..034d44055 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-ccxt==1.18.234
+ccxt==1.18.243
 SQLAlchemy==1.2.17
 python-telegram-bot==11.1.0
 arrow==0.13.1

From 13a16178d27d29ee5bc14ec86b16331fa25e2c3c Mon Sep 17 00:00:00 2001
From: pyup-bot <github-bot@pyup.io>
Date: Sat, 16 Feb 2019 13:32:05 +0100
Subject: [PATCH 09/13] Update sqlalchemy from 1.2.17 to 1.2.18

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 034d44055..4c7e3d486 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 ccxt==1.18.243
-SQLAlchemy==1.2.17
+SQLAlchemy==1.2.18
 python-telegram-bot==11.1.0
 arrow==0.13.1
 cachetools==3.1.0

From 7c651632f1568efad20b80603949483d7cc4cc23 Mon Sep 17 00:00:00 2001
From: pyup-bot <github-bot@pyup.io>
Date: Sat, 16 Feb 2019 13:32:06 +0100
Subject: [PATCH 10/13] Update joblib from 0.13.1 to 0.13.2

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 4c7e3d486..c690b237d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,7 +9,7 @@ wrapt==1.11.1
 numpy==1.16.1
 pandas==0.24.1
 scikit-learn==0.20.2
-joblib==0.13.1
+joblib==0.13.2
 scipy==1.2.1
 jsonschema==2.6.0
 TA-Lib==0.4.17

From ea6d4a9d362e0ac733562e50185bd90bee6e4454 Mon Sep 17 00:00:00 2001
From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
Date: Sat, 16 Feb 2019 19:50:55 +0300
Subject: [PATCH 11/13] fetch amount_reserve_percent from config

---
 freqtrade/freqtradebot.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 4247a6612..3dea34df1 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -290,7 +290,8 @@ class FreqtradeBot(object):
         if not min_stake_amounts:
             return None
 
-        amount_reserve_percent = 1 - 0.05  # reserve 5% + stoploss
+        # reserve some percent defined in config (5% default) + stoploss
+        amount_reserve_percent = 1.0 - self.config.get('amount_reserve_percent', 0.05)
         if self.strategy.stoploss is not None:
             amount_reserve_percent += self.strategy.stoploss
         # it should not be more than 50%

From 66cc600076fa2347a3eec3ca290a6e7b2a4a42f0 Mon Sep 17 00:00:00 2001
From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
Date: Sat, 16 Feb 2019 19:53:35 +0300
Subject: [PATCH 12/13] add amount_reserve_percent into the full config file

---
 config_full.json.example | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config_full.json.example b/config_full.json.example
index 234722f82..23a36dd4c 100644
--- a/config_full.json.example
+++ b/config_full.json.example
@@ -3,6 +3,7 @@
     "stake_currency": "BTC",
     "stake_amount": 0.05,
     "fiat_display_currency": "USD",
+    "amount_reserve_percent" : 0.05,
     "dry_run": false,
     "ticker_interval": "5m",
     "trailing_stop": false,

From df8067d6c4d479424051b0d4b9240e1562049449 Mon Sep 17 00:00:00 2001
From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
Date: Sun, 17 Feb 2019 12:56:28 +0300
Subject: [PATCH 13/13] add description of the new configuration option
 amount_reserve_percent

---
 docs/configuration.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/configuration.md b/docs/configuration.md
index 2caae81f1..108e264c6 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -16,6 +16,7 @@ Mandatory Parameters are marked as **Required**.
 | `max_open_trades` | 3 | **Required.** Number of trades open your bot will have. If -1 then it is ignored (i.e. potentially unlimited open trades)
 | `stake_currency` | BTC | **Required.** Crypto-currency used for trading.
 | `stake_amount` | 0.05 | **Required.** Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to `"unlimited"` to allow the bot to use all available balance.
+| `amount_reserve_percent` | 0.05 | Reserve some amount in min pair stake amount. Default is 5%. The bot will reserve `amount_reserve_percent` + stop-loss value when calculating min pair stake amount in order to avoid possible trade refusals.
 | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes. [Strategy Override](#parameters-in-strategy).
 | `fiat_display_currency` | USD | **Required.** Fiat currency used to show your profits. More information below.
 | `dry_run` | true | **Required.** Define if the bot must be in Dry-run or production mode.