Merge branch 'develop' into ccxt-async

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.6.6-slim-stretch
+FROM python:3.7.0-slim-stretch
 
 # Install TA-lib
 RUN apt-get update && apt-get -y install curl build-essential && apt-get clean

config.json.example

@@ -11,7 +11,18 @@
         "sell": 30
     },
     "bid_strategy": {
-        "ask_last_balance": 0.0
+        "ask_last_balance": 0.0,
+        "use_order_book": false,
+        "order_book_top": 1,
+        "check_depth_of_market": {
+            "enabled": false,
+            "bids_to_ask_delta": 1
+        }
+    },
+    "ask_strategy":{
+        "use_order_book": false,
+        "order_book_min": 1,
+        "order_book_max": 9
     },
     "exchange": {
         "name": "bittrex",

config_full.json.example

@@ -20,7 +20,18 @@
         "sell": 30
     },
     "bid_strategy": {
-        "ask_last_balance": 0.0
+        "ask_last_balance": 0.0,
+        "use_order_book": false,
+        "order_book_top": 1,
+        "check_depth_of_market": {
+            "enabled": false,
+            "bids_to_ask_delta": 1
+        }
+    },
+    "ask_strategy":{
+        "use_order_book": false,
+        "order_book_min": 1,
+        "order_book_max": 9
     },
     "exchange": {
         "name": "bittrex",
@@ -41,7 +52,8 @@
         ],
         "pair_blacklist": [
             "DOGE/BTC"
-        ]
+        ],
+        "outdated_offset": 5
     },
     "experimental": {
         "use_sell_signal": false,

docs/configuration.md

@@ -31,6 +31,13 @@ The table below will list all configuration parameters.
 | `unfilledtimeout.buy` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled.
 | `unfilledtimeout.sell` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled.
 | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below.
+| `bid_strategy.use_order_book` | false | No | Allows buying of pair using the rates in Order Book Bids.
+| `bid_strategy.order_book_top` | 0 | No | Bot will use the top N rate in Order Book Bids. Ie. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids.
+| `bid_strategy.check_depth_of_market.enabled` | false | No | Does not buy if the % difference of buy orders and sell orders is met in Order Book.
+| `bid_strategy.check_depth_of_market.bids_to_ask_delta` | 0 | No | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher.
+| `ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks.
+| `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
+| `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
 | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
 | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode.
 | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode.

docs/sandbox-testing.md

@@ -1,4 +1,5 @@
 # Sandbox API testing
+
 Where an exchange provides a sandbox for risk-free integration, or end-to-end, testing CCXT provides access to these.
 
 This document is a *light overview of configuring Freqtrade and GDAX sandbox.
@@ -11,8 +12,11 @@ https://public.sandbox.gdax.com
 https://api-public.sandbox.gdax.com
 
 ---
+
 # Configure a Sandbox account on Gdax
+
 Aim of this document section
+
 - An sanbox account
 - create 2FA (needed to create an API)
 - Add test 50BTC to account 
@@ -30,122 +34,108 @@ After registration and Email confimation you wil be redirected into your sanbox
 > https://public.sandbox.pro.coinbase.com/
 
 ## Enable 2Fa (a prerequisite to creating sandbox API Keys)
+
 From within sand box site select your profile, top right.
 >Or as a direct link: https://public.sandbox.pro.coinbase.com/profile
 
-From the menu panel to the left of the screen select 
+From the menu panel to the left of the screen select
+
 > Security: "*View or Update*"
 
-In the new site select "enable authenticator" as typical google Authenticator. 
+In the new site select "enable authenticator" as typical google Authenticator.
-- open Google Authenticator on your phone
+
-- scan barcode 
+- open Google Authenticator on your phone
-- enter your generated 2fa 
+- scan barcode
+- enter your generated 2fa
+
+## Enable API Access
 
-## Enable API Access 
 From within sandbox select profile>api>create api-keys
 >or as a direct link: https://public.sandbox.pro.coinbase.com/profile/api
 
-Click on "create one" and ensure  **view** and **trade**  are "checked" and sumbit your 2Fa
+Click on "create one" and ensure  **view** and **trade**  are "checked" and sumbit your 2FA
+
 - **Copy and paste the Passphase** into a notepade this will be needed later
 - **Copy and paste the API Secret** popup into a notepad this will needed later
 - **Copy and paste the API Key** into a notepad this will needed later
 
 ## Add 50 BTC test funds
-To add funds, use the web interface deposit and withdraw buttons.
 
+To add funds, use the web interface deposit and withdraw buttons.
 
 To begin select 'Wallets' from the top menu.
 > Or as a direct link: https://public.sandbox.pro.coinbase.com/wallets
 
 - Deposits (bottom left of screen)
-- - Deposit Funds Bitcoin 
+- - Deposit Funds Bitcoin
-- - - Coinbase BTC Wallet 
+- - - Coinbase BTC Wallet
-- - - - Max (50 BTC) 
+- - - - Max (50 BTC)
 - - - - - Deposit
 
 *This process may be repeated for other currencies, ETH as example*
+
 ---
+
 # Configure Freqtrade to use Gax Sandbox 
 
 The aim of this document section
- - Enable sandbox URLs in Freqtrade
+
- - Configure API 
+- Enable sandbox URLs in Freqtrade
- - - secret
+- Configure API
- - - key
+- - secret
- - - passphrase
+- - key
+- - passphrase
  
 ## Sandbox URLs
-Freqtrade makes use of CCXT which in turn provides a list of URLs to Freqtrade. 
+
-These include `['test']` and `['api']`. 
+Freqtrade makes use of CCXT which in turn provides a list of URLs to Freqtrade.
+These include `['test']` and `['api']`.
+
 - `[Test]` if available will point to an Exchanges sandbox. 
 - `[Api]` normally used, and resolves to live API target on the exchange 
 
 To make use of sandbox / test add "sandbox": true, to your config.json
-```
+
+```json
   "exchange": {
         "name": "gdax",
         "sandbox": true,
         "key": "5wowfxemogxeowo;heiohgmd",
         "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==",
         "password": "1bkjfkhfhfu6sr",
+        "outdated_offset": 5
         "pair_whitelist": [
             "BTC/USD"
 ```
+
 Also insert your 
+
 - api-key  (noted earlier)
 - api-secret (noted earlier)
 - password (the passphrase - noted earlier)
 
 ---
-## You should now be ready to test your sandbox!
+
+## You should now be ready to test your sandbox
+
 Ensure Freqtrade logs show the sandbox URL, and trades made are shown in sandbox.
-** Typically the  BTC/USD has the most activity in sandbox to test against. 
+** Typically the  BTC/USD has the most activity in sandbox to test against.
 
 ## GDAX - Old Candles problem
-It is my experience that GDAX sandbox candles may be 20+- minutes out of date. This can cause trades to fail as one of Freqtrades safety checks 
 
-To disable this check, edit: 
+It is my experience that GDAX sandbox candles may be 20+- minutes out of date. This can cause trades to fail as one of Freqtrades safety checks.
->strategy/interface.py
-Look for the following section:
-```
-      # Check if dataframe is out of date
-        signal_date = arrow.get(latest['date'])
-        interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval]
-        if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))):
-            logger.warning(
-                'Outdated history for pair %s. Last tick is %s minutes old',
-                pair,
-                (arrow.utcnow() - signal_date).seconds // 60
-            )
-            return False, False
-```
 
-You could Hash out the entire check as follows:
+To disable this check, add / change the `"outdated_offset"` parameter in the exchange section of your configuration to adjust for this delay.
+Example based on the above configuration:
+
+```json
+  "exchange": {
+        "name": "gdax",
+        "sandbox": true,
+        "key": "5wowfxemogxeowo;heiohgmd",
+        "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==",
+        "password": "1bkjfkhfhfu6sr",
+        "outdated_offset": 30
+        "pair_whitelist": [
+            "BTC/USD"
 ```
-       # # Check if dataframe is out of date
-        # signal_date = arrow.get(latest['date'])
-        # interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval]
-        # if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))):
-        #     logger.warning(
-        #         'Outdated history for pair %s. Last tick is %s minutes old',
-        #         pair,
-        #         (arrow.utcnow() - signal_date).seconds // 60
-        #     )
-        #     return False, False
- ```
- 
- Or inrease the timeout to offer a level of protection/alignment of this test to freqtrade in live.
- 
- As example, to allow an additional 30 minutes. "(interval_minutes * 2 + 5 + 30)"
- ```
-      # Check if dataframe is out of date
-        signal_date = arrow.get(latest['date'])
-        interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval]
-        if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5 + 30))):
-            logger.warning(
-                'Outdated history for pair %s. Last tick is %s minutes old',
-                pair,
-                (arrow.utcnow() - signal_date).seconds // 60
-            )
-            return False, False
-```

freqtrade/constants.py

@@ -78,18 +78,35 @@ CONF_SCHEMA = {
                     'type': 'number',
                     'minimum': 0,
                     'maximum': 1,
-                    'exclusiveMaximum': False
+                    'exclusiveMaximum': False,
+                    'use_order_book': {'type': 'boolean'},
+                    'order_book_top': {'type': 'number', 'maximum': 20, 'minimum': 1},
+                    'check_depth_of_market': {
+                        'type': 'object',
+                        'properties': {
+                            'enabled': {'type': 'boolean'},
+                            'bids_to_ask_delta': {'type': 'number', 'minimum': 0},
+                        }
+                    },
                 },
             },
             'required': ['ask_last_balance']
         },
+        'ask_strategy': {
+            'type': 'object',
+            'properties': {
+                'use_order_book': {'type': 'boolean'},
+                'order_book_min': {'type': 'number', 'minimum': 1},
+                'order_book_max': {'type': 'number', 'minimum': 1, 'maximum': 50}
+            }
+        },
         'exchange': {'$ref': '#/definitions/exchange'},
         'experimental': {
             'type': 'object',
             'properties': {
                 'use_sell_signal': {'type': 'boolean'},
                 'sell_profit_only': {'type': 'boolean'},
-                "ignore_roi_if_buy_signal_true": {'type': 'boolean'}
+                'ignore_roi_if_buy_signal_true': {'type': 'boolean'}
             }
         },
         'telegram': {
@@ -145,7 +162,8 @@ CONF_SCHEMA = {
                         'pattern': '^[0-9A-Z]+/[0-9A-Z]+$'
                     },
                     'uniqueItems': True
-                }
+                },
+                'outdated_offset': {'type': 'integer', 'minimum': 1}
             },
             'required': ['name', 'key', 'secret', 'pair_whitelist']
         }

freqtrade/exchange/__init__.py

@@ -154,8 +154,8 @@ class Exchange(object):
                 api.urls['api'] = api.urls['test']
                 logger.info("Enabled Sandbox API on %s", name)
             else:
-                logger.warning(self._api.name, "No Sandbox URL in CCXT, exiting. "
+                logger.warning(name, "No Sandbox URL in CCXT, exiting. "
-                                               "Please check your config.json")
+                                     "Please check your config.json")
                 raise OperationalException(f'Exchange {name} does not provide a sandbox api')
 
     def _load_async_markets(self) -> None:
@@ -370,7 +370,7 @@ class Exchange(object):
                 return data
             except (ccxt.NetworkError, ccxt.ExchangeError) as e:
                 raise TemporaryError(
-                    f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}')
+                    f'Could not load ticker due to {e.__class__.__name__}. Message: {e}')
             except ccxt.BaseError as e:
                 raise OperationalException(e)
         else:
@@ -554,6 +554,37 @@ class Exchange(object):
         except ccxt.BaseError as e:
             raise OperationalException(e)
 
+    @retrier
+    def get_order_book(self, pair: str, limit: int = 100) -> dict:
+        """
+        get order book level 2 from exchange
+
+        Notes:
+        20180619: bittrex doesnt support limits -.-
+        20180619: binance support limits but only on specific range
+        """
+        try:
+            if self._api.name == 'Binance':
+                limit_range = [5, 10, 20, 50, 100, 500, 1000]
+                # get next-higher step in the limit_range list
+                limit = min(list(filter(lambda x: limit <= x, limit_range)))
+                # above script works like loop below (but with slightly better performance):
+                #   for limitx in limit_range:
+                #        if limit <= limitx:
+                #           limit = limitx
+                #           break
+
+            return self._api.fetch_l2_order_book(pair, limit)
+        except ccxt.NotSupported as e:
+            raise OperationalException(
+                f'Exchange {self._api.name} does not support fetching order book.'
+                f'Message: {e}')
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not get order book due to {e.__class__.__name__}. Message: {e}')
+        except ccxt.BaseError as e:
+            raise OperationalException(e)
+
     @retrier
     def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List:
         if self._conf['dry_run']:
@@ -607,12 +638,3 @@ class Exchange(object):
                 f'Could not get fee info due to {e.__class__.__name__}. Message: {e}')
         except ccxt.BaseError as e:
             raise OperationalException(e)
-
-    def get_amount_lots(self, pair: str, amount: float) -> float:
-        """
-        get buyable amount rounding, ..
-        """
-        # validate that markets are loaded before trying to get fee
-        if not self._api.markets:
-            self._api.load_markets()
-        return self._api.amount_to_lots(pair, amount)

freqtrade/exchange/exchange_helpers.py

@@ -2,6 +2,7 @@
 Functions to analyze ticker data with indicators and produce buy and sell signals
 """
 import logging
+import pandas as pd
 from pandas import DataFrame, to_datetime
 
 logger = logging.getLogger(__name__)
@@ -31,3 +32,27 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame:
     })
     frame.drop(frame.tail(1).index, inplace=True)     # eliminate partial candle
     return frame
+
+
+def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
+    """
+    Gets order book list, returns dataframe with below format per suggested by creslin
+    -------------------------------------------------------------------
+     b_sum       b_size       bids       asks       a_size       a_sum
+    -------------------------------------------------------------------
+    """
+    cols = ['bids', 'b_size']
+
+    bids_frame = DataFrame(bids, columns=cols)
+    # add cumulative sum column
+    bids_frame['b_sum'] = bids_frame['b_size'].cumsum()
+    cols2 = ['asks', 'a_size']
+    asks_frame = DataFrame(asks, columns=cols2)
+    # add cumulative sum column
+    asks_frame['a_sum'] = asks_frame['a_size'].cumsum()
+
+    frame = pd.concat([bids_frame['b_sum'], bids_frame['b_size'], bids_frame['bids'],
+                       asks_frame['asks'], asks_frame['a_size'], asks_frame['a_sum']], axis=1,
+                      keys=['b_sum', 'b_size', 'bids', 'asks', 'a_size', 'a_sum'])
+    # logger.info('order book %s', frame )
+    return frame

freqtrade/freqtradebot.py

@@ -22,6 +22,7 @@ from freqtrade.rpc import RPCManager, RPCMessageType
 from freqtrade.state import State
 from freqtrade.strategy.interface import SellType
 from freqtrade.strategy.resolver import IStrategy, StrategyResolver
+from freqtrade.exchange.exchange_helpers import order_book_to_dataframe
 
 logger = logging.getLogger(__name__)
 
@@ -271,16 +272,40 @@ class FreqtradeBot(object):
 
         return final_list
 
-    def get_target_bid(self, ticker: Dict[str, float]) -> float:
+    def get_target_bid(self, pair: str, ticker: Dict[str, float]) -> float:
         """
         Calculates bid target between current ask price and last price
         :param ticker: Ticker to use for getting Ask and Last Price
         :return: float: Price
         """
         if ticker['ask'] < ticker['last']:
-            return ticker['ask']
+            ticker_rate = ticker['ask']
-        balance = self.config['bid_strategy']['ask_last_balance']
+        else:
-        return ticker['ask'] + balance * (ticker['last'] - ticker['ask'])
+            balance = self.config['bid_strategy']['ask_last_balance']
+            ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask'])
+
+        used_rate = ticker_rate
+        config_bid_strategy = self.config.get('bid_strategy', {})
+        if 'use_order_book' in config_bid_strategy and\
+                config_bid_strategy.get('use_order_book', False):
+            logger.info('Getting price from order book')
+            order_book_top = config_bid_strategy.get('order_book_top', 1)
+            order_book = self.exchange.get_order_book(pair, order_book_top)
+            logger.debug('order_book %s', order_book)
+            # top 1 = index 0
+            order_book_rate = order_book['bids'][order_book_top - 1][0]
+            # if ticker has lower rate, then use ticker ( usefull if down trending )
+            logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate)
+            if ticker_rate < order_book_rate:
+                logger.info('...using ticker rate instead %0.8f', ticker_rate)
+                used_rate = ticker_rate
+            else:
+                used_rate = order_book_rate
+        else:
+            logger.info('Using Last Ask / Last Price')
+            used_rate = ticker_rate
+
+        return used_rate
 
     def _get_trade_stake_amount(self) -> Optional[float]:
         """
@@ -371,10 +396,35 @@ class FreqtradeBot(object):
         for _pair in whitelist:
             (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair))
             if buy and not sell:
+                bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\
+                    get('check_depth_of_market', {})
+                if (bidstrat_check_depth_of_market.get('enabled', False)) and\
+                        (bidstrat_check_depth_of_market.get('bids_to_ask_delta', 0) > 0):
+                    if self._check_depth_of_market_buy(_pair, bidstrat_check_depth_of_market):
+                        return self.execute_buy(_pair, stake_amount)
+                    else:
+                        return False
                 return self.execute_buy(_pair, stake_amount)
 
         return False
 
+    def _check_depth_of_market_buy(self, pair: str, conf: Dict) -> bool:
+        """
+        Checks depth of market before executing a buy
+        """
+        conf_bids_to_ask_delta = conf.get('bids_to_ask_delta', 0)
+        logger.info('checking depth of market for %s', pair)
+        order_book = self.exchange.get_order_book(pair, 1000)
+        order_book_data_frame = order_book_to_dataframe(order_book['bids'], order_book['asks'])
+        order_book_bids = order_book_data_frame['b_size'].sum()
+        order_book_asks = order_book_data_frame['a_size'].sum()
+        bids_ask_delta = order_book_bids / order_book_asks
+        logger.info('bids: %s, asks: %s, delta: %s', order_book_bids,
+                    order_book_asks, bids_ask_delta)
+        if bids_ask_delta >= conf_bids_to_ask_delta:
+            return True
+        return False
+
     def execute_buy(self, pair: str, stake_amount: float) -> bool:
         """
         Executes a limit buy for the given pair
@@ -387,7 +437,7 @@ class FreqtradeBot(object):
         fiat_currency = self.config.get('fiat_display_currency', None)
 
         # Calculate amount
-        buy_limit = self.get_target_bid(self.exchange.get_ticker(pair))
+        buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair))
 
         min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit)
         if min_stake_amount is not None and min_stake_amount > stake_amount:
@@ -530,7 +580,7 @@ class FreqtradeBot(object):
             raise ValueError(f'attempt to handle closed trade: {trade}')
 
         logger.debug('Handling %s ...', trade)
-        current_rate = self.exchange.get_ticker(trade.pair)['bid']
+        sell_rate = self.exchange.get_ticker(trade.pair)['bid']
 
         (buy, sell) = (False, False)
         experimental = self.config.get('experimental', {})
@@ -539,13 +589,43 @@ class FreqtradeBot(object):
             (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval,
                                                    ticker)
 
-        should_sell = self.strategy.should_sell(trade, current_rate, datetime.utcnow(), buy, sell)
+        config_ask_strategy = self.config.get('ask_strategy', {})
-        if should_sell.sell_flag:
+        if config_ask_strategy.get('use_order_book', False):
-            self.execute_sell(trade, current_rate, should_sell.sell_type)
+            logger.info('Using order book for selling...')
-            return True
+            # logger.debug('Order book %s',orderBook)
+            order_book_min = config_ask_strategy.get('order_book_min', 1)
+            order_book_max = config_ask_strategy.get('order_book_max', 1)
+
+            order_book = self.exchange.get_order_book(trade.pair, order_book_max)
+
+            for i in range(order_book_min, order_book_max + 1):
+                order_book_rate = order_book['asks'][i - 1][0]
+
+                # if orderbook has higher rate (high profit),
+                # use orderbook, otherwise just use bids rate
+                logger.info('  order book asks top %s: %0.8f', i, order_book_rate)
+                if sell_rate < order_book_rate:
+                    sell_rate = order_book_rate
+
+                if self.check_sell(trade, sell_rate, buy, sell):
+                    return True
+                    break
+        else:
+            logger.info('checking sell')
+            if self.check_sell(trade, sell_rate, buy, sell):
+                return True
+
         logger.info('Found no sell signals for whitelisted currencies. Trying again..')
         return False
 
+    def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool:
+        should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell)
+        if should_sell.sell_flag:
+            self.execute_sell(trade, sell_rate, should_sell.sell_type)
+            logger.info('excuted sell')
+            return True
+        return False
+
     def check_handle_timedout(self) -> None:
         """
         Check if any orders are timed out and cancel if neccessary

freqtrade/optimize/__init__.py

@@ -1,7 +1,13 @@
 # pragma pylint: disable=missing-docstring
 
 import gzip
-import json
+try:
+    import ujson as json
+    _UJSON = True
+except ImportError:
+    # see mypy/issues/1153
+    import json  # type: ignore
+    _UJSON = False
 import logging
 import os
 from typing import Optional, List, Dict, Tuple, Any
@@ -14,6 +20,14 @@ from freqtrade.arguments import TimeRange
 logger = logging.getLogger(__name__)
 
 
+def json_load(data):
+    """Try to load data with ujson"""
+    if _UJSON:
+        return json.load(data, precise_float=True)
+    else:
+        return json.load(data)
+
+
 def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
     if not tickerlist:
         return tickerlist
@@ -163,7 +177,7 @@ def load_cached_data_for_updating(filename: str,
     # read the cached file
     if os.path.isfile(filename):
         with open(filename, "rt") as file:
-            data = json.load(file)
+            data = json_load(file)
             # remove the last item, because we are not sure if it is correct
             # it could be fetched when the candle was incompleted
             if data:

freqtrade/persistence.py

@@ -79,10 +79,12 @@ def check_migrate(engine) -> None:
     table_back_name = 'trades_bak'
     for i, table_back_name in enumerate(tabs):
         table_back_name = f'trades_bak{i}'
-        logger.info(f'trying {table_back_name}')
+        logger.debug(f'trying {table_back_name}')
 
     # Check for latest column
     if not has_column(cols, 'ticker_interval'):
+        logger.info(f'Running database migration - backup available as {table_back_name}')
+
         fee_open = get_column_def(cols, 'fee_open', 'fee')
         fee_close = get_column_def(cols, 'fee_close', 'fee')
         open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null')

freqtrade/rpc/rpc.py

@@ -13,6 +13,7 @@ import sqlalchemy as sql
 from numpy import mean, nan_to_num
 from pandas import DataFrame
 
+from freqtrade import TemporaryError
 from freqtrade.fiat_convert import CryptoToFiatConverter
 from freqtrade.misc import shorten_date
 from freqtrade.persistence import Trade
@@ -273,10 +274,13 @@ class RPC(object):
             if coin == 'BTC':
                 rate = 1.0
             else:
-                if coin == 'USDT':
+                try:
-                    rate = 1.0 / self._freqtrade.exchange.get_ticker('BTC/USDT', False)['bid']
+                    if coin == 'USDT':
-                else:
+                        rate = 1.0 / self._freqtrade.exchange.get_ticker('BTC/USDT', False)['bid']
-                    rate = self._freqtrade.exchange.get_ticker(coin + '/BTC', False)['bid']
+                    else:
+                        rate = self._freqtrade.exchange.get_ticker(coin + '/BTC', False)['bid']
+                except TemporaryError:
+                    continue
             est_btc: float = rate * balance['total']
             total = total + est_btc
             output.append({

freqtrade/strategy/__init__.py

@@ -1,4 +1,5 @@
 import logging
+import sys
 from copy import deepcopy
 
 from freqtrade.strategy.interface import IStrategy
@@ -12,8 +13,18 @@ def import_strategy(strategy: IStrategy, config: dict) -> IStrategy:
     Imports given Strategy instance to global scope
     of freqtrade.strategy and returns an instance of it
     """
+
     # Copy all attributes from base class and class
-    attr = deepcopy({**strategy.__class__.__dict__, **strategy.__dict__})
+
+    comb = {**strategy.__class__.__dict__, **strategy.__dict__}
+
+    # Delete '_abc_impl' from dict as deepcopy fails on 3.7 with
+    # `TypeError: can't pickle _abc_data objects``
+    # This will only apply to python 3.7
+    if sys.version_info.major == 3 and sys.version_info.minor == 7 and '_abc_impl' in comb:
+        del comb['_abc_impl']
+
+    attr = deepcopy(comb)
     # Adjust module name
     attr['__module__'] = 'freqtrade.strategy'
 

freqtrade/strategy/interface.py

@@ -156,7 +156,8 @@ class IStrategy(ABC):
         # Check if dataframe is out of date
         signal_date = arrow.get(latest['date'])
         interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval]
-        if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))):
+        offset = self.config.get('exchange', {}).get('outdated_offset', 5)
+        if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + offset))):
             logger.warning(
                 'Outdated history for pair %s. Last tick is %s minutes old',
                 pair,

freqtrade/strategy/resolver.py

@@ -44,14 +44,15 @@ class StrategyResolver(object):
         # Check if we need to override configuration
         if 'minimal_roi' in config:
             self.strategy.minimal_roi = config['minimal_roi']
-            logger.info("Override strategy \'minimal_roi\' with value in config file.")
+            logger.info("Override strategy 'minimal_roi' with value in config file: %s.",
+                        config['minimal_roi'])
         else:
             config['minimal_roi'] = self.strategy.minimal_roi
 
         if 'stoploss' in config:
             self.strategy.stoploss = config['stoploss']
             logger.info(
-                "Override strategy \'stoploss\' with value in config file: %s.", config['stoploss']
+                "Override strategy 'stoploss' with value in config file: %s.", config['stoploss']
             )
         else:
             config['stoploss'] = self.strategy.stoploss
@@ -59,7 +60,7 @@ class StrategyResolver(object):
         if 'ticker_interval' in config:
             self.strategy.ticker_interval = config['ticker_interval']
             logger.info(
-                "Override strategy \'ticker_interval\' with value in config file: %s.",
+                "Override strategy 'ticker_interval' with value in config file: %s.",
                 config['ticker_interval']
             )
         else:

freqtrade/tests/conftest.py

@@ -102,7 +102,18 @@ def default_conf():
             "sell": 30
         },
         "bid_strategy": {
-            "ask_last_balance": 0.0
+            "ask_last_balance": 0.0,
+            "use_order_book": False,
+            "order_book_top": 1,
+            "check_depth_of_market": {
+                "enabled": False,
+                "bids_to_ask_delta": 1
+            }
+        },
+        "ask_strategy": {
+            "use_order_book": False,
+            "order_book_min": 1,
+            "order_book_max": 1
         },
         "exchange": {
             "name": "bittrex",
@@ -403,6 +414,39 @@ def limit_sell_order():
     }
 
 
+@pytest.fixture
+def order_book_l2():
+    return MagicMock(return_value={
+        'bids': [
+            [0.043936, 10.442],
+            [0.043935, 31.865],
+            [0.043933, 11.212],
+            [0.043928, 0.088],
+            [0.043925, 10.0],
+            [0.043921, 10.0],
+            [0.04392, 37.64],
+            [0.043899, 0.066],
+            [0.043885, 0.676],
+            [0.04387, 22.758]
+        ],
+        'asks': [
+            [0.043949, 0.346],
+            [0.04395, 0.608],
+            [0.043951, 3.948],
+            [0.043954, 0.288],
+            [0.043958, 9.277],
+            [0.043995, 1.566],
+            [0.044, 0.588],
+            [0.044002, 0.992],
+            [0.044003, 0.095],
+            [0.04402, 37.64]
+        ],
+        'timestamp': None,
+        'datetime': None,
+        'nonce': 288004540
+    })
+
+
 @pytest.fixture
 def ticker_history():
     return [

freqtrade/tests/exchange/test_exchange.py

@@ -703,6 +703,35 @@ async def test_async_get_candles_history(default_conf, mocker):
     assert exchange._async_get_candle_history.call_count == 2
 
 
+def test_get_order_book(default_conf, mocker, order_book_l2):
+    default_conf['exchange']['name'] = 'binance'
+    api_mock = MagicMock()
+
+    api_mock.fetch_l2_order_book = order_book_l2
+    exchange = get_patched_exchange(mocker, default_conf, api_mock)
+    order_book = exchange.get_order_book(pair='ETH/BTC', limit=10)
+    assert 'bids' in order_book
+    assert 'asks' in order_book
+    assert len(order_book['bids']) == 10
+    assert len(order_book['asks']) == 10
+
+
+def test_get_order_book_exception(default_conf, mocker):
+    api_mock = MagicMock()
+    with pytest.raises(OperationalException):
+        api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.NotSupported)
+        exchange = get_patched_exchange(mocker, default_conf, api_mock)
+        exchange.get_order_book(pair='ETH/BTC', limit=50)
+    with pytest.raises(TemporaryError):
+        api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.NetworkError)
+        exchange = get_patched_exchange(mocker, default_conf, api_mock)
+        exchange.get_order_book(pair='ETH/BTC', limit=50)
+    with pytest.raises(OperationalException):
+        api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.BaseError)
+        exchange = get_patched_exchange(mocker, default_conf, api_mock)
+        exchange.get_order_book(pair='ETH/BTC', limit=50)
+
+
 def make_fetch_ohlcv_mock(data):
     def fetch_ohlcv_mock(pair, timeframe, since):
         if since:
@@ -1011,15 +1040,3 @@ def test_get_fee(default_conf, mocker):
 
     ccxt_exceptionhandlers(mocker, default_conf, api_mock,
                            'get_fee', 'calculate_fee')
-
-
-def test_get_amount_lots(default_conf, mocker):
-    api_mock = MagicMock()
-    api_mock.amount_to_lots = MagicMock(return_value=1.0)
-    api_mock.markets = None
-    marketmock = MagicMock()
-    api_mock.load_markets = marketmock
-    exchange = get_patched_exchange(mocker, default_conf, api_mock)
-
-    assert exchange.get_amount_lots('LTC/BTC', 1.54) == 1
-    assert marketmock.call_count == 1

freqtrade/tests/rpc/test_rpc.py

@@ -6,6 +6,7 @@ from unittest.mock import MagicMock, ANY
 
 import pytest
 
+from freqtrade import TemporaryError
 from freqtrade.fiat_convert import CryptoToFiatConverter
 from freqtrade.freqtradebot import FreqtradeBot
 from freqtrade.persistence import Trade
@@ -285,11 +286,12 @@ def test_rpc_balance_handle(default_conf, mocker):
             'used': 2.0,
         },
         'ETH': {
-            'free': 0.0,
+            'free': 1.0,
-            'total': 0.0,
+            'total': 5.0,
-            'used': 0.0,
+            'used': 4.0,
         }
     }
+    # ETH will be skipped due to mocked Error below
 
     mocker.patch.multiple(
         'freqtrade.fiat_convert.Market',
@@ -301,7 +303,8 @@ def test_rpc_balance_handle(default_conf, mocker):
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         validate_pairs=MagicMock(),
-        get_balances=MagicMock(return_value=mock_balance)
+        get_balances=MagicMock(return_value=mock_balance),
+        get_ticker=MagicMock(side_effect=TemporaryError('Could not load ticker due to xxx'))
     )
 
     freqtradebot = FreqtradeBot(default_conf)
@@ -320,6 +323,7 @@ def test_rpc_balance_handle(default_conf, mocker):
         'pending': 2.0,
         'est_btc': 12.0,
     }]
+    assert result['total'] == 12.0
 
 
 def test_rpc_start(mocker, default_conf) -> None:

freqtrade/tests/strategy/test_interface.py

@@ -8,6 +8,7 @@ from pandas import DataFrame
 
 from freqtrade.arguments import TimeRange
 from freqtrade.optimize.__init__ import load_tickerdata_file
+from freqtrade.persistence import Trade
 from freqtrade.tests.conftest import get_patched_exchange, log_has
 from freqtrade.strategy.default_strategy import DefaultStrategy
 
@@ -104,3 +105,26 @@ def test_tickerdata_to_dataframe(default_conf) -> None:
     tickerlist = {'UNITTEST/BTC': tick}
     data = strategy.tickerdata_to_dataframe(tickerlist)
     assert len(data['UNITTEST/BTC']) == 99       # partial candle was removed
+
+
+def test_min_roi_reached(default_conf, fee) -> None:
+    strategy = DefaultStrategy(default_conf)
+    strategy.minimal_roi = {0: 0.1, 20: 0.05, 55: 0.01}
+    trade = Trade(
+        pair='ETH/BTC',
+        stake_amount=0.001,
+        open_date=arrow.utcnow().shift(hours=-1).datetime,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        exchange='bittrex',
+        open_rate=1,
+    )
+
+    assert not strategy.min_roi_reached(trade, 0.01, arrow.utcnow().shift(minutes=-55).datetime)
+    assert strategy.min_roi_reached(trade, 0.12, arrow.utcnow().shift(minutes=-55).datetime)
+
+    assert not strategy.min_roi_reached(trade, 0.04, arrow.utcnow().shift(minutes=-39).datetime)
+    assert strategy.min_roi_reached(trade, 0.06, arrow.utcnow().shift(minutes=-39).datetime)
+
+    assert not strategy.min_roi_reached(trade, -0.01, arrow.utcnow().shift(minutes=-1).datetime)
+    assert strategy.min_roi_reached(trade, 0.02, arrow.utcnow().shift(minutes=-1).datetime)

freqtrade/tests/strategy/test_strategy.py

@@ -130,7 +130,7 @@ def test_strategy_override_minimal_roi(caplog):
     assert resolver.strategy.minimal_roi[0] == 0.5
     assert ('freqtrade.strategy.resolver',
             logging.INFO,
-            'Override strategy \'minimal_roi\' with value in config file.'
+            "Override strategy 'minimal_roi' with value in config file: {'0': 0.5}."
             ) in caplog.record_tuples
 
 
@@ -145,7 +145,7 @@ def test_strategy_override_stoploss(caplog):
     assert resolver.strategy.stoploss == -0.5
     assert ('freqtrade.strategy.resolver',
             logging.INFO,
-            'Override strategy \'stoploss\' with value in config file: -0.5.'
+            "Override strategy 'stoploss' with value in config file: -0.5."
             ) in caplog.record_tuples
 
 
@@ -161,7 +161,7 @@ def test_strategy_override_ticker_interval(caplog):
     assert resolver.strategy.ticker_interval == 60
     assert ('freqtrade.strategy.resolver',
             logging.INFO,
-            'Override strategy \'ticker_interval\' with value in config file: 60.'
+            "Override strategy 'ticker_interval' with value in config file: 60."
             ) in caplog.record_tuples
 
 

freqtrade/tests/test_freqtradebot.py

@@ -159,6 +159,15 @@ def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None:
     assert whitelist == []
 
 
+def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None:
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers)
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False))
+
+    with pytest.raises(OperationalException):
+        freqtrade._gen_pair_whitelist(base_currency='BTC')
+
+
 @pytest.mark.skip(reason="Test not implemented")
 def test_refresh_whitelist() -> None:
     pass
@@ -663,21 +672,21 @@ def test_balance_fully_ask_side(mocker, default_conf) -> None:
     default_conf['bid_strategy']['ask_last_balance'] = 0.0
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
-    assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 20
+    assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 20
 
 
 def test_balance_fully_last_side(mocker, default_conf) -> None:
     default_conf['bid_strategy']['ask_last_balance'] = 1.0
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
-    assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 10
+    assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 10
 
 
 def test_balance_bigger_last_ask(mocker, default_conf) -> None:
     default_conf['bid_strategy']['ask_last_balance'] = 1.0
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
-    assert freqtrade.get_target_bid({'ask': 5, 'last': 10}) == 5
+    assert freqtrade.get_target_bid('ETH/BTC', {'ask': 5, 'last': 10}) == 5
 
 
 def test_process_maybe_execute_buy(mocker, default_conf) -> None:
@@ -1877,6 +1886,191 @@ def test_get_real_amount_open_trade(default_conf, mocker):
     assert freqtrade.get_real_amount(trade, order) == amount
 
 
+def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, markets, mocker,
+                                    order_book_l2):
+    default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True
+    default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1
+    patch_RPCManager(mocker)
+    mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2)
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        validate_pairs=MagicMock(),
+        get_ticker=ticker,
+        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        get_fee=fee,
+        get_markets=markets
+    )
+
+    # Save state of current whitelist
+    whitelist = deepcopy(default_conf['exchange']['pair_whitelist'])
+    freqtrade = FreqtradeBot(default_conf)
+    patch_get_signal(freqtrade)
+    freqtrade.create_trade()
+
+    trade = Trade.query.first()
+    assert trade is not None
+    assert trade.stake_amount == 0.001
+    assert trade.is_open
+    assert trade.open_date is not None
+    assert trade.exchange == 'bittrex'
+
+    # Simulate fulfilled LIMIT_BUY order for trade
+    trade.update(limit_buy_order)
+
+    assert trade.open_rate == 0.00001099
+    assert whitelist == default_conf['exchange']['pair_whitelist']
+
+
+def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_order,
+                                               fee, markets, mocker, order_book_l2):
+    default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True
+    # delta is 100 which is impossible to reach. hence check_depth_of_market will return false
+    default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100
+    patch_RPCManager(mocker)
+    mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2)
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        validate_pairs=MagicMock(),
+        get_ticker=ticker,
+        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        get_fee=fee,
+        get_markets=markets
+    )
+    # Save state of current whitelist
+    freqtrade = FreqtradeBot(default_conf)
+    patch_get_signal(freqtrade)
+    freqtrade.create_trade()
+
+    trade = Trade.query.first()
+    assert trade is None
+
+
+def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, markets) -> None:
+    """
+    test if function get_target_bid will return the order book price
+    instead of the ask rate
+    """
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        validate_pairs=MagicMock(),
+        get_markets=markets,
+        get_order_book=order_book_l2
+    )
+    default_conf['exchange']['name'] = 'binance'
+    default_conf['bid_strategy']['use_order_book'] = True
+    default_conf['bid_strategy']['order_book_top'] = 2
+    default_conf['bid_strategy']['ask_last_balance'] = 0
+    default_conf['telegram']['enabled'] = False
+
+    freqtrade = FreqtradeBot(default_conf)
+    assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.045, 'last': 0.046}) == 0.043935
+
+
+def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) -> None:
+    """
+    test if function get_target_bid will return the ask rate (since its value is lower)
+    instead of the order book rate (even if enabled)
+    """
+    mocker.patch.multiple(
+           'freqtrade.exchange.Exchange',
+           validate_pairs=MagicMock(),
+           get_markets=markets,
+           get_order_book=order_book_l2
+    )
+    default_conf['exchange']['name'] = 'binance'
+    default_conf['bid_strategy']['use_order_book'] = True
+    default_conf['bid_strategy']['order_book_top'] = 2
+    default_conf['bid_strategy']['ask_last_balance'] = 0
+    default_conf['telegram']['enabled'] = False
+
+    freqtrade = FreqtradeBot(default_conf)
+    assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.042, 'last': 0.046}) == 0.042
+
+
+def test_order_book_bid_strategy3(default_conf, mocker, order_book_l2, markets) -> None:
+    """
+    test if function get_target_bid will return ask rate instead
+    of the order book rate
+    """
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        validate_pairs=MagicMock(),
+        get_markets=markets,
+        get_order_book=order_book_l2
+    )
+    default_conf['exchange']['name'] = 'binance'
+    default_conf['bid_strategy']['use_order_book'] = True
+    default_conf['bid_strategy']['order_book_top'] = 1
+    default_conf['bid_strategy']['ask_last_balance'] = 0
+    default_conf['telegram']['enabled'] = False
+
+    freqtrade = FreqtradeBot(default_conf)
+
+    assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.03, 'last': 0.029}) == 0.03
+
+
+def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2, markets) -> None:
+    """
+    test check depth of market
+    """
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        validate_pairs=MagicMock(),
+        get_markets=markets,
+        get_order_book=order_book_l2
+    )
+    default_conf['telegram']['enabled'] = False
+    default_conf['exchange']['name'] = 'binance'
+    default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True
+    # delta is 100 which is impossible to reach. hence function will return false
+    default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100
+    freqtrade = FreqtradeBot(default_conf)
+
+    conf = default_conf['bid_strategy']['check_depth_of_market']
+    assert freqtrade._check_depth_of_market_buy('ETH/BTC', conf) is False
+
+
+def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order,
+                                 fee, markets, mocker, order_book_l2) -> None:
+    """
+    test order book ask strategy
+    """
+    mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2)
+    default_conf['exchange']['name'] = 'binance'
+    default_conf['ask_strategy']['use_order_book'] = True
+    default_conf['ask_strategy']['order_book_min'] = 1
+    default_conf['ask_strategy']['order_book_max'] = 2
+    default_conf['telegram']['enabled'] = False
+    patch_RPCManager(mocker)
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        validate_pairs=MagicMock(),
+        get_ticker=MagicMock(return_value={
+            'bid': 0.00001172,
+            'ask': 0.00001173,
+            'last': 0.00001172
+        }),
+        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        sell=MagicMock(return_value={'id': limit_sell_order['id']}),
+        get_fee=fee,
+        get_markets=markets
+    )
+    freqtrade = FreqtradeBot(default_conf)
+    patch_get_signal(freqtrade)
+
+    freqtrade.create_trade()
+
+    trade = Trade.query.first()
+    assert trade
+
+    time.sleep(0.01)  # Race condition fix
+    trade.update(limit_buy_order)
+    assert trade.is_open is True
+
+    patch_get_signal(freqtrade, value=(False, True))
+    assert freqtrade.handle_trade(trade) is True
+
+
 def test_startup_messages(default_conf, mocker):
     default_conf['dynamic_whitelist'] = 20
     freqtrade = get_patched_freqtradebot(mocker, default_conf)

freqtrade/tests/test_persistence.py

@@ -1,5 +1,6 @@
 # pragma pylint: disable=missing-docstring, C0103
 from unittest.mock import MagicMock
+import logging
 
 import pytest
 from sqlalchemy import create_engine
@@ -403,6 +404,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     """
     Test Database migration (starting with new pairformat)
     """
+    caplog.set_level(logging.DEBUG)
     amount = 103.223
     # Always create all columns apart from the last!
     create_table_old = """CREATE TABLE IF NOT EXISTS "trades" (
@@ -471,12 +473,15 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert trade.ticker_interval is None
     assert log_has("trying trades_bak1", caplog.record_tuples)
     assert log_has("trying trades_bak2", caplog.record_tuples)
+    assert log_has("Running database migration - backup available as trades_bak2",
+                   caplog.record_tuples)
 
 
 def test_migrate_mid_state(mocker, default_conf, fee, caplog):
     """
     Test Database migration (starting with new pairformat)
     """
+    caplog.set_level(logging.DEBUG)
     amount = 103.223
     create_table_old = """CREATE TABLE IF NOT EXISTS "trades" (
                                 id INTEGER NOT NULL,
@@ -530,6 +535,8 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog):
     assert trade.stop_loss == 0.0
     assert trade.initial_stop_loss == 0.0
     assert log_has("trying trades_bak0", caplog.record_tuples)
+    assert log_has("Running database migration - backup available as trades_bak0",
+                   caplog.record_tuples)
 
 
 def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee):

requirements.txt

@@ -1,5 +1,5 @@
-ccxt==1.17.126
+ccxt==1.17.205
-SQLAlchemy==1.2.10
+SQLAlchemy==1.2.11
 python-telegram-bot==10.1.0
 arrow==0.12.1
 cachetools==2.1.0
@@ -10,9 +10,9 @@ pandas==0.23.4
 scikit-learn==0.19.2
 scipy==1.1.0
 jsonschema==2.6.0
-numpy==1.15.0
+numpy==1.15.1
 TA-Lib==0.4.17
-pytest==3.7.1
+pytest==3.7.3
 pytest-mock==1.10.0
 pytest-cov==2.5.1
 pytest-asyncio==0.9.0

setup.sh

@@ -1,13 +1,31 @@
 #!/usr/bin/env bash
 #encoding=utf8
 
+# Check which python version is installed
+function check_installed_python() {
+    which python3.7
+    if [ $? -eq 0 ]; then
+        echo "using Python 3.7"
+        PYTHON=python3.7
+        return
+    fi
+
+    which python3.6
+    if [ $? -eq 0 ]; then
+        echo "using Python 3.6"
+        PYTHON=python3.6
+        return
+   fi
+
+}
+
 function updateenv () {
     echo "-------------------------"
     echo "Update your virtual env"
     echo "-------------------------"
     source .env/bin/activate
     echo "pip3 install in-progress. Please wait..."
-    pip3.6 install --quiet --upgrade pip
+    pip3 install --quiet --upgrade pip
     pip3 install --quiet -r requirements.txt --upgrade
     pip3 install --quiet -r requirements.txt
     pip3 install --quiet -e .
@@ -79,7 +97,7 @@ function reset () {
     fi
 
     echo
-    python3.6 -m venv .env
+    ${PYTHON} -m venv .env
     updateenv
 }
 
@@ -183,7 +201,7 @@ function install () {
         install_debian
     else
         echo "This script does not support your OS."
-        echo "If you have Python3.6, pip, virtualenv, ta-lib you can continue."
+        echo "If you have Python3.6 or Python3.7, pip, virtualenv, ta-lib you can continue."
         echo "Wait 10 seconds to continue the next install steps or use ctrl+c to interrupt this shell."
         sleep 10
     fi
@@ -193,7 +211,7 @@ function install () {
     echo "-------------------------"
     echo "Run the bot"
     echo "-------------------------"
-    echo "You can now use the bot by executing 'source .env/bin/activate; python3.6 freqtrade/main.py'."
+    echo "You can now use the bot by executing 'source .env/bin/activate; python freqtrade/main.py'."
 }
 
 function plot () {
@@ -214,6 +232,9 @@ function help () {
     echo "	-p,--plot       Install dependencies for Plotting scripts."
 }
 
+# Verify if 3.6 or 3.7 is installed
+check_installed_python
+
 case $* in
 --install|-i)
 install