Merge branch 'develop' into money_mgt

README.md

@@ -200,6 +200,10 @@ to understand the requirements before sending your pull-requests.
 
 ## Requirements
 
+### Uptodate clock
+
+The clock must be accurate, syncronized to a NTP server very frequently to avoid problems with communication to the exchanges.
+
 ### Min hardware required
 
 To run this bot we recommend you a cloud instance with a minimum of:

docs/installation.md

@@ -267,7 +267,7 @@ Official webpage: https://mrjbq7.github.io/ta-lib/install.html
 wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz
 tar xvzf ta-lib-0.4.0-src.tar.gz
 cd ta-lib
-sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h
+sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h
 ./configure --prefix=/usr
 make
 make install

freqtrade/arguments.py

@@ -119,7 +119,6 @@ class Arguments(object):
             help='Override trades database URL, this is useful if dry_run is enabled'
                  ' or in custom deployments (default: %(default)s)',
             dest='db_url',
-            default=constants.DEFAULT_DB_PROD_URL,
             type=str,
             metavar='PATH',
         )

freqtrade/configuration.py

@@ -110,9 +110,12 @@ class Configuration(object):
                 '(not applicable with Backtesting and Hyperopt)'
             )
 
-        if self.args.db_url != constants.DEFAULT_DB_PROD_URL:
+        if self.args.db_url and self.args.db_url != constants.DEFAULT_DB_PROD_URL:
             config.update({'db_url': self.args.db_url})
             logger.info('Parameter --db-url detected ...')
+        else:
+            # Set default here
+            config.update({'db_url': constants.DEFAULT_DB_PROD_URL})
 
         if config.get('dry_run', False):
             logger.info('Dry run is enabled')

freqtrade/exchange/__init__.py

@@ -599,7 +599,8 @@ class Exchange(object):
         if not self.exchange_has('fetchMyTrades'):
             return []
         try:
-            my_trades = self._api.fetch_my_trades(pair, since.timestamp())
+            # Allow 5s offset to catch slight time offsets (discovered in #1185)
+            my_trades = self._api.fetch_my_trades(pair, since.timestamp() - 5)
             matched_trades = [trade for trade in my_trades if trade['order'] == order_id]
 
             return matched_trades

freqtrade/freqtradebot.py

@@ -10,7 +10,7 @@ from datetime import datetime
 from typing import Any, Callable, Dict, List, Optional
 
 import arrow
-import requests
+from requests.exceptions import RequestException
 
 from cachetools import TTLCache, cached
 
@@ -667,7 +667,7 @@ class FreqtradeBot(object):
                 if not trade.open_order_id:
                     continue
                 order = self.exchange.get_order(trade.open_order_id, trade.pair)
-            except requests.exceptions.RequestException:
+            except (RequestException, DependencyException):
                 logger.info(
                     'Cannot query order for %s due to %s',
                     trade,

freqtrade/tests/test_acl_pair.py

@@ -2,38 +2,39 @@
 
 from unittest.mock import MagicMock
 
-import freqtrade.tests.conftest as tt  # test tools
+from freqtrade.tests.conftest import get_patched_freqtradebot
+
+import pytest
 
 # whitelist, blacklist, filtering, all of that will
 # eventually become some rules to run on a generic ACL engine
 # perhaps try to anticipate that by using some python package
 
 
-def whitelist_conf():
+@pytest.fixture(scope="function")
-    config = tt.default_conf()
+def whitelist_conf(default_conf):
-    config['stake_currency'] = 'BTC'
+    default_conf['stake_currency'] = 'BTC'
-    config['exchange']['pair_whitelist'] = [
+    default_conf['exchange']['pair_whitelist'] = [
         'ETH/BTC',
         'TKN/BTC',
         'TRST/BTC',
         'SWT/BTC',
         'BCC/BTC'
     ]
-    config['exchange']['pair_blacklist'] = [
+    default_conf['exchange']['pair_blacklist'] = [
         'BLK/BTC'
     ]
 
-    return config
+    return default_conf
 
 
-def test_refresh_market_pair_not_in_whitelist(mocker, markets):
+def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf):
-    conf = whitelist_conf()
 
-    freqtradebot = tt.get_patched_freqtradebot(mocker, conf)
+    freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf)
 
     mocker.patch('freqtrade.exchange.Exchange.get_markets', markets)
     refreshedwhitelist = freqtradebot._refresh_whitelist(
-        conf['exchange']['pair_whitelist'] + ['XXX/BTC']
+        whitelist_conf['exchange']['pair_whitelist'] + ['XXX/BTC']
     )
     # List ordered by BaseVolume
     whitelist = ['ETH/BTC', 'TKN/BTC']
@@ -41,12 +42,12 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets):
     assert whitelist == refreshedwhitelist
 
 
-def test_refresh_whitelist(mocker, markets):
+def test_refresh_whitelist(mocker, markets, whitelist_conf):
-    conf = whitelist_conf()
+    freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf)
-    freqtradebot = tt.get_patched_freqtradebot(mocker, conf)
 
     mocker.patch('freqtrade.exchange.Exchange.get_markets', markets)
-    refreshedwhitelist = freqtradebot._refresh_whitelist(conf['exchange']['pair_whitelist'])
+    refreshedwhitelist = freqtradebot._refresh_whitelist(
+        whitelist_conf['exchange']['pair_whitelist'])
 
     # List ordered by BaseVolume
     whitelist = ['ETH/BTC', 'TKN/BTC']
@@ -54,9 +55,8 @@ def test_refresh_whitelist(mocker, markets):
     assert whitelist == refreshedwhitelist
 
 
-def test_refresh_whitelist_dynamic(mocker, markets, tickers):
+def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf):
-    conf = whitelist_conf()
+    freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf)
-    freqtradebot = tt.get_patched_freqtradebot(mocker, conf)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         get_markets=markets,
@@ -68,21 +68,20 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers):
     whitelist = ['ETH/BTC', 'TKN/BTC']
 
     refreshedwhitelist = freqtradebot._refresh_whitelist(
-        freqtradebot._gen_pair_whitelist(conf['stake_currency'])
+        freqtradebot._gen_pair_whitelist(whitelist_conf['stake_currency'])
     )
 
     assert whitelist == refreshedwhitelist
 
 
-def test_refresh_whitelist_dynamic_empty(mocker, markets_empty):
+def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf):
-    conf = whitelist_conf()
+    freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf)
-    freqtradebot = tt.get_patched_freqtradebot(mocker, conf)
     mocker.patch('freqtrade.exchange.Exchange.get_markets', markets_empty)
 
     # argument: use the whitelist dynamically by exchange-volume
     whitelist = []
-    conf['exchange']['pair_whitelist'] = []
+    whitelist_conf['exchange']['pair_whitelist'] = []
     freqtradebot._refresh_whitelist(whitelist)
-    pairslist = conf['exchange']['pair_whitelist']
+    pairslist = whitelist_conf['exchange']['pair_whitelist']
 
     assert set(whitelist) == set(pairslist)

freqtrade/tests/test_freqtradebot.py

@@ -993,6 +993,44 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe
     assert nb_trades == 0
 
 
+def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_order_old,
+                                             fee, mocker) -> None:
+    rpc_mock = patch_RPCManager(mocker)
+    cancel_order_mock = MagicMock()
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        validate_pairs=MagicMock(),
+        get_ticker=ticker,
+        get_order=MagicMock(side_effect=DependencyException),
+        cancel_order=cancel_order_mock,
+        get_fee=fee
+    )
+    freqtrade = FreqtradeBot(default_conf)
+
+    trade_buy = Trade(
+        pair='ETH/BTC',
+        open_rate=0.00001099,
+        exchange='bittrex',
+        open_order_id='123456789',
+        amount=90.99181073,
+        fee_open=0.0,
+        fee_close=0.0,
+        stake_amount=1,
+        open_date=arrow.utcnow().shift(minutes=-601).datetime,
+        is_open=True
+    )
+
+    Trade.session.add(trade_buy)
+
+    # check it does cancel buy orders over the time limit
+    freqtrade.check_handle_timedout()
+    assert cancel_order_mock.call_count == 0
+    assert rpc_mock.call_count == 0
+    trades = Trade.query.filter(Trade.open_order_id.is_(trade_buy.open_order_id)).all()
+    nb_trades = len(trades)
+    assert nb_trades == 1
+
+
 def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker) -> None:
     rpc_mock = patch_RPCManager(mocker)
     cancel_order_mock = MagicMock()

install_ta-lib.sh

@@ -1,6 +1,6 @@
 if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then
   tar zxvf ta-lib-0.4.0-src.tar.gz
-  cd ta-lib && sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd ..
+  cd ta-lib && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd ..
 else
   echo "TA-lib already installed, skipping download and build."
   cd ta-lib && sudo make install && cd ..

requirements.txt

@@ -1,4 +1,4 @@
-ccxt==1.17.350
+ccxt==1.17.351
 SQLAlchemy==1.2.12
 python-telegram-bot==11.1.0
 arrow==0.12.1

scripts/plot_dataframe.py

@@ -73,7 +73,7 @@ def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFram
         file = Path(args.exportfilename)
         # must align with columns in backtest.py
         columns = ["pair", "profit", "opents", "closets", "index", "duration",
-                   "open_rate", "close_rate", "open_at_end"]
+                   "open_rate", "close_rate", "open_at_end", "sell_reason"]
         with file.open() as f:
             data = json.load(f)
             trades = pd.DataFrame(data, columns=columns)