Merge remote-tracking branch 'upstream/develop' into StopLossSupport

.dockerignore

@@ -4,3 +4,12 @@ Dockerfile
 .dockerignore
 config.json*
 *.sqlite
+.coveragerc
+.eggs
+.github
+.pylintrc
+.travis.yml
+CONTRIBUTING.md
+MANIFEST.in
+README.md
+freqtrade.service

config.json.example

@@ -3,6 +3,7 @@
     "stake_currency": "BTC",
     "stake_amount": 0.05,
     "fiat_display_currency": "USD",
+    "ticker_interval" : "5m",
     "dry_run": false,
     "trailing_stop": {
         "positive" : 0.005

docs/plotting.md

@@ -43,6 +43,10 @@ python scripts/plot_dataframe.py -p BTC_ETH --timerange=100-200
 ```
 Timerange doesn't work with live data.
 
+To plot trades stored in a database use `--db-url` argument:
+```
+python scripts/plot_dataframe.py --db-url tradesv3.dry_run.sqlite -p BTC_ETH
+```
 
 ## Plot profit
 

docs/sql_cheatsheet.md

@@ -32,9 +32,12 @@ CREATE TABLE trades (
 	exchange VARCHAR NOT NULL,
 	pair VARCHAR NOT NULL,
 	is_open BOOLEAN NOT NULL,
-	fee FLOAT NOT NULL,
+	fee_open FLOAT NOT NULL,
+	fee_close FLOAT NOT NULL,
 	open_rate FLOAT,
+	open_rate_requested FLOAT,
 	close_rate FLOAT,
+	close_rate_requested FLOAT,
 	close_profit FLOAT,
 	stake_amount FLOAT NOT NULL,
 	amount FLOAT,
@@ -71,13 +74,13 @@ WHERE id=31;
 
 ```sql
 INSERT 
-INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date) 
+INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, stake_amount, amount, open_date) 
-VALUES ('BITTREX', 'BTC_<COIN>', 1, 0.0025, <open_rate>, <stake_amount>, <amount>, '<datetime>')
+VALUES ('BITTREX', 'BTC_<COIN>', 1, 0.0025, 0.0025, <open_rate>, <stake_amount>, <amount>, '<datetime>')
 ```
 
 **Example:**
 ```sql
-INSERT INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date) VALUES ('BITTREX', 'BTC_ETC', 1, 0.0025, 0.00258580, 0.002, 0.7715262081, '2017-11-28 12:44:24.000000')
+INSERT INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, stake_amount, amount, open_date) VALUES ('BITTREX', 'BTC_ETC', 1, 0.0025, 0.0025, 0.00258580, 0.002, 0.7715262081, '2017-11-28 12:44:24.000000')
 ```
 
 ## Fix wrong fees in the table

docs/telegram-usage.md

@@ -129,12 +129,8 @@ Day         Profit BTC      Profit USD
 > **Version:** `0.14.3` 
 
 ### using proxy with telegram
-in [freqtrade/freqtrade/rpc/telegram.py](https://github.com/gcarq/freqtrade/blob/develop/freqtrade/rpc/telegram.py) replace
 ```
-self._updater = Updater(token=self._config['telegram']['token'], workers=0)
+$ export HTTP_PROXY="http://addr:port"
-```
+$ export HTTPS_PROXY="http://addr:port"
-
+$ freqtrade
-with
-```
-self._updater = Updater(token=self._config['telegram']['token'], request_kwargs={'proxy_url': 'socks5://127.0.0.1:1080/'}, workers=0)
 ```

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ FreqTrade bot """
-__version__ = '0.16.0'
+__version__ = '0.17.0'
 
 
 class DependencyException(BaseException):

freqtrade/arguments.py

@@ -260,6 +260,13 @@ class Arguments(object):
             default=None
         )
 
+        self.parser.add_argument(
+            '-db', '--db-url',
+            help='Show trades stored in database.',
+            dest='db_url',
+            default=None
+        )
+
     def testdata_dl_options(self) -> None:
         """
         Parses given arguments for testdata download

freqtrade/exchange/__init__.py

@@ -297,9 +297,10 @@ def get_ticker_history(pair: str, tick_interval: str, since_ms: Optional[int] =
             if not data_part:
                 break
 
-            logger.info('Downloaded data for time range [%s, %s]',
+            logger.debug('Downloaded data for %s time range [%s, %s]',
-                        arrow.get(data_part[0][0] / 1000).format(),
+                         pair,
-                        arrow.get(data_part[-1][0] / 1000).format())
+                         arrow.get(data_part[0][0] / 1000).format(),
+                         arrow.get(data_part[-1][0] / 1000).format())
 
             data.extend(data_part)
             since_ms = data[-1][0] + 1

freqtrade/fiat_convert.py

@@ -5,6 +5,7 @@ e.g BTC to USD
 
 import logging
 import time
+from typing import Dict
 
 from coinmarketcap import Market
 
@@ -73,12 +74,7 @@ class CryptoToFiatConverter(object):
         "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD"
     ]
 
-    CRYPTOMAP = {
+    _cryptomap: Dict = {}
-        'BTC': 'bitcoin',
-        'ETH': 'ethereum',
-        'USDT': 'thether',
-        'BNB': 'binance-coin'
-    }
 
     def __new__(cls):
         if CryptoToFiatConverter.__instance is None:
@@ -91,6 +87,15 @@ class CryptoToFiatConverter(object):
 
     def __init__(self) -> None:
         self._pairs = []
+        self._load_cryptomap()
+
+    def _load_cryptomap(self) -> None:
+        try:
+            coinlistings = self._coinmarketcap.listings()
+            self._cryptomap = dict(map(lambda coin: (coin["symbol"], str(coin["id"])),
+                                       coinlistings["data"]))
+        except ValueError:
+            logger.error("Could not load FIAT Cryptocurrency map")
 
     def convert_amount(self, crypto_amount: float, crypto_symbol: str, fiat_symbol: str) -> float:
         """
@@ -182,16 +187,17 @@ class CryptoToFiatConverter(object):
         if not self._is_supported_fiat(fiat=fiat_symbol):
             raise ValueError('The fiat {} is not supported.'.format(fiat_symbol))
 
-        if crypto_symbol not in self.CRYPTOMAP:
+        if crypto_symbol not in self._cryptomap:
             # return 0 for unsupported stake currencies (fiat-convert should not break the bot)
             logger.warning("unsupported crypto-symbol %s - returning 0.0", crypto_symbol)
             return 0.0
         try:
             return float(
                 self._coinmarketcap.ticker(
-                    currency=self.CRYPTOMAP[crypto_symbol],
+                    currency=self._cryptomap[crypto_symbol],
                     convert=fiat_symbol
-                )[0]['price_' + fiat_symbol.lower()]
+                )['data']['quotes'][fiat_symbol.upper()]['price']
             )
-        except BaseException:
+        except BaseException as ex:
+            logger.error("Error in _find_price: %s", ex)
             return 0.0

freqtrade/freqtradebot.py

@@ -330,6 +330,7 @@ class FreqtradeBot(object):
             fee_open=fee,
             fee_close=fee,
             open_rate=buy_limit,
+            open_rate_requested=buy_limit,
             open_date=datetime.utcnow(),
             exchange=exchange.get_id(),
             open_order_id=order_id
@@ -396,7 +397,7 @@ class FreqtradeBot(object):
             return order_amount
 
         # use fee from order-dict if possible
-        if 'fee' in order and order['fee']:
+        if 'fee' in order and order['fee'] and (order['fee'].keys() >= {'currency', 'cost'}):
             if trade.pair.startswith(order['fee']['currency']):
                 new_amount = order_amount - order['fee']['cost']
                 logger.info("Applying fee on amount for %s (from %s to %s) from Order",
@@ -413,7 +414,7 @@ class FreqtradeBot(object):
         fee_abs = 0
         for exectrade in trades:
             amount += exectrade['amount']
-            if "fee" in exectrade:
+            if "fee" in exectrade and (exectrade['fee'].keys() >= {'currency', 'cost'}):
                 # only applies if fee is in quote currency!
                 if trade.pair.startswith(exectrade['fee']['currency']):
                     fee_abs += exectrade['fee']['cost']
@@ -538,6 +539,7 @@ class FreqtradeBot(object):
         # Execute sell and update trade record
         order_id = exchange.sell(str(trade.pair), limit, trade.amount)['id']
         trade.open_order_id = order_id
+        trade.close_rate_requested = limit
 
         fmt_exp_profit = round(trade.calc_profit_percent(rate=limit) * 100, 2)
         profit_trade = trade.calc_profit(rate=limit)

freqtrade/persistence.py

@@ -15,6 +15,8 @@ from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm.scoping import scoped_session
 from sqlalchemy.orm.session import sessionmaker
 from sqlalchemy.pool import StaticPool
+from sqlalchemy import inspect
+
 
 logger = logging.getLogger(__name__)
 
@@ -50,12 +52,61 @@ def init(config: dict, engine: Optional[Engine] = None) -> None:
     Trade.session = session()
     Trade.query = session.query_property()
     _DECL_BASE.metadata.create_all(engine)
+    check_migrate(engine)
 
     # Clean dry_run DB
     if _CONF.get('dry_run', False) and _CONF.get('dry_run_db', False):
         clean_dry_run_db()
 
 
+def has_column(columns, searchname: str) -> bool:
+    return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1
+
+
+def check_migrate(engine) -> None:
+    """
+    Checks if migration is necessary and migrates if necessary
+    """
+    inspector = inspect(engine)
+
+    cols = inspector.get_columns('trades')
+
+    if not has_column(cols, 'fee_open'):
+        # Schema migration necessary
+        engine.execute("alter table trades rename to trades_bak")
+        # let SQLAlchemy create the schema as required
+        _DECL_BASE.metadata.create_all(engine)
+
+        # Copy data back - following the correct schema
+        engine.execute("""insert into trades
+                (id, exchange, pair, is_open, fee_open, fee_close, open_rate,
+                open_rate_requested, close_rate, close_rate_requested, close_profit,
+                stake_amount, amount, open_date, close_date, open_order_id)
+            select id, lower(exchange),
+                case
+                    when instr(pair, '_') != 0 then
+                    substr(pair,    instr(pair, '_') + 1) || '/' ||
+                    substr(pair, 1, instr(pair, '_') - 1)
+                    else pair
+                    end
+                pair,
+                is_open, fee fee_open, fee fee_close,
+                open_rate, null open_rate_requested, close_rate,
+                null close_rate_requested, close_profit,
+                stake_amount, amount, open_date, close_date, open_order_id
+                from trades_bak
+             """)
+
+        # Reread columns - the above recreated the table!
+        inspector = inspect(engine)
+        cols = inspector.get_columns('trades')
+
+    if not has_column(cols, 'open_rate_requested'):
+        engine.execute("alter table trades add open_rate_requested float")
+    if not has_column(cols, 'close_rate_requested'):
+        engine.execute("alter table trades add close_rate_requested float")
+
+
 def cleanup() -> None:
     """
     Flushes all pending operations to disk.
@@ -88,7 +139,9 @@ class Trade(_DECL_BASE):
     fee_open = Column(Float, nullable=False, default=0.0)
     fee_close = Column(Float, nullable=False, default=0.0)
     open_rate = Column(Float)
+    open_rate_requested = Column(Float)
     close_rate = Column(Float)
+    close_rate_requested = Column(Float)
     close_profit = Column(Float)
     stake_amount = Column(Float, nullable=False)
     amount = Column(Float)

freqtrade/rpc/rpc.py

@@ -245,35 +245,34 @@ class RPC(object):
         """
         :return: current account balance per crypto
         """
-        balances = [
-            c for c in exchange.get_balances()
-            if c['Balance'] or c['Available'] or c['Pending']
-        ]
-        if not balances:
-            return True, '`All balances are zero.`'
-
         output = []
         total = 0.0
-        for currency in balances:
+        for coin, balance in exchange.get_balances().items():
-            coin = currency['Currency']
+            if not balance['total']:
+                continue
+
+            rate = None
             if coin == 'BTC':
-                currency["Rate"] = 1.0
+                rate = 1.0
             else:
                 if coin == 'USDT':
-                    currency["Rate"] = 1.0 / exchange.get_ticker('BTC/USDT', False)['bid']
+                    rate = 1.0 / exchange.get_ticker('BTC/USDT', False)['bid']
                 else:
-                    currency["Rate"] = exchange.get_ticker(coin + '/BTC', False)['bid']
+                    rate = exchange.get_ticker(coin + '/BTC', False)['bid']
-            currency['BTC'] = currency["Rate"] * currency["Balance"]
+            est_btc: float = rate * balance['total']
-            total = total + currency['BTC']
+            total = total + est_btc
             output.append(
                 {
-                    'currency': currency['Currency'],
+                    'currency': coin,
-                    'available': currency['Available'],
+                    'available': balance['free'],
-                    'balance': currency['Balance'],
+                    'balance': balance['total'],
-                    'pending': currency['Pending'],
+                    'pending': balance['used'],
-                    'est_btc': currency['BTC']
+                    'est_btc': est_btc
                 }
             )
+        if total == 0.0:
+            return True, '`All balances are zero.`'
+
         fiat = self.freqtrade.fiat_converter
         symbol = fiat_display_currency
         value = fiat.convert_amount(total, 'BTC', symbol)

freqtrade/rpc/telegram.py

@@ -264,17 +264,15 @@ class Telegram(RPC):
         (currencys, total, symbol, value) = result
         output = ''
         for currency in currencys:
-            output += """*Currency*: {currency}
+            output += "*{currency}:*\n" \
-    *Available*: {available}
+                      "\t`Available: {available: .8f}`\n" \
-    *Balance*: {balance}
+                      "\t`Balance: {balance: .8f}`\n" \
-    *Pending*: {pending}
+                      "\t`Pending: {pending: .8f}`\n" \
-    *Est. BTC*: {est_btc: .8f}
+                      "\t`Est. BTC: {est_btc: .8f}`\n".format(**currency)
-    """.format(**currency)
 
-        output += """*Estimated Value*:
+        output += "\n*Estimated Value*:\n" \
-    *BTC*: {0: .8f}
+                  "\t`BTC: {0: .8f}`\n" \
-    *{1}*: {2: .2f}
+                  "\t`{1}: {2: .2f}`\n".format(total, symbol, value)
-    """.format(total, symbol, value)
         self.send_msg(output)
 
     @authorized_only
@@ -357,8 +355,9 @@ class Telegram(RPC):
 
         message = tabulate({
             'current': [len(trades)],
-            'max': [self._config['max_open_trades']]
+            'max': [self._config['max_open_trades']],
-        }, headers=['current', 'max'], tablefmt='simple')
+            'total stake': [sum((trade.open_rate * trade.amount) for trade in trades)]
+        }, headers=['current', 'max', 'total stake'], tablefmt='simple')
         message = "<pre>{}</pre>".format(message)
         logger.debug(message)
         self.send_msg(message, parse_mode=ParseMode.HTML)

freqtrade/tests/conftest.py

@@ -2,6 +2,7 @@
 import json
 import logging
 from datetime import datetime
+from typing import Dict, Optional
 from functools import reduce
 from unittest.mock import MagicMock
 
@@ -34,7 +35,8 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot:
     :param config: Config to pass to the bot
     :return: None
     """
-    mocker.patch('freqtrade.fiat_convert.Market', {'price_usd': 12345.0})
+    # mocker.patch('freqtrade.fiat_convert.Market', {'price_usd': 12345.0})
+    patch_coinmarketcap(mocker, {'price_usd': 12345.0})
     mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock())
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
     mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
@@ -46,6 +48,27 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot:
     return FreqtradeBot(config, create_engine('sqlite://'))
 
 
+def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> None:
+    """
+    Mocker to coinmarketcap to speed up tests
+    :param mocker: mocker to patch coinmarketcap class
+    :return: None
+    """
+
+    tickermock = MagicMock(return_value={'price_usd': 12345.0})
+    listmock = MagicMock(return_value={'data': [{'id': 1, 'name': 'Bitcoin', 'symbol': 'BTC',
+                                                 'website_slug': 'bitcoin'},
+                                                {'id': 1027, 'name': 'Ethereum', 'symbol': 'ETH',
+                                                 'website_slug': 'ethereum'}
+                                                ]})
+    mocker.patch.multiple(
+        'freqtrade.fiat_convert.Market',
+        ticker=tickermock,
+        listings=listmock,
+
+    )
+
+
 @pytest.fixture(scope="function")
 def default_conf():
     """ Returns validated configuration suitable for most tests """

freqtrade/tests/optimize/test_hyperopt.py

@@ -123,7 +123,7 @@ def test_loss_calculation_has_limited_profit(init_hyperopt) -> None:
     assert under > correct
 
 
-def test_log_results_if_loss_improves(capsys) -> None:
+def test_log_results_if_loss_improves(init_hyperopt, capsys) -> None:
     hyperopt = _HYPEROPT
     hyperopt.current_best_loss = 2
     hyperopt.log_results(

freqtrade/tests/rpc/test_rpc.py

@@ -288,22 +288,18 @@ def test_rpc_balance_handle(default_conf, mocker):
     """
     Test rpc_balance() method
     """
-    mock_balance = [
+    mock_balance = {
-        {
+        'BTC': {
-            'Currency': 'BTC',
+            'free': 10.0,
-            'Balance': 10.0,
+            'total': 12.0,
-            'Available': 12.0,
+            'used': 2.0,
-            'Pending': 0.0,
-            'CryptoAddress': 'XXXX',
         },
-        {
+        'ETH': {
-            'Currency': 'ETH',
+            'free': 0.0,
-            'Balance': 0.0,
+            'total': 0.0,
-            'Available': 0.0,
+            'used': 0.0,
-            'Pending': 0.0,
-            'CryptoAddress': 'XXXX',
         }
-    ]
+    }
 
     patch_get_signal(mocker, (True, False))
     mocker.patch.multiple(
@@ -324,15 +320,15 @@ def test_rpc_balance_handle(default_conf, mocker):
     (error, res) = rpc.rpc_balance(default_conf['fiat_display_currency'])
     assert not error
     (trade, x, y, z) = res
-    assert prec_satoshi(x, 10)
+    assert prec_satoshi(x, 12)
-    assert prec_satoshi(z, 150000)
+    assert prec_satoshi(z, 180000)
     assert 'USD' in y
     assert len(trade) == 1
     assert 'BTC' in trade[0]['currency']
-    assert prec_satoshi(trade[0]['available'], 12)
+    assert prec_satoshi(trade[0]['available'], 10)
-    assert prec_satoshi(trade[0]['balance'], 10)
+    assert prec_satoshi(trade[0]['balance'], 12)
-    assert prec_satoshi(trade[0]['pending'], 0)
+    assert prec_satoshi(trade[0]['pending'], 2)
-    assert prec_satoshi(trade[0]['est_btc'], 10)
+    assert prec_satoshi(trade[0]['est_btc'], 12)
 
 
 def test_rpc_start(mocker, default_conf) -> None:

freqtrade/tests/rpc/test_rpc_telegram.py

@@ -554,36 +554,29 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None:
     """
     Test _balance() method
     """
-    mock_balance = [
+
-        {
+    mock_balance = {
-            'Currency': 'BTC',
+        'BTC': {
-            'Balance': 10.0,
+            'total': 12.0,
-            'Available': 12.0,
+            'free': 12.0,
-            'Pending': 0.0,
+            'used': 0.0
-            'CryptoAddress': 'XXXX',
         },
-        {
+        'ETH': {
-            'Currency': 'ETH',
+            'total': 0.0,
-            'Balance': 0.0,
+            'free': 0.0,
-            'Available': 0.0,
+            'used': 0.0
-            'Pending': 0.0,
-            'CryptoAddress': 'XXXX',
         },
-        {
+        'USDT': {
-            'Currency': 'USDT',
+            'total': 10000.0,
-            'Balance': 10000.0,
+            'free': 10000.0,
-            'Available': 0.0,
+            'used': 0.0
-            'Pending': 0.0,
-            'CryptoAddress': 'XXXX',
         },
-        {
+        'LTC': {
-            'Currency': 'LTC',
+            'total': 10.0,
-            'Balance': 10.0,
+            'free': 10.0,
-            'Available': 10.0,
+            'used': 0.0
-            'Pending': 0.0,
-            'CryptoAddress': 'XXXX',
         }
-    ]
+    }
 
     def mock_ticker(symbol, refresh):
         """
@@ -621,12 +614,12 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None:
     telegram._balance(bot=MagicMock(), update=update)
     result = msg_mock.call_args_list[0][0][0]
     assert msg_mock.call_count == 1
-    assert '*Currency*: BTC' in result
+    assert '*BTC:*' in result
-    assert '*Currency*: ETH' not in result
+    assert '*ETH:*' not in result
-    assert '*Currency*: USDT' in result
+    assert '*USDT:*' in result
-    assert 'Balance' in result
+    assert 'Balance:' in result
-    assert 'Est. BTC' in result
+    assert 'Est. BTC:' in result
-    assert '*BTC*:  12.00000000' in result
+    assert 'BTC:  14.00000000' in result
 
 
 def test_zero_balance_handle(default_conf, update, mocker) -> None:
@@ -636,7 +629,7 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None:
     patch_get_signal(mocker, (True, False))
     patch_coinmarketcap(mocker, value={'price_usd': 15000.0})
     mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock())
-    mocker.patch('freqtrade.freqtradebot.exchange.get_balances', return_value=[])
+    mocker.patch('freqtrade.freqtradebot.exchange.get_balances', return_value={})
 
     msg_mock = MagicMock()
     mocker.patch.multiple(
@@ -1013,9 +1006,12 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None:
     msg_mock.reset_mock()
     telegram._count(bot=MagicMock(), update=update)
 
-    msg = '<pre>  current    max\n---------  -----\n        1      {}</pre>'.format(
+    msg = '<pre>  current    max    total stake\n---------  -----  -------------\n' \
-        default_conf['max_open_trades']
+          '        1      {}          {}</pre>'\
-    )
+        .format(
+            default_conf['max_open_trades'],
+            default_conf['stake_amount']
+        )
     assert msg in msg_mock.call_args_list[0][0][0]
 
 

freqtrade/tests/test_fiat_convert.py

@@ -7,6 +7,7 @@ from unittest.mock import MagicMock
 import pytest
 
 from freqtrade.fiat_convert import CryptoFiat, CryptoToFiatConverter
+from freqtrade.tests.conftest import patch_coinmarketcap
 
 
 def test_pair_convertion_object():
@@ -123,12 +124,23 @@ def test_fiat_convert_get_price(mocker):
     assert fiat_convert._pairs[0]._expiration is not expiration
 
 
+def test_loadcryptomap(mocker):
+    patch_coinmarketcap(mocker)
+
+    fiat_convert = CryptoToFiatConverter()
+    assert len(fiat_convert._cryptomap) == 2
+
+    assert fiat_convert._cryptomap["BTC"] == "1"
+
+
 def test_fiat_convert_without_network():
     # Because CryptoToFiatConverter is a Singleton we reset the value of _coinmarketcap
 
     fiat_convert = CryptoToFiatConverter()
 
+    cmc_temp = CryptoToFiatConverter._coinmarketcap
     CryptoToFiatConverter._coinmarketcap = None
 
     assert fiat_convert._coinmarketcap is None
     assert fiat_convert._find_price(crypto_symbol='BTC', fiat_symbol='USD') == 0.0
+    CryptoToFiatConverter._coinmarketcap = cmc_temp

freqtrade/tests/test_freqtradebot.py

@@ -8,7 +8,6 @@ import logging
 import re
 import time
 from copy import deepcopy
-from typing import Dict, Optional
 from unittest.mock import MagicMock
 
 import arrow
@@ -20,7 +19,7 @@ from freqtrade import DependencyException, OperationalException, TemporaryError
 from freqtrade.freqtradebot import FreqtradeBot
 from freqtrade.persistence import Trade
 from freqtrade.state import State
-from freqtrade.tests.conftest import log_has
+from freqtrade.tests.conftest import log_has, patch_coinmarketcap
 
 
 # Functions for recurrent object patching
@@ -64,20 +63,6 @@ def patch_RPCManager(mocker) -> MagicMock:
     return rpc_mock
 
 
-def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> None:
-    """
-    Mocker to coinmarketcap to speed up tests
-    :param mocker: mocker to patch coinmarketcap class
-    :return: None
-    """
-    mock = MagicMock()
-
-    if value:
-        mock.ticker = {'price_usd': 12345.0}
-
-    mocker.patch('freqtrade.fiat_convert.Market', mock)
-
-
 # Unit tests
 def test_freqtradebot_object() -> None:
     """
@@ -1458,7 +1443,7 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee
     patch_RPCManager(mocker)
     patch_coinmarketcap(mocker)
     mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True))
-    mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=trades_for_order)
+    mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=[trades_for_order])
     amount = float(sum(x['amount'] for x in trades_for_order))
     trade = Trade(
         pair='LTC/ETH',
@@ -1473,3 +1458,53 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee
     assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
                    'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996) from Order',
                    caplog.record_tuples)
+
+
+def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, mocker):
+    """
+    Test get_real_amount with split trades (multiple trades for this order)
+    """
+    limit_buy_order = deepcopy(buy_order_fee)
+    limit_buy_order['fee'] = {'cost': 0.004}
+
+    patch_get_signal(mocker)
+    patch_RPCManager(mocker)
+    patch_coinmarketcap(mocker)
+    mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True))
+    mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=[])
+    amount = float(sum(x['amount'] for x in trades_for_order))
+    trade = Trade(
+        pair='LTC/ETH',
+        amount=amount,
+        exchange='binance',
+        open_rate=0.245441,
+        open_order_id="123456"
+    )
+    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    # Amount does not change
+    assert freqtrade.get_real_amount(trade, limit_buy_order) == amount
+
+
+def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, mocker):
+    """
+    Test get_real_amount - fees in Stake currency
+    """
+    # Remove "Currency" from fee dict
+    trades_for_order[0]['fee'] = {'cost': 0.008}
+
+    patch_get_signal(mocker)
+    patch_RPCManager(mocker)
+    patch_coinmarketcap(mocker)
+    mocker.patch('freqtrade.exchange.validate_pairs', MagicMock(return_value=True))
+    mocker.patch('freqtrade.exchange.get_trades_for_order', return_value=trades_for_order)
+    amount = sum(x['amount'] for x in trades_for_order)
+    trade = Trade(
+        pair='LTC/ETH',
+        amount=amount,
+        exchange='binance',
+        open_rate=0.245441,
+        open_order_id="123456"
+    )
+    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    # Amount does not change
+    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount

freqtrade/tests/test_persistence.py

@@ -375,3 +375,105 @@ def test_clean_dry_run_db(default_conf, fee):
 
     # We have now only the prod
     assert len(Trade.query.filter(Trade.open_order_id.isnot(None)).all()) == 1
+
+
+def test_migrate_old(default_conf, fee):
+    """
+    Test Database migration(starting with old pairformat)
+    """
+    amount = 103.223
+    create_table_old = """CREATE TABLE IF NOT EXISTS "trades" (
+                                id INTEGER NOT NULL,
+                                exchange VARCHAR NOT NULL,
+                                pair VARCHAR NOT NULL,
+                                is_open BOOLEAN NOT NULL,
+                                fee FLOAT NOT NULL,
+                                open_rate FLOAT,
+                                close_rate FLOAT,
+                                close_profit FLOAT,
+                                stake_amount FLOAT NOT NULL,
+                                amount FLOAT,
+                                open_date DATETIME NOT NULL,
+                                close_date DATETIME,
+                                open_order_id VARCHAR,
+                                PRIMARY KEY (id),
+                                CHECK (is_open IN (0, 1))
+                                );"""
+    insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee,
+                          open_rate, stake_amount, amount, open_date)
+                          VALUES ('BITTREX', 'BTC_ETC', 1, {fee},
+                          0.00258580, {stake}, {amount},
+                          '2017-11-28 12:44:24.000000')
+                          """.format(fee=fee.return_value,
+                                     stake=default_conf.get("stake_amount"),
+                                     amount=amount
+                                     )
+    engine = create_engine('sqlite://')
+    # Create table using the old format
+    engine.execute(create_table_old)
+    engine.execute(insert_table_old)
+    # Run init to test migration
+    init(default_conf, engine)
+
+    assert len(Trade.query.filter(Trade.id == 1).all()) == 1
+    trade = Trade.query.filter(Trade.id == 1).first()
+    assert trade.fee_open == fee.return_value
+    assert trade.fee_close == fee.return_value
+    assert trade.open_rate_requested is None
+    assert trade.close_rate_requested is None
+    assert trade.is_open == 1
+    assert trade.amount == amount
+    assert trade.stake_amount == default_conf.get("stake_amount")
+    assert trade.pair == "ETC/BTC"
+    assert trade.exchange == "bittrex"
+
+
+def test_migrate_new(default_conf, fee):
+    """
+    Test Database migration (starting with new pairformat)
+    """
+    amount = 103.223
+    create_table_old = """CREATE TABLE IF NOT EXISTS "trades" (
+                                id INTEGER NOT NULL,
+                                exchange VARCHAR NOT NULL,
+                                pair VARCHAR NOT NULL,
+                                is_open BOOLEAN NOT NULL,
+                                fee FLOAT NOT NULL,
+                                open_rate FLOAT,
+                                close_rate FLOAT,
+                                close_profit FLOAT,
+                                stake_amount FLOAT NOT NULL,
+                                amount FLOAT,
+                                open_date DATETIME NOT NULL,
+                                close_date DATETIME,
+                                open_order_id VARCHAR,
+                                PRIMARY KEY (id),
+                                CHECK (is_open IN (0, 1))
+                                );"""
+    insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee,
+                          open_rate, stake_amount, amount, open_date)
+                          VALUES ('binance', 'ETC/BTC', 1, {fee},
+                          0.00258580, {stake}, {amount},
+                          '2019-11-28 12:44:24.000000')
+                          """.format(fee=fee.return_value,
+                                     stake=default_conf.get("stake_amount"),
+                                     amount=amount
+                                     )
+    engine = create_engine('sqlite://')
+    # Create table using the old format
+    engine.execute(create_table_old)
+    engine.execute(insert_table_old)
+    # Run init to test migration
+    init(default_conf, engine)
+
+    assert len(Trade.query.filter(Trade.id == 1).all()) == 1
+    trade = Trade.query.filter(Trade.id == 1).first()
+    assert trade.fee_open == fee.return_value
+    assert trade.fee_close == fee.return_value
+    assert trade.open_rate_requested is None
+    assert trade.close_rate_requested is None
+    assert trade.is_open == 1
+    assert trade.amount == amount
+    assert trade.stake_amount == default_conf.get("stake_amount")
+    assert trade.pair == "ETC/BTC"
+    assert trade.exchange == "binance"

requirements.txt

@@ -1,14 +1,14 @@
-ccxt==1.11.149
+ccxt==1.14.24
 SQLAlchemy==1.2.7
-python-telegram-bot==10.0.2
+python-telegram-bot==10.1.0
 arrow==0.12.1
-cachetools==2.0.1
+cachetools==2.1.0
 requests==2.18.4
 urllib3==1.22
 wrapt==1.10.11
-pandas==0.22.0
+pandas==0.23.0
 scikit-learn==0.19.1
-scipy==1.0.1
+scipy==1.1.0
 jsonschema==2.6.0
 numpy==1.14.3
 TA-Lib==0.4.17
@@ -19,7 +19,7 @@ hyperopt==0.1
 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325
 networkx==1.11
 tabulate==0.8.2
-coinmarketcap==4.2.1
+coinmarketcap==5.0.3
 
 # Required for plotting data
 #plotly==2.3.0

scripts/plot_dataframe.py

@@ -10,6 +10,7 @@ Optional Cli parameters
 -d / --datadir: path to pair backtest data
 --timerange: specify what timerange of data to use.
 -l / --live: Live, to download the latest ticker for the pair
+-db / --db-url: Show trades stored in database
 """
 import logging
 import sys
@@ -21,18 +22,18 @@ from plotly import tools
 from plotly.offline import plot
 import plotly.graph_objs as go
 
+from typing import Dict, List, Any
+from sqlalchemy import create_engine
+
 from freqtrade.arguments import Arguments
 from freqtrade.analyze import Analyze
 from freqtrade import exchange
 import freqtrade.optimize as optimize
+from freqtrade import persistence
+from freqtrade.persistence import Trade
 
-
-<<<<<<< HEAD
-logger = logging.getLogger('freqtrade')
-=======
 logger = logging.getLogger(__name__)
->>>>>>> bddf009a2b6d0e1a19cca558887ce972e99a6238
+_CONF: Dict[str, Any] = {}
-
 
 def plot_analyzed_dataframe(args: Namespace) -> None:
     """
@@ -73,6 +74,12 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
     dataframe = analyze.populate_buy_trend(dataframe)
     dataframe = analyze.populate_sell_trend(dataframe)
 
+    trades = []
+    if args.db_url:
+        engine = create_engine('sqlite:///' + args.db_url)
+        persistence.init(_CONF, engine)
+        trades = Trade.query.filter(Trade.pair.is_(pair)).all()
+
     if len(dataframe.index) > 750:
         logger.warning('Ticker contained more than 750 candles, clipping.')
     data = dataframe.tail(750)
@@ -113,6 +120,31 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
         )
     )
 
+    trade_buys = go.Scattergl(
+        x=[t.open_date.isoformat() for t in trades],
+        y=[t.open_rate for t in trades],
+        mode='markers',
+        name='trade_buy',
+        marker=dict(
+            symbol='square-open',
+            size=11,
+            line=dict(width=2),
+            color='green'
+        )
+    )
+    trade_sells = go.Scattergl(
+        x=[t.close_date.isoformat() for t in trades],
+        y=[t.close_rate for t in trades],
+        mode='markers',
+        name='trade_sell',
+        marker=dict(
+            symbol='square-open',
+            size=11,
+            line=dict(width=2),
+            color='red'
+        )
+    )
+
     bb_lower = go.Scatter(
         x=data.date,
         y=data.bb_lowerband,
@@ -147,6 +179,8 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
     fig.append_trace(volume, 2, 1)
     fig.append_trace(macd, 3, 1)
     fig.append_trace(macdsignal, 3, 1)
+    fig.append_trace(trade_buys, 1, 1)
+    fig.append_trace(trade_sells, 1, 1)
 
     fig['layout'].update(title=args.pair)
     fig['layout']['yaxis1'].update(title='Price')

scripts/plot_profit.py

@@ -24,8 +24,7 @@ import plotly.graph_objs as go
 from freqtrade.arguments import Arguments
 from freqtrade.configuration import Configuration
 from freqtrade.analyze import Analyze
-from freqtradeimport constants
+from freqtrade import constants
-
 
 import freqtrade.optimize as optimize
 import freqtrade.misc as misc
@@ -33,11 +32,12 @@ import freqtrade.misc as misc
 
 logger = logging.getLogger(__name__)
 
+
 # data:: [ pair,      profit-%,  enter,         exit,        time, duration]
 # data:: ["ETH/BTC", 0.0023975, "1515598200", "1515602100", "2018-01-10 07:30:00+00:00", 65]
-def make_profit_array(
+def make_profit_array(data: List, px: int, min_date: int,
-        data: List, px: int, min_date: int,
+                      interval: int,
-        interval: int, filter_pairs: Optional[List] = None) -> np.ndarray:
+                      filter_pairs: Optional[List] = None) -> np.ndarray:
     pg = np.zeros(px)
     filter_pairs = filter_pairs or []
     # Go through the trades