Merge branch 'develop' into time_in_force

freqtrade/exchange/__init__.py

@@ -499,7 +499,9 @@ class Exchange(object):
             # Because some exchange sort Tickers ASC and other DESC.
             # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last)
             # when GDAX returns a list of tickers DESC (newest first, oldest last)
-            data = sorted(data, key=lambda x: x[0])
+            # Only sort if necessary to save computing time
+            if data and data[0][0] > data[-1][0]:
+                data = sorted(data, key=lambda x: x[0])
 
             # keeping last candle time as last refreshed time of the pair
             if data:
@@ -521,51 +523,6 @@ class Exchange(object):
         except ccxt.BaseError as e:
             raise OperationalException(f'Could not fetch ticker data. Msg: {e}')
 
-    @retrier
-    def get_candle_history(self, pair: str, tick_interval: str,
-                           since_ms: Optional[int] = None) -> List[Dict]:
-        try:
-            # last item should be in the time interval [now - tick_interval, now]
-            till_time_ms = arrow.utcnow().shift(
-                            minutes=-constants.TICKER_INTERVAL_MINUTES[tick_interval]
-                        ).timestamp * 1000
-            # it looks as if some exchanges return cached data
-            # and they update it one in several minute, so 10 mins interval
-            # is necessary to skeep downloading of an empty array when all
-            # chached data was already downloaded
-            till_time_ms = min(till_time_ms, arrow.utcnow().shift(minutes=-10).timestamp * 1000)
-
-            data: List[Dict[Any, Any]] = []
-            while not since_ms or since_ms < till_time_ms:
-                data_part = self._api.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms)
-
-                # Because some exchange sort Tickers ASC and other DESC.
-                # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last)
-                # when GDAX returns a list of tickers DESC (newest first, oldest last)
-                data_part = sorted(data_part, key=lambda x: x[0])
-
-                if not data_part:
-                    break
-
-                logger.debug('Downloaded data for %s time range [%s, %s]',
-                             pair,
-                             arrow.get(data_part[0][0] / 1000).format(),
-                             arrow.get(data_part[-1][0] / 1000).format())
-
-                data.extend(data_part)
-                since_ms = data[-1][0] + 1
-
-            return data
-        except ccxt.NotSupported as e:
-            raise OperationalException(
-                f'Exchange {self._api.name} does not support fetching historical candlestick data.'
-                f'Message: {e}')
-        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
-            raise TemporaryError(
-                f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}')
-        except ccxt.BaseError as e:
-            raise OperationalException(f'Could not fetch ticker data. Msg: {e}')
-
     @retrier
     def cancel_order(self, order_id: str, pair: str) -> None:
         if self._conf['dry_run']:

freqtrade/exchange/exchange_helpers.py

@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
 def parse_ticker_dataframe(ticker: list) -> DataFrame:
     """
     Analyses the trend for the given ticker history
-    :param ticker: See exchange.get_candle_history
+    :param ticker: ticker list, as returned by exchange.async_get_candle_history
     :return: DataFrame
     """
     cols = ['date', 'open', 'high', 'low', 'close', 'volume']

freqtrade/tests/conftest.py

@@ -4,7 +4,6 @@ import logging
 from datetime import datetime
 from functools import reduce
 from typing import Dict, Optional
-from collections import namedtuple
 from unittest.mock import MagicMock, PropertyMock
 
 import arrow
@@ -13,7 +12,7 @@ from telegram import Chat, Message, Update
 
 from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe
 from freqtrade.exchange import Exchange
-from freqtrade.edge import Edge
+from freqtrade.edge import Edge, PairInfo
 from freqtrade.freqtradebot import FreqtradeBot
 
 logging.getLogger('').setLevel(logging.INFO)
@@ -56,13 +55,11 @@ def patch_edge(mocker) -> None:
     # "LTC/BTC",
     # "XRP/BTC",
     # "NEO/BTC"
-    pair_info = namedtuple(
+
-        'pair_info',
-        'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy')
     mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock(
         return_value={
-            'NEO/BTC': pair_info(-0.20, 0.66, 3.71, 0.50, 1.71),
+            'NEO/BTC': PairInfo(-0.20, 0.66, 3.71, 0.50, 1.71, 10, 25),
-            'LTC/BTC': pair_info(-0.21, 0.66, 3.71, 0.50, 1.71),
+            'LTC/BTC': PairInfo(-0.21, 0.66, 3.71, 0.50, 1.71, 11, 20),
         }
     ))
     mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20))

freqtrade/tests/exchange/test_exchange.py

@@ -875,65 +875,10 @@ def make_fetch_ohlcv_mock(data):
     return fetch_ohlcv_mock
 
 
-def test_get_candle_history(default_conf, mocker):
+@pytest.mark.asyncio
-    api_mock = MagicMock()
+async def test___async_get_candle_history_sort(default_conf, mocker):
-    tick = [
+    def sort_data(data, key):
-        [
+        return sorted(data, key=key)
-            1511686200000,  # unix timestamp ms
-            1,  # open
-            2,  # high
-            3,  # low
-            4,  # close
-            5,  # volume (in quote currency)
-        ]
-    ]
-    type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True})
-    api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick))
-    exchange = get_patched_exchange(mocker, default_conf, api_mock)
-
-    # retrieve original ticker
-    ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval'])
-    assert ticks[0][0] == 1511686200000
-    assert ticks[0][1] == 1
-    assert ticks[0][2] == 2
-    assert ticks[0][3] == 3
-    assert ticks[0][4] == 4
-    assert ticks[0][5] == 5
-
-    # change ticker and ensure tick changes
-    new_tick = [
-        [
-            1511686210000,  # unix timestamp ms
-            6,  # open
-            7,  # high
-            8,  # low
-            9,  # close
-            10,  # volume (in quote currency)
-        ]
-    ]
-    api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(new_tick))
-    exchange = get_patched_exchange(mocker, default_conf, api_mock)
-
-    ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval'])
-    assert ticks[0][0] == 1511686210000
-    assert ticks[0][1] == 6
-    assert ticks[0][2] == 7
-    assert ticks[0][3] == 8
-    assert ticks[0][4] == 9
-    assert ticks[0][5] == 10
-
-    ccxt_exceptionhandlers(mocker, default_conf, api_mock,
-                           "get_candle_history", "fetch_ohlcv",
-                           pair='ABCD/BTC', tick_interval=default_conf['ticker_interval'])
-
-    with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'):
-        api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported)
-        exchange = get_patched_exchange(mocker, default_conf, api_mock)
-        exchange.get_candle_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval'])
-
-
-def test_get_candle_history_sort(default_conf, mocker):
-    api_mock = MagicMock()
 
     # GDAX use-case (real data from GDAX)
     # This ticker history is ordered DESC (newest first, oldest last)
@@ -949,13 +894,15 @@ def test_get_candle_history_sort(default_conf, mocker):
         [1527830700000, 0.07652, 0.07652, 0.07651, 0.07652, 10.04822687],
         [1527830400000, 0.07649, 0.07651, 0.07649, 0.07651, 2.5734867]
     ]
-    type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True})
+    exchange = get_patched_exchange(mocker, default_conf)
-    api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick))
+    exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
-
+    sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data))
-    exchange = get_patched_exchange(mocker, default_conf, api_mock)
-
     # Test the ticker history sort
-    ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval'])
+    res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval'])
+    assert res[0] == 'ETH/BTC'
+    ticks = res[1]
+
+    assert sort_mock.call_count == 1
     assert ticks[0][0] == 1527830400000
     assert ticks[0][1] == 0.07649
     assert ticks[0][2] == 0.07651
@@ -984,11 +931,15 @@ def test_get_candle_history_sort(default_conf, mocker):
         [1527830100000, 0.076695, 0.07671, 0.07624171, 0.07671, 1.80689244],
         [1527830400000, 0.07671, 0.07674399, 0.07629216, 0.07655213, 2.31452783]
     ]
-    type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True})
+    exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
-    api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick))
+    # Reset sort mock
-    exchange = get_patched_exchange(mocker, default_conf, api_mock)
+    sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data))
     # Test the ticker history sort
-    ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval'])
+    res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval'])
+    assert res[0] == 'ETH/BTC'
+    ticks = res[1]
+    # Sorted not called again - data is already in order
+    assert sort_mock.call_count == 0
     assert ticks[0][0] == 1527827700000
     assert ticks[0][1] == 0.07659999
     assert ticks[0][2] == 0.0766

freqtrade/tests/test_freqtradebot.py

@@ -292,7 +292,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker,
     freqtrade = FreqtradeBot(edge_conf)
     freqtrade.active_pair_whitelist = ['NEO/BTC']
     patch_get_signal(freqtrade)
-    freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False
+    freqtrade.strategy.min_roi_reached = MagicMock(return_value=False)
     freqtrade.create_trade()
     trade = Trade.query.first()
     trade.update(limit_buy_order)
@@ -332,7 +332,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets,
     freqtrade = FreqtradeBot(edge_conf)
     freqtrade.active_pair_whitelist = ['NEO/BTC']
     patch_get_signal(freqtrade)
-    freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False
+    freqtrade.strategy.min_roi_reached = MagicMock(return_value=False)
     freqtrade.create_trade()
     trade = Trade.query.first()
     trade.update(limit_buy_order)
@@ -1010,7 +1010,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order,
 
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade, value=(True, True))
-    freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False
+    freqtrade.strategy.min_roi_reached = MagicMock(return_value=False)
 
     freqtrade.create_trade()
 
@@ -1066,7 +1066,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order,
 
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade, value=(True, False))
-    freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True
+    freqtrade.strategy.min_roi_reached = MagicMock(return_value=True)
 
     freqtrade.create_trade()
 
@@ -1099,7 +1099,7 @@ def test_handle_trade_experimental(
 
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade)
-    freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False
+    freqtrade.strategy.min_roi_reached = MagicMock(return_value=False)
     freqtrade.create_trade()
 
     trade = Trade.query.first()
@@ -1580,7 +1580,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order,
     }
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade)
-    freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False
+    freqtrade.strategy.min_roi_reached = MagicMock(return_value=False)
 
     freqtrade.create_trade()
 
@@ -1612,7 +1612,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order,
     }
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade)
-    freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False
+    freqtrade.strategy.min_roi_reached = MagicMock(return_value=False)
     freqtrade.create_trade()
 
     trade = Trade.query.first()
@@ -1674,7 +1674,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke
 
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade)
-    freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False
+    freqtrade.strategy.min_roi_reached = MagicMock(return_value=False)
 
     freqtrade.create_trade()
 
@@ -1704,7 +1704,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m
     }
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade)
-    freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True
+    freqtrade.strategy.min_roi_reached = MagicMock(return_value=True)
 
     freqtrade.create_trade()
 
@@ -1736,7 +1736,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog,
     default_conf['trailing_stop'] = True
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade)
-    freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False
+    freqtrade.strategy.min_roi_reached = MagicMock(return_value=False)
 
     freqtrade.create_trade()
 
@@ -1771,7 +1771,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets
     default_conf['trailing_stop_positive'] = 0.01
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade)
-    freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False
+    freqtrade.strategy.min_roi_reached = MagicMock(return_value=False)
     freqtrade.create_trade()
 
     trade = Trade.query.first()
@@ -1831,7 +1831,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee,
     default_conf['trailing_stop_positive_offset'] = 0.011
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade)
-    freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False
+    freqtrade.strategy.min_roi_reached = MagicMock(return_value=False)
     freqtrade.create_trade()
 
     trade = Trade.query.first()
@@ -1890,7 +1890,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order,
     }
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade)
-    freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True
+    freqtrade.strategy.min_roi_reached = MagicMock(return_value=True)
 
     freqtrade.create_trade()
 

freqtrade/tests/test_wallets.py

@@ -30,6 +30,7 @@ def test_sync_wallet_at_boot(mocker, default_conf):
     assert freqtrade.wallets.wallets['GAS'].free == 0.260739
     assert freqtrade.wallets.wallets['GAS'].used == 0.0
     assert freqtrade.wallets.wallets['GAS'].total == 0.260739
+    assert freqtrade.wallets.get_free('BNT') == 1.0
 
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
@@ -56,6 +57,7 @@ def test_sync_wallet_at_boot(mocker, default_conf):
     assert freqtrade.wallets.wallets['GAS'].free == 0.270739
     assert freqtrade.wallets.wallets['GAS'].used == 0.1
     assert freqtrade.wallets.wallets['GAS'].total == 0.260439
+    assert freqtrade.wallets.get_free('GAS') == 0.270739
 
 
 def test_sync_wallet_missing_data(mocker, default_conf):
@@ -84,3 +86,4 @@ def test_sync_wallet_missing_data(mocker, default_conf):
     assert freqtrade.wallets.wallets['GAS'].free == 0.260739
     assert freqtrade.wallets.wallets['GAS'].used is None
     assert freqtrade.wallets.wallets['GAS'].total == 0.260739
+    assert freqtrade.wallets.get_free('GAS') == 0.260739

freqtrade/wallets.py

@@ -35,8 +35,8 @@ class Wallets(object):
             return 999.9
 
         balance = self.wallets.get(currency)
-        if balance and balance['free']:
+        if balance and balance.free:
-            return balance['free']
+            return balance.free
         else:
             return 0
 

requirements.txt

@@ -1,4 +1,4 @@
-ccxt==1.17.536
+ccxt==1.17.539
 SQLAlchemy==1.2.14
 python-telegram-bot==11.1.0
 arrow==0.12.1