diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index ae07e36e9..350c730a4 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -478,7 +478,9 @@ class Exchange(object): # Because some exchange sort Tickers ASC and other DESC. # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) # when GDAX returns a list of tickers DESC (newest first, oldest last) - data = sorted(data, key=lambda x: x[0]) + # Only sort if necessary to save computing time + if data and data[0][0] > data[-1][0]: + data = sorted(data, key=lambda x: x[0]) # keeping last candle time as last refreshed time of the pair if data: @@ -500,51 +502,6 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(f'Could not fetch ticker data. Msg: {e}') - @retrier - def get_candle_history(self, pair: str, tick_interval: str, - since_ms: Optional[int] = None) -> List[Dict]: - try: - # last item should be in the time interval [now - tick_interval, now] - till_time_ms = arrow.utcnow().shift( - minutes=-constants.TICKER_INTERVAL_MINUTES[tick_interval] - ).timestamp * 1000 - # it looks as if some exchanges return cached data - # and they update it one in several minute, so 10 mins interval - # is necessary to skeep downloading of an empty array when all - # chached data was already downloaded - till_time_ms = min(till_time_ms, arrow.utcnow().shift(minutes=-10).timestamp * 1000) - - data: List[Dict[Any, Any]] = [] - while not since_ms or since_ms < till_time_ms: - data_part = self._api.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) - - # Because some exchange sort Tickers ASC and other DESC. - # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) - # when GDAX returns a list of tickers DESC (newest first, oldest last) - data_part = sorted(data_part, key=lambda x: x[0]) - - if not data_part: - break - - logger.debug('Downloaded data for %s time range [%s, %s]', - pair, - arrow.get(data_part[0][0] / 1000).format(), - arrow.get(data_part[-1][0] / 1000).format()) - - data.extend(data_part) - since_ms = data[-1][0] + 1 - - return data - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching historical candlestick data.' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(f'Could not fetch ticker data. Msg: {e}') - @retrier def cancel_order(self, order_id: str, pair: str) -> None: if self._conf['dry_run']: diff --git a/freqtrade/exchange/exchange_helpers.py b/freqtrade/exchange/exchange_helpers.py index 8f4b03daf..84e68d4bb 100644 --- a/freqtrade/exchange/exchange_helpers.py +++ b/freqtrade/exchange/exchange_helpers.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) def parse_ticker_dataframe(ticker: list) -> DataFrame: """ Analyses the trend for the given ticker history - :param ticker: See exchange.get_candle_history + :param ticker: ticker list, as returned by exchange.async_get_candle_history :return: DataFrame """ cols = ['date', 'open', 'high', 'low', 'close', 'volume'] diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 63655126c..f7fe697b8 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -4,7 +4,6 @@ import logging from datetime import datetime from functools import reduce from typing import Dict, Optional -from collections import namedtuple from unittest.mock import MagicMock, PropertyMock import arrow @@ -13,7 +12,7 @@ from telegram import Chat, Message, Update from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange -from freqtrade.edge import Edge +from freqtrade.edge import Edge, PairInfo from freqtrade.freqtradebot import FreqtradeBot logging.getLogger('').setLevel(logging.INFO) @@ -56,13 +55,11 @@ def patch_edge(mocker) -> None: # "LTC/BTC", # "XRP/BTC", # "NEO/BTC" - pair_info = namedtuple( - 'pair_info', - 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'NEO/BTC': pair_info(-0.20, 0.66, 3.71, 0.50, 1.71), - 'LTC/BTC': pair_info(-0.21, 0.66, 3.71, 0.50, 1.71), + 'NEO/BTC': PairInfo(-0.20, 0.66, 3.71, 0.50, 1.71, 10, 25), + 'LTC/BTC': PairInfo(-0.21, 0.66, 3.71, 0.50, 1.71, 11, 20), } )) mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 207f14efe..dbb8d4ec2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -875,65 +875,10 @@ def make_fetch_ohlcv_mock(data): return fetch_ohlcv_mock -def test_get_candle_history(default_conf, mocker): - api_mock = MagicMock() - tick = [ - [ - 1511686200000, # unix timestamp ms - 1, # open - 2, # high - 3, # low - 4, # close - 5, # volume (in quote currency) - ] - ] - type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - - # retrieve original ticker - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) - assert ticks[0][0] == 1511686200000 - assert ticks[0][1] == 1 - assert ticks[0][2] == 2 - assert ticks[0][3] == 3 - assert ticks[0][4] == 4 - assert ticks[0][5] == 5 - - # change ticker and ensure tick changes - new_tick = [ - [ - 1511686210000, # unix timestamp ms - 6, # open - 7, # high - 8, # low - 9, # close - 10, # volume (in quote currency) - ] - ] - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(new_tick)) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) - assert ticks[0][0] == 1511686210000 - assert ticks[0][1] == 6 - assert ticks[0][2] == 7 - assert ticks[0][3] == 8 - assert ticks[0][4] == 9 - assert ticks[0][5] == 10 - - ccxt_exceptionhandlers(mocker, default_conf, api_mock, - "get_candle_history", "fetch_ohlcv", - pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) - - with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_candle_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) - - -def test_get_candle_history_sort(default_conf, mocker): - api_mock = MagicMock() +@pytest.mark.asyncio +async def test___async_get_candle_history_sort(default_conf, mocker): + def sort_data(data, key): + return sorted(data, key=key) # GDAX use-case (real data from GDAX) # This ticker history is ordered DESC (newest first, oldest last) @@ -949,13 +894,15 @@ def test_get_candle_history_sort(default_conf, mocker): [1527830700000, 0.07652, 0.07652, 0.07651, 0.07652, 10.04822687], [1527830400000, 0.07649, 0.07651, 0.07649, 0.07651, 2.5734867] ] - type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - - exchange = get_patched_exchange(mocker, default_conf, api_mock) - + exchange = get_patched_exchange(mocker, default_conf) + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data)) # Test the ticker history sort - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) + res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) + assert res[0] == 'ETH/BTC' + ticks = res[1] + + assert sort_mock.call_count == 1 assert ticks[0][0] == 1527830400000 assert ticks[0][1] == 0.07649 assert ticks[0][2] == 0.07651 @@ -984,11 +931,15 @@ def test_get_candle_history_sort(default_conf, mocker): [1527830100000, 0.076695, 0.07671, 0.07624171, 0.07671, 1.80689244], [1527830400000, 0.07671, 0.07674399, 0.07629216, 0.07655213, 2.31452783] ] - type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + # Reset sort mock + sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data)) # Test the ticker history sort - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) + res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) + assert res[0] == 'ETH/BTC' + ticks = res[1] + # Sorted not called again - data is already in order + assert sort_mock.call_count == 0 assert ticks[0][0] == 1527827700000 assert ticks[0][1] == 0.07659999 assert ticks[0][2] == 0.0766 diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index a36ae2cd8..eb5336c61 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -292,7 +292,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, freqtrade = FreqtradeBot(edge_conf) freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) @@ -332,7 +332,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, freqtrade = FreqtradeBot(edge_conf) freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) @@ -1010,7 +1010,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade, value=(True, True)) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() @@ -1066,7 +1066,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade, value=(True, False)) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) freqtrade.create_trade() @@ -1099,7 +1099,7 @@ def test_handle_trade_experimental( freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() @@ -1580,7 +1580,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, } freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() @@ -1612,7 +1612,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, } freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() @@ -1674,7 +1674,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() @@ -1704,7 +1704,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m } freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) freqtrade.create_trade() @@ -1736,7 +1736,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, default_conf['trailing_stop'] = True freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() @@ -1771,7 +1771,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets default_conf['trailing_stop_positive'] = 0.01 freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() @@ -1831,7 +1831,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, default_conf['trailing_stop_positive_offset'] = 0.011 freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() @@ -1890,7 +1890,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, } freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) freqtrade.create_trade() diff --git a/freqtrade/tests/test_wallets.py b/freqtrade/tests/test_wallets.py index e6a17ecbf..88366a869 100644 --- a/freqtrade/tests/test_wallets.py +++ b/freqtrade/tests/test_wallets.py @@ -30,6 +30,7 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets.wallets['GAS'].free == 0.260739 assert freqtrade.wallets.wallets['GAS'].used == 0.0 assert freqtrade.wallets.wallets['GAS'].total == 0.260739 + assert freqtrade.wallets.get_free('BNT') == 1.0 mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -56,6 +57,7 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets.wallets['GAS'].free == 0.270739 assert freqtrade.wallets.wallets['GAS'].used == 0.1 assert freqtrade.wallets.wallets['GAS'].total == 0.260439 + assert freqtrade.wallets.get_free('GAS') == 0.270739 def test_sync_wallet_missing_data(mocker, default_conf): @@ -84,3 +86,4 @@ def test_sync_wallet_missing_data(mocker, default_conf): assert freqtrade.wallets.wallets['GAS'].free == 0.260739 assert freqtrade.wallets.wallets['GAS'].used is None assert freqtrade.wallets.wallets['GAS'].total == 0.260739 + assert freqtrade.wallets.get_free('GAS') == 0.260739 diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index b8b37907d..bf6f8b027 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -35,8 +35,8 @@ class Wallets(object): return 999.9 balance = self.wallets.get(currency) - if balance and balance['free']: - return balance['free'] + if balance and balance.free: + return balance.free else: return 0 diff --git a/requirements.txt b/requirements.txt index fa550195d..6f969be06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.536 +ccxt==1.17.539 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1