From fd7184718baa99cd328c4f0dacac98c7f79cd3c0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Nov 2018 14:31:46 +0100 Subject: [PATCH 1/8] replace lambda with Magicmock in test --- freqtrade/tests/test_freqtradebot.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index a36ae2cd8..eb5336c61 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -292,7 +292,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, freqtrade = FreqtradeBot(edge_conf) freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) @@ -332,7 +332,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, freqtrade = FreqtradeBot(edge_conf) freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) @@ -1010,7 +1010,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade, value=(True, True)) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() @@ -1066,7 +1066,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade, value=(True, False)) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) freqtrade.create_trade() @@ -1099,7 +1099,7 @@ def test_handle_trade_experimental( freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() @@ -1580,7 +1580,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, } freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() @@ -1612,7 +1612,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, } freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() @@ -1674,7 +1674,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() @@ -1704,7 +1704,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m } freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) freqtrade.create_trade() @@ -1736,7 +1736,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, default_conf['trailing_stop'] = True freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() @@ -1771,7 +1771,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets default_conf['trailing_stop_positive'] = 0.01 freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() @@ -1831,7 +1831,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, default_conf['trailing_stop_positive_offset'] = 0.011 freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() @@ -1890,7 +1890,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, } freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) freqtrade.create_trade() From 317eba2139f7fcc352b97c1de333a76ef18d27eb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Nov 2018 14:36:02 +0100 Subject: [PATCH 2/8] Remove dual instanciation of pairinfo named tuple --- freqtrade/tests/conftest.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 63655126c..f7fe697b8 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -4,7 +4,6 @@ import logging from datetime import datetime from functools import reduce from typing import Dict, Optional -from collections import namedtuple from unittest.mock import MagicMock, PropertyMock import arrow @@ -13,7 +12,7 @@ from telegram import Chat, Message, Update from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange -from freqtrade.edge import Edge +from freqtrade.edge import Edge, PairInfo from freqtrade.freqtradebot import FreqtradeBot logging.getLogger('').setLevel(logging.INFO) @@ -56,13 +55,11 @@ def patch_edge(mocker) -> None: # "LTC/BTC", # "XRP/BTC", # "NEO/BTC" - pair_info = namedtuple( - 'pair_info', - 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'NEO/BTC': pair_info(-0.20, 0.66, 3.71, 0.50, 1.71), - 'LTC/BTC': pair_info(-0.21, 0.66, 3.71, 0.50, 1.71), + 'NEO/BTC': PairInfo(-0.20, 0.66, 3.71, 0.50, 1.71, 10, 25), + 'LTC/BTC': PairInfo(-0.21, 0.66, 3.71, 0.50, 1.71, 11, 20), } )) mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) From 745a5177958ad16e220dd0ba2d8dc982fddfc016 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Nov 2018 14:40:21 +0100 Subject: [PATCH 3/8] Fix comment pointing to wrong column --- freqtrade/exchange/exchange_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange_helpers.py b/freqtrade/exchange/exchange_helpers.py index 8f4b03daf..84e68d4bb 100644 --- a/freqtrade/exchange/exchange_helpers.py +++ b/freqtrade/exchange/exchange_helpers.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) def parse_ticker_dataframe(ticker: list) -> DataFrame: """ Analyses the trend for the given ticker history - :param ticker: See exchange.get_candle_history + :param ticker: ticker list, as returned by exchange.async_get_candle_history :return: DataFrame """ cols = ['date', 'open', 'high', 'low', 'close', 'volume'] From 8a4361199266c3f558e6eb6b98947155ace77d57 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Nov 2018 14:48:15 +0100 Subject: [PATCH 4/8] Remove get_candle_history (it's now async) convert sort-test to async --- freqtrade/exchange/__init__.py | 45 ------------- freqtrade/tests/exchange/test_exchange.py | 78 ++++------------------- 2 files changed, 12 insertions(+), 111 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index ae07e36e9..0f8d8895c 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -500,51 +500,6 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(f'Could not fetch ticker data. Msg: {e}') - @retrier - def get_candle_history(self, pair: str, tick_interval: str, - since_ms: Optional[int] = None) -> List[Dict]: - try: - # last item should be in the time interval [now - tick_interval, now] - till_time_ms = arrow.utcnow().shift( - minutes=-constants.TICKER_INTERVAL_MINUTES[tick_interval] - ).timestamp * 1000 - # it looks as if some exchanges return cached data - # and they update it one in several minute, so 10 mins interval - # is necessary to skeep downloading of an empty array when all - # chached data was already downloaded - till_time_ms = min(till_time_ms, arrow.utcnow().shift(minutes=-10).timestamp * 1000) - - data: List[Dict[Any, Any]] = [] - while not since_ms or since_ms < till_time_ms: - data_part = self._api.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) - - # Because some exchange sort Tickers ASC and other DESC. - # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) - # when GDAX returns a list of tickers DESC (newest first, oldest last) - data_part = sorted(data_part, key=lambda x: x[0]) - - if not data_part: - break - - logger.debug('Downloaded data for %s time range [%s, %s]', - pair, - arrow.get(data_part[0][0] / 1000).format(), - arrow.get(data_part[-1][0] / 1000).format()) - - data.extend(data_part) - since_ms = data[-1][0] + 1 - - return data - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching historical candlestick data.' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(f'Could not fetch ticker data. Msg: {e}') - @retrier def cancel_order(self, order_id: str, pair: str) -> None: if self._conf['dry_run']: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 207f14efe..7fadb4cc1 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -875,64 +875,8 @@ def make_fetch_ohlcv_mock(data): return fetch_ohlcv_mock -def test_get_candle_history(default_conf, mocker): - api_mock = MagicMock() - tick = [ - [ - 1511686200000, # unix timestamp ms - 1, # open - 2, # high - 3, # low - 4, # close - 5, # volume (in quote currency) - ] - ] - type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - - # retrieve original ticker - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) - assert ticks[0][0] == 1511686200000 - assert ticks[0][1] == 1 - assert ticks[0][2] == 2 - assert ticks[0][3] == 3 - assert ticks[0][4] == 4 - assert ticks[0][5] == 5 - - # change ticker and ensure tick changes - new_tick = [ - [ - 1511686210000, # unix timestamp ms - 6, # open - 7, # high - 8, # low - 9, # close - 10, # volume (in quote currency) - ] - ] - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(new_tick)) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) - assert ticks[0][0] == 1511686210000 - assert ticks[0][1] == 6 - assert ticks[0][2] == 7 - assert ticks[0][3] == 8 - assert ticks[0][4] == 9 - assert ticks[0][5] == 10 - - ccxt_exceptionhandlers(mocker, default_conf, api_mock, - "get_candle_history", "fetch_ohlcv", - pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) - - with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_candle_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) - - -def test_get_candle_history_sort(default_conf, mocker): +@pytest.mark.asyncio +async def test___async_get_candle_history_sort(default_conf, mocker): api_mock = MagicMock() # GDAX use-case (real data from GDAX) @@ -949,13 +893,14 @@ def test_get_candle_history_sort(default_conf, mocker): [1527830700000, 0.07652, 0.07652, 0.07651, 0.07652, 10.04822687], [1527830400000, 0.07649, 0.07651, 0.07649, 0.07651, 2.5734867] ] - type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - - exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange = get_patched_exchange(mocker, default_conf) + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) # Test the ticker history sort - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) + res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) + assert res[0] == 'ETH/BTC' + ticks = res[1] assert ticks[0][0] == 1527830400000 assert ticks[0][1] == 0.07649 assert ticks[0][2] == 0.07651 @@ -984,11 +929,12 @@ def test_get_candle_history_sort(default_conf, mocker): [1527830100000, 0.076695, 0.07671, 0.07624171, 0.07671, 1.80689244], [1527830400000, 0.07671, 0.07674399, 0.07629216, 0.07655213, 2.31452783] ] - type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) # Test the ticker history sort - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) + res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) + assert res[0] == 'ETH/BTC' + ticks = res[1] + assert ticks[0][0] == 1527827700000 assert ticks[0][1] == 0.07659999 assert ticks[0][2] == 0.0766 From ebaf58b0fe465be049d46ef891c57746946ed4a0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Nov 2018 15:00:50 +0100 Subject: [PATCH 5/8] Only sort data if necessary --- freqtrade/exchange/__init__.py | 4 +++- freqtrade/tests/exchange/test_exchange.py | 13 +++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 0f8d8895c..350c730a4 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -478,7 +478,9 @@ class Exchange(object): # Because some exchange sort Tickers ASC and other DESC. # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) # when GDAX returns a list of tickers DESC (newest first, oldest last) - data = sorted(data, key=lambda x: x[0]) + # Only sort if necessary to save computing time + if data and data[0][0] > data[-1][0]: + data = sorted(data, key=lambda x: x[0]) # keeping last candle time as last refreshed time of the pair if data: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 7fadb4cc1..dbb8d4ec2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -877,7 +877,8 @@ def make_fetch_ohlcv_mock(data): @pytest.mark.asyncio async def test___async_get_candle_history_sort(default_conf, mocker): - api_mock = MagicMock() + def sort_data(data, key): + return sorted(data, key=key) # GDAX use-case (real data from GDAX) # This ticker history is ordered DESC (newest first, oldest last) @@ -893,14 +894,15 @@ async def test___async_get_candle_history_sort(default_conf, mocker): [1527830700000, 0.07652, 0.07652, 0.07651, 0.07652, 10.04822687], [1527830400000, 0.07649, 0.07651, 0.07649, 0.07651, 2.5734867] ] - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) exchange = get_patched_exchange(mocker, default_conf) exchange._api_async.fetch_ohlcv = get_mock_coro(tick) - + sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data)) # Test the ticker history sort res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert res[0] == 'ETH/BTC' ticks = res[1] + + assert sort_mock.call_count == 1 assert ticks[0][0] == 1527830400000 assert ticks[0][1] == 0.07649 assert ticks[0][2] == 0.07651 @@ -930,11 +932,14 @@ async def test___async_get_candle_history_sort(default_conf, mocker): [1527830400000, 0.07671, 0.07674399, 0.07629216, 0.07655213, 2.31452783] ] exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + # Reset sort mock + sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data)) # Test the ticker history sort res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert res[0] == 'ETH/BTC' ticks = res[1] - + # Sorted not called again - data is already in order + assert sort_mock.call_count == 0 assert ticks[0][0] == 1527827700000 assert ticks[0][1] == 0.07659999 assert ticks[0][2] == 0.0766 From 16eec078d78fcdc8e5948d45998dac2536e9a902 Mon Sep 17 00:00:00 2001 From: Pan Long Date: Mon, 26 Nov 2018 09:18:29 +0800 Subject: [PATCH 6/8] Use dot to access attribute in NamedTuple This should fix the crash in #1359 --- freqtrade/wallets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index b8b37907d..bf6f8b027 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -35,8 +35,8 @@ class Wallets(object): return 999.9 balance = self.wallets.get(currency) - if balance and balance['free']: - return balance['free'] + if balance and balance.free: + return balance.free else: return 0 From ad8592f3168c6d4fa908e3d9fae615f9490f9ae8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 26 Nov 2018 06:22:53 +0100 Subject: [PATCH 7/8] Test live mode of get_free --- freqtrade/tests/test_wallets.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/tests/test_wallets.py b/freqtrade/tests/test_wallets.py index e6a17ecbf..88366a869 100644 --- a/freqtrade/tests/test_wallets.py +++ b/freqtrade/tests/test_wallets.py @@ -30,6 +30,7 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets.wallets['GAS'].free == 0.260739 assert freqtrade.wallets.wallets['GAS'].used == 0.0 assert freqtrade.wallets.wallets['GAS'].total == 0.260739 + assert freqtrade.wallets.get_free('BNT') == 1.0 mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -56,6 +57,7 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets.wallets['GAS'].free == 0.270739 assert freqtrade.wallets.wallets['GAS'].used == 0.1 assert freqtrade.wallets.wallets['GAS'].total == 0.260439 + assert freqtrade.wallets.get_free('GAS') == 0.270739 def test_sync_wallet_missing_data(mocker, default_conf): @@ -84,3 +86,4 @@ def test_sync_wallet_missing_data(mocker, default_conf): assert freqtrade.wallets.wallets['GAS'].free == 0.260739 assert freqtrade.wallets.wallets['GAS'].used is None assert freqtrade.wallets.wallets['GAS'].total == 0.260739 + assert freqtrade.wallets.get_free('GAS') == 0.260739 From d3712c6e4035552e62490e31d8e422944a36abc6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 26 Nov 2018 13:34:05 +0100 Subject: [PATCH 8/8] Update ccxt from 1.17.536 to 1.17.539 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fa550195d..6f969be06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.536 +ccxt==1.17.539 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1