From 9ff52c0a93666daeaf00b46c64a82bc8a953f38d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 11 Mar 2022 08:00:18 +0100 Subject: [PATCH 1/8] Add test for emergencysell behaviour --- tests/test_freqtradebot.py | 47 +++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e5f8d3694..1aeb56cdd 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -926,12 +926,10 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, }), create_order=MagicMock(side_effect=[ {'id': limit_buy_order_usdt['id']}, - {'id': limit_sell_order_usdt['id']}, + limit_sell_order_usdt, + # {'id': limit_sell_order_usdt['id']}, ]), get_fee=fee, - ) - mocker.patch.multiple( - 'freqtrade.exchange.Binance', stoploss=stoploss ) freqtrade = FreqtradeBot(default_conf_usdt) @@ -956,7 +954,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, trade.stoploss_order_id = 100 hanging_stoploss_order = MagicMock(return_value={'status': 'open'}) - mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', hanging_stoploss_order) + mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', hanging_stoploss_order) assert freqtrade.handle_stoploss_on_exchange(trade) is False assert trade.stoploss_order_id == 100 @@ -969,7 +967,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, trade.stoploss_order_id = 100 canceled_stoploss_order = MagicMock(return_value={'status': 'canceled'}) - mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', canceled_stoploss_order) + mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', canceled_stoploss_order) stoploss.reset_mock() assert freqtrade.handle_stoploss_on_exchange(trade) is False @@ -1001,7 +999,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, 'average': 2, 'amount': limit_buy_order_usdt['amount'], }) - mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', stoploss_order_hit) + mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_hit) assert freqtrade.handle_stoploss_on_exchange(trade) is True assert log_has_re(r'STOP_LOSS_LIMIT is hit for Trade\(id=1, .*\)\.', caplog) assert trade.stoploss_order_id is None @@ -1009,7 +1007,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, caplog.clear() mocker.patch( - 'freqtrade.exchange.Binance.stoploss', + 'freqtrade.exchange.Exchange.stoploss', side_effect=ExchangeError() ) trade.is_open = True @@ -1021,9 +1019,9 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, # It should try to add stoploss order trade.stoploss_order_id = 100 stoploss.reset_mock() - mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', + mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', side_effect=InvalidOrderException()) - mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss) freqtrade.handle_stoploss_on_exchange(trade) assert stoploss.call_count == 1 @@ -1033,10 +1031,37 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, trade.is_open = False stoploss.reset_mock() mocker.patch('freqtrade.exchange.Exchange.fetch_order') - mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss) assert freqtrade.handle_stoploss_on_exchange(trade) is False assert stoploss.call_count == 0 + # Seventh case: emergency exit triggered + # Trailing stop should not act anymore + stoploss_order_cancelled = MagicMock(side_effect=[{ + 'id': "100", + 'status': 'canceled', + 'type': 'stop_loss_limit', + 'price': 3, + 'average': 2, + 'amount': limit_buy_order_usdt['amount'], + 'info': {'stopPrice': 22}, + }]) + trade.stoploss_order_id = 100 + trade.is_open = True + trade.stoploss_last_update = arrow.utcnow().shift(hours=-1).datetime + trade.stop_loss = 24 + freqtrade.config['trailing_stop'] = True + stoploss = MagicMock(side_effect=InvalidOrderException()) + + mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order_with_result', + side_effect=InvalidOrderException()) + mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_cancelled) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss) + assert freqtrade.handle_stoploss_on_exchange(trade) is False + assert trade.stoploss_order_id is None + assert trade.is_open is False + assert trade.sell_reason == str(SellType.EMERGENCY_SELL) + def test_handle_sle_cancel_cant_recreate(mocker, default_conf_usdt, fee, caplog, limit_buy_order_usdt, limit_sell_order_usdt) -> None: From 24f480b4ced4a9e0ea9ec09c93bf38911d57feef Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 11 Mar 2022 08:27:42 +0100 Subject: [PATCH 2/8] Double-check stoploss behaviour closes #6508 --- freqtrade/freqtradebot.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ac6ad65de..16864f814 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -873,11 +873,15 @@ class FreqtradeBot(LoggingMixin): stop_price = trade.open_rate * (1 + stoploss) if self.create_stoploss_order(trade=trade, stop_price=stop_price): + # The above will return False if the placement failed and the trade was force-sold. + # in which case the trade will be closed - which we must check below. trade.stoploss_last_update = datetime.utcnow() return False # If stoploss order is canceled for some reason we add it - if stoploss_order and stoploss_order['status'] in ('canceled', 'cancelled'): + if (trade.is_open + and stoploss_order + and stoploss_order['status'] in ('canceled', 'cancelled')): if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss): return False else: @@ -887,7 +891,7 @@ class FreqtradeBot(LoggingMixin): # Finally we check if stoploss on exchange should be moved up because of trailing. # Triggered Orders are now real orders - so don't replace stoploss anymore if ( - stoploss_order + trade.is_open and stoploss_order and stoploss_order.get('status_stop') != 'triggered' and (self.config.get('trailing_stop', False) or self.config.get('use_custom_stoploss', False)) From 11c76c3c89ed2b9090997161a764080f6d27ff1f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 11 Mar 2022 17:59:57 +0100 Subject: [PATCH 3/8] Check if timeframe is available before calling exchange closes #6517 --- freqtrade/exchange/exchange.py | 39 ++++++++++++++++++++------------- tests/conftest.py | 2 ++ tests/exchange/test_exchange.py | 7 ++++++ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index da89a7c8a..cfd12622b 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -9,7 +9,7 @@ import logging from copy import deepcopy from datetime import datetime, timedelta, timezone from math import ceil -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Coroutine, Dict, List, Optional, Tuple import arrow import ccxt @@ -1371,6 +1371,22 @@ class Exchange: data = sorted(data, key=lambda x: x[0]) return pair, timeframe, data + def _build_coroutine(self, pair: str, timeframe: str, since_ms: Optional[int]) -> Coroutine: + if not since_ms and self.required_candle_call_count > 1: + # Multiple calls for one pair - to get more history + one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) + move_to = one_call * self.required_candle_call_count + now = timeframe_to_next_date(timeframe) + since_ms = int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000) + + if since_ms: + return self._async_get_historic_ohlcv( + pair, timeframe, since_ms=since_ms, raise_=True) + else: + # One call ... "regular" refresh + return self._async_get_candle_history( + pair, timeframe, since_ms=since_ms) + def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, since_ms: Optional[int] = None, cache: bool = True ) -> Dict[Tuple[str, str], DataFrame]: @@ -1389,22 +1405,15 @@ class Exchange: cached_pairs = [] # Gather coroutines to run for pair, timeframe in set(pair_list): + if timeframe not in self.timeframes: + logger.warning( + f"Cannot download ({pair}, {timeframe}) combination as this timeframe is " + f"not available on {self.name}. Available timeframes are " + f"{', '.join(self.timeframes)}.") + continue if ((pair, timeframe) not in self._klines or not cache or self._now_is_time_to_refresh(pair, timeframe)): - if not since_ms and self.required_candle_call_count > 1: - # Multiple calls for one pair - to get more history - one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) - move_to = one_call * self.required_candle_call_count - now = timeframe_to_next_date(timeframe) - since_ms = int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000) - - if since_ms: - input_coroutines.append(self._async_get_historic_ohlcv( - pair, timeframe, since_ms=since_ms, raise_=True)) - else: - # One call ... "regular" refresh - input_coroutines.append(self._async_get_candle_history( - pair, timeframe, since_ms=since_ms)) + input_coroutines.append(self._build_coroutine(pair, timeframe, since_ms)) else: logger.debug( "Using cached candle (OHLCV) data for pair %s, timeframe %s ...", diff --git a/tests/conftest.py b/tests/conftest.py index ae35b0326..57122c01c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -107,6 +107,8 @@ def patch_exchange(mocker, api_mock=None, id='binance', mock_markets=True) -> No mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) else: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.timeframes', PropertyMock( + return_value=['5m', '15m', '1h', '1d'])) def get_patched_exchange(mocker, config, api_mock=None, id='binance', diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 527e8050b..2bd84942d 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1692,6 +1692,13 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: cache=False) assert len(res) == 3 assert exchange._api_async.fetch_ohlcv.call_count == 3 + exchange._api_async.fetch_ohlcv.reset_mock() + caplog.clear() + # Call with invalid timeframe + res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '3m')],cache=False) + assert not res + assert len(res) == 0 + assert log_has_re(r'Cannot download \(IOTA\/ETH, 3m\).*', caplog) @pytest.mark.asyncio From 7825d855cd10e589103afd614199c042e85fc3a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 11 Mar 2022 19:28:15 +0100 Subject: [PATCH 4/8] Fix flake8 error in tests --- tests/exchange/test_exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 2bd84942d..ff8383997 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1695,7 +1695,7 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: exchange._api_async.fetch_ohlcv.reset_mock() caplog.clear() # Call with invalid timeframe - res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '3m')],cache=False) + res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '3m')], cache=False) assert not res assert len(res) == 0 assert log_has_re(r'Cannot download \(IOTA\/ETH, 3m\).*', caplog) From 9107819c9518a9075e8feeab43b896171ead0bfe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Mar 2022 14:42:15 +0100 Subject: [PATCH 5/8] Fix order migration "forgetting" average --- freqtrade/persistence/migrations.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index ef64a2b27..2da24b748 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -174,16 +174,17 @@ def drop_orders_table(engine, table_back_name: str): def migrate_orders_table(engine, table_back_name: str, cols_order: List): ft_fee_base = get_column_def(cols_order, 'ft_fee_base', 'null') + average = get_column_def(cols_order, 'average', 'null') # let SQLAlchemy create the schema as required with engine.begin() as connection: connection.execute(text(f""" insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, - status, symbol, order_type, side, price, amount, filled, average, remaining, cost, - order_date, order_filled_date, order_update_date, ft_fee_base) + status, symbol, order_type, side, price, amount, filled, average, remaining, + cost, order_date, order_filled_date, order_update_date, ft_fee_base) select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, - status, symbol, order_type, side, price, amount, filled, null average, remaining, cost, - order_date, order_filled_date, order_update_date, {ft_fee_base} + status, symbol, order_type, side, price, amount, filled, {average} average, remaining, + cost, order_date, order_filled_date, order_update_date, {ft_fee_base} ft_fee_base from {table_back_name} """)) From b8b56d95f39164256f61f7aee5bb146816e5dfc9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Mar 2022 14:57:32 +0100 Subject: [PATCH 6/8] Update missleading docstring --- freqtrade/plugins/pairlist/VolatilityFilter.py | 2 +- freqtrade/plugins/pairlist/rangestabilityfilter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 20b899c5f..8a7eeeca8 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -90,7 +90,7 @@ class VolatilityFilter(IPairList): """ Validate trading range :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.fetch_tickers() + :param daily_candles: Downloaded daily candles :return: True if the pair can stay, false if it should be removed """ # Check symbol in cache diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 314056fbb..e17ec2dab 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -88,7 +88,7 @@ class RangeStabilityFilter(IPairList): """ Validate trading range :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.fetch_tickers() + :param daily_candles: Downloaded daily candles :return: True if the pair can stay, false if it should be removed """ # Check symbol in cache From 7146122f4adc97b4534428300a5e34c3b025879e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Mar 2022 15:06:32 +0100 Subject: [PATCH 7/8] Update docstring --- freqtrade/plugins/pairlist/AgeFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 5627d82ce..a6d5ec79b 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -98,7 +98,7 @@ class AgeFilter(IPairList): """ Validate age for the ticker :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.fetch_tickers() + :param daily_candles: Downloaded daily candles :return: True if the pair can stay, false if it should be removed """ # Check symbol in cache From 3133be19e33077968cd069344f4e860b10132091 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Mar 2022 15:23:01 +0100 Subject: [PATCH 8/8] Update precisionfilter to use last instead of ask or bid. --- freqtrade/plugins/pairlist/PrecisionFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index a3c262e8c..521f38635 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -51,7 +51,7 @@ class PrecisionFilter(IPairList): :param ticker: ticker dict as returned from ccxt.fetch_tickers() :return: True if the pair can stay, false if it should be removed """ - stop_price = ticker['ask'] * self._stoploss + stop_price = ticker['last'] * self._stoploss # Adjust stop-prices to precision sp = self._exchange.price_to_precision(pair, stop_price)