diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d561fe4cc..fc0d52caf 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -9,7 +9,7 @@ import logging from copy import deepcopy from datetime import datetime, timedelta, timezone from math import ceil -from typing import Any, Dict, List, Literal, Optional, Tuple, Union +from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union import arrow import ccxt @@ -1639,6 +1639,24 @@ class Exchange: data = sorted(data, key=lambda x: x[0]) return pair, timeframe, candle_type, data + def _build_coroutine(self, pair: str, timeframe: str, candle_type: CandleType, + since_ms: Optional[int]) -> Coroutine: + + if not since_ms and self.required_candle_call_count > 1: + # Multiple calls for one pair - to get more history + one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) + move_to = one_call * self.required_candle_call_count + now = timeframe_to_next_date(timeframe) + since_ms = int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000) + + if since_ms: + return self._async_get_historic_ohlcv( + pair, timeframe, since_ms=since_ms, raise_=True, candle_type=candle_type) + else: + # One call ... "regular" refresh + return self._async_get_candle_history( + pair, timeframe, since_ms=since_ms, candle_type=candle_type) + def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, since_ms: Optional[int] = None, cache: bool = True, drop_incomplete: bool = None @@ -1660,22 +1678,17 @@ class Exchange: cached_pairs = [] # Gather coroutines to run for pair, timeframe, candle_type in set(pair_list): + if timeframe not in self.timeframes: + logger.warning( + f"Cannot download ({pair}, {timeframe}) combination as this timeframe is " + f"not available on {self.name}. Available timeframes are " + f"{', '.join(self.timeframes)}.") + continue if ((pair, timeframe, candle_type) not in self._klines or not cache or self._now_is_time_to_refresh(pair, timeframe, candle_type)): - if not since_ms and self.required_candle_call_count > 1: - # Multiple calls for one pair - to get more history - one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) - move_to = one_call * self.required_candle_call_count - now = timeframe_to_next_date(timeframe) - since_ms = int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000) + input_coroutines.append(self._build_coroutine( + pair, timeframe, candle_type=candle_type, since_ms=since_ms)) - if since_ms: - input_coroutines.append(self._async_get_historic_ohlcv( - pair, timeframe, since_ms=since_ms, raise_=True, candle_type=candle_type)) - else: - # One call ... "regular" refresh - input_coroutines.append(self._async_get_candle_history( - pair, timeframe, since_ms=since_ms, candle_type=candle_type)) else: logger.debug( f"Using cached candle (OHLCV) data for {pair}, {timeframe}, {candle_type} ..." diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 23804bfec..c0af6e0e7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1042,11 +1042,15 @@ class FreqtradeBot(LoggingMixin): stop_price = trade.open_rate * (1 + stoploss) if self.create_stoploss_order(trade=trade, stop_price=stop_price): + # The above will return False if the placement failed and the trade was force-sold. + # in which case the trade will be closed - which we must check below. trade.stoploss_last_update = datetime.utcnow() return False # If stoploss order is canceled for some reason we add it - if stoploss_order and stoploss_order['status'] in ('canceled', 'cancelled'): + if (trade.is_open + and stoploss_order + and stoploss_order['status'] in ('canceled', 'cancelled')): if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss): return False else: @@ -1056,7 +1060,7 @@ class FreqtradeBot(LoggingMixin): # Finally we check if stoploss on exchange should be moved up because of trailing. # Triggered Orders are now real orders - so don't replace stoploss anymore if ( - stoploss_order + trade.is_open and stoploss_order and stoploss_order.get('status_stop') != 'triggered' and (self.config.get('trailing_stop', False) or self.config.get('use_custom_stoploss', False)) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 112538570..7ef8eab94 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -195,6 +195,7 @@ def drop_orders_table(engine, table_back_name: str): def migrate_orders_table(engine, table_back_name: str, cols_order: List): ft_fee_base = get_column_def(cols_order, 'ft_fee_base', 'null') + average = get_column_def(cols_order, 'average', 'null') # let SQLAlchemy create the schema as required leverage = get_column_def(cols_order, 'leverage', '1.0') @@ -205,8 +206,8 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List): status, symbol, order_type, side, price, amount, filled, average, remaining, cost, order_date, order_filled_date, order_update_date, ft_fee_base, leverage) select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, - status, symbol, order_type, side, price, amount, filled, null average, remaining, cost, - order_date, order_filled_date, order_update_date, {ft_fee_base} ft_fee_base, + status, symbol, order_type, side, price, amount, filled, {average} average, remaining, + cost, order_date, order_filled_date, order_update_date, {ft_fee_base} ft_fee_base, {leverage} leverage from {table_back_name} """)) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index f5507d0a6..bb6f75012 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -100,7 +100,7 @@ class AgeFilter(IPairList): """ Validate age for the ticker :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.fetch_tickers() + :param daily_candles: Downloaded daily candles :return: True if the pair can stay, false if it should be removed """ # Check symbol in cache diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index a3c262e8c..521f38635 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -51,7 +51,7 @@ class PrecisionFilter(IPairList): :param ticker: ticker dict as returned from ccxt.fetch_tickers() :return: True if the pair can stay, false if it should be removed """ - stop_price = ticker['ask'] * self._stoploss + stop_price = ticker['last'] * self._stoploss # Adjust stop-prices to precision sp = self._exchange.price_to_precision(pair, stop_price) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 13c6e7306..7a355c291 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -94,7 +94,7 @@ class VolatilityFilter(IPairList): """ Validate trading range :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.fetch_tickers() + :param daily_candles: Downloaded daily candles :return: True if the pair can stay, false if it should be removed """ # Check symbol in cache diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 3ce30347a..c9edfd13d 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -92,7 +92,7 @@ class RangeStabilityFilter(IPairList): """ Validate trading range :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.fetch_tickers() + :param daily_candles: Downloaded daily candles :return: True if the pair can stay, false if it should be removed """ # Check symbol in cache diff --git a/tests/conftest.py b/tests/conftest.py index 035c74068..698cdc7b4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -129,6 +129,8 @@ def patch_exchange( mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) else: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.timeframes', PropertyMock( + return_value=['5m', '15m', '1h', '1d'])) def get_patched_exchange(mocker, config, api_mock=None, id='binance', diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index bb2408b5c..b47c11b80 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1882,6 +1882,13 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog, candle_type) -> None res = exchange.refresh_latest_ohlcv(pairlist, cache=False) assert len(res) == 3 assert exchange._api_async.fetch_ohlcv.call_count == 3 + exchange._api_async.fetch_ohlcv.reset_mock() + caplog.clear() + # Call with invalid timeframe + res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '3m', candle_type)], cache=False) + assert not res + assert len(res) == 0 + assert log_has_re(r'Cannot download \(IOTA\/ETH, 3m\).*', caplog) @pytest.mark.asyncio @@ -3833,6 +3840,8 @@ def test__fetch_and_calculate_funding_fees( type(api_mock).has = PropertyMock(return_value={'fetchFundingRateHistory': True}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange) + mocker.patch('freqtrade.exchange.Exchange.timeframes', PropertyMock( + return_value=['1h', '4h', '8h'])) funding_fees = exchange._fetch_and_calculate_funding_fees( pair='ADA/USDT', amount=amount, is_short=True, open_date=d1, close_date=d2) assert pytest.approx(funding_fees) == expected_fees @@ -3861,7 +3870,7 @@ def test__fetch_and_calculate_funding_fees_datetime_called( return_value=funding_rate_history_octohourly) type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) type(api_mock).has = PropertyMock(return_value={'fetchFundingRateHistory': True}) - + mocker.patch('freqtrade.exchange.Exchange.timeframes', PropertyMock(return_value=['4h', '8h'])) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange) d1 = datetime.strptime("2021-09-01 00:00:00 +0000", '%Y-%m-%d %H:%M:%S %z') diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 85c105672..637cae83f 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1043,12 +1043,9 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ }), create_order=MagicMock(side_effect=[ {'id': enter_order['id']}, - {'id': exit_order['id']}, + exit_order, ]), get_fee=fee, - ) - mocker.patch.multiple( - 'freqtrade.exchange.Binance', stoploss=stoploss ) freqtrade = FreqtradeBot(default_conf_usdt) @@ -1075,7 +1072,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ trade.stoploss_order_id = 100 hanging_stoploss_order = MagicMock(return_value={'status': 'open'}) - mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', hanging_stoploss_order) + mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', hanging_stoploss_order) assert freqtrade.handle_stoploss_on_exchange(trade) is False assert trade.stoploss_order_id == 100 @@ -1088,7 +1085,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ trade.stoploss_order_id = 100 canceled_stoploss_order = MagicMock(return_value={'status': 'canceled'}) - mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', canceled_stoploss_order) + mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', canceled_stoploss_order) stoploss.reset_mock() assert freqtrade.handle_stoploss_on_exchange(trade) is False @@ -1121,7 +1118,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ 'average': 2, 'amount': enter_order['amount'], }) - mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', stoploss_order_hit) + mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_hit) assert freqtrade.handle_stoploss_on_exchange(trade) is True assert log_has_re(r'STOP_LOSS_LIMIT is hit for Trade\(id=1, .*\)\.', caplog) assert trade.stoploss_order_id is None @@ -1129,7 +1126,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ caplog.clear() mocker.patch( - 'freqtrade.exchange.Binance.stoploss', + 'freqtrade.exchange.Exchange.stoploss', side_effect=ExchangeError() ) trade.is_open = True @@ -1141,9 +1138,9 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ # It should try to add stoploss order trade.stoploss_order_id = 100 stoploss.reset_mock() - mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', + mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', side_effect=InvalidOrderException()) - mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss) freqtrade.handle_stoploss_on_exchange(trade) assert stoploss.call_count == 1 @@ -1153,10 +1150,37 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ trade.is_open = False stoploss.reset_mock() mocker.patch('freqtrade.exchange.Exchange.fetch_order') - mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss) assert freqtrade.handle_stoploss_on_exchange(trade) is False assert stoploss.call_count == 0 + # Seventh case: emergency exit triggered + # Trailing stop should not act anymore + stoploss_order_cancelled = MagicMock(side_effect=[{ + 'id': "100", + 'status': 'canceled', + 'type': 'stop_loss_limit', + 'price': 3, + 'average': 2, + 'amount': enter_order['amount'], + 'info': {'stopPrice': 22}, + }]) + trade.stoploss_order_id = 100 + trade.is_open = True + trade.stoploss_last_update = arrow.utcnow().shift(hours=-1).datetime + trade.stop_loss = 24 + freqtrade.config['trailing_stop'] = True + stoploss = MagicMock(side_effect=InvalidOrderException()) + + mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order_with_result', + side_effect=InvalidOrderException()) + mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_cancelled) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss) + assert freqtrade.handle_stoploss_on_exchange(trade) is False + assert trade.stoploss_order_id is None + assert trade.is_open is False + assert trade.sell_reason == str(SellType.EMERGENCY_SELL) + @pytest.mark.parametrize("is_short", [False, True]) def test_handle_sle_cancel_cant_recreate(mocker, default_conf_usdt, fee, caplog, is_short,