Merge branch 'develop' into feat/short

freqtrade/exchange/exchange.py

@@ -9,7 +9,7 @@ import logging
 from copy import deepcopy
 from datetime import datetime, timedelta, timezone
 from math import ceil
-from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union
 
 import arrow
 import ccxt
@@ -1639,6 +1639,24 @@ class Exchange:
         data = sorted(data, key=lambda x: x[0])
         return pair, timeframe, candle_type, data
 
+    def _build_coroutine(self, pair: str, timeframe: str, candle_type: CandleType,
+                         since_ms: Optional[int]) -> Coroutine:
+
+        if not since_ms and self.required_candle_call_count > 1:
+            # Multiple calls for one pair - to get more history
+            one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe)
+            move_to = one_call * self.required_candle_call_count
+            now = timeframe_to_next_date(timeframe)
+            since_ms = int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000)
+
+        if since_ms:
+            return self._async_get_historic_ohlcv(
+                pair, timeframe, since_ms=since_ms, raise_=True, candle_type=candle_type)
+        else:
+            # One call ... "regular" refresh
+            return self._async_get_candle_history(
+                pair, timeframe, since_ms=since_ms, candle_type=candle_type)
+
     def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *,
                              since_ms: Optional[int] = None, cache: bool = True,
                              drop_incomplete: bool = None
@@ -1660,22 +1678,17 @@ class Exchange:
         cached_pairs = []
         # Gather coroutines to run
         for pair, timeframe, candle_type in set(pair_list):
+            if timeframe not in self.timeframes:
+                logger.warning(
+                    f"Cannot download ({pair}, {timeframe}) combination as this timeframe is "
+                    f"not available on {self.name}. Available timeframes are "
+                    f"{', '.join(self.timeframes)}.")
+                continue
             if ((pair, timeframe, candle_type) not in self._klines or not cache
                     or self._now_is_time_to_refresh(pair, timeframe, candle_type)):
-                if not since_ms and self.required_candle_call_count > 1:
+                input_coroutines.append(self._build_coroutine(
-                    # Multiple calls for one pair - to get more history
+                    pair, timeframe, candle_type=candle_type, since_ms=since_ms))
-                    one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe)
-                    move_to = one_call * self.required_candle_call_count
-                    now = timeframe_to_next_date(timeframe)
-                    since_ms = int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000)
 
-                if since_ms:
-                    input_coroutines.append(self._async_get_historic_ohlcv(
-                        pair, timeframe, since_ms=since_ms, raise_=True, candle_type=candle_type))
-                else:
-                    # One call ... "regular" refresh
-                    input_coroutines.append(self._async_get_candle_history(
-                        pair, timeframe, since_ms=since_ms, candle_type=candle_type))
             else:
                 logger.debug(
                     f"Using cached candle (OHLCV) data for {pair}, {timeframe}, {candle_type} ..."

freqtrade/freqtradebot.py

@@ -1042,11 +1042,15 @@ class FreqtradeBot(LoggingMixin):
                 stop_price = trade.open_rate * (1 + stoploss)
 
             if self.create_stoploss_order(trade=trade, stop_price=stop_price):
+                # The above will return False if the placement failed and the trade was force-sold.
+                # in which case the trade will be closed - which we must check below.
                 trade.stoploss_last_update = datetime.utcnow()
                 return False
 
         # If stoploss order is canceled for some reason we add it
-        if stoploss_order and stoploss_order['status'] in ('canceled', 'cancelled'):
+        if (trade.is_open
+                and stoploss_order
+                and stoploss_order['status'] in ('canceled', 'cancelled')):
             if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss):
                 return False
             else:
@@ -1056,7 +1060,7 @@ class FreqtradeBot(LoggingMixin):
         # Finally we check if stoploss on exchange should be moved up because of trailing.
         # Triggered Orders are now real orders - so don't replace stoploss anymore
         if (
-            stoploss_order
+            trade.is_open and stoploss_order
             and stoploss_order.get('status_stop') != 'triggered'
             and (self.config.get('trailing_stop', False)
                  or self.config.get('use_custom_stoploss', False))

freqtrade/persistence/migrations.py

@@ -195,6 +195,7 @@ def drop_orders_table(engine, table_back_name: str):
 def migrate_orders_table(engine, table_back_name: str, cols_order: List):
 
     ft_fee_base = get_column_def(cols_order, 'ft_fee_base', 'null')
+    average = get_column_def(cols_order, 'average', 'null')
 
     # let SQLAlchemy create the schema as required
     leverage = get_column_def(cols_order, 'leverage', '1.0')
@@ -205,8 +206,8 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List):
             status, symbol, order_type, side, price, amount, filled, average, remaining, cost,
             order_date, order_filled_date, order_update_date, ft_fee_base, leverage)
             select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
-            status, symbol, order_type, side, price, amount, filled, null average, remaining, cost,
+            status, symbol, order_type, side, price, amount, filled, {average} average, remaining,
-            order_date, order_filled_date, order_update_date, {ft_fee_base} ft_fee_base,
+            cost, order_date, order_filled_date, order_update_date, {ft_fee_base} ft_fee_base,
             {leverage} leverage
             from {table_back_name}
             """))

freqtrade/plugins/pairlist/AgeFilter.py

@@ -100,7 +100,7 @@ class AgeFilter(IPairList):
         """
         Validate age for the ticker
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
+        :param daily_candles: Downloaded daily candles
         :return: True if the pair can stay, false if it should be removed
         """
         # Check symbol in cache

freqtrade/plugins/pairlist/PrecisionFilter.py

@@ -51,7 +51,7 @@ class PrecisionFilter(IPairList):
         :param ticker: ticker dict as returned from ccxt.fetch_tickers()
         :return: True if the pair can stay, false if it should be removed
         """
-        stop_price = ticker['ask'] * self._stoploss
+        stop_price = ticker['last'] * self._stoploss
 
         # Adjust stop-prices to precision
         sp = self._exchange.price_to_precision(pair, stop_price)

freqtrade/plugins/pairlist/VolatilityFilter.py

@@ -94,7 +94,7 @@ class VolatilityFilter(IPairList):
         """
         Validate trading range
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
+        :param daily_candles: Downloaded daily candles
         :return: True if the pair can stay, false if it should be removed
         """
         # Check symbol in cache

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -92,7 +92,7 @@ class RangeStabilityFilter(IPairList):
         """
         Validate trading range
         :param pair: Pair that's currently validated
-        :param ticker: ticker dict as returned from ccxt.fetch_tickers()
+        :param daily_candles: Downloaded daily candles
         :return: True if the pair can stay, false if it should be removed
         """
         # Check symbol in cache

tests/conftest.py

@@ -129,6 +129,8 @@ def patch_exchange(
         mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
     else:
         mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock())
+        mocker.patch('freqtrade.exchange.Exchange.timeframes', PropertyMock(
+                return_value=['5m', '15m', '1h', '1d']))
 
 
 def get_patched_exchange(mocker, config, api_mock=None, id='binance',

tests/exchange/test_exchange.py

@@ -1882,6 +1882,13 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog, candle_type) -> None
     res = exchange.refresh_latest_ohlcv(pairlist, cache=False)
     assert len(res) == 3
     assert exchange._api_async.fetch_ohlcv.call_count == 3
+    exchange._api_async.fetch_ohlcv.reset_mock()
+    caplog.clear()
+    # Call with invalid timeframe
+    res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '3m', candle_type)], cache=False)
+    assert not res
+    assert len(res) == 0
+    assert log_has_re(r'Cannot download \(IOTA\/ETH, 3m\).*', caplog)
 
 
 @pytest.mark.asyncio
@@ -3833,6 +3840,8 @@ def test__fetch_and_calculate_funding_fees(
     type(api_mock).has = PropertyMock(return_value={'fetchFundingRateHistory': True})
 
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange)
+    mocker.patch('freqtrade.exchange.Exchange.timeframes', PropertyMock(
+                return_value=['1h', '4h', '8h']))
     funding_fees = exchange._fetch_and_calculate_funding_fees(
         pair='ADA/USDT', amount=amount, is_short=True, open_date=d1, close_date=d2)
     assert pytest.approx(funding_fees) == expected_fees
@@ -3861,7 +3870,7 @@ def test__fetch_and_calculate_funding_fees_datetime_called(
         return_value=funding_rate_history_octohourly)
     type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True})
     type(api_mock).has = PropertyMock(return_value={'fetchFundingRateHistory': True})
-
+    mocker.patch('freqtrade.exchange.Exchange.timeframes', PropertyMock(return_value=['4h', '8h']))
     exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange)
     d1 = datetime.strptime("2021-09-01 00:00:00 +0000", '%Y-%m-%d %H:%M:%S %z')
 

tests/test_freqtradebot.py

@@ -1043,12 +1043,9 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_
         }),
         create_order=MagicMock(side_effect=[
             {'id': enter_order['id']},
-            {'id': exit_order['id']},
+            exit_order,
         ]),
         get_fee=fee,
-    )
-    mocker.patch.multiple(
-        'freqtrade.exchange.Binance',
         stoploss=stoploss
     )
     freqtrade = FreqtradeBot(default_conf_usdt)
@@ -1075,7 +1072,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_
     trade.stoploss_order_id = 100
 
     hanging_stoploss_order = MagicMock(return_value={'status': 'open'})
-    mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', hanging_stoploss_order)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', hanging_stoploss_order)
 
     assert freqtrade.handle_stoploss_on_exchange(trade) is False
     assert trade.stoploss_order_id == 100
@@ -1088,7 +1085,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_
     trade.stoploss_order_id = 100
 
     canceled_stoploss_order = MagicMock(return_value={'status': 'canceled'})
-    mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', canceled_stoploss_order)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', canceled_stoploss_order)
     stoploss.reset_mock()
 
     assert freqtrade.handle_stoploss_on_exchange(trade) is False
@@ -1121,7 +1118,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_
         'average': 2,
         'amount': enter_order['amount'],
     })
-    mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', stoploss_order_hit)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_hit)
     assert freqtrade.handle_stoploss_on_exchange(trade) is True
     assert log_has_re(r'STOP_LOSS_LIMIT is hit for Trade\(id=1, .*\)\.', caplog)
     assert trade.stoploss_order_id is None
@@ -1129,7 +1126,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_
     caplog.clear()
 
     mocker.patch(
-        'freqtrade.exchange.Binance.stoploss',
+        'freqtrade.exchange.Exchange.stoploss',
         side_effect=ExchangeError()
     )
     trade.is_open = True
@@ -1141,9 +1138,9 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_
     # It should try to add stoploss order
     trade.stoploss_order_id = 100
     stoploss.reset_mock()
-    mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order',
+    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order',
                  side_effect=InvalidOrderException())
-    mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss)
+    mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss)
     freqtrade.handle_stoploss_on_exchange(trade)
     assert stoploss.call_count == 1
 
@@ -1153,10 +1150,37 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_
     trade.is_open = False
     stoploss.reset_mock()
     mocker.patch('freqtrade.exchange.Exchange.fetch_order')
-    mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss)
+    mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss)
     assert freqtrade.handle_stoploss_on_exchange(trade) is False
     assert stoploss.call_count == 0
 
+    # Seventh case: emergency exit triggered
+    # Trailing stop should not act anymore
+    stoploss_order_cancelled = MagicMock(side_effect=[{
+        'id': "100",
+        'status': 'canceled',
+        'type': 'stop_loss_limit',
+        'price': 3,
+        'average': 2,
+        'amount': enter_order['amount'],
+        'info': {'stopPrice': 22},
+    }])
+    trade.stoploss_order_id = 100
+    trade.is_open = True
+    trade.stoploss_last_update = arrow.utcnow().shift(hours=-1).datetime
+    trade.stop_loss = 24
+    freqtrade.config['trailing_stop'] = True
+    stoploss = MagicMock(side_effect=InvalidOrderException())
+
+    mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order_with_result',
+                 side_effect=InvalidOrderException())
+    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_cancelled)
+    mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss)
+    assert freqtrade.handle_stoploss_on_exchange(trade) is False
+    assert trade.stoploss_order_id is None
+    assert trade.is_open is False
+    assert trade.sell_reason == str(SellType.EMERGENCY_SELL)
+
 
 @pytest.mark.parametrize("is_short", [False, True])
 def test_handle_sle_cancel_cant_recreate(mocker, default_conf_usdt, fee, caplog, is_short,