Merge branch 'freqtrade:feat/short' into feat/short

freqtrade/data/history/hdf5datahandler.py

@@ -206,7 +206,9 @@ class HDF5DataHandler(IDataHandler):
 
     @classmethod
     def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str:
-        return f"{pair}/ohlcv/tf_{timeframe}"
+        # Escape futures pairs to avoid warnings
+        pair_esc = pair.replace(':', '_')
+        return f"{pair_esc}/ohlcv/tf_{timeframe}"
 
     @classmethod
     def _pair_trades_key(cls, pair: str) -> str:

freqtrade/exchange/exchange.py

@@ -75,6 +75,7 @@ class Exchange:
         "mark_ohlcv_price": "mark",
         "mark_ohlcv_timeframe": "8h",
         "ccxt_futures_name": "swap",
+        "needs_trading_fees": False,  # use fetch_trading_fees to cache fees
     }
     _ft_has: Dict = {}
     _ft_has_futures: Dict = {}
@@ -92,6 +93,7 @@ class Exchange:
         self._api: ccxt.Exchange = None
         self._api_async: ccxt_async.Exchange = None
         self._markets: Dict = {}
+        self._trading_fees: Dict[str, Any] = {}
         self._leverage_tiers: Dict[str, List[Dict]] = {}
         self.loop = asyncio.new_event_loop()
         asyncio.set_event_loop(self.loop)
@@ -451,6 +453,9 @@ class Exchange:
             self._markets = self._api.load_markets()
             self._load_async_markets()
             self._last_markets_refresh = arrow.utcnow().int_timestamp
+            if self._ft_has['needs_trading_fees']:
+                self._trading_fees = self.fetch_trading_fees()
+
         except ccxt.BaseError:
             logger.exception('Unable to initialize markets.')
 
@@ -1299,6 +1304,27 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
+    @retrier
+    def fetch_trading_fees(self) -> Dict[str, Any]:
+        """
+        Fetch user account trading fees
+        Can be cached, should not update often.
+        """
+        if (self._config['dry_run'] or self.trading_mode != TradingMode.FUTURES
+                or not self.exchange_has('fetchTradingFees')):
+            return {}
+        try:
+            trading_fees: Dict[str, Any] = self._api.fetch_trading_fees()
+            self._log_exchange_response('fetch_trading_fees', trading_fees)
+            return trading_fees
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not fetch trading fees due to {e.__class__.__name__}. Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(e) from e
+
     @retrier
     def fetch_bids_asks(self, symbols: List[str] = None, cached: bool = False) -> Dict:
         """

freqtrade/exchange/gateio.py

@@ -1,6 +1,7 @@
 """ Gate.io exchange subclass """
 import logging
-from typing import Dict, List, Tuple
+from datetime import datetime
+from typing import Dict, List, Optional, Tuple
 
 from freqtrade.enums import MarginMode, TradingMode
 from freqtrade.exceptions import OperationalException
@@ -27,6 +28,10 @@ class Gateio(Exchange):
         "stoploss_on_exchange": True,
     }
 
+    _ft_has_futures: Dict = {
+        "needs_trading_fees": True
+    }
+
     _supported_trading_mode_margin_pairs: List[Tuple[TradingMode, MarginMode]] = [
         # TradingMode.SPOT always supported and not required in this list
         # (TradingMode.MARGIN, MarginMode.CROSS),
@@ -42,6 +47,30 @@ class Gateio(Exchange):
                 raise OperationalException(
                     f'Exchange {self.name} does not support market orders.')
 
+    def get_trades_for_order(self, order_id: str, pair: str, since: datetime,
+                             params: Optional[Dict] = None) -> List:
+        trades = super().get_trades_for_order(order_id, pair, since, params)
+
+        if self.trading_mode == TradingMode.FUTURES:
+            # Futures usually don't contain fees in the response.
+            # As such, futures orders on gateio will not contain a fee, which causes
+            # a repeated "update fee" cycle and wrong calculations.
+            # Therefore we patch the response with fees if it's not available.
+            # An alternative also contianing fees would be
+            # privateFuturesGetSettleAccountBook({"settle": "usdt"})
+            pair_fees = self._trading_fees.get(pair, {})
+            if pair_fees:
+                for idx, trade in enumerate(trades):
+                    if trade.get('fee', {}).get('cost') is None:
+                        takerOrMaker = trade.get('takerOrMaker', 'taker')
+                        if pair_fees.get(takerOrMaker) is not None:
+                            trades[idx]['fee'] = {
+                                'currency': self.get_pair_quote_currency(pair),
+                                'cost': trade['cost'] * pair_fees[takerOrMaker],
+                                'rate': pair_fees[takerOrMaker],
+                            }
+        return trades
+
     def fetch_stoploss_order(self, order_id: str, pair: str, params={}) -> Dict:
         return self.fetch_order(
             order_id=order_id,

freqtrade/freqtradebot.py

@@ -511,7 +511,7 @@ class FreqtradeBot(LoggingMixin):
                 return
         else:
             logger.debug("Max adjustment entries is set to unlimited.")
-        current_rate = self.exchange.get_rate(trade.pair, refresh=True, side="buy")
+        current_rate = self.exchange.get_rate(trade.pair, refresh=True, side=trade.enter_side)
         current_profit = trade.calc_profit_ratio(current_rate)
 
         min_stake_amount = self.exchange.get_min_pair_stake_amount(trade.pair,
@@ -536,12 +536,7 @@ class FreqtradeBot(LoggingMixin):
             logger.error(f"Unable to decrease trade position / sell partially"
                          f" for pair {trade.pair}, feature not implemented.")
 
-    def _check_depth_of_market(
+    def _check_depth_of_market(self, pair: str, conf: Dict, side: SignalDirection) -> bool:
-        self,
-        pair: str,
-        conf: Dict,
-        side: SignalDirection
-    ) -> bool:
         """
         Checks depth of market before executing a buy
         """
@@ -1564,6 +1559,7 @@ class FreqtradeBot(LoggingMixin):
         if not order_obj:
             raise DependencyException(
                 f"Order_obj not found for {order_id}. This should not have happened.")
+
         self.handle_order_fee(trade, order_obj, order)
 
         trade.update_trade(order_obj)

requirements.txt

@@ -2,7 +2,7 @@ numpy==1.22.3
 pandas==1.4.1
 pandas-ta==0.3.14b
 
-ccxt==1.76.65
+ccxt==1.77.29
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==36.0.2
 aiohttp==3.8.1

setup.py

@@ -42,7 +42,7 @@ setup(
     ],
     install_requires=[
         # from requirements.txt
-        'ccxt>=1.76.5',
+        'ccxt>=1.77.29',
         'SQLAlchemy',
         'python-telegram-bot>=13.4',
         'arrow>=0.17.0',

tests/data/test_history.py

@@ -683,7 +683,7 @@ def test_datahandler_ohlcv_get_pairs(testdatadir):
     assert set(pairs) == {'XRP/USDT'}
 
     pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
-    assert set(pairs) == {'UNITTEST/USDT'}
+    assert set(pairs) == {'UNITTEST/USDT:USDT'}
 
 
 @pytest.mark.parametrize('filename,pair,timeframe,candletype', [
@@ -914,7 +914,7 @@ def test_hdf5datahandler_trades_purge(mocker, testdatadir):
     # Data goes from 2018-01-10 - 2018-01-30
     ('UNITTEST/BTC', '5m', 'spot',  '', '2018-01-15', '2018-01-19'),
     # Mark data goes from to 2021-11-15 2021-11-19
-    ('UNITTEST/USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'),
+    ('UNITTEST/USDT:USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'),
 ])
 def test_hdf5datahandler_ohlcv_load_and_resave(
     testdatadir,

tests/exchange/test_ccxt_compat.py

@@ -134,7 +134,7 @@ def exchange_futures(request, exchange_conf, class_mocker):
 
         class_mocker.patch(
             'freqtrade.exchange.binance.Binance.fill_leverage_tiers')
-
+        class_mocker.patch('freqtrade.exchange.exchange.Exchange.fetch_trading_fees')
         exchange = ExchangeResolver.load_exchange(request.param, exchange_conf, validate=True)
 
         yield exchange, request.param

tests/exchange/test_exchange.py

@@ -1624,6 +1624,62 @@ def test_fetch_positions(default_conf, mocker, exchange_name):
                            "fetch_positions", "fetch_positions")
 
 
+def test_fetch_trading_fees(default_conf, mocker):
+    api_mock = MagicMock()
+    tick = {
+        '1INCH/USDT:USDT': {
+            'info': {'user_id': '',
+                     'taker_fee': '0.0018',
+                     'maker_fee': '0.0018',
+                     'gt_discount': False,
+                     'gt_taker_fee': '0',
+                     'gt_maker_fee': '0',
+                     'loan_fee': '0.18',
+                     'point_type': '1',
+                     'futures_taker_fee': '0.0005',
+                     'futures_maker_fee': '0'},
+            'symbol': '1INCH/USDT:USDT',
+            'maker': 0.0,
+            'taker': 0.0005},
+        'ETH/USDT:USDT': {
+            'info': {'user_id': '',
+                     'taker_fee': '0.0018',
+                     'maker_fee': '0.0018',
+                     'gt_discount': False,
+                     'gt_taker_fee': '0',
+                     'gt_maker_fee': '0',
+                     'loan_fee': '0.18',
+                     'point_type': '1',
+                     'futures_taker_fee': '0.0005',
+                     'futures_maker_fee': '0'},
+            'symbol': 'ETH/USDT:USDT',
+            'maker': 0.0,
+            'taker': 0.0005}
+    }
+    exchange_name = 'gateio'
+    default_conf['dry_run'] = False
+    default_conf['trading_mode'] = TradingMode.FUTURES
+    default_conf['margin_mode'] = MarginMode.ISOLATED
+    api_mock.fetch_trading_fees = MagicMock(return_value=tick)
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True)
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+
+    assert '1INCH/USDT:USDT' in exchange._trading_fees
+    assert 'ETH/USDT:USDT' in exchange._trading_fees
+    assert api_mock.fetch_trading_fees.call_count == 1
+
+    api_mock.fetch_trading_fees.reset_mock()
+
+    ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name,
+                           "fetch_trading_fees", "fetch_trading_fees")
+
+    api_mock.fetch_trading_fees = MagicMock(return_value={})
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+    exchange.fetch_trading_fees()
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True)
+    assert exchange.fetch_trading_fees() == {}
+
+
 def test_fetch_bids_asks(default_conf, mocker):
     api_mock = MagicMock()
     tick = {'ETH/BTC': {

tests/exchange/test_gateio.py

@@ -1,7 +1,9 @@
+from datetime import datetime, timezone
 from unittest.mock import MagicMock
 
 import pytest
 
+from freqtrade.enums import MarginMode, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Gateio
 from freqtrade.resolvers.exchange_resolver import ExchangeResolver
@@ -70,3 +72,47 @@ def test_stoploss_adjust_gateio(mocker, default_conf, sl1, sl2, sl3, side):
     }
     assert exchange.stoploss_adjust(sl1, order, side)
     assert not exchange.stoploss_adjust(sl2, order, side)
+
+
+@pytest.mark.parametrize('takerormaker,rate,cost', [
+    ('taker', 0.0005, 0.0001554325),
+    ('maker', 0.0, 0.0),
+])
+def test_fetch_my_trades_gateio(mocker, default_conf, takerormaker, rate, cost):
+    mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True)
+    tick = {'ETH/USDT:USDT': {
+        'info': {'user_id': '',
+                 'taker_fee': '0.0018',
+                 'maker_fee': '0.0018',
+                 'gt_discount': False,
+                 'gt_taker_fee': '0',
+                 'gt_maker_fee': '0',
+                 'loan_fee': '0.18',
+                 'point_type': '1',
+                 'futures_taker_fee': '0.0005',
+                 'futures_maker_fee': '0'},
+            'symbol': 'ETH/USDT:USDT',
+            'maker': 0.0,
+            'taker': 0.0005}
+            }
+    default_conf['dry_run'] = False
+    default_conf['trading_mode'] = TradingMode.FUTURES
+    default_conf['margin_mode'] = MarginMode.ISOLATED
+
+    api_mock = MagicMock()
+    api_mock.fetch_my_trades = MagicMock(return_value=[{
+        'fee': {'cost': None},
+        'price': 3108.65,
+        'cost': 0.310865,
+        'order': '22255',
+        'takerOrMaker': takerormaker,
+        'amount': 1,  # 1 contract
+    }])
+    exchange = get_patched_exchange(mocker, default_conf, api_mock=api_mock, id='gateio')
+    exchange._trading_fees = tick
+    trades = exchange.get_trades_for_order('22255', 'ETH/USDT:USDT', datetime.now(timezone.utc))
+    trade = trades[0]
+    assert trade['fee']
+    assert trade['fee']['rate'] == rate
+    assert trade['fee']['currency'] == 'USDT'
+    assert trade['fee']['cost'] == cost

tests/test_integration.py

@@ -311,8 +311,7 @@ def test_dca_short(default_conf_usdt, ticker_usdt, fee, mocker) -> None:
 
     # Reduce bid amount
     ticker_usdt_modif = ticker_usdt.return_value
-    ticker_usdt_modif['ask'] = ticker_usdt_modif['ask'] * 1.015
+    ticker_usdt_modif['ask'] = ticker_usdt_modif['ask'] * 1.004
-    ticker_usdt_modif['bid'] = ticker_usdt_modif['bid'] * 1.0125
     mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value=ticker_usdt_modif)
 
     # additional buy order

tests/test_misc.py

@@ -73,6 +73,8 @@ def test_file_load_json(mocker, testdatadir) -> None:
     ("ETH/BTC", 'ETH_BTC'),
     ("ETH/USDT", 'ETH_USDT'),
     ("ETH/USDT:USDT", 'ETH_USDT_USDT'),  # swap with USDT as settlement currency
+    ("ETH/USD:USD", 'ETH_USD_USD'),  # swap with USD as settlement currency
+    ("AAVE/USD:USD", 'AAVE_USD_USD'),  # swap with USDT as settlement currency
     ("ETH/USDT:USDT-210625", 'ETH_USDT_USDT-210625'),  # expiring futures
     ("Fabric Token/ETH", 'Fabric_Token_ETH'),
     ("ETHH20", 'ETHH20'),