Merge branch 'freqtrade:develop' into develop

.github/workflows/ci.yml

@@ -360,6 +360,8 @@ jobs:
         pip install -e .
 
     - name: Tests incl. ccxt compatibility tests
+      env:
+        CI_WEB_PROXY: http://152.67.78.211:13128
       run: |
         pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun
 

docs/advanced-backtesting.md

@@ -32,7 +32,7 @@ To analyze the entry/exit tags, we now need to use the `freqtrade backtesting-an
 with `--analysis-groups` option provided with space-separated arguments (default `0 1 2`):
 
 ``` bash
-freqtrade backtesting-analysis -c <config.json> --analysis-groups 0 1 2 3 4
+freqtrade backtesting-analysis -c <config.json> --analysis-groups 0 1 2 3 4 5
 ```
 
 This command will read from the last backtesting results. The `--analysis-groups` option is
@@ -43,6 +43,7 @@ ranging from the simplest (0) to the most detailed per pair, per buy and per sel
 * 2: profit summaries grouped by enter_tag and exit_tag
 * 3: profit summaries grouped by pair and enter_tag
 * 4: profit summaries grouped by pair, enter_ and exit_tag (this can get quite large)
+* 5: profit summaries grouped by exit_tag
 
 More options are available by running with the `-h` option.
 

docs/leverage.md

@@ -67,8 +67,6 @@ You will also have to pick a "margin mode" (explanation below) - with freqtrade
 Freqtrade follows the [ccxt naming conventions for futures](https://docs.ccxt.com/en/latest/manual.html?#perpetual-swap-perpetual-future).
 A futures pair will therefore have the naming of `base/quote:settle` (e.g. `ETH/USDT:USDT`).
 
-Binance is currently still an exception to this naming scheme, where pairs are named `ETH/USDT` also for futures markets, but will be aligned as soon as CCXT is ready.
-
 ### Margin mode
 
 On top of `trading_mode` - you will also have to configure your `margin_mode`.

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.3.7
 mkdocs==1.4.2
-mkdocs-material==9.0.3
+mkdocs-material==9.0.5
 mdx_truly_sane_lists==1.3
-pymdown-extensions==9.9
+pymdown-extensions==9.9.1
 jinja2==3.1.2

freqtrade/commands/cli_options.py

@@ -632,10 +632,11 @@ AVAILABLE_CLI_OPTIONS = {
               "1: by enter_tag, "
               "2: by enter_tag and exit_tag, "
               "3: by pair and enter_tag, "
-              "4: by pair, enter_ and exit_tag (this can get quite large)"),
+              "4: by pair, enter_ and exit_tag (this can get quite large), "
+              "5: by exit_tag"),
         nargs='+',
         default=['0', '1', '2'],
-        choices=['0', '1', '2', '3', '4'],
+        choices=['0', '1', '2', '3', '4', '5'],
     ),
     "enter_reason_list": Arg(
         "--enter-reason-list",

freqtrade/commands/data_commands.py

@@ -14,6 +14,7 @@ from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import market_is_active, timeframe_to_minutes
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
+from freqtrade.util.binance_mig import migrate_binance_futures_data
 
 
 logger = logging.getLogger(__name__)
@@ -86,6 +87,7 @@ def start_download_data(args: Dict[str, Any]) -> None:
                     "Please use `--dl-trades` instead for this exchange "
                     "(will unfortunately take a long time)."
                     )
+            migrate_binance_futures_data(config)
             pairs_not_available = refresh_backtest_ohlcv_data(
                 exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
                 datadir=config['datadir'], timerange=timerange,
@@ -145,6 +147,7 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
     """
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
     if ohlcv:
+        migrate_binance_futures_data(config)
         candle_types = [CandleType.from_string(ct) for ct in config.get('candle_types', ['spot'])]
         for candle_type in candle_types:
             convert_ohlcv_format(config,

freqtrade/data/entryexitanalysis.py

@@ -141,6 +141,12 @@ def _do_group_table_output(bigdf, glist):
             # 4: profit summaries grouped by pair, enter_ and exit_tag (this can get quite large)
             if g == "4":
                 group_mask = ['pair', 'enter_reason', 'exit_reason']
+
+            # 5: profit summaries grouped by exit_tag
+            if g == "5":
+                group_mask = ['exit_reason']
+                sortcols = ['exit_reason']
+
             if group_mask:
                 new = bigdf.groupby(group_mask).agg(agg_mask).reset_index()
                 new.columns = group_mask + agg_cols

freqtrade/data/history/idatahandler.py

@@ -374,6 +374,21 @@ class IDataHandler(ABC):
                 logger.warning(f"{pair}, {candle_type}, {timeframe}, "
                                f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}")
 
+    def rename_futures_data(
+            self, pair: str, new_pair: str, timeframe: str, candle_type: CandleType):
+        """
+        Temporary method to migrate data from old naming to new naming (BTC/USDT -> BTC/USDT:USDT)
+        Only used for binance to support the binance futures naming unification.
+        """
+
+        file_old = self._pair_data_filename(self._datadir, pair, timeframe, candle_type)
+        file_new = self._pair_data_filename(self._datadir, new_pair, timeframe, candle_type)
+        # print(file_old, file_new)
+        if file_new.exists():
+            logger.warning(f"{file_new} exists already, can't migrate {pair}.")
+            return
+        file_old.rename(file_new)
+
 
 def get_datahandlerclass(datatype: str) -> Type[IDataHandler]:
     """

freqtrade/exchange/binance.py

@@ -28,7 +28,7 @@ class Binance(Exchange):
         "trades_pagination": "id",
         "trades_pagination_arg": "fromId",
         "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000],
-        "ccxt_futures_name": "future"
+        "ccxt_futures_name": "swap"
     }
     _ft_has_futures: Dict = {
         "stoploss_order_types": {"limit": "stop", "market": "stop_market"},

freqtrade/freqtradebot.py

@@ -33,6 +33,7 @@ from freqtrade.rpc.external_message_consumer import ExternalMessageConsumer
 from freqtrade.strategy.interface import IStrategy
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 from freqtrade.util import FtPrecise
+from freqtrade.util.binance_mig import migrate_binance_futures_names
 from freqtrade.wallets import Wallets
 
 
@@ -177,6 +178,8 @@ class FreqtradeBot(LoggingMixin):
         Called on startup and after reloading the bot - triggers notifications and
         performs startup tasks
         """
+        migrate_binance_futures_names(self.config)
+
         self.rpc.startup_messages(self.config, self.pairlists, self.protections)
         # Update older trades with precision and precision mode
         self.startup_backpopulate_precision()

freqtrade/optimize/backtesting.py

@@ -37,6 +37,7 @@ from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.strategy.interface import IStrategy
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
+from freqtrade.util.binance_mig import migrate_binance_futures_data
 from freqtrade.wallets import Wallets
 
 
@@ -157,6 +158,7 @@ class Backtesting:
         self._can_short = self.trading_mode != TradingMode.SPOT
         self._position_stacking: bool = self.config.get('position_stacking', False)
         self.enable_protections: bool = self.config.get('enable_protections', False)
+        migrate_binance_futures_data(config)
 
         self.init_backtest()
 

freqtrade/resolvers/iresolver.py

@@ -89,7 +89,8 @@ class IResolver:
             module = importlib.util.module_from_spec(spec)
             try:
                 spec.loader.exec_module(module)  # type: ignore # importlib does not use typehints
-            except (ModuleNotFoundError, SyntaxError, ImportError, NameError) as err:
+            except (AttributeError, ModuleNotFoundError, SyntaxError,
+                    ImportError, NameError) as err:
                 # Catch errors in case a specific module is not installed
                 logger.warning(f"Could not import {module_path} due to '{err}'")
                 if enum_failed:

freqtrade/util/binance_mig.py

@@ -0,0 +1,78 @@
+import logging
+
+from packaging import version
+
+from freqtrade.constants import Config
+from freqtrade.enums.tradingmode import TradingMode
+from freqtrade.exceptions import OperationalException
+from freqtrade.persistence.pairlock import PairLock
+from freqtrade.persistence.trade_model import Trade
+
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_binance_futures_names(config: Config):
+
+    if (
+        not (config.get('trading_mode', TradingMode.SPOT) == TradingMode.FUTURES
+             and config['exchange']['name'] == 'binance')
+    ):
+        # only act on new futures
+        return
+    import ccxt
+    if version.parse("2.6.26") > version.parse(ccxt.__version__):
+        raise OperationalException(
+            "Please follow the update instructions in the docs "
+            "(https://www.freqtrade.io/en/latest/updating/) to install a compatible ccxt version.")
+    _migrate_binance_futures_db(config)
+    migrate_binance_futures_data(config)
+
+
+def _migrate_binance_futures_db(config: Config):
+    logger.warning('Migrating binance futures pairs in database.')
+    trades = Trade.get_trades([Trade.exchange == 'binance', Trade.trading_mode == 'FUTURES']).all()
+    for trade in trades:
+        if ':' in trade.pair:
+            # already migrated
+            continue
+        new_pair = f"{trade.pair}:{trade.stake_currency}"
+        trade.pair = new_pair
+
+        for order in trade.orders:
+            order.ft_pair = new_pair
+            # Should symbol be migrated too?
+            # order.symbol = new_pair
+    Trade.commit()
+    pls = PairLock.query.filter(PairLock.pair.notlike('%:%'))
+    for pl in pls:
+        pl.pair = f"{pl.pair}:{config['stake_currency']}"
+    # print(pls)
+    # pls.update({'pair': concat(PairLock.pair,':USDT')})
+    Trade.commit()
+    logger.warning('Done migrating binance futures pairs in database.')
+
+
+def migrate_binance_futures_data(config: Config):
+
+    if (
+        not (config.get('trading_mode', TradingMode.SPOT) == TradingMode.FUTURES
+             and config['exchange']['name'] == 'binance')
+    ):
+        # only act on new futures
+        return
+
+    from freqtrade.data.history.idatahandler import get_datahandler
+    dhc = get_datahandler(config['datadir'], config.get('dataformat_ohlcv', 'json'))
+
+    paircombs = dhc.ohlcv_get_available_data(
+        config['datadir'],
+        config.get('trading_mode', TradingMode.SPOT)
+        )
+
+    for pair, timeframe, candle_type in paircombs:
+        if ':' in pair:
+            # already migrated
+            continue
+        new_pair = f"{pair}:{config['stake_currency']}"
+        dhc.rename_futures_data(pair, new_pair, timeframe, candle_type)

requirements-dev.txt

@@ -11,7 +11,7 @@ flake8==6.0.0
 flake8-tidy-imports==4.8.0
 mypy==0.991
 pre-commit==2.21.0
-pytest==7.2.0
+pytest==7.2.1
 pytest-asyncio==0.20.3
 pytest-cov==4.0.0
 pytest-mock==3.10.0

requirements-freqai.txt

@@ -7,5 +7,5 @@ scikit-learn==1.1.3
 joblib==1.2.0
 catboost==1.1.1; platform_machine != 'aarch64'
 lightgbm==3.3.4
-xgboost==1.7.2
+xgboost==1.7.3
-tensorboard==2.11.0
+tensorboard==2.11.2

requirements.txt

@@ -2,17 +2,17 @@ numpy==1.24.1
 pandas==1.5.2
 pandas-ta==0.3.14b
 
-ccxt==2.5.56
+ccxt==2.6.26
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==38.0.1; platform_machine == 'armv7l'
-cryptography==38.0.4; platform_machine != 'armv7l'
+cryptography==39.0.0; platform_machine != 'armv7l'
 aiohttp==3.8.3
 SQLAlchemy==1.4.46
 python-telegram-bot==13.15
 arrow==1.2.3
 cachetools==4.2.2
-requests==2.28.1
+requests==2.28.2
-urllib3==1.26.13
+urllib3==1.26.14
 jsonschema==4.17.3
 TA-Lib==0.4.25
 technical==1.3.0
@@ -30,13 +30,13 @@ py_find_1st==1.1.5
 # Load ticker files 30% faster
 python-rapidjson==1.9
 # Properly format api responses
-orjson==3.8.4
+orjson==3.8.5
 
 # Notify systemd
 sdnotify==0.3.2
 
 # API Server
-fastapi==0.89.0
+fastapi==0.89.1
 pydantic==1.10.4
 uvicorn==0.20.0
 pyjwt==2.6.0

setup.py

@@ -60,7 +60,7 @@ setup(
     ],
     install_requires=[
         # from requirements.txt
-        'ccxt>=1.92.9',
+        'ccxt>=2.6.26',
         'SQLAlchemy',
         'python-telegram-bot>=13.4',
         'arrow>=0.17.0',

tests/commands/test_commands.py

@@ -1451,9 +1451,9 @@ def test_start_list_data(testdatadir, capsys):
     captured = capsys.readouterr()
 
     assert "Found 5 pair / timeframe combinations." in captured.out
-    assert "\n|          Pair |   Timeframe |         Type |\n" in captured.out
+    assert "\n|               Pair |   Timeframe |         Type |\n" in captured.out
-    assert "\n|      XRP/USDT |          1h |      futures |\n" in captured.out
+    assert "\n|      XRP/USDT:USDT |          1h |      futures |\n" in captured.out
-    assert "\n|      XRP/USDT |      1h, 8h |         mark |\n" in captured.out
+    assert "\n|      XRP/USDT:USDT |      1h, 8h |         mark |\n" in captured.out
 
     args = [
         "list-data",

tests/conftest.py

@@ -3109,7 +3109,7 @@ def funding_rate_history_octohourly():
 @pytest.fixture(scope='function')
 def leverage_tiers():
     return {
-        "1000SHIB/USDT": [
+        "1000SHIB/USDT:USDT": [
             {
                 'minNotional': 0,
                 'maxNotional': 50000,
@@ -3160,7 +3160,7 @@ def leverage_tiers():
                 'maintAmt': 654500.0
             },
         ],
-        "1INCH/USDT": [
+        "1INCH/USDT:USDT": [
             {
                 'minNotional': 0,
                 'maxNotional': 5000,
@@ -3204,7 +3204,7 @@ def leverage_tiers():
                 'maintAmt': 386940.0
             },
         ],
-        "AAVE/USDT": [
+        "AAVE/USDT:USDT": [
             {
                 'minNotional': 0,
                 'maxNotional': 5000,
@@ -3248,7 +3248,7 @@ def leverage_tiers():
                 'maintAmt': 386950.0
             },
         ],
-        "ADA/BUSD": [
+        "ADA/BUSD:BUSD": [
             {
                 "minNotional": 0,
                 "maxNotional": 100000,
@@ -3292,7 +3292,7 @@ def leverage_tiers():
                 "maintAmt": 1527500.0
             },
         ],
-        'BNB/BUSD': [
+        'BNB/BUSD:BUSD': [
             {
                 "minNotional": 0,       # stake(before leverage) = 0
                 "maxNotional": 100000,  # max stake(before leverage) = 5000
@@ -3336,7 +3336,7 @@ def leverage_tiers():
                 "maintAmt": 1527500.0
             }
         ],
-        'BNB/USDT': [
+        'BNB/USDT:USDT': [
             {
                 "minNotional": 0,      # stake = 0.0
                 "maxNotional": 10000,  # max_stake = 133.33333333333334
@@ -3401,7 +3401,7 @@ def leverage_tiers():
                 "maintAmt": 6233035.0
             },
         ],
-        'BTC/USDT': [
+        'BTC/USDT:USDT': [
             {
                 "minNotional": 0,      # stake = 0.0
                 "maxNotional": 50000,  # max_stake = 400.0
@@ -3473,7 +3473,7 @@ def leverage_tiers():
                 "maintAmt": 1.997038E8
             },
         ],
-        "ZEC/USDT": [
+        "ZEC/USDT:USDT": [
             {
                 'minNotional': 0,
                 'maxNotional': 50000,

tests/data/test_converter.py

@@ -294,8 +294,8 @@ def test_convert_trades_format(default_conf, testdatadir, tmpdir):
 
 @pytest.mark.parametrize('file_base,candletype', [
     (['XRP_ETH-5m', 'XRP_ETH-1m'], CandleType.SPOT),
-    (['UNITTEST_USDT-1h-mark', 'XRP_USDT-1h-mark'], CandleType.MARK),
+    (['UNITTEST_USDT_USDT-1h-mark', 'XRP_USDT_USDT-1h-mark'], CandleType.MARK),
-    (['XRP_USDT-1h-futures'], CandleType.FUTURES),
+    (['XRP_USDT_USDT-1h-futures'], CandleType.FUTURES),
 ])
 def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, candletype):
     tmpdir1 = Path(tmpdir)
@@ -315,7 +315,10 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, cand
         files_new.append(file_new)
 
     default_conf['datadir'] = tmpdir1
-    default_conf['pairs'] = ['XRP_ETH', 'XRP_USDT', 'UNITTEST_USDT']
+    if candletype == CandleType.SPOT:
+        default_conf['pairs'] = ['XRP/ETH', 'XRP/USDT', 'UNITTEST/USDT']
+    else:
+        default_conf['pairs'] = ['XRP/ETH:ETH', 'XRP/USDT:USDT', 'UNITTEST/USDT:USDT']
     default_conf['timeframes'] = ['1m', '5m', '1h']
 
     assert not file_new.exists()

tests/data/test_datahandler.py

@@ -33,10 +33,10 @@ def test_datahandler_ohlcv_get_pairs(testdatadir):
     assert set(pairs) == {'UNITTEST/BTC'}
 
     pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
-    assert set(pairs) == {'UNITTEST/USDT', 'XRP/USDT'}
+    assert set(pairs) == {'UNITTEST/USDT:USDT', 'XRP/USDT:USDT'}
 
     pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.FUTURES)
-    assert set(pairs) == {'XRP/USDT'}
+    assert set(pairs) == {'XRP/USDT:USDT'}
 
     pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK)
     assert set(pairs) == {'UNITTEST/USDT:USDT'}
@@ -104,11 +104,11 @@ def test_datahandler_ohlcv_get_available_data(testdatadir):
     paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.FUTURES)
     # Convert to set to avoid failures due to sorting
     assert set(paircombs) == {
-        ('UNITTEST/USDT', '1h', 'mark'),
+        ('UNITTEST/USDT:USDT', '1h', 'mark'),
-        ('XRP/USDT', '1h', 'futures'),
+        ('XRP/USDT:USDT', '1h', 'futures'),
-        ('XRP/USDT', '1h', 'mark'),
+        ('XRP/USDT:USDT', '1h', 'mark'),
-        ('XRP/USDT', '8h', 'mark'),
+        ('XRP/USDT:USDT', '8h', 'mark'),
-        ('XRP/USDT', '8h', 'funding_rate'),
+        ('XRP/USDT:USDT', '8h', 'funding_rate'),
     }
 
     paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir, TradingMode.SPOT)
@@ -142,7 +142,7 @@ def test_jsondatahandler_ohlcv_load(testdatadir, caplog):
     df = dh.ohlcv_load('XRP/ETH', '5m', 'spot')
     assert len(df) == 712
 
-    df_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', candle_type="mark")
+    df_mark = dh.ohlcv_load('UNITTEST/USDT:USDT', '1h', candle_type="mark")
     assert len(df_mark) == 100
 
     df_no_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', 'spot')
@@ -424,7 +424,7 @@ def test_hdf5datahandler_ohlcv_load_and_resave(
     # Data goes from 2018-01-10 - 2018-01-30
     ('UNITTEST/BTC', '5m', 'spot',  '', '2018-01-15', '2018-01-19'),
     # Mark data goes from to 2021-11-15 2021-11-19
-    ('UNITTEST/USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'),
+    ('UNITTEST/USDT:USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'),
 ])
 @pytest.mark.parametrize('datahandler', ['hdf5', 'feather', 'parquet'])
 def test_generic_datahandler_ohlcv_load_and_resave(

tests/data/test_entryexitanalysis.py

@@ -190,6 +190,15 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp
     assert '1' in captured.out
     assert '2.5' in captured.out
 
+    # test group 5
+    args = get_args(base_args + ['--analysis-groups', "5"])
+    start_analysis_entries_exits(args)
+    captured = capsys.readouterr()
+    assert 'exit_signal' in captured.out
+    assert 'roi' in captured.out
+    assert 'stop_loss' in captured.out
+    assert 'trailing_stop_loss' in captured.out
+
     # test date filtering
     args = get_args(base_args + ['--timerange', "20180129-20180130"])
     start_analysis_entries_exits(args)

tests/data/test_history.py

@@ -78,11 +78,11 @@ def test_load_data_1min_timeframe(ohlcv_history, mocker, caplog, testdatadir) ->
 
 def test_load_data_mark(ohlcv_history, mocker, caplog, testdatadir) -> None:
     mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history)
-    file = testdatadir / 'futures/UNITTEST_USDT-1h-mark.json'
+    file = testdatadir / 'futures/UNITTEST_USDT_USDT-1h-mark.json'
     load_data(datadir=testdatadir, timeframe='1h', pairs=['UNITTEST/BTC'], candle_type='mark')
     assert file.is_file()
     assert not log_has(
-        'Download history data for pair: "UNITTEST/USDT", interval: 1m '
+        'Download history data for pair: "UNITTEST/USDT:USDT", interval: 1m '
         'and store in None.', caplog
     )
 

tests/exchange/test_binance.py

@@ -575,25 +575,13 @@ async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog, c
     assert log_has_re(r"Candle-data for ETH/BTC available starting with .*", caplog)
 
 
-@pytest.mark.parametrize("trading_mode,margin_mode,config", [
-    ("spot", "", {}),
-    ("margin", "cross", {"options": {"defaultType": "margin"}}),
-    ("futures", "isolated", {"options": {"defaultType": "future"}}),
-])
-def test__ccxt_config(default_conf, mocker, trading_mode, margin_mode, config):
-    default_conf['trading_mode'] = trading_mode
-    default_conf['margin_mode'] = margin_mode
-    exchange = get_patched_exchange(mocker, default_conf, id="binance")
-    assert exchange._ccxt_config == config
-
-
 @pytest.mark.parametrize('pair,nominal_value,mm_ratio,amt', [
-    ("BNB/BUSD", 0.0, 0.025, 0),
+    ("BNB/BUSD:BUSD", 0.0, 0.025, 0),
-    ("BNB/USDT", 100.0, 0.0065, 0),
+    ("BNB/USDT:USDT", 100.0, 0.0065, 0),
-    ("BTC/USDT", 170.30, 0.004, 0),
+    ("BTC/USDT:USDT", 170.30, 0.004, 0),
-    ("BNB/BUSD", 999999.9, 0.1, 27500.0),
+    ("BNB/BUSD:BUSD", 999999.9, 0.1, 27500.0),
-    ("BNB/USDT", 5000000.0, 0.15, 233035.0),
+    ("BNB/USDT:USDT", 5000000.0, 0.15, 233035.0),
-    ("BTC/USDT", 600000000, 0.5, 1.997038E8),
+    ("BTC/USDT:USDT", 600000000, 0.5, 1.997038E8),
 ])
 def test_get_maintenance_ratio_and_amt_binance(
     default_conf,

tests/exchange/test_ccxt_compat.py

@@ -12,6 +12,7 @@ from typing import Tuple
 
 import pytest
 
+from freqtrade.constants import Config
 from freqtrade.enums import CandleType
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
 from freqtrade.exchange.exchange import Exchange, timeframe_to_msecs
@@ -31,15 +32,18 @@ EXCHANGES = {
         'leverage_tiers_public': False,
         'leverage_in_spot_market': False,
     },
-    # 'binance': {
+    'binance': {
-    #     'pair': 'BTC/USDT',
+        'pair': 'BTC/USDT',
-    #     'stake_currency': 'USDT',
+        'stake_currency': 'USDT',
-    #     'hasQuoteVolume': True,
+        'use_ci_proxy': True,
-    #     'timeframe': '5m',
+        'hasQuoteVolume': True,
-    #     'futures': True,
+        'timeframe': '5m',
-    #     'leverage_tiers_public': False,
+        'futures': True,
-    #     'leverage_in_spot_market': False,
+        'futures_pair': 'BTC/USDT:USDT',
-    # },
+        'hasQuoteVolumeFutures': True,
+        'leverage_tiers_public': False,
+        'leverage_in_spot_market': False,
+    },
     'kraken': {
         'pair': 'BTC/USDT',
         'stake_currency': 'USDT',
@@ -63,6 +67,7 @@ EXCHANGES = {
         'timeframe': '5m',
         'futures': True,
         'futures_pair': 'BTC/USDT:USDT',
+        'hasQuoteVolumeFutures': True,
         'leverage_tiers_public': True,
         'leverage_in_spot_market': True,
     },
@@ -71,8 +76,9 @@ EXCHANGES = {
         'stake_currency': 'USDT',
         'hasQuoteVolume': True,
         'timeframe': '5m',
-        'futures_pair': 'BTC/USDT:USDT',
         'futures': True,
+        'futures_pair': 'BTC/USDT:USDT',
+        'hasQuoteVolumeFutures': False,
         'leverage_tiers_public': True,
         'leverage_in_spot_market': True,
     },
@@ -106,8 +112,27 @@ def exchange_conf():
     return config
 
 
+def set_test_proxy(config: Config, use_proxy: bool) -> Config:
+    # Set proxy to test in CI.
+    import os
+    if use_proxy and (proxy := os.environ.get('CI_WEB_PROXY')):
+        config1 = deepcopy(config)
+        config1['exchange']['ccxt_config'] = {
+            "aiohttp_proxy": proxy,
+            'proxies': {
+                'https': proxy,
+                'http': proxy,
+            }
+        }
+        return config1
+
+    return config
+
+
 @pytest.fixture(params=EXCHANGES, scope="class")
 def exchange(request, exchange_conf):
+    exchange_conf = set_test_proxy(
+        exchange_conf, EXCHANGES[request.param].get('use_ci_proxy', False))
     exchange_conf['exchange']['name'] = request.param
     exchange_conf['stake_currency'] = EXCHANGES[request.param]['stake_currency']
     exchange = ExchangeResolver.load_exchange(request.param, exchange_conf, validate=True)
@@ -120,6 +145,8 @@ def exchange_futures(request, exchange_conf, class_mocker):
     if not EXCHANGES[request.param].get('futures') is True:
         yield None, request.param
     else:
+        exchange_conf = set_test_proxy(
+            exchange_conf, EXCHANGES[request.param].get('use_ci_proxy', False))
         exchange_conf = deepcopy(exchange_conf)
         exchange_conf['exchange']['name'] = request.param
         exchange_conf['trading_mode'] = 'futures'
@@ -198,6 +225,25 @@ class TestCCXTExchange():
         if EXCHANGES[exchangename].get('hasQuoteVolume'):
             assert tickers[pair]['quoteVolume'] is not None
 
+    def test_ccxt_fetch_tickers_futures(self, exchange_futures: EXCHANGE_FIXTURE_TYPE):
+        exch, exchangename = exchange_futures
+        if not exch or exchangename in ('gateio'):
+            # exchange_futures only returns values for supported exchanges
+            return
+
+        pair = EXCHANGES[exchangename]['pair']
+        pair = EXCHANGES[exchangename].get('futures_pair', pair)
+
+        tickers = exch.get_tickers()
+        assert pair in tickers
+        assert 'ask' in tickers[pair]
+        assert tickers[pair]['ask'] is not None
+        assert 'bid' in tickers[pair]
+        assert tickers[pair]['bid'] is not None
+        assert 'quoteVolume' in tickers[pair]
+        if EXCHANGES[exchangename].get('hasQuoteVolumeFutures'):
+            assert tickers[pair]['quoteVolume'] is not None
+
     def test_ccxt_fetch_ticker(self, exchange: EXCHANGE_FIXTURE_TYPE):
         exch, exchangename = exchange
         pair = EXCHANGES[exchangename]['pair']

tests/exchange/test_exchange.py

@@ -3957,7 +3957,7 @@ def test_validate_trading_mode_and_margin_mode(
 @pytest.mark.parametrize("exchange_name,trading_mode,ccxt_config", [
     ("binance", "spot", {}),
     ("binance", "margin", {"options": {"defaultType": "margin"}}),
-    ("binance", "futures", {"options": {"defaultType": "future"}}),
+    ("binance", "futures", {"options": {"defaultType": "swap"}}),
     ("bybit", "spot", {"options": {"defaultType": "spot"}}),
     ("bybit", "futures", {"options": {"defaultType": "linear"}}),
     ("gateio", "futures", {"options": {"defaultType": "swap"}}),
@@ -4898,22 +4898,22 @@ def test_get_maintenance_ratio_and_amt_exceptions(mocker, default_conf, leverage
         OperationalException,
         match='nominal value can not be lower than 0',
     ):
-        exchange.get_maintenance_ratio_and_amt('1000SHIB/USDT', -1)
+        exchange.get_maintenance_ratio_and_amt('1000SHIB/USDT:USDT', -1)
 
     exchange._leverage_tiers = {}
 
     with pytest.raises(
         InvalidOrderException,
-        match="Maintenance margin rate for 1000SHIB/USDT is unavailable for",
+        match="Maintenance margin rate for 1000SHIB/USDT:USDT is unavailable for",
     ):
-        exchange.get_maintenance_ratio_and_amt('1000SHIB/USDT', 10000)
+        exchange.get_maintenance_ratio_and_amt('1000SHIB/USDT:USDT', 10000)
 
 
 @pytest.mark.parametrize('pair,value,mmr,maintAmt', [
-    ('ADA/BUSD', 500, 0.025, 0.0),
+    ('ADA/BUSD:BUSD', 500, 0.025, 0.0),
-    ('ADA/BUSD', 20000000, 0.5, 1527500.0),
+    ('ADA/BUSD:BUSD', 20000000, 0.5, 1527500.0),
-    ('ZEC/USDT', 500, 0.01, 0.0),
+    ('ZEC/USDT:USDT', 500, 0.01, 0.0),
-    ('ZEC/USDT', 20000000, 0.5, 654500.0),
+    ('ZEC/USDT:USDT', 20000000, 0.5, 654500.0),
 ])
 def test_get_maintenance_ratio_and_amt(
     mocker,
@@ -4946,21 +4946,21 @@ def test_get_max_leverage_futures(default_conf, mocker, leverage_tiers):
 
     exchange._leverage_tiers = leverage_tiers
 
-    assert exchange.get_max_leverage("BNB/BUSD", 1.0) == 20.0
+    assert exchange.get_max_leverage("BNB/BUSD:BUSD", 1.0) == 20.0
-    assert exchange.get_max_leverage("BNB/USDT", 100.0) == 75.0
+    assert exchange.get_max_leverage("BNB/USDT:USDT", 100.0) == 75.0
-    assert exchange.get_max_leverage("BTC/USDT", 170.30) == 125.0
+    assert exchange.get_max_leverage("BTC/USDT:USDT", 170.30) == 125.0
-    assert pytest.approx(exchange.get_max_leverage("BNB/BUSD", 99999.9)) == 5.000005
+    assert pytest.approx(exchange.get_max_leverage("BNB/BUSD:BUSD", 99999.9)) == 5.000005
-    assert pytest.approx(exchange.get_max_leverage("BNB/USDT", 1500)) == 33.333333333333333
+    assert pytest.approx(exchange.get_max_leverage("BNB/USDT:USDT", 1500)) == 33.333333333333333
-    assert exchange.get_max_leverage("BTC/USDT", 300000000) == 2.0
+    assert exchange.get_max_leverage("BTC/USDT:USDT", 300000000) == 2.0
-    assert exchange.get_max_leverage("BTC/USDT", 600000000) == 1.0  # Last tier
+    assert exchange.get_max_leverage("BTC/USDT:USDT", 600000000) == 1.0  # Last tier
 
-    assert exchange.get_max_leverage("SPONGE/USDT", 200) == 1.0    # Pair not in leverage_tiers
+    assert exchange.get_max_leverage("SPONGE/USDT:USDT", 200) == 1.0    # Pair not in leverage_tiers
-    assert exchange.get_max_leverage("BTC/USDT", 0.0) == 125.0  # No stake amount
+    assert exchange.get_max_leverage("BTC/USDT:USDT", 0.0) == 125.0  # No stake amount
     with pytest.raises(
         InvalidOrderException,
-        match=r'Amount 1000000000.01 too high for BTC/USDT'
+        match=r'Amount 1000000000.01 too high for BTC/USDT:USDT'
     ):
-        exchange.get_max_leverage("BTC/USDT", 1000000000.01)
+        exchange.get_max_leverage("BTC/USDT:USDT", 1000000000.01)
 
 
 @pytest.mark.parametrize("exchange_name", ['bittrex', 'binance', 'kraken', 'gateio', 'okx'])

tests/exchange/test_okx.py

@@ -195,12 +195,12 @@ def test_get_max_pair_stake_amount_okx(default_conf, mocker, leverage_tiers):
     exchange = get_patched_exchange(mocker, default_conf, id="okx")
     exchange._leverage_tiers = leverage_tiers
 
-    assert exchange.get_max_pair_stake_amount('BNB/BUSD', 1.0) == 30000000
+    assert exchange.get_max_pair_stake_amount('BNB/BUSD:BUSD', 1.0) == 30000000
-    assert exchange.get_max_pair_stake_amount('BNB/USDT', 1.0) == 50000000
+    assert exchange.get_max_pair_stake_amount('BNB/USDT:USDT', 1.0) == 50000000
-    assert exchange.get_max_pair_stake_amount('BTC/USDT', 1.0) == 1000000000
+    assert exchange.get_max_pair_stake_amount('BTC/USDT:USDT', 1.0) == 1000000000
-    assert exchange.get_max_pair_stake_amount('BTC/USDT', 1.0, 10.0) == 100000000
+    assert exchange.get_max_pair_stake_amount('BTC/USDT:USDT', 1.0, 10.0) == 100000000
 
-    assert exchange.get_max_pair_stake_amount('TTT/USDT', 1.0) == float('inf')  # Not in tiers
+    assert exchange.get_max_pair_stake_amount('TTT/USDT:USDT', 1.0) == float('inf')  # Not in tiers
 
 
 @pytest.mark.parametrize('mode,side,reduceonly,result', [

tests/optimize/test_backtesting.py

@@ -549,6 +549,7 @@ def test_backtest__enter_trade_futures(default_conf_usdt, fee, mocker) -> None:
     default_conf_usdt['trading_mode'] = 'futures'
     default_conf_usdt['margin_mode'] = 'isolated'
     default_conf_usdt['stake_currency'] = 'USDT'
+    default_conf_usdt['datadir'] = Path(default_conf_usdt['datadir'])
     default_conf_usdt['exchange']['pair_whitelist'] = ['.*']
     backtesting = Backtesting(default_conf_usdt)
     backtesting._set_strategy(backtesting.strategylist[0])
@@ -1460,7 +1461,7 @@ def test_backtest_start_futures_noliq(default_conf_usdt, mocker,
     patch_exchange(mocker)
 
     mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
-                 PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT']))
+                 PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT:USDT']))
     # mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
 
     patched_configuration_load_config_file(mocker, default_conf_usdt)
@@ -1491,7 +1492,7 @@ def test_backtest_start_nomock_futures(default_conf_usdt, mocker,
         "strategy": CURRENT_TEST_STRATEGY,
     })
     patch_exchange(mocker)
-    result1 = pd.DataFrame({'pair': ['XRP/USDT', 'XRP/USDT'],
+    result1 = pd.DataFrame({'pair': ['XRP/USDT:USDT', 'XRP/USDT:USDT'],
                             'profit_ratio': [0.0, 0.0],
                             'profit_abs': [0.0, 0.0],
                             'open_date': pd.to_datetime(['2021-11-18 18:00:00',
@@ -1507,7 +1508,7 @@ def test_backtest_start_nomock_futures(default_conf_usdt, mocker,
                             'close_rate': [0.104969, 0.103541],
                             'exit_reason': [ExitType.ROI, ExitType.ROI]
                             })
-    result2 = pd.DataFrame({'pair': ['XRP/USDT', 'XRP/USDT', 'XRP/USDT'],
+    result2 = pd.DataFrame({'pair': ['XRP/USDT:USDT', 'XRP/USDT:USDT', 'XRP/USDT:USDT'],
                             'profit_ratio': [0.03, 0.01, 0.1],
                             'profit_abs': [0.01, 0.02, 0.2],
                             'open_date': pd.to_datetime(['2021-11-19 18:00:00',
@@ -1552,7 +1553,7 @@ def test_backtest_start_nomock_futures(default_conf_usdt, mocker,
         }
     ])
     mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
-                 PropertyMock(return_value=['XRP/USDT']))
+                 PropertyMock(return_value=['XRP/USDT:USDT']))
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
 
     patched_configuration_load_config_file(mocker, default_conf_usdt)
@@ -1575,8 +1576,8 @@ def test_backtest_start_nomock_futures(default_conf_usdt, mocker,
         'up to 2021-11-21 04:00:00 (4 days).',
         'Backtesting with data from 2021-11-17 21:00:00 '
         'up to 2021-11-21 04:00:00 (3 days).',
-        'XRP/USDT, funding_rate, 8h, data starts at 2021-11-18 00:00:00',
+        'XRP/USDT:USDT, funding_rate, 8h, data starts at 2021-11-18 00:00:00',
-        'XRP/USDT, mark, 8h, data starts at 2021-11-18 00:00:00',
+        'XRP/USDT:USDT, mark, 8h, data starts at 2021-11-18 00:00:00',
         f'Running backtesting for Strategy {CURRENT_TEST_STRATEGY}',
     ]
 

tests/rpc/test_rpc_apiserver.py

@@ -1553,13 +1553,13 @@ def test_list_available_pairs(botclient):
         client, f"{BASE_URI}/available_pairs?timeframe=1h")
     assert_response(rc)
     assert rc.json()['length'] == 1
-    assert rc.json()['pairs'] == ['XRP/USDT']
+    assert rc.json()['pairs'] == ['XRP/USDT:USDT']
 
     rc = client_get(
         client, f"{BASE_URI}/available_pairs?timeframe=1h&candletype=mark")
     assert_response(rc)
     assert rc.json()['length'] == 2
-    assert rc.json()['pairs'] == ['UNITTEST/USDT', 'XRP/USDT']
+    assert rc.json()['pairs'] == ['UNITTEST/USDT:USDT', 'XRP/USDT:USDT']
     assert len(rc.json()['pair_interval']) == 2
 
 

tests/test_binance_mig.py

@@ -0,0 +1,59 @@
+
+
+import shutil
+from pathlib import Path
+
+import pytest
+
+from freqtrade.persistence import Trade
+from freqtrade.util.binance_mig import migrate_binance_futures_data, migrate_binance_futures_names
+from tests.conftest import create_mock_trades_usdt, log_has
+
+
+def test_binance_mig_data_conversion(default_conf_usdt, tmpdir, testdatadir):
+
+    # call doing nothing (spot mode)
+    migrate_binance_futures_data(default_conf_usdt)
+    default_conf_usdt['trading_mode'] = 'futures'
+    pair_old = 'XRP_USDT'
+    pair_unified = 'XRP_USDT_USDT'
+    futures_src = testdatadir / 'futures'
+    futures_dst = tmpdir / 'futures'
+    futures_dst.mkdir()
+    files = [
+        '-1h-mark.json',
+        '-1h-futures.json',
+        '-8h-funding_rate.json',
+        '-8h-mark.json',
+    ]
+
+    # Copy files to tmpdir and rename to old naming
+    for file in files:
+        fn_after = futures_dst / f'{pair_old}{file}'
+        shutil.copy(futures_src / f'{pair_unified}{file}', fn_after)
+
+    default_conf_usdt['datadir'] = Path(tmpdir)
+    # Migrate files to unified namings
+    migrate_binance_futures_data(default_conf_usdt)
+
+    for file in files:
+        fn_after = futures_dst / f'{pair_unified}{file}'
+        assert fn_after.exists()
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_binance_mig_db_conversion(default_conf_usdt, fee, caplog):
+    # Does nothing in spot mode
+    migrate_binance_futures_names(default_conf_usdt)
+
+    create_mock_trades_usdt(fee, None)
+
+    for t in Trade.get_trades():
+        t.trading_mode = 'FUTURES'
+        t.exchange = 'binance'
+    Trade.commit()
+
+    default_conf_usdt['datadir'] = Path(default_conf_usdt['datadir'])
+    default_conf_usdt['trading_mode'] = 'futures'
+    migrate_binance_futures_names(default_conf_usdt)
+    assert log_has('Migrating binance futures pairs in database.', caplog)