Added ability to keep invalid pairs while expanding expand_pairlist

docs/configuration.md

@@ -81,7 +81,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
 | `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
 | `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
-| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used by VolumePairList (see [below](#pairlists-and-pairlist-handlers)). <br> **Datatype:** List
+| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting.Supports regex pairs as */BTC. Not used by VolumePairList (see [below](#pairlists-and-pairlist-handlers)). <br> **Datatype:** List
 | `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#pairlists-and-pairlist-handlers)). <br> **Datatype:** List
 | `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) <br> **Datatype:** Dict
 | `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) <br> **Datatype:** Dict

docs/includes/pairlists.md

@@ -35,7 +35,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
 
 #### Static Pair List
 
-By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration.
+By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. Also the pairlist  does support wildcards (in regex-style) - so `*/BTC` will include all pairs with BTC as a stake.
 
 It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`.
 

freqtrade/exchange/exchange.py

@@ -336,7 +336,7 @@ class Exchange:
         if not self.markets:
             logger.warning('Unable to validate pairs (assuming they are correct).')
             return
-        extended_pairs = expand_pairlist(pairs, list(self.markets))
+        extended_pairs = expand_pairlist(pairs, list(self.markets), keep_invalid=True)
         invalid_pairs = []
         for pair in extended_pairs:
             # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs

freqtrade/plugins/pairlist/IPairList.py

@@ -124,19 +124,21 @@ class IPairList(LoggingMixin, ABC):
         """
         return self._pairlistmanager.verify_blacklist(pairlist, logmethod)
 
-    def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]:
+    def verify_whitelist(self, pairlist: List[str], logmethod,
+                         keep_invalid: bool = False) -> List[str]:
         """
         Proxy method to verify_whitelist for easy access for child classes.
         :param pairlist: Pairlist to validate
-        :param logmethod: Function that'll be called, `logger.info` or `logger.warning`.
+        :param logmethod: Function that'll be called, `logger.info` or `logger.warning`
+        :param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes.
         :return: pairlist - whitelisted pairs
         """
-        return self._pairlistmanager.verify_whitelist(pairlist, logmethod)
+        return self._pairlistmanager.verify_whitelist(pairlist, logmethod, keep_invalid)
 
     def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]:
         """
         Check available markets and remove pair from whitelist if necessary
-        :param whitelist: the sorted list of pairs the user might want to trade
+        :param pairlist: the sorted list of pairs the user might want to trade
         :return: the list of pairs the user wants to trade without those unavailable or
         black_listed
         """

freqtrade/plugins/pairlist/StaticPairList.py

@@ -50,7 +50,9 @@ class StaticPairList(IPairList):
         :return: List of pairs
         """
         if self._allow_inactive:
-            return self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info)
+            return self.verify_whitelist(
+                self._config['exchange']['pair_whitelist'], logger.info, keep_invalid=True
+            )
         else:
             return self._whitelist_for_active_markets(
                 self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info))

freqtrade/plugins/pairlist/pairlist_helpers.py

@@ -2,22 +2,41 @@ import re
 from typing import List
 
 
-def expand_pairlist(wildcardpl: List[str], available_pairs: List[str]) -> List[str]:
+def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
+                    keep_invalid: bool = False) -> List[str]:
     """
     Expand pairlist potentially containing wildcards based on available markets.
     This will implicitly filter all pairs in the wildcard-list which are not in available_pairs.
     :param wildcardpl: List of Pairlists, which may contain regex
     :param available_pairs: List of all available pairs (`exchange.get_markets().keys()`)
+    :param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes
     :return expanded pairlist, with Regexes from wildcardpl applied to match all available pairs.
     :raises: ValueError if a wildcard is invalid (like '*/BTC' - which should be `.*/BTC`)
     """
     result = []
-    for pair_wc in wildcardpl:
+    if keep_invalid:
-        try:
+        for pair_wc in wildcardpl:
-            comp = re.compile(pair_wc)
+            try:
-            result += [
+                comp = re.compile(pair_wc)
-                pair for pair in available_pairs if re.match(comp, pair)
+                result_partial = [
-            ]
+                    pair for pair in available_pairs if re.match(comp, pair)
-        except re.error as err:
+                ]
-            raise ValueError(f"Wildcard error in {pair_wc}, {err}")
+                # Add all matching pairs.
+                # If there are no matching pairs (Pair not on exchange) keep it.
+                result += result_partial or [pair_wc]
+            except re.error as err:
+                raise ValueError(f"Wildcard error in {pair_wc}, {err}")
+
+        for element in result:
+            if not re.fullmatch(r'^[A-Za-z0-9/-]+$', element):
+                result.remove(element)
+    else:
+        for pair_wc in wildcardpl:
+            try:
+                comp = re.compile(pair_wc)
+                result += [
+                    pair for pair in available_pairs if re.match(comp, pair)
+                ]
+            except re.error as err:
+                raise ValueError(f"Wildcard error in {pair_wc}, {err}")
     return result

freqtrade/plugins/pairlistmanager.py

@@ -59,9 +59,15 @@ class PairListManager():
         """The expanded blacklist (including wildcard expansion)"""
         return expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
 
+    @property
+    def expanded_whitelist_keep_invalid(self) -> List[str]:
+        """The expanded whitelist (including wildcard expansion), maintaining invalid pairs"""
+        return expand_pairlist(self._whitelist, self._exchange.get_markets().keys(),
+                               keep_invalid=True)
+
     @property
     def expanded_whitelist(self) -> List[str]:
-        """The expanded whitelist (including wildcard expansion)"""
+        """The expanded whitelist (including wildcard expansion), filtering invalid pairs"""
         return expand_pairlist(self._whitelist, self._exchange.get_markets().keys())
 
     @property
@@ -134,19 +140,30 @@ class PairListManager():
                 pairlist.remove(pair)
         return pairlist
 
-    def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]:
+    def verify_whitelist(self, pairlist: List[str], logmethod,
+                         keep_invalid: bool = False) -> List[str]:
         """
         Verify and remove items from pairlist - returning a filtered pairlist.
         Logs a warning or info depending on `aswarning`.
         Pairlist Handlers explicitly using this method shall use
         `logmethod=logger.info` to avoid spamming with warning messages
-        :return: pairlist - blacklisted pairs
+        :param pairlist: Pairlist to validate
+        :param logmethod: Function that'll be called, `logger.info` or `logger.warning`
+        :param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes.
+        :return: pairlist - whitelisted pairs
         """
-        try:
+        if keep_invalid:
-            whitelist = self.expanded_whitelist
+            try:
-        except ValueError as err:
+                whitelist = self.expanded_whitelist_keep_invalid
-            logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
+            except ValueError as err:
-            return []
+                logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
+                return []
+        else:
+            try:
+                whitelist = self.expanded_whitelist
+            except ValueError as err:
+                logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
+                return []
         return whitelist
 
     def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes:

tests/exchange/test_exchange.py

@@ -505,38 +505,37 @@ def test_validate_pairs(default_conf, mocker):  # test exchange.validate_pairs d
     Exchange(default_conf)
 
 
-# This cannot happen anymore as expand_pairlist implicitly filters out unavaliablie pairs
+def test_validate_pairs_not_available(default_conf, mocker):
-# def test_validate_pairs_not_available(default_conf, mocker):
+    api_mock = MagicMock()
-#     api_mock = MagicMock()
+    type(api_mock).markets = PropertyMock(return_value={
-#     type(api_mock).markets = PropertyMock(return_value={
+        'XRP/BTC': {'inactive': True, 'base': 'XRP', 'quote': 'BTC'}
-#         'XRP/BTC': {'inactive': True, 'base': 'XRP', 'quote': 'BTC'}
+    })
-#     })
+    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
-#     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
+    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
-#     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
+    mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
-#     mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
-#     mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
+
-#
+    with pytest.raises(OperationalException, match=r'not available'):
-#     with pytest.raises(OperationalException, match=r'not available'):
+        Exchange(default_conf)
-#         Exchange(default_conf)
+
-#
+
-#
+def test_validate_pairs_exception(default_conf, mocker, caplog):
-# def test_validate_pairs_exception(default_conf, mocker, caplog):
+    caplog.set_level(logging.INFO)
-#     caplog.set_level(logging.INFO)
+    api_mock = MagicMock()
-#     api_mock = MagicMock()
+    mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance'))
-#     mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance'))
+
-#
+    type(api_mock).markets = PropertyMock(return_value={})
-#     type(api_mock).markets = PropertyMock(return_value={})
+    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
-#     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
+    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
-#     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
+    mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
-#     mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
-#     mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
+
-#
+    with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'):
-#     with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'):
+        Exchange(default_conf)
-#         Exchange(default_conf)
+
-#
+    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}))
-#     mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}))
+    Exchange(default_conf)
-#     Exchange(default_conf)
+    assert log_has('Unable to validate pairs (assuming they are correct).', caplog)
-#     assert log_has('Unable to validate pairs (assuming they are correct).', caplog)
 
 
 def test_validate_pairs_restricted(default_conf, mocker, caplog):

tests/plugins/test_pairlist.py

@@ -853,3 +853,35 @@ def test_expand_pairlist(wildcardlist, pairs, expected):
             expand_pairlist(wildcardlist, pairs)
     else:
         assert sorted(expand_pairlist(wildcardlist, pairs)) == sorted(expected)
+
+
+@pytest.mark.parametrize('wildcardlist,pairs,expected', [
+    (['BTC/USDT'],
+     ['BTC/USDT'],
+     ['BTC/USDT']),
+    (['BTC/USDT', 'ETH/USDT'],
+     ['BTC/USDT', 'ETH/USDT'],
+     ['BTC/USDT', 'ETH/USDT']),
+    (['BTC/USDT', 'ETH/USDT'],
+     ['BTC/USDT'], ['BTC/USDT', 'ETH/USDT']),  # Test one too many
+    (['.*/USDT'],
+     ['BTC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETH/USDT']),  # Wildcard simple
+    (['.*C/USDT'],
+     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETC/USDT']),  # Wildcard exclude one
+    (['.*UP/USDT', 'BTC/USDT', 'ETH/USDT'],
+     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'],
+     ['BTC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT']),  # Wildcard exclude one
+    (['BTC/.*', 'ETH/.*'],
+     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP'],
+     ['BTC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP']),  # Wildcard exclude one
+    (['*UP/USDT', 'BTC/USDT', 'ETH/USDT'],
+     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'],
+     None),
+    (['HELLO/WORLD'], [], ['HELLO/WORLD'])  # Invalid pair kept
+])
+def test_expand_pairlist_keep_invalid(wildcardlist, pairs, expected):
+    if expected is None:
+        with pytest.raises(ValueError, match=r'Wildcard error in \*UP/USDT,'):
+            expand_pairlist(wildcardlist, pairs, keep_invalid=True)
+    else:
+        assert sorted(expand_pairlist(wildcardlist, pairs, keep_invalid=True)) == sorted(expected)