Added suppoort for regex in whitelist

freqtrade/commands/data_commands.py

@@ -10,6 +10,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_oh
                                     refresh_backtest_trades_data)
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.state import RunMode
 
@@ -42,15 +43,17 @@ def start_download_data(args: Dict[str, Any]) -> None:
             "Downloading data requires a list of pairs. "
             "Please check the documentation on how to configure this.")
 
-    logger.info(f"About to download pairs: {config['pairs']}, "
-                f"intervals: {config['timeframes']} to {config['datadir']}")
-
     pairs_not_available: List[str] = []
 
     # Init exchange
     exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
     # Manual validations of relevant settings
     exchange.validate_pairs(config['pairs'])
+    expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
+
+    logger.info(f"About to download pairs: {expanded_pairs}, "
+                f"intervals: {config['timeframes']} to {config['datadir']}")
+
     for timeframe in config['timeframes']:
         exchange.validate_timeframes(timeframe)
 
@@ -58,20 +61,20 @@ def start_download_data(args: Dict[str, Any]) -> None:
 
         if config.get('download_trades'):
             pairs_not_available = refresh_backtest_trades_data(
-                exchange, pairs=config['pairs'], datadir=config['datadir'],
+                exchange, pairs=expanded_pairs, datadir=config['datadir'],
                 timerange=timerange, erase=bool(config.get('erase')),
                 data_format=config['dataformat_trades'])
 
             # Convert downloaded trade data to different timeframes
             convert_trades_to_ohlcv(
-                pairs=config['pairs'], timeframes=config['timeframes'],
+                pairs=expanded_pairs, timeframes=config['timeframes'],
                 datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
                 data_format_ohlcv=config['dataformat_ohlcv'],
                 data_format_trades=config['dataformat_trades'],
-                )
+            )
         else:
             pairs_not_available = refresh_backtest_ohlcv_data(
-                exchange, pairs=config['pairs'], timeframes=config['timeframes'],
+                exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
                 datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
                 data_format=config['dataformat_ohlcv'])
 

freqtrade/edge/edge_positioning.py

@@ -12,6 +12,7 @@ from freqtrade.configuration import TimeRange
 from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT
 from freqtrade.data.history import get_timerange, load_data, refresh_data
 from freqtrade.exceptions import OperationalException
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.strategy.interface import SellType
 
 
@@ -80,10 +81,12 @@ class Edge:
         if config.get('fee'):
             self.fee = config['fee']
         else:
-            self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0])
+            self.fee = self.exchange.get_fee(symbol=expand_pairlist(
+                self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0])
 
     def calculate(self) -> bool:
-        pairs = self.config['exchange']['pair_whitelist']
+        pairs = expand_pairlist(self.config['exchange']['pair_whitelist'],
+                                list(self.exchange.markets))
         heartbeat = self.edge_config.get('process_throttle_secs')
 
         if (self._last_updated > 0) and (

freqtrade/exchange/exchange.py

@@ -25,6 +25,7 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun
 from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, retrier,
                                        retrier_async)
 from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 
 
 CcxtModuleType = Any
@@ -335,8 +336,9 @@ class Exchange:
         if not self.markets:
             logger.warning('Unable to validate pairs (assuming they are correct).')
             return
+        extended_pairs = expand_pairlist(pairs, list(self.markets))
         invalid_pairs = []
-        for pair in pairs:
+        for pair in extended_pairs:
             # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs
             # TODO: add a support for having coins in BTC/USDT format
             if self.markets and pair not in self.markets:

freqtrade/plot/plotting.py

@@ -13,6 +13,7 @@ from freqtrade.data.history import get_timerange, load_data
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds
 from freqtrade.misc import pair_to_filename
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.strategy import IStrategy
 
@@ -29,16 +30,16 @@ except ImportError:
     exit(1)
 
 
-def init_plotscript(config, startup_candles: int = 0):
+def init_plotscript(config, markets: List, startup_candles: int = 0):
     """
     Initialize objects needed for plotting
     :return: Dict with candle (OHLCV) data, trades and pairs
     """
 
     if "pairs" in config:
-        pairs = config['pairs']
+        pairs = expand_pairlist(config['pairs'], markets)
     else:
-        pairs = config['exchange']['pair_whitelist']
+        pairs = expand_pairlist(config['exchange']['pair_whitelist'], markets)
 
     # Set timerange to use
     timerange = TimeRange.parse_timerange(config.get('timerange'))
@@ -177,7 +178,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
         trades['desc'] = trades.apply(lambda row: f"{round(row['profit_percent'] * 100, 1)}%, "
                                                   f"{row['sell_reason']}, "
                                                   f"{row['trade_duration']} min",
-                                                  axis=1)
+                                      axis=1)
         trade_buys = go.Scatter(
             x=trades["open_date"],
             y=trades["open_rate"],
@@ -527,7 +528,7 @@ def load_and_plot_trades(config: Dict[str, Any]):
 
     exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
     IStrategy.dp = DataProvider(config, exchange)
-    plot_elements = init_plotscript(config, strategy.startup_candle_count)
+    plot_elements = init_plotscript(config, list(exchange.markets), strategy.startup_candle_count)
     timerange = plot_elements['timerange']
     trades = plot_elements['trades']
     pair_counter = 0
@@ -562,7 +563,8 @@ def plot_profit(config: Dict[str, Any]) -> None:
     But should be somewhat proportional, and therefor useful
     in helping out to find a good algorithm.
     """
-    plot_elements = init_plotscript(config)
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
+    plot_elements = init_plotscript(config, list(exchange.markets))
     trades = plot_elements['trades']
     # Filter trades to relevant pairs
     # Remove open pairs - we don't know the profit yet so can't calculate profit for these.

freqtrade/plugins/pairlist/IPairList.py

@@ -124,6 +124,15 @@ class IPairList(LoggingMixin, ABC):
         """
         return self._pairlistmanager.verify_blacklist(pairlist, logmethod)
 
+    def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]:
+        """
+        Proxy method to verify_whitelist for easy access for child classes.
+        :param pairlist: Pairlist to validate
+        :param logmethod: Function that'll be called, `logger.info` or `logger.warning`.
+        :return: pairlist - whitelisted pairs
+        """
+        return self._pairlistmanager.verify_whitelist(pairlist, logmethod)
+
     def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]:
         """
         Check available markets and remove pair from whitelist if necessary

freqtrade/plugins/pairlist/StaticPairList.py

@@ -50,9 +50,10 @@ class StaticPairList(IPairList):
         :return: List of pairs
         """
         if self._allow_inactive:
-            return self._config['exchange']['pair_whitelist']
+            return self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info)
         else:
-            return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist'])
+            return self._whitelist_for_active_markets(
+                self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info))
 
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """

freqtrade/plugins/pairlistmanager.py

@@ -59,6 +59,11 @@ class PairListManager():
         """The expanded blacklist (including wildcard expansion)"""
         return expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
 
+    @property
+    def expanded_whitelist(self) -> List[str]:
+        """The expanded whitelist (including wildcard expansion)"""
+        return expand_pairlist(self._whitelist, self._exchange.get_markets().keys())
+
     @property
     def name_list(self) -> List[str]:
         """Get list of loaded Pairlist Handler names"""
@@ -129,6 +134,21 @@ class PairListManager():
                 pairlist.remove(pair)
         return pairlist
 
+    def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]:
+        """
+        Verify and remove items from pairlist - returning a filtered pairlist.
+        Logs a warning or info depending on `aswarning`.
+        Pairlist Handlers explicitly using this method shall use
+        `logmethod=logger.info` to avoid spamming with warning messages
+        :return: pairlist - blacklisted pairs
+        """
+        try:
+            whitelist = self.expanded_whitelist
+        except ValueError as err:
+            logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
+            return []
+        return whitelist
+
     def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes:
         """
         Create list of pair tuples with (pair, timeframe)

tests/exchange/test_exchange.py

@@ -505,37 +505,38 @@ def test_validate_pairs(default_conf, mocker):  # test exchange.validate_pairs d
     Exchange(default_conf)
 
 
-def test_validate_pairs_not_available(default_conf, mocker):
+# This cannot happen anymore as expand_pairlist implicitly filters out unavaliablie pairs
-    api_mock = MagicMock()
+# def test_validate_pairs_not_available(default_conf, mocker):
-    type(api_mock).markets = PropertyMock(return_value={
+#     api_mock = MagicMock()
-        'XRP/BTC': {'inactive': True}
+#     type(api_mock).markets = PropertyMock(return_value={
-    })
+#         'XRP/BTC': {'inactive': True, 'base': 'XRP', 'quote': 'BTC'}
-    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
+#     })
-    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
+#     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
-    mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
+#     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
-    mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
+#     mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
-
+#     mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
-    with pytest.raises(OperationalException, match=r'not available'):
+#
-        Exchange(default_conf)
+#     with pytest.raises(OperationalException, match=r'not available'):
-
+#         Exchange(default_conf)
-
+#
-def test_validate_pairs_exception(default_conf, mocker, caplog):
+#
-    caplog.set_level(logging.INFO)
+# def test_validate_pairs_exception(default_conf, mocker, caplog):
-    api_mock = MagicMock()
+#     caplog.set_level(logging.INFO)
-    mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance'))
+#     api_mock = MagicMock()
-
+#     mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance'))
-    type(api_mock).markets = PropertyMock(return_value={})
+#
-    mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
+#     type(api_mock).markets = PropertyMock(return_value={})
-    mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
+#     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
-    mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
+#     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
-    mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
+#     mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
-
+#     mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
-    with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'):
+#
-        Exchange(default_conf)
+#     with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'):
-
+#         Exchange(default_conf)
-    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}))
+#
-    Exchange(default_conf)
+#     mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}))
-    assert log_has('Unable to validate pairs (assuming they are correct).', caplog)
+#     Exchange(default_conf)
+#     assert log_has('Unable to validate pairs (assuming they are correct).', caplog)
 
 
 def test_validate_pairs_restricted(default_conf, mocker, caplog):

tests/test_plotting.py

@@ -47,14 +47,15 @@ def test_init_plotscript(default_conf, mocker, testdatadir):
     default_conf['timeframe'] = "5m"
     default_conf["datadir"] = testdatadir
     default_conf['exportfilename'] = testdatadir / "backtest-result_test.json"
-    ret = init_plotscript(default_conf)
+    supported_markets = ["TRX/BTC", "ADA/BTC"]
+    ret = init_plotscript(default_conf, supported_markets)
     assert "ohlcv" in ret
     assert "trades" in ret
     assert "pairs" in ret
     assert 'timerange' in ret
 
     default_conf['pairs'] = ["TRX/BTC", "ADA/BTC"]
-    ret = init_plotscript(default_conf, 20)
+    ret = init_plotscript(default_conf, supported_markets, 20)
     assert "ohlcv" in ret
     assert "TRX/BTC" in ret["ohlcv"]
     assert "ADA/BTC" in ret["ohlcv"]