From 4d7ffa8c810e20b95ca746e61b5e7f2e964ac5ad Mon Sep 17 00:00:00 2001 From: nas- Date: Tue, 12 Jan 2021 01:13:58 +0100 Subject: [PATCH 1/3] Added suppoort for regex in whitelist --- freqtrade/commands/data_commands.py | 17 +++--- freqtrade/edge/edge_positioning.py | 7 ++- freqtrade/exchange/exchange.py | 4 +- freqtrade/plot/plotting.py | 14 +++-- freqtrade/plugins/pairlist/IPairList.py | 9 +++ freqtrade/plugins/pairlist/StaticPairList.py | 5 +- freqtrade/plugins/pairlistmanager.py | 20 +++++++ tests/exchange/test_exchange.py | 63 ++++++++++---------- tests/test_plotting.py | 5 +- 9 files changed, 93 insertions(+), 51 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 25c7d0436..1ce02eee5 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -10,6 +10,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_oh refresh_backtest_trades_data) from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode @@ -42,15 +43,17 @@ def start_download_data(args: Dict[str, Any]) -> None: "Downloading data requires a list of pairs. " "Please check the documentation on how to configure this.") - logger.info(f"About to download pairs: {config['pairs']}, " - f"intervals: {config['timeframes']} to {config['datadir']}") - pairs_not_available: List[str] = [] # Init exchange exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) # Manual validations of relevant settings exchange.validate_pairs(config['pairs']) + expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets)) + + logger.info(f"About to download pairs: {expanded_pairs}, " + f"intervals: {config['timeframes']} to {config['datadir']}") + for timeframe in config['timeframes']: exchange.validate_timeframes(timeframe) @@ -58,20 +61,20 @@ def start_download_data(args: Dict[str, Any]) -> None: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( - exchange, pairs=config['pairs'], datadir=config['datadir'], + exchange, pairs=expanded_pairs, datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format=config['dataformat_trades']) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( - pairs=config['pairs'], timeframes=config['timeframes'], + pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], - ) + ) else: pairs_not_available = refresh_backtest_ohlcv_data( - exchange, pairs=config['pairs'], timeframes=config['timeframes'], + exchange, pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv']) diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 037717c68..e549a3701 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -12,6 +12,7 @@ from freqtrade.configuration import TimeRange from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT from freqtrade.data.history import get_timerange, load_data, refresh_data from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.strategy.interface import SellType @@ -80,10 +81,12 @@ class Edge: if config.get('fee'): self.fee = config['fee'] else: - self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0]) + self.fee = self.exchange.get_fee(symbol=expand_pairlist( + self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0]) def calculate(self) -> bool: - pairs = self.config['exchange']['pair_whitelist'] + pairs = expand_pairlist(self.config['exchange']['pair_whitelist'], + list(self.exchange.markets)) heartbeat = self.edge_config.get('process_throttle_secs') if (self._last_updated > 0) and ( diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b610b28f4..489f70528 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -25,6 +25,7 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, retrier, retrier_async) from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist CcxtModuleType = Any @@ -335,8 +336,9 @@ class Exchange: if not self.markets: logger.warning('Unable to validate pairs (assuming they are correct).') return + extended_pairs = expand_pairlist(pairs, list(self.markets)) invalid_pairs = [] - for pair in pairs: + for pair in extended_pairs: # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs # TODO: add a support for having coins in BTC/USDT format if self.markets and pair not in self.markets: diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 40e3da9c9..996c5276c 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -13,6 +13,7 @@ from freqtrade.data.history import get_timerange, load_data from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds from freqtrade.misc import pair_to_filename +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.strategy import IStrategy @@ -29,16 +30,16 @@ except ImportError: exit(1) -def init_plotscript(config, startup_candles: int = 0): +def init_plotscript(config, markets: List, startup_candles: int = 0): """ Initialize objects needed for plotting :return: Dict with candle (OHLCV) data, trades and pairs """ if "pairs" in config: - pairs = config['pairs'] + pairs = expand_pairlist(config['pairs'], markets) else: - pairs = config['exchange']['pair_whitelist'] + pairs = expand_pairlist(config['exchange']['pair_whitelist'], markets) # Set timerange to use timerange = TimeRange.parse_timerange(config.get('timerange')) @@ -177,7 +178,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: trades['desc'] = trades.apply(lambda row: f"{round(row['profit_percent'] * 100, 1)}%, " f"{row['sell_reason']}, " f"{row['trade_duration']} min", - axis=1) + axis=1) trade_buys = go.Scatter( x=trades["open_date"], y=trades["open_rate"], @@ -527,7 +528,7 @@ def load_and_plot_trades(config: Dict[str, Any]): exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) IStrategy.dp = DataProvider(config, exchange) - plot_elements = init_plotscript(config, strategy.startup_candle_count) + plot_elements = init_plotscript(config, list(exchange.markets), strategy.startup_candle_count) timerange = plot_elements['timerange'] trades = plot_elements['trades'] pair_counter = 0 @@ -562,7 +563,8 @@ def plot_profit(config: Dict[str, Any]) -> None: But should be somewhat proportional, and therefor useful in helping out to find a good algorithm. """ - plot_elements = init_plotscript(config) + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) + plot_elements = init_plotscript(config, list(exchange.markets)) trades = plot_elements['trades'] # Filter trades to relevant pairs # Remove open pairs - we don't know the profit yet so can't calculate profit for these. diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 865aa90d6..f9809aeb1 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -124,6 +124,15 @@ class IPairList(LoggingMixin, ABC): """ return self._pairlistmanager.verify_blacklist(pairlist, logmethod) + def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]: + """ + Proxy method to verify_whitelist for easy access for child classes. + :param pairlist: Pairlist to validate + :param logmethod: Function that'll be called, `logger.info` or `logger.warning`. + :return: pairlist - whitelisted pairs + """ + return self._pairlistmanager.verify_whitelist(pairlist, logmethod) + def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index dd592e0ca..c216a322d 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -50,9 +50,10 @@ class StaticPairList(IPairList): :return: List of pairs """ if self._allow_inactive: - return self._config['exchange']['pair_whitelist'] + return self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info) else: - return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist']) + return self._whitelist_for_active_markets( + self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info)) def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index ad7b46cb8..c471d7c51 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -59,6 +59,11 @@ class PairListManager(): """The expanded blacklist (including wildcard expansion)""" return expand_pairlist(self._blacklist, self._exchange.get_markets().keys()) + @property + def expanded_whitelist(self) -> List[str]: + """The expanded whitelist (including wildcard expansion)""" + return expand_pairlist(self._whitelist, self._exchange.get_markets().keys()) + @property def name_list(self) -> List[str]: """Get list of loaded Pairlist Handler names""" @@ -129,6 +134,21 @@ class PairListManager(): pairlist.remove(pair) return pairlist + def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]: + """ + Verify and remove items from pairlist - returning a filtered pairlist. + Logs a warning or info depending on `aswarning`. + Pairlist Handlers explicitly using this method shall use + `logmethod=logger.info` to avoid spamming with warning messages + :return: pairlist - blacklisted pairs + """ + try: + whitelist = self.expanded_whitelist + except ValueError as err: + logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") + return [] + return whitelist + def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes: """ Create list of pair tuples with (pair, timeframe) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 7d9954cb9..8cd2a9bca 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -505,37 +505,38 @@ def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs d Exchange(default_conf) -def test_validate_pairs_not_available(default_conf, mocker): - api_mock = MagicMock() - type(api_mock).markets = PropertyMock(return_value={ - 'XRP/BTC': {'inactive': True} - }) - mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') - - with pytest.raises(OperationalException, match=r'not available'): - Exchange(default_conf) - - -def test_validate_pairs_exception(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - api_mock = MagicMock() - mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) - - type(api_mock).markets = PropertyMock(return_value={}) - mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') - - with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'): - Exchange(default_conf) - - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})) - Exchange(default_conf) - assert log_has('Unable to validate pairs (assuming they are correct).', caplog) +# This cannot happen anymore as expand_pairlist implicitly filters out unavaliablie pairs +# def test_validate_pairs_not_available(default_conf, mocker): +# api_mock = MagicMock() +# type(api_mock).markets = PropertyMock(return_value={ +# 'XRP/BTC': {'inactive': True, 'base': 'XRP', 'quote': 'BTC'} +# }) +# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) +# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') +# mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') +# mocker.patch('freqtrade.exchange.Exchange._load_async_markets') +# +# with pytest.raises(OperationalException, match=r'not available'): +# Exchange(default_conf) +# +# +# def test_validate_pairs_exception(default_conf, mocker, caplog): +# caplog.set_level(logging.INFO) +# api_mock = MagicMock() +# mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) +# +# type(api_mock).markets = PropertyMock(return_value={}) +# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) +# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') +# mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') +# mocker.patch('freqtrade.exchange.Exchange._load_async_markets') +# +# with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'): +# Exchange(default_conf) +# +# mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})) +# Exchange(default_conf) +# assert log_has('Unable to validate pairs (assuming they are correct).', caplog) def test_validate_pairs_restricted(default_conf, mocker, caplog): diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 8e7b0ef7c..8f3ac464e 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -47,14 +47,15 @@ def test_init_plotscript(default_conf, mocker, testdatadir): default_conf['timeframe'] = "5m" default_conf["datadir"] = testdatadir default_conf['exportfilename'] = testdatadir / "backtest-result_test.json" - ret = init_plotscript(default_conf) + supported_markets = ["TRX/BTC", "ADA/BTC"] + ret = init_plotscript(default_conf, supported_markets) assert "ohlcv" in ret assert "trades" in ret assert "pairs" in ret assert 'timerange' in ret default_conf['pairs'] = ["TRX/BTC", "ADA/BTC"] - ret = init_plotscript(default_conf, 20) + ret = init_plotscript(default_conf, supported_markets, 20) assert "ohlcv" in ret assert "TRX/BTC" in ret["ohlcv"] assert "ADA/BTC" in ret["ohlcv"] From f72d53351cd9c10530db8e5f613685d9009abbcf Mon Sep 17 00:00:00 2001 From: nas- Date: Fri, 15 Jan 2021 00:13:11 +0100 Subject: [PATCH 2/3] Added ability to keep invalid pairs while expanding expand_pairlist --- docs/configuration.md | 2 +- docs/includes/pairlists.md | 2 +- freqtrade/exchange/exchange.py | 2 +- freqtrade/plugins/pairlist/IPairList.py | 10 +-- freqtrade/plugins/pairlist/StaticPairList.py | 4 +- .../plugins/pairlist/pairlist_helpers.py | 37 ++++++++--- freqtrade/plugins/pairlistmanager.py | 33 +++++++--- tests/exchange/test_exchange.py | 63 +++++++++---------- tests/plugins/test_pairlist.py | 32 ++++++++++ 9 files changed, 128 insertions(+), 57 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 13c7eb47b..33d117c91 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -81,7 +81,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String -| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used by VolumePairList (see [below](#pairlists-and-pairlist-handlers)).
**Datatype:** List +| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting.Supports regex pairs as */BTC. Not used by VolumePairList (see [below](#pairlists-and-pairlist-handlers)).
**Datatype:** List | `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#pairlists-and-pairlist-handlers)).
**Datatype:** List | `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict | `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 8919c4e3d..1ad38cc73 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -35,7 +35,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged #### Static Pair List -By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. +By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. Also the pairlist does support wildcards (in regex-style) - so `*/BTC` will include all pairs with BTC as a stake. It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`. diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 489f70528..436c8e4e9 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -336,7 +336,7 @@ class Exchange: if not self.markets: logger.warning('Unable to validate pairs (assuming they are correct).') return - extended_pairs = expand_pairlist(pairs, list(self.markets)) + extended_pairs = expand_pairlist(pairs, list(self.markets), keep_invalid=True) invalid_pairs = [] for pair in extended_pairs: # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index f9809aeb1..95d776ae6 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -124,19 +124,21 @@ class IPairList(LoggingMixin, ABC): """ return self._pairlistmanager.verify_blacklist(pairlist, logmethod) - def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]: + def verify_whitelist(self, pairlist: List[str], logmethod, + keep_invalid: bool = False) -> List[str]: """ Proxy method to verify_whitelist for easy access for child classes. :param pairlist: Pairlist to validate - :param logmethod: Function that'll be called, `logger.info` or `logger.warning`. + :param logmethod: Function that'll be called, `logger.info` or `logger.warning` + :param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes. :return: pairlist - whitelisted pairs """ - return self._pairlistmanager.verify_whitelist(pairlist, logmethod) + return self._pairlistmanager.verify_whitelist(pairlist, logmethod, keep_invalid) def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary - :param whitelist: the sorted list of pairs the user might want to trade + :param pairlist: the sorted list of pairs the user might want to trade :return: the list of pairs the user wants to trade without those unavailable or black_listed """ diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index c216a322d..c5ced48c9 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -50,7 +50,9 @@ class StaticPairList(IPairList): :return: List of pairs """ if self._allow_inactive: - return self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info) + return self.verify_whitelist( + self._config['exchange']['pair_whitelist'], logger.info, keep_invalid=True + ) else: return self._whitelist_for_active_markets( self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info)) diff --git a/freqtrade/plugins/pairlist/pairlist_helpers.py b/freqtrade/plugins/pairlist/pairlist_helpers.py index 3352777f0..04320fe16 100644 --- a/freqtrade/plugins/pairlist/pairlist_helpers.py +++ b/freqtrade/plugins/pairlist/pairlist_helpers.py @@ -2,22 +2,41 @@ import re from typing import List -def expand_pairlist(wildcardpl: List[str], available_pairs: List[str]) -> List[str]: +def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], + keep_invalid: bool = False) -> List[str]: """ Expand pairlist potentially containing wildcards based on available markets. This will implicitly filter all pairs in the wildcard-list which are not in available_pairs. :param wildcardpl: List of Pairlists, which may contain regex :param available_pairs: List of all available pairs (`exchange.get_markets().keys()`) + :param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes :return expanded pairlist, with Regexes from wildcardpl applied to match all available pairs. :raises: ValueError if a wildcard is invalid (like '*/BTC' - which should be `.*/BTC`) """ result = [] - for pair_wc in wildcardpl: - try: - comp = re.compile(pair_wc) - result += [ - pair for pair in available_pairs if re.match(comp, pair) - ] - except re.error as err: - raise ValueError(f"Wildcard error in {pair_wc}, {err}") + if keep_invalid: + for pair_wc in wildcardpl: + try: + comp = re.compile(pair_wc) + result_partial = [ + pair for pair in available_pairs if re.match(comp, pair) + ] + # Add all matching pairs. + # If there are no matching pairs (Pair not on exchange) keep it. + result += result_partial or [pair_wc] + except re.error as err: + raise ValueError(f"Wildcard error in {pair_wc}, {err}") + + for element in result: + if not re.fullmatch(r'^[A-Za-z0-9/-]+$', element): + result.remove(element) + else: + for pair_wc in wildcardpl: + try: + comp = re.compile(pair_wc) + result += [ + pair for pair in available_pairs if re.match(comp, pair) + ] + except re.error as err: + raise ValueError(f"Wildcard error in {pair_wc}, {err}") return result diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index c471d7c51..15e186e6c 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -59,9 +59,15 @@ class PairListManager(): """The expanded blacklist (including wildcard expansion)""" return expand_pairlist(self._blacklist, self._exchange.get_markets().keys()) + @property + def expanded_whitelist_keep_invalid(self) -> List[str]: + """The expanded whitelist (including wildcard expansion), maintaining invalid pairs""" + return expand_pairlist(self._whitelist, self._exchange.get_markets().keys(), + keep_invalid=True) + @property def expanded_whitelist(self) -> List[str]: - """The expanded whitelist (including wildcard expansion)""" + """The expanded whitelist (including wildcard expansion), filtering invalid pairs""" return expand_pairlist(self._whitelist, self._exchange.get_markets().keys()) @property @@ -134,19 +140,30 @@ class PairListManager(): pairlist.remove(pair) return pairlist - def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]: + def verify_whitelist(self, pairlist: List[str], logmethod, + keep_invalid: bool = False) -> List[str]: """ Verify and remove items from pairlist - returning a filtered pairlist. Logs a warning or info depending on `aswarning`. Pairlist Handlers explicitly using this method shall use `logmethod=logger.info` to avoid spamming with warning messages - :return: pairlist - blacklisted pairs + :param pairlist: Pairlist to validate + :param logmethod: Function that'll be called, `logger.info` or `logger.warning` + :param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes. + :return: pairlist - whitelisted pairs """ - try: - whitelist = self.expanded_whitelist - except ValueError as err: - logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") - return [] + if keep_invalid: + try: + whitelist = self.expanded_whitelist_keep_invalid + except ValueError as err: + logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") + return [] + else: + try: + whitelist = self.expanded_whitelist + except ValueError as err: + logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") + return [] return whitelist def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes: diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 8cd2a9bca..9d655997f 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -505,38 +505,37 @@ def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs d Exchange(default_conf) -# This cannot happen anymore as expand_pairlist implicitly filters out unavaliablie pairs -# def test_validate_pairs_not_available(default_conf, mocker): -# api_mock = MagicMock() -# type(api_mock).markets = PropertyMock(return_value={ -# 'XRP/BTC': {'inactive': True, 'base': 'XRP', 'quote': 'BTC'} -# }) -# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) -# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') -# mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') -# mocker.patch('freqtrade.exchange.Exchange._load_async_markets') -# -# with pytest.raises(OperationalException, match=r'not available'): -# Exchange(default_conf) -# -# -# def test_validate_pairs_exception(default_conf, mocker, caplog): -# caplog.set_level(logging.INFO) -# api_mock = MagicMock() -# mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) -# -# type(api_mock).markets = PropertyMock(return_value={}) -# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) -# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') -# mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') -# mocker.patch('freqtrade.exchange.Exchange._load_async_markets') -# -# with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'): -# Exchange(default_conf) -# -# mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})) -# Exchange(default_conf) -# assert log_has('Unable to validate pairs (assuming they are correct).', caplog) +def test_validate_pairs_not_available(default_conf, mocker): + api_mock = MagicMock() + type(api_mock).markets = PropertyMock(return_value={ + 'XRP/BTC': {'inactive': True, 'base': 'XRP', 'quote': 'BTC'} + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + + with pytest.raises(OperationalException, match=r'not available'): + Exchange(default_conf) + + +def test_validate_pairs_exception(default_conf, mocker, caplog): + caplog.set_level(logging.INFO) + api_mock = MagicMock() + mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) + + type(api_mock).markets = PropertyMock(return_value={}) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + + with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'): + Exchange(default_conf) + + mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})) + Exchange(default_conf) + assert log_has('Unable to validate pairs (assuming they are correct).', caplog) def test_validate_pairs_restricted(default_conf, mocker, caplog): diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index d822f8319..e20e42c60 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -853,3 +853,35 @@ def test_expand_pairlist(wildcardlist, pairs, expected): expand_pairlist(wildcardlist, pairs) else: assert sorted(expand_pairlist(wildcardlist, pairs)) == sorted(expected) + + +@pytest.mark.parametrize('wildcardlist,pairs,expected', [ + (['BTC/USDT'], + ['BTC/USDT'], + ['BTC/USDT']), + (['BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETH/USDT']), + (['BTC/USDT', 'ETH/USDT'], + ['BTC/USDT'], ['BTC/USDT', 'ETH/USDT']), # Test one too many + (['.*/USDT'], + ['BTC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETH/USDT']), # Wildcard simple + (['.*C/USDT'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETC/USDT']), # Wildcard exclude one + (['.*UP/USDT', 'BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'], + ['BTC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT']), # Wildcard exclude one + (['BTC/.*', 'ETH/.*'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP'], + ['BTC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP']), # Wildcard exclude one + (['*UP/USDT', 'BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'], + None), + (['HELLO/WORLD'], [], ['HELLO/WORLD']) # Invalid pair kept +]) +def test_expand_pairlist_keep_invalid(wildcardlist, pairs, expected): + if expected is None: + with pytest.raises(ValueError, match=r'Wildcard error in \*UP/USDT,'): + expand_pairlist(wildcardlist, pairs, keep_invalid=True) + else: + assert sorted(expand_pairlist(wildcardlist, pairs, keep_invalid=True)) == sorted(expected) From bf5868c96da6b0194c6bafa319f09bddef00d350 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 15 Jan 2021 06:56:15 +0100 Subject: [PATCH 3/3] Add testcase for nonexisting pairs on whitelist --- docs/configuration.md | 2 +- docs/includes/pairlists.md | 2 +- freqtrade/plugins/pairlistmanager.py | 16 ++++++---------- tests/plugins/test_pairlist.py | 26 +++++++++++++++++++++++++- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 33d117c91..e655182b8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -81,7 +81,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String -| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting.Supports regex pairs as */BTC. Not used by VolumePairList (see [below](#pairlists-and-pairlist-handlers)).
**Datatype:** List +| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Supports regex pairs as `.*/BTC`. Not used by VolumePairList (see [below](#pairlists-and-pairlist-handlers)).
**Datatype:** List | `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#pairlists-and-pairlist-handlers)).
**Datatype:** List | `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict | `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 1ad38cc73..2653406e7 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -35,7 +35,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged #### Static Pair List -By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. Also the pairlist does support wildcards (in regex-style) - so `*/BTC` will include all pairs with BTC as a stake. +By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. The pairlist also supports wildcards (in regex-style) - so `.*/BTC` will include all pairs with BTC as a stake. It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`. diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 15e186e6c..7ce77da59 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -152,18 +152,14 @@ class PairListManager(): :param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes. :return: pairlist - whitelisted pairs """ - if keep_invalid: - try: + try: + if keep_invalid: whitelist = self.expanded_whitelist_keep_invalid - except ValueError as err: - logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") - return [] - else: - try: + else: whitelist = self.expanded_whitelist - except ValueError as err: - logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") - return [] + except ValueError as err: + logger.error(f"Pair whitelist contains an invalid Wildcard: {err}") + return [] return whitelist def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes: diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index e20e42c60..910a9580c 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -156,6 +156,31 @@ def test_refresh_static_pairlist(mocker, markets, static_pl_conf): assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist +@pytest.mark.parametrize('pairs,expected', [ + (['NOEXIST/BTC', r'\+WHAT/BTC'], + ['ETH/BTC', 'TKN/BTC', 'TRST/BTC', 'NOEXIST/BTC', 'SWT/BTC', 'BCC/BTC', 'HOT/BTC']), + (['NOEXIST/BTC', r'*/BTC'], # This is an invalid regex + []), +]) +def test_refresh_static_pairlist_noexist(mocker, markets, static_pl_conf, pairs, expected, caplog): + + static_pl_conf['pairlists'][0]['allow_inactive'] = True + static_pl_conf['exchange']['pair_whitelist'] += pairs + freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + markets=PropertyMock(return_value=markets), + ) + freqtrade.pairlists.refresh_pairlist() + + # Ensure all except those in whitelist are removed + assert set(expected) == set(freqtrade.pairlists.whitelist) + assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist + if not expected: + assert log_has_re(r'Pair whitelist contains an invalid Wildcard: Wildcard error.*', caplog) + + def test_invalid_blacklist(mocker, markets, static_pl_conf, caplog): static_pl_conf['exchange']['pair_blacklist'] = ['*/BTC'] freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) @@ -165,7 +190,6 @@ def test_invalid_blacklist(mocker, markets, static_pl_conf, caplog): markets=PropertyMock(return_value=markets), ) freqtrade.pairlists.refresh_pairlist() - # List ordered by BaseVolume whitelist = [] # Ensure all except those in whitelist are removed assert set(whitelist) == set(freqtrade.pairlists.whitelist)