From 4d7ffa8c810e20b95ca746e61b5e7f2e964ac5ad Mon Sep 17 00:00:00 2001 From: nas- Date: Tue, 12 Jan 2021 01:13:58 +0100 Subject: [PATCH] Added suppoort for regex in whitelist --- freqtrade/commands/data_commands.py | 17 +++--- freqtrade/edge/edge_positioning.py | 7 ++- freqtrade/exchange/exchange.py | 4 +- freqtrade/plot/plotting.py | 14 +++-- freqtrade/plugins/pairlist/IPairList.py | 9 +++ freqtrade/plugins/pairlist/StaticPairList.py | 5 +- freqtrade/plugins/pairlistmanager.py | 20 +++++++ tests/exchange/test_exchange.py | 63 ++++++++++---------- tests/test_plotting.py | 5 +- 9 files changed, 93 insertions(+), 51 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 25c7d0436..1ce02eee5 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -10,6 +10,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_oh refresh_backtest_trades_data) from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode @@ -42,15 +43,17 @@ def start_download_data(args: Dict[str, Any]) -> None: "Downloading data requires a list of pairs. " "Please check the documentation on how to configure this.") - logger.info(f"About to download pairs: {config['pairs']}, " - f"intervals: {config['timeframes']} to {config['datadir']}") - pairs_not_available: List[str] = [] # Init exchange exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) # Manual validations of relevant settings exchange.validate_pairs(config['pairs']) + expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets)) + + logger.info(f"About to download pairs: {expanded_pairs}, " + f"intervals: {config['timeframes']} to {config['datadir']}") + for timeframe in config['timeframes']: exchange.validate_timeframes(timeframe) @@ -58,20 +61,20 @@ def start_download_data(args: Dict[str, Any]) -> None: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( - exchange, pairs=config['pairs'], datadir=config['datadir'], + exchange, pairs=expanded_pairs, datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format=config['dataformat_trades']) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( - pairs=config['pairs'], timeframes=config['timeframes'], + pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], - ) + ) else: pairs_not_available = refresh_backtest_ohlcv_data( - exchange, pairs=config['pairs'], timeframes=config['timeframes'], + exchange, pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv']) diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 037717c68..e549a3701 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -12,6 +12,7 @@ from freqtrade.configuration import TimeRange from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT from freqtrade.data.history import get_timerange, load_data, refresh_data from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.strategy.interface import SellType @@ -80,10 +81,12 @@ class Edge: if config.get('fee'): self.fee = config['fee'] else: - self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0]) + self.fee = self.exchange.get_fee(symbol=expand_pairlist( + self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0]) def calculate(self) -> bool: - pairs = self.config['exchange']['pair_whitelist'] + pairs = expand_pairlist(self.config['exchange']['pair_whitelist'], + list(self.exchange.markets)) heartbeat = self.edge_config.get('process_throttle_secs') if (self._last_updated > 0) and ( diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b610b28f4..489f70528 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -25,6 +25,7 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, retrier, retrier_async) from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist CcxtModuleType = Any @@ -335,8 +336,9 @@ class Exchange: if not self.markets: logger.warning('Unable to validate pairs (assuming they are correct).') return + extended_pairs = expand_pairlist(pairs, list(self.markets)) invalid_pairs = [] - for pair in pairs: + for pair in extended_pairs: # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs # TODO: add a support for having coins in BTC/USDT format if self.markets and pair not in self.markets: diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 40e3da9c9..996c5276c 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -13,6 +13,7 @@ from freqtrade.data.history import get_timerange, load_data from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds from freqtrade.misc import pair_to_filename +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.strategy import IStrategy @@ -29,16 +30,16 @@ except ImportError: exit(1) -def init_plotscript(config, startup_candles: int = 0): +def init_plotscript(config, markets: List, startup_candles: int = 0): """ Initialize objects needed for plotting :return: Dict with candle (OHLCV) data, trades and pairs """ if "pairs" in config: - pairs = config['pairs'] + pairs = expand_pairlist(config['pairs'], markets) else: - pairs = config['exchange']['pair_whitelist'] + pairs = expand_pairlist(config['exchange']['pair_whitelist'], markets) # Set timerange to use timerange = TimeRange.parse_timerange(config.get('timerange')) @@ -177,7 +178,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: trades['desc'] = trades.apply(lambda row: f"{round(row['profit_percent'] * 100, 1)}%, " f"{row['sell_reason']}, " f"{row['trade_duration']} min", - axis=1) + axis=1) trade_buys = go.Scatter( x=trades["open_date"], y=trades["open_rate"], @@ -527,7 +528,7 @@ def load_and_plot_trades(config: Dict[str, Any]): exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) IStrategy.dp = DataProvider(config, exchange) - plot_elements = init_plotscript(config, strategy.startup_candle_count) + plot_elements = init_plotscript(config, list(exchange.markets), strategy.startup_candle_count) timerange = plot_elements['timerange'] trades = plot_elements['trades'] pair_counter = 0 @@ -562,7 +563,8 @@ def plot_profit(config: Dict[str, Any]) -> None: But should be somewhat proportional, and therefor useful in helping out to find a good algorithm. """ - plot_elements = init_plotscript(config) + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) + plot_elements = init_plotscript(config, list(exchange.markets)) trades = plot_elements['trades'] # Filter trades to relevant pairs # Remove open pairs - we don't know the profit yet so can't calculate profit for these. diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 865aa90d6..f9809aeb1 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -124,6 +124,15 @@ class IPairList(LoggingMixin, ABC): """ return self._pairlistmanager.verify_blacklist(pairlist, logmethod) + def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]: + """ + Proxy method to verify_whitelist for easy access for child classes. + :param pairlist: Pairlist to validate + :param logmethod: Function that'll be called, `logger.info` or `logger.warning`. + :return: pairlist - whitelisted pairs + """ + return self._pairlistmanager.verify_whitelist(pairlist, logmethod) + def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index dd592e0ca..c216a322d 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -50,9 +50,10 @@ class StaticPairList(IPairList): :return: List of pairs """ if self._allow_inactive: - return self._config['exchange']['pair_whitelist'] + return self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info) else: - return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist']) + return self._whitelist_for_active_markets( + self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info)) def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index ad7b46cb8..c471d7c51 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -59,6 +59,11 @@ class PairListManager(): """The expanded blacklist (including wildcard expansion)""" return expand_pairlist(self._blacklist, self._exchange.get_markets().keys()) + @property + def expanded_whitelist(self) -> List[str]: + """The expanded whitelist (including wildcard expansion)""" + return expand_pairlist(self._whitelist, self._exchange.get_markets().keys()) + @property def name_list(self) -> List[str]: """Get list of loaded Pairlist Handler names""" @@ -129,6 +134,21 @@ class PairListManager(): pairlist.remove(pair) return pairlist + def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]: + """ + Verify and remove items from pairlist - returning a filtered pairlist. + Logs a warning or info depending on `aswarning`. + Pairlist Handlers explicitly using this method shall use + `logmethod=logger.info` to avoid spamming with warning messages + :return: pairlist - blacklisted pairs + """ + try: + whitelist = self.expanded_whitelist + except ValueError as err: + logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") + return [] + return whitelist + def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes: """ Create list of pair tuples with (pair, timeframe) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 7d9954cb9..8cd2a9bca 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -505,37 +505,38 @@ def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs d Exchange(default_conf) -def test_validate_pairs_not_available(default_conf, mocker): - api_mock = MagicMock() - type(api_mock).markets = PropertyMock(return_value={ - 'XRP/BTC': {'inactive': True} - }) - mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') - - with pytest.raises(OperationalException, match=r'not available'): - Exchange(default_conf) - - -def test_validate_pairs_exception(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - api_mock = MagicMock() - mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) - - type(api_mock).markets = PropertyMock(return_value={}) - mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') - - with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'): - Exchange(default_conf) - - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})) - Exchange(default_conf) - assert log_has('Unable to validate pairs (assuming they are correct).', caplog) +# This cannot happen anymore as expand_pairlist implicitly filters out unavaliablie pairs +# def test_validate_pairs_not_available(default_conf, mocker): +# api_mock = MagicMock() +# type(api_mock).markets = PropertyMock(return_value={ +# 'XRP/BTC': {'inactive': True, 'base': 'XRP', 'quote': 'BTC'} +# }) +# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) +# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') +# mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') +# mocker.patch('freqtrade.exchange.Exchange._load_async_markets') +# +# with pytest.raises(OperationalException, match=r'not available'): +# Exchange(default_conf) +# +# +# def test_validate_pairs_exception(default_conf, mocker, caplog): +# caplog.set_level(logging.INFO) +# api_mock = MagicMock() +# mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) +# +# type(api_mock).markets = PropertyMock(return_value={}) +# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) +# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') +# mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') +# mocker.patch('freqtrade.exchange.Exchange._load_async_markets') +# +# with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'): +# Exchange(default_conf) +# +# mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})) +# Exchange(default_conf) +# assert log_has('Unable to validate pairs (assuming they are correct).', caplog) def test_validate_pairs_restricted(default_conf, mocker, caplog): diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 8e7b0ef7c..8f3ac464e 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -47,14 +47,15 @@ def test_init_plotscript(default_conf, mocker, testdatadir): default_conf['timeframe'] = "5m" default_conf["datadir"] = testdatadir default_conf['exportfilename'] = testdatadir / "backtest-result_test.json" - ret = init_plotscript(default_conf) + supported_markets = ["TRX/BTC", "ADA/BTC"] + ret = init_plotscript(default_conf, supported_markets) assert "ohlcv" in ret assert "trades" in ret assert "pairs" in ret assert 'timerange' in ret default_conf['pairs'] = ["TRX/BTC", "ADA/BTC"] - ret = init_plotscript(default_conf, 20) + ret = init_plotscript(default_conf, supported_markets, 20) assert "ohlcv" in ret assert "TRX/BTC" in ret["ohlcv"] assert "ADA/BTC" in ret["ohlcv"]