diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 0f55c1b79..7dff75a02 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -22,6 +22,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged * [`StaticPairList`](#static-pair-list) (default, if not configured differently) * [`VolumePairList`](#volume-pair-list) +* [`ProducerPairList`](#producerpairlist) * [`AgeFilter`](#agefilter) * [`OffsetFilter`](#offsetfilter) * [`PerformanceFilter`](#performancefilter) @@ -84,7 +85,7 @@ Filtering instances (not the first position in the list) will not apply any cach You can define a minimum volume with `min_value` - which will filter out pairs with a volume lower than the specified value in the specified timerange. -### VolumePairList Advanced mode +##### VolumePairList Advanced mode `VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles. @@ -146,6 +147,32 @@ More sophisticated approach can be used, by using `lookback_timeframe` for candl !!! Note `VolumePairList` does not support backtesting mode. +#### ProducerPairList + +With `ProducerPairList`, you can reuse the pairlist from a [Producer](producer-consumer.md) without explicitly defining the pairlist on each consumer. + +[Consumer mode](producer-consumer.md) is required for this pairlist to work. + +The pairlist will perform a check on active pairs against the current exchange configuration to avoid attempting to trade on invalid markets. + +You can limit the length of the pairlist with the optional parameter `number_assets`. Using `"number_assets"=0` or omitting this key will result in the reuse of all producer pairs valid for the current setup. + +```json +"pairlists": [ + { + "method": "ProducerPairList", + "number_assets": 5, + "producer_name": "default", + } +], +``` + + +!!! Tip "Combining pairlists" + This pairlist can be combined with all other pairlists and filters for further pairlist reduction, and can also act as an "additional" pairlist, on top of already defined pairs. + `ProducerPairList` can also be used multiple times in sequence, combining the pairs from multiple producers. + Obviously in complex such configurations, the Producer may not provide data for all pairs, so the strategy must be fit for this. + #### AgeFilter Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity). diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e14e81343..494bb4fcb 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -31,7 +31,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'CalmarHyperOptLoss', 'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss', 'ProfitDrawDownHyperOptLoss'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'AgeFilter', 'OffsetFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index b1c95a721..e912fa832 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -82,7 +82,10 @@ class FreqtradeBot(LoggingMixin): # Keep this at the end of this initialization method. self.rpc: RPCManager = RPCManager(self) - self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists, self.rpc) + self.dataprovider = DataProvider(self.config, self.exchange, rpc=self.rpc) + self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider) + + self.dataprovider.add_pairlisthandler(self.pairlists) # Attach Dataprovider to strategy instance self.strategy.dp = self.dataprovider diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index efe199bdf..376c2de7c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -110,7 +110,7 @@ class Backtesting: self.timeframe = str(self.config.get('timeframe')) self.timeframe_min = timeframe_to_minutes(self.timeframe) self.init_backtest_detail() - self.pairlists = PairListManager(self.exchange, self.config) + self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider) if 'VolumePairList' in self.pairlists.name_list: raise OperationalException("VolumePairList not allowed for backtesting. " "Please use StaticPairList instead.") diff --git a/freqtrade/plugins/pairlist/ProducerPairList.py b/freqtrade/plugins/pairlist/ProducerPairList.py new file mode 100644 index 000000000..50b674e60 --- /dev/null +++ b/freqtrade/plugins/pairlist/ProducerPairList.py @@ -0,0 +1,90 @@ +""" +External Pair List provider + +Provides pair list from Leader data +""" +import logging +from typing import Any, Dict, List, Optional + +from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class ProducerPairList(IPairList): + """ + PairList plugin for use with external_message_consumer. + Will use pairs given from leader data. + + Usage: + "pairlists": [ + { + "method": "ProducerPairList", + "number_assets": 5, + "producer_name": "default", + } + ], + """ + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._num_assets: int = self._pairlistconfig.get('number_assets', 0) + self._producer_name = self._pairlistconfig.get('producer_name', 'default') + if not config.get('external_message_consumer', {}).get('enabled'): + raise OperationalException( + "ProducerPairList requires external_message_consumer to be enabled.") + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty Dict is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + -> Please overwrite in subclasses + """ + return f"{self.name} - {self._producer_name}" + + def _filter_pairlist(self, pairlist: Optional[List[str]]): + upstream_pairlist = self._pairlistmanager._dataprovider.get_producer_pairs( + self._producer_name) + + if pairlist is None: + pairlist = self._pairlistmanager._dataprovider.get_producer_pairs(self._producer_name) + + pairs = list(dict.fromkeys(pairlist + upstream_pairlist)) + if self._num_assets: + pairs = pairs[:self._num_assets] + + return pairs + + def gen_pairlist(self, tickers: Dict) -> List[str]: + """ + Generate the pairlist + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: List of pairs + """ + pairs = self._filter_pairlist(None) + self.log_once(f"Received pairs: {pairs}", logger.debug) + pairs = self._whitelist_for_active_markets(self.verify_whitelist(pairs, logger.info)) + return pairs + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist + """ + return self._filter_pairlist(pairlist) diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 9dcada291..b290f76aa 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -232,6 +232,4 @@ class VolumePairList(IPairList): # Limit pairlist to the requested number of pairs pairs = pairs[:self._number_pairs] - self.log_once(f"Searching {self._number_pairs} pairs: {pairs}", logger.info) - return pairs diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index e01abb297..5ed319e93 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -3,11 +3,12 @@ PairList manager class """ import logging from functools import partial -from typing import Dict, List +from typing import Dict, List, Optional from cachetools import TTLCache, cached from freqtrade.constants import Config, ListPairsWithTimeframes +from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.mixins import LoggingMixin @@ -21,13 +22,14 @@ logger = logging.getLogger(__name__) class PairListManager(LoggingMixin): - def __init__(self, exchange, config: Config) -> None: + def __init__(self, exchange, config: Config, dataprovider: DataProvider = None) -> None: self._exchange = exchange self._config = config self._whitelist = self._config['exchange'].get('pair_whitelist') self._blacklist = self._config['exchange'].get('pair_blacklist', []) self._pairlist_handlers: List[IPairList] = [] self._tickers_needed = False + self._dataprovider: Optional[DataProvider] = dataprovider for pairlist_handler_config in self._config.get('pairlists', []): pairlist_handler = PairListResolver.load_pairlist( pairlist_handler_config['method'], @@ -96,6 +98,8 @@ class PairListManager(LoggingMixin): # to ensure blacklist is respected. pairlist = self.verify_blacklist(pairlist, logger.warning) + self.log_once(f"Whitelist with {len(pairlist)} pairs: {pairlist}", logger.info) + self._whitelist = pairlist def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]: diff --git a/tests/conftest.py b/tests/conftest.py index 51b1b03e3..a9eeb481e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -200,6 +200,8 @@ def patch_freqtradebot(mocker, config) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) patch_whitelist(mocker, config) + mocker.patch('freqtrade.freqtradebot.ExternalMessageConsumer') + mocker.patch('freqtrade.configuration.config_validation._validate_consumers') def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index d6f074edb..f0b983063 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -9,6 +9,7 @@ import pytest import time_machine from freqtrade.constants import AVAILABLE_PAIRLISTS +from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import CandleType, RunMode from freqtrade.exceptions import OperationalException from freqtrade.persistence import Trade @@ -40,6 +41,12 @@ def whitelist_conf(default_conf): "sort_key": "quoteVolume", }, ] + default_conf.update({ + "external_message_consumer": { + "enabled": True, + "producers": [], + } + }) return default_conf @@ -126,7 +133,7 @@ def test_log_cached(mocker, static_pl_conf, markets, tickers): def test_load_pairlist_noexist(mocker, markets, default_conf): freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - plm = PairListManager(freqtrade.exchange, default_conf) + plm = PairListManager(freqtrade.exchange, default_conf, MagicMock()) with pytest.raises(OperationalException, match=r"Impossible to load Pairlist 'NonexistingPairList'. " r"This class does not exist or contains Python code errors."): @@ -137,7 +144,7 @@ def test_load_pairlist_noexist(mocker, markets, default_conf): def test_load_pairlist_verify_multi(mocker, markets_static, default_conf): freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets_static)) - plm = PairListManager(freqtrade.exchange, default_conf) + plm = PairListManager(freqtrade.exchange, default_conf, MagicMock()) # Call different versions one after the other, should always consider what was passed in # and have no side-effects (therefore the same check multiple times) assert plm.verify_whitelist(['ETH/BTC', 'XRP/BTC', ], print) == ['ETH/BTC', 'XRP/BTC'] @@ -269,7 +276,7 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co with pytest.raises(OperationalException, match=r'`number_assets` not specified. Please check your configuration ' r'for "pairlist.config.number_assets"'): - PairListManager(freqtrade.exchange, whitelist_conf) + PairListManager(freqtrade.exchange, whitelist_conf, MagicMock()) def test_refresh_pairlist_dynamic_2(mocker, shitcoinmarkets, tickers, whitelist_conf_2): @@ -694,7 +701,7 @@ def test_PrecisionFilter_error(mocker, whitelist_conf) -> None: with pytest.raises(OperationalException, match=r"PrecisionFilter can only work with stoploss defined\..*"): - PairListManager(MagicMock, whitelist_conf) + PairListManager(MagicMock, whitelist_conf, MagicMock()) def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None: @@ -703,7 +710,7 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None: del Trade.query mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) exchange = get_patched_exchange(mocker, whitelist_conf) - pm = PairListManager(exchange, whitelist_conf) + pm = PairListManager(exchange, whitelist_conf, MagicMock()) pm.refresh_pairlist() assert log_has("PerformanceFilter is not available in this mode.", caplog) @@ -1167,6 +1174,10 @@ def test_spreadfilter_invalid_data(mocker, default_conf, markets, tickers, caplo "[{'OffsetFilter': 'OffsetFilter - Taking 10 Pairs, starting from 5.'}]", None ), + ({"method": "ProducerPairList"}, + "[{'ProducerPairList': 'ProducerPairList - default'}]", + None + ), ]) def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig, desc_expected, exception_expected): @@ -1341,3 +1352,77 @@ def test_expand_pairlist_keep_invalid(wildcardlist, pairs, expected): expand_pairlist(wildcardlist, pairs, keep_invalid=True) else: assert sorted(expand_pairlist(wildcardlist, pairs, keep_invalid=True)) == sorted(expected) + + +def test_ProducerPairlist_no_emc(mocker, whitelist_conf): + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + whitelist_conf['pairlists'] = [ + { + "method": "ProducerPairList", + "number_assets": 10, + "producer_name": "hello_world", + } + ] + del whitelist_conf['external_message_consumer'] + + with pytest.raises(OperationalException, + match=r"ProducerPairList requires external_message_consumer to be enabled."): + get_patched_freqtradebot(mocker, whitelist_conf) + + +def test_ProducerPairlist(mocker, whitelist_conf, markets): + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + ) + whitelist_conf['pairlists'] = [ + { + "method": "ProducerPairList", + "number_assets": 2, + "producer_name": "hello_world", + } + ] + whitelist_conf.update({ + "external_message_consumer": { + "enabled": True, + "producers": [ + { + "name": "hello_world", + "host": "null", + "port": 9891, + "ws_token": "dummy", + } + ] + } + }) + + exchange = get_patched_exchange(mocker, whitelist_conf) + dp = DataProvider(whitelist_conf, exchange, None) + pairs = ['ETH/BTC', 'LTC/BTC', 'XRP/BTC'] + # different producer + dp._set_producer_pairs(pairs + ['MEEP/USDT'], 'default') + pm = PairListManager(exchange, whitelist_conf, dp) + pm.refresh_pairlist() + assert pm.whitelist == [] + # proper producer + dp._set_producer_pairs(pairs, 'hello_world') + pm.refresh_pairlist() + + # Pairlist reduced to 2 + assert pm.whitelist == pairs[:2] + assert len(pm.whitelist) == 2 + whitelist_conf['exchange']['pair_whitelist'] = ['TKN/BTC'] + + whitelist_conf['pairlists'] = [ + {"method": "StaticPairList"}, + { + "method": "ProducerPairList", + "producer_name": "hello_world", + } + ] + pm = PairListManager(exchange, whitelist_conf, dp) + pm.refresh_pairlist() + assert len(pm.whitelist) == 4 + assert pm.whitelist == ['TKN/BTC'] + pairs