Merge pull request #7473 from freqtrade/feat/producerpairlist
Producerpairlist
This commit is contained in:
Matthias
GitHub
commit
2ce265bed3
|
|
@ -22,6 +22,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
|
|||
|
||||
* [`StaticPairList`](#static-pair-list) (default, if not configured differently)
|
||||
* [`VolumePairList`](#volume-pair-list)
|
||||
* [`ProducerPairList`](#producerpairlist)
|
||||
* [`AgeFilter`](#agefilter)
|
||||
* [`OffsetFilter`](#offsetfilter)
|
||||
* [`PerformanceFilter`](#performancefilter)
|
||||
|
|
@ -84,7 +85,7 @@ Filtering instances (not the first position in the list) will not apply any cach
|
|||
|
||||
You can define a minimum volume with `min_value` - which will filter out pairs with a volume lower than the specified value in the specified timerange.
|
||||
|
||||
### VolumePairList Advanced mode
|
||||
##### VolumePairList Advanced mode
|
||||
|
||||
`VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
|
||||
|
||||
|
|
@ -146,6 +147,32 @@ More sophisticated approach can be used, by using `lookback_timeframe` for candl
|
|||
!!! Note
|
||||
`VolumePairList` does not support backtesting mode.
|
||||
|
||||
#### ProducerPairList
|
||||
|
||||
With `ProducerPairList`, you can reuse the pairlist from a [Producer](producer-consumer.md) without explicitly defining the pairlist on each consumer.
|
||||
|
||||
[Consumer mode](producer-consumer.md) is required for this pairlist to work.
|
||||
|
||||
The pairlist will perform a check on active pairs against the current exchange configuration to avoid attempting to trade on invalid markets.
|
||||
|
||||
You can limit the length of the pairlist with the optional parameter `number_assets`. Using `"number_assets"=0` or omitting this key will result in the reuse of all producer pairs valid for the current setup.
|
||||
|
||||
```json
|
||||
"pairlists": [
|
||||
{
|
||||
"method": "ProducerPairList",
|
||||
"number_assets": 5,
|
||||
"producer_name": "default",
|
||||
}
|
||||
],
|
||||
```
|
||||
|
||||
|
||||
!!! Tip "Combining pairlists"
|
||||
This pairlist can be combined with all other pairlists and filters for further pairlist reduction, and can also act as an "additional" pairlist, on top of already defined pairs.
|
||||
`ProducerPairList` can also be used multiple times in sequence, combining the pairs from multiple producers.
|
||||
Obviously in complex such configurations, the Producer may not provide data for all pairs, so the strategy must be fit for this.
|
||||
|
||||
#### AgeFilter
|
||||
|
||||
Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity).
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
|
|||
'CalmarHyperOptLoss',
|
||||
'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
|
||||
'ProfitDrawDownHyperOptLoss']
|
||||
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
|
||||
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList',
|
||||
'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
|
||||
'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
|
||||
'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
|
||||
|
|
|
|||
|
|
@ -82,7 +82,10 @@ class FreqtradeBot(LoggingMixin):
|
|||
# Keep this at the end of this initialization method.
|
||||
self.rpc: RPCManager = RPCManager(self)
|
||||
|
||||
self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists, self.rpc)
|
||||
self.dataprovider = DataProvider(self.config, self.exchange, rpc=self.rpc)
|
||||
self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider)
|
||||
|
||||
self.dataprovider.add_pairlisthandler(self.pairlists)
|
||||
|
||||
# Attach Dataprovider to strategy instance
|
||||
self.strategy.dp = self.dataprovider
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ class Backtesting:
|
|||
self.timeframe = str(self.config.get('timeframe'))
|
||||
self.timeframe_min = timeframe_to_minutes(self.timeframe)
|
||||
self.init_backtest_detail()
|
||||
self.pairlists = PairListManager(self.exchange, self.config)
|
||||
self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider)
|
||||
if 'VolumePairList' in self.pairlists.name_list:
|
||||
raise OperationalException("VolumePairList not allowed for backtesting. "
|
||||
"Please use StaticPairList instead.")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,90 @@
|
|||
"""
|
||||
External Pair List provider
|
||||
|
||||
Provides pair list from Leader data
|
||||
"""
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.plugins.pairlist.IPairList import IPairList
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ProducerPairList(IPairList):
|
||||
"""
|
||||
PairList plugin for use with external_message_consumer.
|
||||
Will use pairs given from leader data.
|
||||
|
||||
Usage:
|
||||
"pairlists": [
|
||||
{
|
||||
"method": "ProducerPairList",
|
||||
"number_assets": 5,
|
||||
"producer_name": "default",
|
||||
}
|
||||
],
|
||||
"""
|
||||
|
||||
def __init__(self, exchange, pairlistmanager,
|
||||
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
|
||||
pairlist_pos: int) -> None:
|
||||
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
|
||||
|
||||
self._num_assets: int = self._pairlistconfig.get('number_assets', 0)
|
||||
self._producer_name = self._pairlistconfig.get('producer_name', 'default')
|
||||
if not config.get('external_message_consumer', {}).get('enabled'):
|
||||
raise OperationalException(
|
||||
"ProducerPairList requires external_message_consumer to be enabled.")
|
||||
|
||||
@property
|
||||
def needstickers(self) -> bool:
|
||||
"""
|
||||
Boolean property defining if tickers are necessary.
|
||||
If no Pairlist requires tickers, an empty Dict is passed
|
||||
as tickers argument to filter_pairlist
|
||||
"""
|
||||
return False
|
||||
|
||||
def short_desc(self) -> str:
|
||||
"""
|
||||
Short whitelist method description - used for startup-messages
|
||||
-> Please overwrite in subclasses
|
||||
"""
|
||||
return f"{self.name} - {self._producer_name}"
|
||||
|
||||
def _filter_pairlist(self, pairlist: Optional[List[str]]):
|
||||
upstream_pairlist = self._pairlistmanager._dataprovider.get_producer_pairs(
|
||||
self._producer_name)
|
||||
|
||||
if pairlist is None:
|
||||
pairlist = self._pairlistmanager._dataprovider.get_producer_pairs(self._producer_name)
|
||||
|
||||
pairs = list(dict.fromkeys(pairlist + upstream_pairlist))
|
||||
if self._num_assets:
|
||||
pairs = pairs[:self._num_assets]
|
||||
|
||||
return pairs
|
||||
|
||||
def gen_pairlist(self, tickers: Dict) -> List[str]:
|
||||
"""
|
||||
Generate the pairlist
|
||||
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
|
||||
:return: List of pairs
|
||||
"""
|
||||
pairs = self._filter_pairlist(None)
|
||||
self.log_once(f"Received pairs: {pairs}", logger.debug)
|
||||
pairs = self._whitelist_for_active_markets(self.verify_whitelist(pairs, logger.info))
|
||||
return pairs
|
||||
|
||||
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
|
||||
"""
|
||||
Filters and sorts pairlist and returns the whitelist again.
|
||||
Called on each bot iteration - please use internal caching if necessary
|
||||
:param pairlist: pairlist to filter or sort
|
||||
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
|
||||
:return: new whitelist
|
||||
"""
|
||||
return self._filter_pairlist(pairlist)
|
||||
|
|
@ -232,6 +232,4 @@ class VolumePairList(IPairList):
|
|||
# Limit pairlist to the requested number of pairs
|
||||
pairs = pairs[:self._number_pairs]
|
||||
|
||||
self.log_once(f"Searching {self._number_pairs} pairs: {pairs}", logger.info)
|
||||
|
||||
return pairs
|
||||
|
|
|
|||
|
|
@ -3,11 +3,12 @@ PairList manager class
|
|||
"""
|
||||
import logging
|
||||
from functools import partial
|
||||
from typing import Dict, List
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from cachetools import TTLCache, cached
|
||||
|
||||
from freqtrade.constants import Config, ListPairsWithTimeframes
|
||||
from freqtrade.data.dataprovider import DataProvider
|
||||
from freqtrade.enums import CandleType
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.mixins import LoggingMixin
|
||||
|
|
@ -21,13 +22,14 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
class PairListManager(LoggingMixin):
|
||||
|
||||
def __init__(self, exchange, config: Config) -> None:
|
||||
def __init__(self, exchange, config: Config, dataprovider: DataProvider = None) -> None:
|
||||
self._exchange = exchange
|
||||
self._config = config
|
||||
self._whitelist = self._config['exchange'].get('pair_whitelist')
|
||||
self._blacklist = self._config['exchange'].get('pair_blacklist', [])
|
||||
self._pairlist_handlers: List[IPairList] = []
|
||||
self._tickers_needed = False
|
||||
self._dataprovider: Optional[DataProvider] = dataprovider
|
||||
for pairlist_handler_config in self._config.get('pairlists', []):
|
||||
pairlist_handler = PairListResolver.load_pairlist(
|
||||
pairlist_handler_config['method'],
|
||||
|
|
@ -96,6 +98,8 @@ class PairListManager(LoggingMixin):
|
|||
# to ensure blacklist is respected.
|
||||
pairlist = self.verify_blacklist(pairlist, logger.warning)
|
||||
|
||||
self.log_once(f"Whitelist with {len(pairlist)} pairs: {pairlist}", logger.info)
|
||||
|
||||
self._whitelist = pairlist
|
||||
|
||||
def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:
|
||||
|
|
|
|||
|
|
@ -200,6 +200,8 @@ def patch_freqtradebot(mocker, config) -> None:
|
|||
mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock())
|
||||
mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock())
|
||||
patch_whitelist(mocker, config)
|
||||
mocker.patch('freqtrade.freqtradebot.ExternalMessageConsumer')
|
||||
mocker.patch('freqtrade.configuration.config_validation._validate_consumers')
|
||||
|
||||
|
||||
def get_patched_freqtradebot(mocker, config) -> FreqtradeBot:
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import pytest
|
|||
import time_machine
|
||||
|
||||
from freqtrade.constants import AVAILABLE_PAIRLISTS
|
||||
from freqtrade.data.dataprovider import DataProvider
|
||||
from freqtrade.enums import CandleType, RunMode
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.persistence import Trade
|
||||
|
|
@ -40,6 +41,12 @@ def whitelist_conf(default_conf):
|
|||
"sort_key": "quoteVolume",
|
||||
},
|
||||
]
|
||||
default_conf.update({
|
||||
"external_message_consumer": {
|
||||
"enabled": True,
|
||||
"producers": [],
|
||||
}
|
||||
})
|
||||
return default_conf
|
||||
|
||||
|
||||
|
|
@ -126,7 +133,7 @@ def test_log_cached(mocker, static_pl_conf, markets, tickers):
|
|||
def test_load_pairlist_noexist(mocker, markets, default_conf):
|
||||
freqtrade = get_patched_freqtradebot(mocker, default_conf)
|
||||
mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
|
||||
plm = PairListManager(freqtrade.exchange, default_conf)
|
||||
plm = PairListManager(freqtrade.exchange, default_conf, MagicMock())
|
||||
with pytest.raises(OperationalException,
|
||||
match=r"Impossible to load Pairlist 'NonexistingPairList'. "
|
||||
r"This class does not exist or contains Python code errors."):
|
||||
|
|
@ -137,7 +144,7 @@ def test_load_pairlist_noexist(mocker, markets, default_conf):
|
|||
def test_load_pairlist_verify_multi(mocker, markets_static, default_conf):
|
||||
freqtrade = get_patched_freqtradebot(mocker, default_conf)
|
||||
mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets_static))
|
||||
plm = PairListManager(freqtrade.exchange, default_conf)
|
||||
plm = PairListManager(freqtrade.exchange, default_conf, MagicMock())
|
||||
# Call different versions one after the other, should always consider what was passed in
|
||||
# and have no side-effects (therefore the same check multiple times)
|
||||
assert plm.verify_whitelist(['ETH/BTC', 'XRP/BTC', ], print) == ['ETH/BTC', 'XRP/BTC']
|
||||
|
|
@ -269,7 +276,7 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co
|
|||
with pytest.raises(OperationalException,
|
||||
match=r'`number_assets` not specified. Please check your configuration '
|
||||
r'for "pairlist.config.number_assets"'):
|
||||
PairListManager(freqtrade.exchange, whitelist_conf)
|
||||
PairListManager(freqtrade.exchange, whitelist_conf, MagicMock())
|
||||
|
||||
|
||||
def test_refresh_pairlist_dynamic_2(mocker, shitcoinmarkets, tickers, whitelist_conf_2):
|
||||
|
|
@ -694,7 +701,7 @@ def test_PrecisionFilter_error(mocker, whitelist_conf) -> None:
|
|||
|
||||
with pytest.raises(OperationalException,
|
||||
match=r"PrecisionFilter can only work with stoploss defined\..*"):
|
||||
PairListManager(MagicMock, whitelist_conf)
|
||||
PairListManager(MagicMock, whitelist_conf, MagicMock())
|
||||
|
||||
|
||||
def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None:
|
||||
|
|
@ -703,7 +710,7 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None:
|
|||
del Trade.query
|
||||
mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
|
||||
exchange = get_patched_exchange(mocker, whitelist_conf)
|
||||
pm = PairListManager(exchange, whitelist_conf)
|
||||
pm = PairListManager(exchange, whitelist_conf, MagicMock())
|
||||
pm.refresh_pairlist()
|
||||
|
||||
assert log_has("PerformanceFilter is not available in this mode.", caplog)
|
||||
|
|
@ -1167,6 +1174,10 @@ def test_spreadfilter_invalid_data(mocker, default_conf, markets, tickers, caplo
|
|||
"[{'OffsetFilter': 'OffsetFilter - Taking 10 Pairs, starting from 5.'}]",
|
||||
None
|
||||
),
|
||||
({"method": "ProducerPairList"},
|
||||
"[{'ProducerPairList': 'ProducerPairList - default'}]",
|
||||
None
|
||||
),
|
||||
])
|
||||
def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig,
|
||||
desc_expected, exception_expected):
|
||||
|
|
@ -1341,3 +1352,77 @@ def test_expand_pairlist_keep_invalid(wildcardlist, pairs, expected):
|
|||
expand_pairlist(wildcardlist, pairs, keep_invalid=True)
|
||||
else:
|
||||
assert sorted(expand_pairlist(wildcardlist, pairs, keep_invalid=True)) == sorted(expected)
|
||||
|
||||
|
||||
def test_ProducerPairlist_no_emc(mocker, whitelist_conf):
|
||||
mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
|
||||
|
||||
whitelist_conf['pairlists'] = [
|
||||
{
|
||||
"method": "ProducerPairList",
|
||||
"number_assets": 10,
|
||||
"producer_name": "hello_world",
|
||||
}
|
||||
]
|
||||
del whitelist_conf['external_message_consumer']
|
||||
|
||||
with pytest.raises(OperationalException,
|
||||
match=r"ProducerPairList requires external_message_consumer to be enabled."):
|
||||
get_patched_freqtradebot(mocker, whitelist_conf)
|
||||
|
||||
|
||||
def test_ProducerPairlist(mocker, whitelist_conf, markets):
|
||||
mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
|
||||
mocker.patch.multiple('freqtrade.exchange.Exchange',
|
||||
markets=PropertyMock(return_value=markets),
|
||||
exchange_has=MagicMock(return_value=True),
|
||||
)
|
||||
whitelist_conf['pairlists'] = [
|
||||
{
|
||||
"method": "ProducerPairList",
|
||||
"number_assets": 2,
|
||||
"producer_name": "hello_world",
|
||||
}
|
||||
]
|
||||
whitelist_conf.update({
|
||||
"external_message_consumer": {
|
||||
"enabled": True,
|
||||
"producers": [
|
||||
{
|
||||
"name": "hello_world",
|
||||
"host": "null",
|
||||
"port": 9891,
|
||||
"ws_token": "dummy",
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
exchange = get_patched_exchange(mocker, whitelist_conf)
|
||||
dp = DataProvider(whitelist_conf, exchange, None)
|
||||
pairs = ['ETH/BTC', 'LTC/BTC', 'XRP/BTC']
|
||||
# different producer
|
||||
dp._set_producer_pairs(pairs + ['MEEP/USDT'], 'default')
|
||||
pm = PairListManager(exchange, whitelist_conf, dp)
|
||||
pm.refresh_pairlist()
|
||||
assert pm.whitelist == []
|
||||
# proper producer
|
||||
dp._set_producer_pairs(pairs, 'hello_world')
|
||||
pm.refresh_pairlist()
|
||||
|
||||
# Pairlist reduced to 2
|
||||
assert pm.whitelist == pairs[:2]
|
||||
assert len(pm.whitelist) == 2
|
||||
whitelist_conf['exchange']['pair_whitelist'] = ['TKN/BTC']
|
||||
|
||||
whitelist_conf['pairlists'] = [
|
||||
{"method": "StaticPairList"},
|
||||
{
|
||||
"method": "ProducerPairList",
|
||||
"producer_name": "hello_world",
|
||||
}
|
||||
]
|
||||
pm = PairListManager(exchange, whitelist_conf, dp)
|
||||
pm.refresh_pairlist()
|
||||
assert len(pm.whitelist) == 4
|
||||
assert pm.whitelist == ['TKN/BTC'] + pairs
|
||||
|
|
|
|||
Loading…
Reference in New Issue