adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #7473 from freqtrade/feat/producerpairlist

Browse Source 
Producerpairlist
This commit is contained in:
Matthias 2022-09-30 06:54:15 +02:00 committed by GitHub
commit 2ce265bed3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 222 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
docs/includes/pairlists.md
View File

@ -22,6 +22,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
* [`StaticPairList`](#static-pair-list) (default, if not configured differently)
* [`VolumePairList`](#volume-pair-list)
* [`ProducerPairList`](#producerpairlist)
* [`AgeFilter`](#agefilter)
* [`OffsetFilter`](#offsetfilter)
* [`PerformanceFilter`](#performancefilter)
@ -84,7 +85,7 @@ Filtering instances (not the first position in the list) will not apply any cach
You can define a minimum volume with `min_value` - which will filter out pairs with a volume lower than the specified value in the specified timerange.
### VolumePairList Advanced mode
##### VolumePairList Advanced mode
`VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
@ -146,6 +147,32 @@ More sophisticated approach can be used, by using `lookback_timeframe` for candl
!!! Note
`VolumePairList` does not support backtesting mode.
#### ProducerPairList
With `ProducerPairList`, you can reuse the pairlist from a [Producer](producer-consumer.md) without explicitly defining the pairlist on each consumer.
[Consumer mode](producer-consumer.md) is required for this pairlist to work.
The pairlist will perform a check on active pairs against the current exchange configuration to avoid attempting to trade on invalid markets.
You can limit the length of the pairlist with the optional parameter `number_assets`. Using `"number_assets"=0` or omitting this key will result in the reuse of all producer pairs valid for the current setup.
```json
"pairlists": [
{
"method": "ProducerPairList",
"number_assets": 5,
"producer_name": "default",
}
],
```
!!! Tip "Combining pairlists"
This pairlist can be combined with all other pairlists and filters for further pairlist reduction, and can also act as an "additional" pairlist, on top of already defined pairs.
`ProducerPairList` can also be used multiple times in sequence, combining the pairs from multiple producers.
Obviously in complex such configurations, the Producer may not provide data for all pairs, so the strategy must be fit for this.
#### AgeFilter
Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity).

2
freqtrade/constants.py
View File

@ -31,7 +31,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
'CalmarHyperOptLoss',
'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
'ProfitDrawDownHyperOptLoss']
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList',
'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']

5
freqtrade/freqtradebot.py
View File

@ -82,7 +82,10 @@ class FreqtradeBot(LoggingMixin):
# Keep this at the end of this initialization method.
self.rpc: RPCManager = RPCManager(self)
self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists, self.rpc)
self.dataprovider = DataProvider(self.config, self.exchange, rpc=self.rpc)
self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider)
self.dataprovider.add_pairlisthandler(self.pairlists)
# Attach Dataprovider to strategy instance
self.strategy.dp = self.dataprovider

2
freqtrade/optimize/backtesting.py
View File

@ -110,7 +110,7 @@ class Backtesting:
self.timeframe = str(self.config.get('timeframe'))
self.timeframe_min = timeframe_to_minutes(self.timeframe)
self.init_backtest_detail()
self.pairlists = PairListManager(self.exchange, self.config)
self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider)
if 'VolumePairList' in self.pairlists.name_list:
raise OperationalException("VolumePairList not allowed for backtesting. "
"Please use StaticPairList instead.")

90
freqtrade/plugins/pairlist/ProducerPairList.py Normal file
View File

@ -0,0 +1,90 @@
"""
External Pair List provider
Provides pair list from Leader data
"""
import logging
from typing import Any, Dict, List, Optional
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)
class ProducerPairList(IPairList):
"""
PairList plugin for use with external_message_consumer.
Will use pairs given from leader data.
Usage:
"pairlists": [
{
"method": "ProducerPairList",
"number_assets": 5,
"producer_name": "default",
}
],
"""
def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
self._num_assets: int = self._pairlistconfig.get('number_assets', 0)
self._producer_name = self._pairlistconfig.get('producer_name', 'default')
if not config.get('external_message_consumer', {}).get('enabled'):
raise OperationalException(
"ProducerPairList requires external_message_consumer to be enabled.")
@property
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requires tickers, an empty Dict is passed
as tickers argument to filter_pairlist
"""
return False
def short_desc(self) -> str:
"""
Short whitelist method description - used for startup-messages
-> Please overwrite in subclasses
"""
return f"{self.name} - {self._producer_name}"
def _filter_pairlist(self, pairlist: Optional[List[str]]):
upstream_pairlist = self._pairlistmanager._dataprovider.get_producer_pairs(
self._producer_name)
if pairlist is None:
pairlist = self._pairlistmanager._dataprovider.get_producer_pairs(self._producer_name)
pairs = list(dict.fromkeys(pairlist + upstream_pairlist))
if self._num_assets:
pairs = pairs[:self._num_assets]
return pairs
def gen_pairlist(self, tickers: Dict) -> List[str]:
"""
Generate the pairlist
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: List of pairs
"""
pairs = self._filter_pairlist(None)
self.log_once(f"Received pairs: {pairs}", logger.debug)
pairs = self._whitelist_for_active_markets(self.verify_whitelist(pairs, logger.info))
return pairs
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
Filters and sorts pairlist and returns the whitelist again.
Called on each bot iteration - please use internal caching if necessary
:param pairlist: pairlist to filter or sort
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new whitelist
"""
return self._filter_pairlist(pairlist)

2
freqtrade/plugins/pairlist/VolumePairList.py
View File

@ -232,6 +232,4 @@ class VolumePairList(IPairList):
# Limit pairlist to the requested number of pairs
pairs = pairs[:self._number_pairs]
self.log_once(f"Searching {self._number_pairs} pairs: {pairs}", logger.info)
return pairs

8
freqtrade/plugins/pairlistmanager.py
View File

@ -3,11 +3,12 @@ PairList manager class
"""
import logging
from functools import partial
from typing import Dict, List
from typing import Dict, List, Optional
from cachetools import TTLCache, cached
from freqtrade.constants import Config, ListPairsWithTimeframes
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import CandleType
from freqtrade.exceptions import OperationalException
from freqtrade.mixins import LoggingMixin
@ -21,13 +22,14 @@ logger = logging.getLogger(__name__)
class PairListManager(LoggingMixin):
def __init__(self, exchange, config: Config) -> None:
def __init__(self, exchange, config: Config, dataprovider: DataProvider = None) -> None:
self._exchange = exchange
self._config = config
self._whitelist = self._config['exchange'].get('pair_whitelist')
self._blacklist = self._config['exchange'].get('pair_blacklist', [])
self._pairlist_handlers: List[IPairList] = []
self._tickers_needed = False
self._dataprovider: Optional[DataProvider] = dataprovider
for pairlist_handler_config in self._config.get('pairlists', []):
pairlist_handler = PairListResolver.load_pairlist(
pairlist_handler_config['method'],
@ -96,6 +98,8 @@ class PairListManager(LoggingMixin):
# to ensure blacklist is respected.
pairlist = self.verify_blacklist(pairlist, logger.warning)
self.log_once(f"Whitelist with {len(pairlist)} pairs: {pairlist}", logger.info)
self._whitelist = pairlist
def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:

2
tests/conftest.py
View File

@ -200,6 +200,8 @@ def patch_freqtradebot(mocker, config) -> None:
mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock())
mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock())
patch_whitelist(mocker, config)
mocker.patch('freqtrade.freqtradebot.ExternalMessageConsumer')
mocker.patch('freqtrade.configuration.config_validation._validate_consumers')
def get_patched_freqtradebot(mocker, config) -> FreqtradeBot:

95
tests/plugins/test_pairlist.py
View File

@ -9,6 +9,7 @@ import pytest
import time_machine
from freqtrade.constants import AVAILABLE_PAIRLISTS
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import CandleType, RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.persistence import Trade
@ -40,6 +41,12 @@ def whitelist_conf(default_conf):
"sort_key": "quoteVolume",
},
]
default_conf.update({
"external_message_consumer": {
"enabled": True,
"producers": [],
}
})
return default_conf
@ -126,7 +133,7 @@ def test_log_cached(mocker, static_pl_conf, markets, tickers):
def test_load_pairlist_noexist(mocker, markets, default_conf):
freqtrade = get_patched_freqtradebot(mocker, default_conf)
mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
plm = PairListManager(freqtrade.exchange, default_conf)
plm = PairListManager(freqtrade.exchange, default_conf, MagicMock())
with pytest.raises(OperationalException,
match=r"Impossible to load Pairlist 'NonexistingPairList'. "
r"This class does not exist or contains Python code errors."):
@ -137,7 +144,7 @@ def test_load_pairlist_noexist(mocker, markets, default_conf):
def test_load_pairlist_verify_multi(mocker, markets_static, default_conf):
freqtrade = get_patched_freqtradebot(mocker, default_conf)
mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets_static))
plm = PairListManager(freqtrade.exchange, default_conf)
plm = PairListManager(freqtrade.exchange, default_conf, MagicMock())
# Call different versions one after the other, should always consider what was passed in
# and have no side-effects (therefore the same check multiple times)
assert plm.verify_whitelist(['ETH/BTC', 'XRP/BTC', ], print) == ['ETH/BTC', 'XRP/BTC']
@ -269,7 +276,7 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co
with pytest.raises(OperationalException,
match=r'`number_assets` not specified. Please check your configuration '
r'for "pairlist.config.number_assets"'):
PairListManager(freqtrade.exchange, whitelist_conf)
PairListManager(freqtrade.exchange, whitelist_conf, MagicMock())
def test_refresh_pairlist_dynamic_2(mocker, shitcoinmarkets, tickers, whitelist_conf_2):
@ -694,7 +701,7 @@ def test_PrecisionFilter_error(mocker, whitelist_conf) -> None:
with pytest.raises(OperationalException,
match=r"PrecisionFilter can only work with stoploss defined\..*"):
PairListManager(MagicMock, whitelist_conf)
PairListManager(MagicMock, whitelist_conf, MagicMock())
def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None:
@ -703,7 +710,7 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None:
del Trade.query
mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
exchange = get_patched_exchange(mocker, whitelist_conf)
pm = PairListManager(exchange, whitelist_conf)
pm = PairListManager(exchange, whitelist_conf, MagicMock())
pm.refresh_pairlist()
assert log_has("PerformanceFilter is not available in this mode.", caplog)
@ -1167,6 +1174,10 @@ def test_spreadfilter_invalid_data(mocker, default_conf, markets, tickers, caplo
"[{'OffsetFilter': 'OffsetFilter - Taking 10 Pairs, starting from 5.'}]",
None
),
({"method": "ProducerPairList"},
"[{'ProducerPairList': 'ProducerPairList - default'}]",
None
),
])
def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig,
desc_expected, exception_expected):
@ -1341,3 +1352,77 @@ def test_expand_pairlist_keep_invalid(wildcardlist, pairs, expected):
expand_pairlist(wildcardlist, pairs, keep_invalid=True)
else:
assert sorted(expand_pairlist(wildcardlist, pairs, keep_invalid=True)) == sorted(expected)
def test_ProducerPairlist_no_emc(mocker, whitelist_conf):
mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
whitelist_conf['pairlists'] = [
{
"method": "ProducerPairList",
"number_assets": 10,
"producer_name": "hello_world",
}
]
del whitelist_conf['external_message_consumer']
with pytest.raises(OperationalException,
match=r"ProducerPairList requires external_message_consumer to be enabled."):
get_patched_freqtradebot(mocker, whitelist_conf)
def test_ProducerPairlist(mocker, whitelist_conf, markets):
mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
mocker.patch.multiple('freqtrade.exchange.Exchange',
markets=PropertyMock(return_value=markets),
exchange_has=MagicMock(return_value=True),
)
whitelist_conf['pairlists'] = [
{
"method": "ProducerPairList",
"number_assets": 2,
"producer_name": "hello_world",
}
]
whitelist_conf.update({
"external_message_consumer": {
"enabled": True,
"producers": [
{
"name": "hello_world",
"host": "null",
"port": 9891,
"ws_token": "dummy",
}
]
}
})
exchange = get_patched_exchange(mocker, whitelist_conf)
dp = DataProvider(whitelist_conf, exchange, None)
pairs = ['ETH/BTC', 'LTC/BTC', 'XRP/BTC']
# different producer
dp._set_producer_pairs(pairs + ['MEEP/USDT'], 'default')
pm = PairListManager(exchange, whitelist_conf, dp)
pm.refresh_pairlist()
assert pm.whitelist == []
# proper producer
dp._set_producer_pairs(pairs, 'hello_world')
pm.refresh_pairlist()
# Pairlist reduced to 2
assert pm.whitelist == pairs[:2]
assert len(pm.whitelist) == 2
whitelist_conf['exchange']['pair_whitelist'] = ['TKN/BTC']
whitelist_conf['pairlists'] = [
{"method": "StaticPairList"},
{
"method": "ProducerPairList",
"producer_name": "hello_world",
}
]
pm = PairListManager(exchange, whitelist_conf, dp)
pm.refresh_pairlist()
assert len(pm.whitelist) == 4
assert pm.whitelist == ['TKN/BTC'] + pairs