Merge pull request #7867 from Bloodhunter4rc/remotepairlist
Add Remotepairlist
This commit is contained in:
Matthias
GitHub
commit
3012c55ec5
|
|
@ -23,6 +23,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
|
|||
* [`StaticPairList`](#static-pair-list) (default, if not configured differently)
|
||||
* [`VolumePairList`](#volume-pair-list)
|
||||
* [`ProducerPairList`](#producerpairlist)
|
||||
* [`RemotePairList`](#remotepairlist)
|
||||
* [`AgeFilter`](#agefilter)
|
||||
* [`OffsetFilter`](#offsetfilter)
|
||||
* [`PerformanceFilter`](#performancefilter)
|
||||
|
|
@ -173,6 +174,48 @@ You can limit the length of the pairlist with the optional parameter `number_ass
|
|||
`ProducerPairList` can also be used multiple times in sequence, combining the pairs from multiple producers.
|
||||
Obviously in complex such configurations, the Producer may not provide data for all pairs, so the strategy must be fit for this.
|
||||
|
||||
#### RemotePairList
|
||||
|
||||
It allows the user to fetch a pairlist from a remote server or a locally stored json file within the freqtrade directory, enabling dynamic updates and customization of the trading pairlist.
|
||||
|
||||
The RemotePairList is defined in the pairlists section of the configuration settings. It uses the following configuration options:
|
||||
|
||||
```json
|
||||
"pairlists": [
|
||||
{
|
||||
"method": "RemotePairList",
|
||||
"pairlist_url": "https://example.com/pairlist",
|
||||
"number_assets": 10,
|
||||
"refresh_period": 1800,
|
||||
"keep_pairlist_on_failure": true,
|
||||
"read_timeout": 60,
|
||||
"bearer_token": "my-bearer-token"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
The `pairlist_url` option specifies the URL of the remote server where the pairlist is located, or the path to a local file (if file:/// is prepended). This allows the user to use either a remote server or a local file as the source for the pairlist.
|
||||
|
||||
The user is responsible for providing a server or local file that returns a JSON object with the following structure:
|
||||
|
||||
```json
|
||||
{
|
||||
"pairs": ["XRP/USDT", "ETH/USDT", "LTC/USDT"],
|
||||
"refresh_period": 1800,
|
||||
}
|
||||
```
|
||||
|
||||
The `pairs` property should contain a list of strings with the trading pairs to be used by the bot. The `refresh_period` property is optional and specifies the number of seconds that the pairlist should be cached before being refreshed.
|
||||
|
||||
The optional `keep_pairlist_on_failure` specifies whether the previous received pairlist should be used if the remote server is not reachable or returns an error. The default value is true.
|
||||
|
||||
The optional `read_timeout` specifies the maximum amount of time (in seconds) to wait for a response from the remote source, The default value is 60.
|
||||
|
||||
The optional `bearer_token` will be included in the requests Authorization Header.
|
||||
|
||||
!!! Note
|
||||
In case of a server error the last received pairlist will be kept if `keep_pairlist_on_failure` is set to true, when set to false a empty pairlist is returned.
|
||||
|
||||
#### AgeFilter
|
||||
|
||||
Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity).
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
|
|||
'CalmarHyperOptLoss',
|
||||
'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
|
||||
'ProfitDrawDownHyperOptLoss']
|
||||
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList',
|
||||
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList',
|
||||
'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
|
||||
'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
|
||||
'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
|
||||
|
|
|
|||
|
|
@ -0,0 +1,206 @@
|
|||
"""
|
||||
Remote PairList provider
|
||||
|
||||
Provides pair list fetched from a remote source
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import requests
|
||||
from cachetools import TTLCache
|
||||
|
||||
from freqtrade import __version__
|
||||
from freqtrade.constants import Config
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange.types import Tickers
|
||||
from freqtrade.plugins.pairlist.IPairList import IPairList
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RemotePairList(IPairList):
|
||||
|
||||
def __init__(self, exchange, pairlistmanager,
|
||||
config: Config, pairlistconfig: Dict[str, Any],
|
||||
pairlist_pos: int) -> None:
|
||||
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
|
||||
|
||||
if 'number_assets' not in self._pairlistconfig:
|
||||
raise OperationalException(
|
||||
'`number_assets` not specified. Please check your configuration '
|
||||
'for "pairlist.config.number_assets"')
|
||||
|
||||
if 'pairlist_url' not in self._pairlistconfig:
|
||||
raise OperationalException(
|
||||
'`pairlist_url` not specified. Please check your configuration '
|
||||
'for "pairlist.config.pairlist_url"')
|
||||
|
||||
self._number_pairs = self._pairlistconfig['number_assets']
|
||||
self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800)
|
||||
self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True)
|
||||
self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
|
||||
self._pairlist_url = self._pairlistconfig.get('pairlist_url', '')
|
||||
self._read_timeout = self._pairlistconfig.get('read_timeout', 60)
|
||||
self._bearer_token = self._pairlistconfig.get('bearer_token', '')
|
||||
self._init_done = False
|
||||
self._last_pairlist: List[Any] = list()
|
||||
|
||||
@property
|
||||
def needstickers(self) -> bool:
|
||||
"""
|
||||
Boolean property defining if tickers are necessary.
|
||||
If no Pairlist requires tickers, an empty Dict is passed
|
||||
as tickers argument to filter_pairlist
|
||||
"""
|
||||
return False
|
||||
|
||||
def short_desc(self) -> str:
|
||||
"""
|
||||
Short whitelist method description - used for startup-messages
|
||||
"""
|
||||
return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist."
|
||||
|
||||
def process_json(self, jsonparse) -> List[str]:
|
||||
|
||||
pairlist = jsonparse.get('pairs', [])
|
||||
remote_refresh_period = int(jsonparse.get('refresh_period', self._refresh_period))
|
||||
|
||||
if self._refresh_period < remote_refresh_period:
|
||||
self.log_once(f'Refresh Period has been increased from {self._refresh_period}'
|
||||
f' to minimum allowed: {remote_refresh_period} from Remote.', logger.info)
|
||||
|
||||
self._refresh_period = remote_refresh_period
|
||||
self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period)
|
||||
|
||||
self._init_done = True
|
||||
|
||||
return pairlist
|
||||
|
||||
def return_last_pairlist(self) -> List[str]:
|
||||
if self._keep_pairlist_on_failure:
|
||||
pairlist = self._last_pairlist
|
||||
self.log_once('Keeping last fetched pairlist', logger.info)
|
||||
else:
|
||||
pairlist = []
|
||||
|
||||
return pairlist
|
||||
|
||||
def fetch_pairlist(self) -> Tuple[List[str], float]:
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Freqtrade/' + __version__ + ' Remotepairlist'
|
||||
}
|
||||
|
||||
if self._bearer_token:
|
||||
headers['Authorization'] = f'Bearer {self._bearer_token}'
|
||||
|
||||
try:
|
||||
response = requests.get(self._pairlist_url, headers=headers,
|
||||
timeout=self._read_timeout)
|
||||
content_type = response.headers.get('content-type')
|
||||
time_elapsed = response.elapsed.total_seconds()
|
||||
|
||||
if "application/json" in str(content_type):
|
||||
jsonparse = response.json()
|
||||
|
||||
try:
|
||||
pairlist = self.process_json(jsonparse)
|
||||
except Exception as e:
|
||||
|
||||
if self._init_done:
|
||||
pairlist = self.return_last_pairlist()
|
||||
logger.warning(f'Error while processing JSON data: {type(e)}')
|
||||
else:
|
||||
raise OperationalException(f'Error while processing JSON data: {type(e)}')
|
||||
|
||||
else:
|
||||
if self._init_done:
|
||||
self.log_once(f'Error: RemotePairList is not of type JSON: '
|
||||
f' {self._pairlist_url}', logger.info)
|
||||
pairlist = self.return_last_pairlist()
|
||||
else:
|
||||
raise OperationalException('RemotePairList is not of type JSON, abort.')
|
||||
|
||||
except requests.exceptions.RequestException:
|
||||
self.log_once(f'Was not able to fetch pairlist from:'
|
||||
f' {self._pairlist_url}', logger.info)
|
||||
|
||||
pairlist = self.return_last_pairlist()
|
||||
|
||||
time_elapsed = 0
|
||||
|
||||
return pairlist, time_elapsed
|
||||
|
||||
def gen_pairlist(self, tickers: Tickers) -> List[str]:
|
||||
"""
|
||||
Generate the pairlist
|
||||
:param tickers: Tickers (from exchange.get_tickers). May be cached.
|
||||
:return: List of pairs
|
||||
"""
|
||||
|
||||
if self._init_done:
|
||||
pairlist = self._pair_cache.get('pairlist')
|
||||
else:
|
||||
pairlist = []
|
||||
|
||||
time_elapsed = 0.0
|
||||
|
||||
if pairlist:
|
||||
# Item found - no refresh necessary
|
||||
return pairlist.copy()
|
||||
else:
|
||||
if self._pairlist_url.startswith("file:///"):
|
||||
filename = self._pairlist_url.split("file:///", 1)[1]
|
||||
file_path = Path(filename)
|
||||
|
||||
if file_path.exists():
|
||||
with open(filename) as json_file:
|
||||
# Load the JSON data into a dictionary
|
||||
jsonparse = json.load(json_file)
|
||||
|
||||
try:
|
||||
pairlist = self.process_json(jsonparse)
|
||||
except Exception as e:
|
||||
if self._init_done:
|
||||
pairlist = self.return_last_pairlist()
|
||||
logger.warning(f'Error while processing JSON data: {type(e)}')
|
||||
else:
|
||||
raise OperationalException('Error while processing'
|
||||
f'JSON data: {type(e)}')
|
||||
else:
|
||||
raise ValueError(f"{self._pairlist_url} does not exist.")
|
||||
else:
|
||||
# Fetch Pairlist from Remote URL
|
||||
pairlist, time_elapsed = self.fetch_pairlist()
|
||||
|
||||
self.log_once(f"Fetched pairs: {pairlist}", logger.debug)
|
||||
|
||||
pairlist = self._whitelist_for_active_markets(pairlist)
|
||||
pairlist = pairlist[:self._number_pairs]
|
||||
|
||||
self._pair_cache['pairlist'] = pairlist.copy()
|
||||
|
||||
if time_elapsed != 0.0:
|
||||
self.log_once(f'Pairlist Fetched in {time_elapsed} seconds.', logger.info)
|
||||
else:
|
||||
self.log_once('Fetched Pairlist.', logger.info)
|
||||
|
||||
self._last_pairlist = list(pairlist)
|
||||
|
||||
return pairlist
|
||||
|
||||
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
|
||||
"""
|
||||
Filters and sorts pairlist and returns the whitelist again.
|
||||
Called on each bot iteration - please use internal caching if necessary
|
||||
:param pairlist: pairlist to filter or sort
|
||||
:param tickers: Tickers (from exchange.get_tickers). May be cached.
|
||||
:return: new whitelist
|
||||
"""
|
||||
rpl_pairlist = self.gen_pairlist(tickers)
|
||||
merged_list = pairlist + rpl_pairlist
|
||||
merged_list = sorted(set(merged_list), key=merged_list.index)
|
||||
return merged_list
|
||||
|
|
@ -22,6 +22,11 @@ from tests.conftest import (create_mock_trades_usdt, get_patched_exchange, get_p
|
|||
log_has, log_has_re, num_log_has)
|
||||
|
||||
|
||||
# Exclude RemotePairList from tests.
|
||||
# It has a mandatory parameter, and requires special handling, which happens in test_remotepairlist.
|
||||
TESTABLE_PAIRLISTS = [p for p in AVAILABLE_PAIRLISTS if p not in ['RemotePairList']]
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def whitelist_conf(default_conf):
|
||||
default_conf['stake_currency'] = 'BTC'
|
||||
|
|
@ -824,7 +829,7 @@ def test_pair_whitelist_not_supported_Spread(mocker, default_conf, tickers) -> N
|
|||
get_patched_freqtradebot(mocker, default_conf)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS)
|
||||
@pytest.mark.parametrize("pairlist", TESTABLE_PAIRLISTS)
|
||||
def test_pairlist_class(mocker, whitelist_conf, markets, pairlist):
|
||||
whitelist_conf['pairlists'][0]['method'] = pairlist
|
||||
mocker.patch.multiple('freqtrade.exchange.Exchange',
|
||||
|
|
@ -839,7 +844,7 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist):
|
|||
assert isinstance(freqtrade.pairlists.blacklist, list)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS)
|
||||
@pytest.mark.parametrize("pairlist", TESTABLE_PAIRLISTS)
|
||||
@pytest.mark.parametrize("whitelist,log_message", [
|
||||
(['ETH/BTC', 'TKN/BTC'], ""),
|
||||
# TRX/ETH not in markets
|
||||
|
|
@ -872,7 +877,7 @@ def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist
|
|||
assert log_message in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS)
|
||||
@pytest.mark.parametrize("pairlist", TESTABLE_PAIRLISTS)
|
||||
def test__whitelist_for_active_markets_empty(mocker, whitelist_conf, pairlist, tickers):
|
||||
whitelist_conf['pairlists'][0]['method'] = pairlist
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,185 @@
|
|||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.plugins.pairlist.RemotePairList import RemotePairList
|
||||
from freqtrade.plugins.pairlistmanager import PairListManager
|
||||
from tests.conftest import get_patched_exchange, get_patched_freqtradebot, log_has
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def rpl_config(default_conf):
|
||||
default_conf['stake_currency'] = 'USDT'
|
||||
|
||||
default_conf['exchange']['pair_whitelist'] = [
|
||||
'ETH/USDT',
|
||||
'BTC/USDT',
|
||||
]
|
||||
default_conf['exchange']['pair_blacklist'] = [
|
||||
'BLK/USDT'
|
||||
]
|
||||
return default_conf
|
||||
|
||||
|
||||
def test_gen_pairlist_with_local_file(mocker, rpl_config):
|
||||
|
||||
mock_file = MagicMock()
|
||||
mock_file.read.return_value = '{"pairs": ["TKN/USDT","ETH/USDT","NANO/USDT"]}'
|
||||
mocker.patch('freqtrade.plugins.pairlist.RemotePairList.open', return_value=mock_file)
|
||||
|
||||
mock_file_path = mocker.patch('freqtrade.plugins.pairlist.RemotePairList.Path')
|
||||
mock_file_path.exists.return_value = True
|
||||
|
||||
jsonparse = json.loads(mock_file.read.return_value)
|
||||
mocker.patch('freqtrade.plugins.pairlist.RemotePairList.json.load', return_value=jsonparse)
|
||||
|
||||
rpl_config['pairlists'] = [
|
||||
{
|
||||
"method": "RemotePairList",
|
||||
'number_assets': 2,
|
||||
'refresh_period': 1800,
|
||||
'keep_pairlist_on_failure': True,
|
||||
'pairlist_url': 'file:///pairlist.json',
|
||||
'bearer_token': '',
|
||||
'read_timeout': 60
|
||||
}
|
||||
]
|
||||
|
||||
exchange = get_patched_exchange(mocker, rpl_config)
|
||||
pairlistmanager = PairListManager(exchange, rpl_config)
|
||||
|
||||
remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config,
|
||||
rpl_config['pairlists'][0], 0)
|
||||
|
||||
result = remote_pairlist.gen_pairlist([])
|
||||
|
||||
assert result == ['TKN/USDT', 'ETH/USDT']
|
||||
|
||||
|
||||
def test_fetch_pairlist_mock_response_html(mocker, rpl_config):
|
||||
mock_response = MagicMock()
|
||||
mock_response.headers = {'content-type': 'text/html'}
|
||||
|
||||
rpl_config['pairlists'] = [
|
||||
{
|
||||
"method": "RemotePairList",
|
||||
"pairlist_url": "http://example.com/pairlist",
|
||||
"number_assets": 10,
|
||||
"read_timeout": 10,
|
||||
"keep_pairlist_on_failure": True,
|
||||
}
|
||||
]
|
||||
|
||||
exchange = get_patched_exchange(mocker, rpl_config)
|
||||
pairlistmanager = PairListManager(exchange, rpl_config)
|
||||
|
||||
mocker.patch("freqtrade.plugins.pairlist.RemotePairList.requests.get",
|
||||
return_value=mock_response)
|
||||
remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config,
|
||||
rpl_config['pairlists'][0], 0)
|
||||
|
||||
with pytest.raises(OperationalException, match='RemotePairList is not of type JSON, abort.'):
|
||||
remote_pairlist.fetch_pairlist()
|
||||
|
||||
|
||||
def test_fetch_pairlist_timeout_keep_last_pairlist(mocker, rpl_config, caplog):
|
||||
rpl_config['pairlists'] = [
|
||||
{
|
||||
"method": "RemotePairList",
|
||||
"pairlist_url": "http://example.com/pairlist",
|
||||
"number_assets": 10,
|
||||
"read_timeout": 10,
|
||||
"keep_pairlist_on_failure": True,
|
||||
}
|
||||
]
|
||||
|
||||
exchange = get_patched_exchange(mocker, rpl_config)
|
||||
pairlistmanager = PairListManager(exchange, rpl_config)
|
||||
|
||||
mocker.patch("freqtrade.plugins.pairlist.RemotePairList.requests.get",
|
||||
side_effect=requests.exceptions.RequestException)
|
||||
|
||||
remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config,
|
||||
rpl_config['pairlists'][0], 0)
|
||||
|
||||
remote_pairlist._last_pairlist = ["BTC/USDT", "ETH/USDT", "LTC/USDT"]
|
||||
|
||||
pairs, time_elapsed = remote_pairlist.fetch_pairlist()
|
||||
assert log_has(f"Was not able to fetch pairlist from: {remote_pairlist._pairlist_url}", caplog)
|
||||
assert log_has("Keeping last fetched pairlist", caplog)
|
||||
assert pairs == ["BTC/USDT", "ETH/USDT", "LTC/USDT"]
|
||||
|
||||
|
||||
def test_remote_pairlist_init_no_pairlist_url(mocker, rpl_config):
|
||||
|
||||
rpl_config['pairlists'] = [
|
||||
{
|
||||
"method": "RemotePairList",
|
||||
"number_assets": 10,
|
||||
"keep_pairlist_on_failure": True,
|
||||
}
|
||||
]
|
||||
|
||||
get_patched_exchange(mocker, rpl_config)
|
||||
with pytest.raises(OperationalException, match=r'`pairlist_url` not specified.'
|
||||
r' Please check your configuration for "pairlist.config.pairlist_url"'):
|
||||
get_patched_freqtradebot(mocker, rpl_config)
|
||||
|
||||
|
||||
def test_remote_pairlist_init_no_number_assets(mocker, rpl_config):
|
||||
|
||||
rpl_config['pairlists'] = [
|
||||
{
|
||||
"method": "RemotePairList",
|
||||
"pairlist_url": "http://example.com/pairlist",
|
||||
"keep_pairlist_on_failure": True,
|
||||
}
|
||||
]
|
||||
|
||||
get_patched_exchange(mocker, rpl_config)
|
||||
|
||||
with pytest.raises(OperationalException, match=r'`number_assets` not specified. '
|
||||
'Please check your configuration for "pairlist.config.number_assets"'):
|
||||
get_patched_freqtradebot(mocker, rpl_config)
|
||||
|
||||
|
||||
def test_fetch_pairlist_mock_response_valid(mocker, rpl_config):
|
||||
|
||||
rpl_config['pairlists'] = [
|
||||
{
|
||||
"method": "RemotePairList",
|
||||
"pairlist_url": "http://example.com/pairlist",
|
||||
"number_assets": 10,
|
||||
"refresh_period": 10,
|
||||
"read_timeout": 10,
|
||||
"keep_pairlist_on_failure": True,
|
||||
}
|
||||
]
|
||||
|
||||
mock_response = MagicMock()
|
||||
|
||||
mock_response.json.return_value = {
|
||||
"pairs": ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"],
|
||||
"refresh_period": 60
|
||||
}
|
||||
|
||||
mock_response.headers = {
|
||||
"content-type": "application/json"
|
||||
}
|
||||
|
||||
mock_response.elapsed.total_seconds.return_value = 0.4
|
||||
mocker.patch("freqtrade.plugins.pairlist.RemotePairList.requests.get",
|
||||
return_value=mock_response)
|
||||
|
||||
exchange = get_patched_exchange(mocker, rpl_config)
|
||||
pairlistmanager = PairListManager(exchange, rpl_config)
|
||||
remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config,
|
||||
rpl_config['pairlists'][0], 0)
|
||||
pairs, time_elapsed = remote_pairlist.fetch_pairlist()
|
||||
|
||||
assert pairs == ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"]
|
||||
assert time_elapsed == 0.4
|
||||
assert remote_pairlist._refresh_period == 60
|
||||
Loading…
Reference in New Issue