adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

parse exception handling, remove info, cache change

Browse Source
This commit is contained in:
Bloodhunter4rc 2022-12-19 15:36:28 +01:00
parent 6380c3d462
commit 43f5a16006
3 changed files with 41 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/includes/pairlists.md
View File

@ -202,11 +202,10 @@ The user is responsible for providing a server or local file that returns a JSON
{
"pairs": ["XRP/USDT", "ETH/USDT", "LTC/USDT"],
"refresh_period": 1800,
"info": "Pairlist updated on 2022-12-12 at 12:12" // Maximum Length: 256 Characters, Charset: Alphanumeric + "+-.,%:"
}
```
The `pairs` property should contain a list of strings with the trading pairs to be used by the bot. The `refresh_period` property is optional and specifies the number of seconds that the pairlist should be cached before being refreshed. The `info` property is also optional and can be used to provide any additional information about the pairlist.
The `pairs` property should contain a list of strings with the trading pairs to be used by the bot. The `refresh_period` property is optional and specifies the number of seconds that the pairlist should be cached before being refreshed.
The optional `keep_pairlist_on_failure` specifies whether the previous received pairlist should be used if the remote server is not reachable or returns an error. The default value is true.

68
freqtrade/plugins/pairlist/RemotePairList.py
View File

@ -6,7 +6,7 @@ Provides pair list fetched from a remote source
import json
import logging
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, List, Tuple
import requests
from cachetools import TTLCache
@ -41,7 +41,7 @@ class RemotePairList(IPairList):
self._number_pairs = self._pairlistconfig['number_assets']
self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800)
self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True)
self._pair_cache: Optional[TTLCache] = None
self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
self._pairlist_url = self._pairlistconfig.get('pairlist_url', '')
self._read_timeout = self._pairlistconfig.get('read_timeout', 60)
self._bearer_token = self._pairlistconfig.get('bearer_token', '')
@ -63,28 +63,20 @@ class RemotePairList(IPairList):
"""
return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist."
def process_json(self, jsonparse) -> Tuple[List[str], str]:
def process_json(self, jsonparse) -> List[str]:
pairlist = jsonparse.get('pairs', [])
remote_info = jsonparse.get('info', '')[:256].strip()
remote_refresh_period = jsonparse.get('refresh_period', self._refresh_period)
remote_refresh_period = int(jsonparse.get('refresh_period', self._refresh_period))
info = "".join(char if char.isalnum() or
char in " +-.,%:" else "-" for char in remote_info)
if not self._init_done:
if self._refresh_period < remote_refresh_period:
self.log_once(f'Refresh Period has been increased from {self._refresh_period}'
f' to {remote_refresh_period} from Remote.', logger.info)
self._refresh_period = remote_refresh_period
self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period)
else:
self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period)
if self._refresh_period < remote_refresh_period:
self.log_once(f'Refresh Period has been increased from {self._refresh_period}'
f' to minimum allowed: {remote_refresh_period} from Remote.', logger.info)
self._refresh_period = remote_refresh_period
self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period)
self._init_done = True
return pairlist, info
return pairlist
def return_last_pairlist(self) -> List[str]:
if self._keep_pairlist_on_failure:
@ -95,7 +87,7 @@ class RemotePairList(IPairList):
return pairlist
def fetch_pairlist(self) -> Tuple[List[str], float, str]:
def fetch_pairlist(self) -> Tuple[List[str], float]:
headers = {
'User-Agent': 'Freqtrade/' + __version__ + ' Remotepairlist'
@ -104,8 +96,6 @@ class RemotePairList(IPairList):
if self._bearer_token:
headers['Authorization'] = f'Bearer {self._bearer_token}'
info = "Pairlist"
try:
response = requests.get(self._pairlist_url, headers=headers,
timeout=self._read_timeout)
@ -114,7 +104,17 @@ class RemotePairList(IPairList):
if "application/json" in str(content_type):
jsonparse = response.json()
pairlist, info = self.process_json(jsonparse)
try:
pairlist = self.process_json(jsonparse)
except Exception as e:
if self._init_done:
pairlist = self.return_last_pairlist()
logger.warning(f'Error while processing JSON data: {type(e)}')
else:
raise OperationalException(f'Error while processing JSON data: {type(e)}')
else:
if self._init_done:
self.log_once(f'Error: RemotePairList is not of type JSON: '
@ -131,7 +131,7 @@ class RemotePairList(IPairList):
time_elapsed = 0
return pairlist, time_elapsed, info
return pairlist, time_elapsed
def gen_pairlist(self, tickers: Tickers) -> List[str]:
"""
@ -140,7 +140,7 @@ class RemotePairList(IPairList):
:return: List of pairs
"""
if self._init_done and self._pair_cache is not None:
if self._init_done:
pairlist = self._pair_cache.get('pairlist')
else:
pairlist = []
@ -159,25 +159,33 @@ class RemotePairList(IPairList):
with open(filename) as json_file:
# Load the JSON data into a dictionary
jsonparse = json.load(json_file)
pairlist, info = self.process_json(jsonparse)
try:
pairlist = self.process_json(jsonparse)
except Exception as e:
if self._init_done:
pairlist = self.return_last_pairlist()
logger.warning(f'Error while processing JSON data: {type(e)}')
else:
raise OperationalException('Error while processing'
f'JSON data: {type(e)}')
else:
raise ValueError(f"{self._pairlist_url} does not exist.")
else:
# Fetch Pairlist from Remote URL
pairlist, time_elapsed, info = self.fetch_pairlist()
pairlist, time_elapsed = self.fetch_pairlist()
self.log_once(f"Fetched pairs: {pairlist}", logger.debug)
pairlist = self._whitelist_for_active_markets(pairlist)
pairlist = pairlist[:self._number_pairs]
if self._pair_cache is not None:
self._pair_cache['pairlist'] = pairlist.copy()
self._pair_cache['pairlist'] = pairlist.copy()
if time_elapsed != 0.0:
self.log_once(f'{info} Fetched in {time_elapsed} seconds.', logger.info)
self.log_once(f'Pairlist Fetched in {time_elapsed} seconds.', logger.info)
else:
self.log_once(f'{info} Fetched Pairlist.', logger.info)
self.log_once('Fetched Pairlist.', logger.info)
self._last_pairlist = list(pairlist)

6
tests/plugins/test_remotepairlist.py
View File

@ -107,7 +107,7 @@ def test_fetch_pairlist_timeout_keep_last_pairlist(mocker, rpl_config, caplog):
remote_pairlist._last_pairlist = ["BTC/USDT", "ETH/USDT", "LTC/USDT"]
pairs, time_elapsed, info = remote_pairlist.fetch_pairlist()
pairs, time_elapsed = remote_pairlist.fetch_pairlist()
assert log_has(f"Was not able to fetch pairlist from: {remote_pairlist._pairlist_url}", caplog)
assert log_has("Keeping last fetched pairlist", caplog)
assert pairs == ["BTC/USDT", "ETH/USDT", "LTC/USDT"]
@ -163,7 +163,6 @@ def test_fetch_pairlist_mock_response_valid(mocker, rpl_config):
mock_response.json.return_value = {
"pairs": ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"],
"info": "Mock pairlist response",
"refresh_period": 60
}
@ -179,9 +178,8 @@ def test_fetch_pairlist_mock_response_valid(mocker, rpl_config):
pairlistmanager = PairListManager(exchange, rpl_config)
remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config,
rpl_config['pairlists'][0], 0)
pairs, time_elapsed, info = remote_pairlist.fetch_pairlist()
pairs, time_elapsed = remote_pairlist.fetch_pairlist()
assert pairs == ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"]
assert time_elapsed == 0.4
assert info == "Mock pairlist response"
assert remote_pairlist._refresh_period == 60