adelorenzo
/
stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

init cache on first iteration, init checks, limit length and charmap to info replace if invalid, move filter logic

This commit is contained in:
Bloodhunter4rc 2022-12-18 22:28:12 +01:00
parent 6fa3db3a1d
commit bb33b96ba7
2 changed files with 60 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/includes/pairlists.md
View File

@ -202,7 +202,7 @@ The user is responsible for providing a server or local file that returns a JSON
{
"pairs": ["XRP/USDT", "ETH/USDT", "LTC/USDT"],
"refresh_period": 1800,
"info": "Pairlist updated on 2022-12-12 at 12:12"
"info": "Pairlist updated on 2022-12-12 at 12:12" // Maximum Length: 256 Characters, Charset: Alphanumeric + "+-.,%:"
}
```

87
freqtrade/plugins/pairlist/RemotePairList.py
View File

@ -6,7 +6,7 @@ Provides pair list fetched from a remote source
import json
import logging
from pathlib import Path
from typing import Any, Dict, List, Tuple
from typing import Any, Dict, List, Optional, Tuple
import requests
from cachetools import TTLCache
@ -39,12 +39,13 @@ class RemotePairList(IPairList):
'for "pairlist.config.pairlist_url"')
self._number_pairs = self._pairlistconfig['number_assets']
self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800)
self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True)
self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
self._pair_cache: Optional[TTLCache] = None
self._pairlist_url = self._pairlistconfig.get('pairlist_url', '')
self._read_timeout = self._pairlistconfig.get('read_timeout', 60)
self._bearer_token = self._pairlistconfig.get('bearer_token', '')
self._init_done = False
self._last_pairlist: List[Any] = list()
@property
@ -62,6 +63,15 @@ class RemotePairList(IPairList):
"""
return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist."
def return_last_pairlist(self) -> List[str]:
if self._keep_pairlist_on_failure:
pairlist = self._last_pairlist
self.log_once('Keeping last fetched pairlist', logger.info)
else:
pairlist = []
return pairlist
def fetch_pairlist(self) -> Tuple[List[str], float, str]:
headers = {
@ -81,23 +91,35 @@ class RemotePairList(IPairList):
if "application/json" in str(content_type):
jsonparse = response.json()
pairlist = jsonparse['pairs']
info = jsonparse.get('info', '')
else:
raise OperationalException('RemotePairList is not of type JSON abort ')
pairlist = jsonparse.get('pairs', [])
remote_info = jsonparse.get('info', '')[:256].strip()
remote_refresh_period = jsonparse.get('refresh_period', self._refresh_period)
self._refresh_period = jsonparse.get('refresh_period', self._refresh_period)
self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period)
info = "".join(char if char.isalnum() or
char in " +-.,%:" else "-" for char in remote_info)
if not self._init_done and self._refresh_period < remote_refresh_period:
self.log_once(f'Refresh Period has been increased from {self._refresh_period}'
f' to {remote_refresh_period} from Remote.', logger.info)
self._refresh_period = remote_refresh_period
self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period)
self._init_done = True
else:
if self._init_done:
self.log_once(f'Error: RemotePairList is not of type JSON: '
f' {self._pairlist_url}', logger.info)
pairlist = self.return_last_pairlist()
else:
raise OperationalException('RemotePairList is not of type JSON abort ')
except requests.exceptions.RequestException:
self.log_once(f'Was not able to fetch pairlist from:'
f' {self._pairlist_url}', logger.info)
if self._keep_pairlist_on_failure:
pairlist = self._last_pairlist
self.log_once('Keeping last fetched pairlist', logger.info)
else:
pairlist = []
pairlist = self.return_last_pairlist()
time_elapsed = 0
@ -110,12 +132,17 @@ class RemotePairList(IPairList):
:return: List of pairs
"""
pairlist = self._pair_cache.get('pairlist')
if self._init_done and self._pair_cache:
pairlist = self._pair_cache.get('pairlist')
else:
pairlist = []
time_elapsed = 0.0
if pairlist:
# Item found - no refresh necessary
return pairlist.copy()
self._init_done = True
else:
if self._pairlist_url.startswith("file:///"):
filename = self._pairlist_url.split("file:///", 1)[1]
@ -127,17 +154,25 @@ class RemotePairList(IPairList):
jsonparse = json.load(json_file)
pairlist = jsonparse['pairs']
info = jsonparse.get('info', '')
self._refresh_period = jsonparse.get('refresh_period', self._refresh_period)
self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period)
if not self._init_done:
self._refresh_period = jsonparse.get('refresh_period',
self._refresh_period)
self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period)
self._init_done = True
else:
raise ValueError(f"{self._pairlist_url} does not exist.")
else:
# Fetch Pairlist from Remote URL
pairlist, time_elapsed, info = self.fetch_pairlist()
pairlist = self.filter_pairlist(pairlist, tickers)
self._pair_cache['pairlist'] = pairlist.copy()
self.log_once(f"Fetched pairs: {pairlist}", logger.debug)
pairlist = self._whitelist_for_active_markets(pairlist)
pairlist = pairlist[:self._number_pairs]
if self._pair_cache:
self._pair_cache['pairlist'] = pairlist.copy()
if time_elapsed != 0.0:
self.log_once(f'{info} Fetched in {time_elapsed} seconds.', logger.info)
@ -145,6 +180,7 @@ class RemotePairList(IPairList):
self.log_once(f'{info} Fetched Pairlist.', logger.info)
self._last_pairlist = list(pairlist)
return pairlist
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
@ -155,12 +191,7 @@ class RemotePairList(IPairList):
:param tickers: Tickers (from exchange.get_tickers). May be cached.
:return: new whitelist
"""
# Validate whitelist to only have active market pairs
pairlist = self._whitelist_for_active_markets(pairlist)
pairlist = self.verify_blacklist(pairlist, logger.info)
# Limit pairlist to the requested number of pairs
pairlist = pairlist[:self._number_pairs]
self.log_once(f"Searching {self._number_pairs} pairs: {pairlist}", logger.info)
return pairlist
rpl_pairlist = self.gen_pairlist(tickers)
merged_list = pairlist + rpl_pairlist
merged_list = sorted(set(merged_list), key=merged_list.index)
return merged_list