207 lines
7.6 KiB
Python
207 lines
7.6 KiB
Python
"""
|
|
Remote PairList provider
|
|
|
|
Provides pair list fetched from a remote source
|
|
"""
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Tuple
|
|
|
|
import requests
|
|
from cachetools import TTLCache
|
|
|
|
from freqtrade import __version__
|
|
from freqtrade.constants import Config
|
|
from freqtrade.exceptions import OperationalException
|
|
from freqtrade.exchange.types import Tickers
|
|
from freqtrade.plugins.pairlist.IPairList import IPairList
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class RemotePairList(IPairList):
|
|
|
|
def __init__(self, exchange, pairlistmanager,
|
|
config: Config, pairlistconfig: Dict[str, Any],
|
|
pairlist_pos: int) -> None:
|
|
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
|
|
|
|
if 'number_assets' not in self._pairlistconfig:
|
|
raise OperationalException(
|
|
'`number_assets` not specified. Please check your configuration '
|
|
'for "pairlist.config.number_assets"')
|
|
|
|
if 'pairlist_url' not in self._pairlistconfig:
|
|
raise OperationalException(
|
|
'`pairlist_url` not specified. Please check your configuration '
|
|
'for "pairlist.config.pairlist_url"')
|
|
|
|
self._number_pairs = self._pairlistconfig['number_assets']
|
|
self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800)
|
|
self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True)
|
|
self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
|
|
self._pairlist_url = self._pairlistconfig.get('pairlist_url', '')
|
|
self._read_timeout = self._pairlistconfig.get('read_timeout', 60)
|
|
self._bearer_token = self._pairlistconfig.get('bearer_token', '')
|
|
self._init_done = False
|
|
self._last_pairlist: List[Any] = list()
|
|
|
|
@property
|
|
def needstickers(self) -> bool:
|
|
"""
|
|
Boolean property defining if tickers are necessary.
|
|
If no Pairlist requires tickers, an empty Dict is passed
|
|
as tickers argument to filter_pairlist
|
|
"""
|
|
return False
|
|
|
|
def short_desc(self) -> str:
|
|
"""
|
|
Short whitelist method description - used for startup-messages
|
|
"""
|
|
return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist."
|
|
|
|
def process_json(self, jsonparse) -> List[str]:
|
|
|
|
pairlist = jsonparse.get('pairs', [])
|
|
remote_refresh_period = int(jsonparse.get('refresh_period', self._refresh_period))
|
|
|
|
if self._refresh_period < remote_refresh_period:
|
|
self.log_once(f'Refresh Period has been increased from {self._refresh_period}'
|
|
f' to minimum allowed: {remote_refresh_period} from Remote.', logger.info)
|
|
|
|
self._refresh_period = remote_refresh_period
|
|
self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period)
|
|
|
|
self._init_done = True
|
|
|
|
return pairlist
|
|
|
|
def return_last_pairlist(self) -> List[str]:
|
|
if self._keep_pairlist_on_failure:
|
|
pairlist = self._last_pairlist
|
|
self.log_once('Keeping last fetched pairlist', logger.info)
|
|
else:
|
|
pairlist = []
|
|
|
|
return pairlist
|
|
|
|
def fetch_pairlist(self) -> Tuple[List[str], float]:
|
|
|
|
headers = {
|
|
'User-Agent': 'Freqtrade/' + __version__ + ' Remotepairlist'
|
|
}
|
|
|
|
if self._bearer_token:
|
|
headers['Authorization'] = f'Bearer {self._bearer_token}'
|
|
|
|
try:
|
|
response = requests.get(self._pairlist_url, headers=headers,
|
|
timeout=self._read_timeout)
|
|
content_type = response.headers.get('content-type')
|
|
time_elapsed = response.elapsed.total_seconds()
|
|
|
|
if "application/json" in str(content_type):
|
|
jsonparse = response.json()
|
|
|
|
try:
|
|
pairlist = self.process_json(jsonparse)
|
|
except Exception as e:
|
|
|
|
if self._init_done:
|
|
pairlist = self.return_last_pairlist()
|
|
logger.warning(f'Error while processing JSON data: {type(e)}')
|
|
else:
|
|
raise OperationalException(f'Error while processing JSON data: {type(e)}')
|
|
|
|
else:
|
|
if self._init_done:
|
|
self.log_once(f'Error: RemotePairList is not of type JSON: '
|
|
f' {self._pairlist_url}', logger.info)
|
|
pairlist = self.return_last_pairlist()
|
|
else:
|
|
raise OperationalException('RemotePairList is not of type JSON, abort.')
|
|
|
|
except requests.exceptions.RequestException:
|
|
self.log_once(f'Was not able to fetch pairlist from:'
|
|
f' {self._pairlist_url}', logger.info)
|
|
|
|
pairlist = self.return_last_pairlist()
|
|
|
|
time_elapsed = 0
|
|
|
|
return pairlist, time_elapsed
|
|
|
|
def gen_pairlist(self, tickers: Tickers) -> List[str]:
|
|
"""
|
|
Generate the pairlist
|
|
:param tickers: Tickers (from exchange.get_tickers). May be cached.
|
|
:return: List of pairs
|
|
"""
|
|
|
|
if self._init_done:
|
|
pairlist = self._pair_cache.get('pairlist')
|
|
else:
|
|
pairlist = []
|
|
|
|
time_elapsed = 0.0
|
|
|
|
if pairlist:
|
|
# Item found - no refresh necessary
|
|
return pairlist.copy()
|
|
else:
|
|
if self._pairlist_url.startswith("file:///"):
|
|
filename = self._pairlist_url.split("file:///", 1)[1]
|
|
file_path = Path(filename)
|
|
|
|
if file_path.exists():
|
|
with file_path.open() as json_file:
|
|
# Load the JSON data into a dictionary
|
|
jsonparse = json.load(json_file)
|
|
|
|
try:
|
|
pairlist = self.process_json(jsonparse)
|
|
except Exception as e:
|
|
if self._init_done:
|
|
pairlist = self.return_last_pairlist()
|
|
logger.warning(f'Error while processing JSON data: {type(e)}')
|
|
else:
|
|
raise OperationalException('Error while processing'
|
|
f'JSON data: {type(e)}')
|
|
else:
|
|
raise ValueError(f"{self._pairlist_url} does not exist.")
|
|
else:
|
|
# Fetch Pairlist from Remote URL
|
|
pairlist, time_elapsed = self.fetch_pairlist()
|
|
|
|
self.log_once(f"Fetched pairs: {pairlist}", logger.debug)
|
|
|
|
pairlist = self._whitelist_for_active_markets(pairlist)
|
|
pairlist = pairlist[:self._number_pairs]
|
|
|
|
self._pair_cache['pairlist'] = pairlist.copy()
|
|
|
|
if time_elapsed != 0.0:
|
|
self.log_once(f'Pairlist Fetched in {time_elapsed} seconds.', logger.info)
|
|
else:
|
|
self.log_once('Fetched Pairlist.', logger.info)
|
|
|
|
self._last_pairlist = list(pairlist)
|
|
|
|
return pairlist
|
|
|
|
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
|
|
"""
|
|
Filters and sorts pairlist and returns the whitelist again.
|
|
Called on each bot iteration - please use internal caching if necessary
|
|
:param pairlist: pairlist to filter or sort
|
|
:param tickers: Tickers (from exchange.get_tickers). May be cached.
|
|
:return: new whitelist
|
|
"""
|
|
rpl_pairlist = self.gen_pairlist(tickers)
|
|
merged_list = pairlist + rpl_pairlist
|
|
merged_list = sorted(set(merged_list), key=merged_list.index)
|
|
return merged_list
|