diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 3a6ab7a3c..0bff9b29b 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -202,7 +202,7 @@ The user is responsible for providing a server or local file that returns a JSON { "pairs": ["XRP/USDT", "ETH/USDT", "LTC/USDT"], "refresh_period": 1800, - "info": "Pairlist updated on 2022-12-12 at 12:12" + "info": "Pairlist updated on 2022-12-12 at 12:12" // Maximum Length: 256 Characters, Charset: Alphanumeric + "+-.,%:" } ``` diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index e46ac0419..a0e140b42 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -6,7 +6,7 @@ Provides pair list fetched from a remote source import json import logging from pathlib import Path -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Optional, Tuple import requests from cachetools import TTLCache @@ -39,12 +39,13 @@ class RemotePairList(IPairList): 'for "pairlist.config.pairlist_url"') self._number_pairs = self._pairlistconfig['number_assets'] - self._refresh_period = self._pairlistconfig.get('refresh_period', 1800) + self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800) self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True) - self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) + self._pair_cache: Optional[TTLCache] = None self._pairlist_url = self._pairlistconfig.get('pairlist_url', '') self._read_timeout = self._pairlistconfig.get('read_timeout', 60) self._bearer_token = self._pairlistconfig.get('bearer_token', '') + self._init_done = False self._last_pairlist: List[Any] = list() @property @@ -62,6 +63,15 @@ class RemotePairList(IPairList): """ return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." + def return_last_pairlist(self) -> List[str]: + if self._keep_pairlist_on_failure: + pairlist = self._last_pairlist + self.log_once('Keeping last fetched pairlist', logger.info) + else: + pairlist = [] + + return pairlist + def fetch_pairlist(self) -> Tuple[List[str], float, str]: headers = { @@ -81,23 +91,35 @@ class RemotePairList(IPairList): if "application/json" in str(content_type): jsonparse = response.json() - pairlist = jsonparse['pairs'] - info = jsonparse.get('info', '') - else: - raise OperationalException('RemotePairList is not of type JSON abort ') + pairlist = jsonparse.get('pairs', []) + remote_info = jsonparse.get('info', '')[:256].strip() + remote_refresh_period = jsonparse.get('refresh_period', self._refresh_period) - self._refresh_period = jsonparse.get('refresh_period', self._refresh_period) - self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) + info = "".join(char if char.isalnum() or + char in " +-.,%:" else "-" for char in remote_info) + + if not self._init_done and self._refresh_period < remote_refresh_period: + self.log_once(f'Refresh Period has been increased from {self._refresh_period}' + f' to {remote_refresh_period} from Remote.', logger.info) + + self._refresh_period = remote_refresh_period + self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) + + self._init_done = True + else: + if self._init_done: + self.log_once(f'Error: RemotePairList is not of type JSON: ' + f' {self._pairlist_url}', logger.info) + pairlist = self.return_last_pairlist() + + else: + raise OperationalException('RemotePairList is not of type JSON abort ') except requests.exceptions.RequestException: self.log_once(f'Was not able to fetch pairlist from:' f' {self._pairlist_url}', logger.info) - if self._keep_pairlist_on_failure: - pairlist = self._last_pairlist - self.log_once('Keeping last fetched pairlist', logger.info) - else: - pairlist = [] + pairlist = self.return_last_pairlist() time_elapsed = 0 @@ -110,12 +132,17 @@ class RemotePairList(IPairList): :return: List of pairs """ - pairlist = self._pair_cache.get('pairlist') + if self._init_done and self._pair_cache: + pairlist = self._pair_cache.get('pairlist') + else: + pairlist = [] + time_elapsed = 0.0 if pairlist: # Item found - no refresh necessary return pairlist.copy() + self._init_done = True else: if self._pairlist_url.startswith("file:///"): filename = self._pairlist_url.split("file:///", 1)[1] @@ -127,17 +154,25 @@ class RemotePairList(IPairList): jsonparse = json.load(json_file) pairlist = jsonparse['pairs'] info = jsonparse.get('info', '') - self._refresh_period = jsonparse.get('refresh_period', self._refresh_period) - self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) + if not self._init_done: + self._refresh_period = jsonparse.get('refresh_period', + self._refresh_period) + self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) + self._init_done = True else: raise ValueError(f"{self._pairlist_url} does not exist.") else: # Fetch Pairlist from Remote URL pairlist, time_elapsed, info = self.fetch_pairlist() - pairlist = self.filter_pairlist(pairlist, tickers) - self._pair_cache['pairlist'] = pairlist.copy() + self.log_once(f"Fetched pairs: {pairlist}", logger.debug) + + pairlist = self._whitelist_for_active_markets(pairlist) + pairlist = pairlist[:self._number_pairs] + + if self._pair_cache: + self._pair_cache['pairlist'] = pairlist.copy() if time_elapsed != 0.0: self.log_once(f'{info} Fetched in {time_elapsed} seconds.', logger.info) @@ -145,6 +180,7 @@ class RemotePairList(IPairList): self.log_once(f'{info} Fetched Pairlist.', logger.info) self._last_pairlist = list(pairlist) + return pairlist def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: @@ -155,12 +191,7 @@ class RemotePairList(IPairList): :param tickers: Tickers (from exchange.get_tickers). May be cached. :return: new whitelist """ - - # Validate whitelist to only have active market pairs - pairlist = self._whitelist_for_active_markets(pairlist) - pairlist = self.verify_blacklist(pairlist, logger.info) - # Limit pairlist to the requested number of pairs - pairlist = pairlist[:self._number_pairs] - self.log_once(f"Searching {self._number_pairs} pairs: {pairlist}", logger.info) - - return pairlist + rpl_pairlist = self.gen_pairlist(tickers) + merged_list = pairlist + rpl_pairlist + merged_list = sorted(set(merged_list), key=merged_list.index) + return merged_list