init cache on first iteration, init checks, limit length and charmap to info replace if invalid, move filter logic

docs/includes/pairlists.md

@@ -202,7 +202,7 @@ The user is responsible for providing a server or local file that returns a JSON
 {
     "pairs": ["XRP/USDT", "ETH/USDT", "LTC/USDT"],
     "refresh_period": 1800,
-    "info": "Pairlist updated on 2022-12-12 at 12:12"
+    "info": "Pairlist updated on 2022-12-12 at 12:12" // Maximum Length: 256 Characters, Charset: Alphanumeric + "+-.,%:"
 }
 ```
 

freqtrade/plugins/pairlist/RemotePairList.py

@@ -6,7 +6,7 @@ Provides pair list fetched from a remote source
 import json
 import logging
 from pathlib import Path
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 
 import requests
 from cachetools import TTLCache
@@ -39,12 +39,13 @@ class RemotePairList(IPairList):
                 'for "pairlist.config.pairlist_url"')
 
         self._number_pairs = self._pairlistconfig['number_assets']
-        self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
+        self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800)
         self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True)
-        self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
+        self._pair_cache: Optional[TTLCache] = None
         self._pairlist_url = self._pairlistconfig.get('pairlist_url', '')
         self._read_timeout = self._pairlistconfig.get('read_timeout', 60)
         self._bearer_token = self._pairlistconfig.get('bearer_token', '')
+        self._init_done = False
         self._last_pairlist: List[Any] = list()
 
     @property
@@ -62,6 +63,15 @@ class RemotePairList(IPairList):
         """
         return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist."
 
+    def return_last_pairlist(self) -> List[str]:
+        if self._keep_pairlist_on_failure:
+            pairlist = self._last_pairlist
+            self.log_once('Keeping last fetched pairlist', logger.info)
+        else:
+            pairlist = []
+
+        return pairlist
+
     def fetch_pairlist(self) -> Tuple[List[str], float, str]:
 
         headers = {
@@ -81,23 +91,35 @@ class RemotePairList(IPairList):
 
             if "application/json" in str(content_type):
                 jsonparse = response.json()
-                pairlist = jsonparse['pairs']
+                pairlist = jsonparse.get('pairs', [])
-                info = jsonparse.get('info', '')
+                remote_info = jsonparse.get('info', '')[:256].strip()
+                remote_refresh_period = jsonparse.get('refresh_period', self._refresh_period)
+
+                info = "".join(char if char.isalnum() or
+                               char in " +-.,%:" else "-" for char in remote_info)
+
+                if not self._init_done and self._refresh_period < remote_refresh_period:
+                    self.log_once(f'Refresh Period has been increased from {self._refresh_period}'
+                                  f' to {remote_refresh_period} from Remote.', logger.info)
+
+                    self._refresh_period = remote_refresh_period
+                    self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period)
+
+                self._init_done = True
+            else:
+                if self._init_done:
+                    self.log_once(f'Error: RemotePairList is not of type JSON: '
+                                  f' {self._pairlist_url}', logger.info)
+                    pairlist = self.return_last_pairlist()
+
                 else:
                     raise OperationalException('RemotePairList is not of type JSON abort ')
 
-            self._refresh_period = jsonparse.get('refresh_period', self._refresh_period)
-            self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period)
-
         except requests.exceptions.RequestException:
             self.log_once(f'Was not able to fetch pairlist from:'
                           f' {self._pairlist_url}', logger.info)
 
-            if self._keep_pairlist_on_failure:
+            pairlist = self.return_last_pairlist()
-                pairlist = self._last_pairlist
-                self.log_once('Keeping last fetched pairlist', logger.info)
-            else:
-                pairlist = []
 
             time_elapsed = 0
 
@@ -110,12 +132,17 @@ class RemotePairList(IPairList):
         :return: List of pairs
         """
 
+        if self._init_done and self._pair_cache:
             pairlist = self._pair_cache.get('pairlist')
+        else:
+            pairlist = []
+
         time_elapsed = 0.0
 
         if pairlist:
             # Item found - no refresh necessary
             return pairlist.copy()
+            self._init_done = True
         else:
             if self._pairlist_url.startswith("file:///"):
                 filename = self._pairlist_url.split("file:///", 1)[1]
@@ -127,16 +154,24 @@ class RemotePairList(IPairList):
                         jsonparse = json.load(json_file)
                         pairlist = jsonparse['pairs']
                         info = jsonparse.get('info', '')
-                        self._refresh_period = jsonparse.get('refresh_period', self._refresh_period)
-                        self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period)
 
+                        if not self._init_done:
+                            self._refresh_period = jsonparse.get('refresh_period',
+                                                                 self._refresh_period)
+                            self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period)
+                        self._init_done = True
                 else:
                     raise ValueError(f"{self._pairlist_url} does not exist.")
             else:
                 # Fetch Pairlist from Remote URL
                 pairlist, time_elapsed, info = self.fetch_pairlist()
 
-        pairlist = self.filter_pairlist(pairlist, tickers)
+        self.log_once(f"Fetched pairs: {pairlist}", logger.debug)
+
+        pairlist = self._whitelist_for_active_markets(pairlist)
+        pairlist = pairlist[:self._number_pairs]
+
+        if self._pair_cache:
             self._pair_cache['pairlist'] = pairlist.copy()
 
         if time_elapsed != 0.0:
@@ -145,6 +180,7 @@ class RemotePairList(IPairList):
             self.log_once(f'{info} Fetched Pairlist.', logger.info)
 
         self._last_pairlist = list(pairlist)
+
         return pairlist
 
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
@@ -155,12 +191,7 @@ class RemotePairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers). May be cached.
         :return: new whitelist
         """
-
+        rpl_pairlist = self.gen_pairlist(tickers)
-        # Validate whitelist to only have active market pairs
+        merged_list = pairlist + rpl_pairlist
-        pairlist = self._whitelist_for_active_markets(pairlist)
+        merged_list = sorted(set(merged_list), key=merged_list.index)
-        pairlist = self.verify_blacklist(pairlist, logger.info)
+        return merged_list
-        # Limit pairlist to the requested number of pairs
-        pairlist = pairlist[:self._number_pairs]
-        self.log_once(f"Searching {self._number_pairs} pairs: {pairlist}", logger.info)
-
-        return pairlist