Merge pull request #7867 from Bloodhunter4rc/remotepairlist

Add Remotepairlist

freqtrade/constants.py

@@ -31,7 +31,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
                          'CalmarHyperOptLoss',
                          'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
                          'ProfitDrawDownHyperOptLoss']
-AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList',
+AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList',
                        'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
                        'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
                        'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']

freqtrade/plugins/pairlist/RemotePairList.py

@@ -0,0 +1,206 @@
+"""
+Remote PairList provider
+
+Provides pair list fetched from a remote source
+"""
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Tuple
+
+import requests
+from cachetools import TTLCache
+
+from freqtrade import __version__
+from freqtrade.constants import Config
+from freqtrade.exceptions import OperationalException
+from freqtrade.exchange.types import Tickers
+from freqtrade.plugins.pairlist.IPairList import IPairList
+
+
+logger = logging.getLogger(__name__)
+
+
+class RemotePairList(IPairList):
+
+    def __init__(self, exchange, pairlistmanager,
+                 config: Config, pairlistconfig: Dict[str, Any],
+                 pairlist_pos: int) -> None:
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+
+        if 'number_assets' not in self._pairlistconfig:
+            raise OperationalException(
+                '`number_assets` not specified. Please check your configuration '
+                'for "pairlist.config.number_assets"')
+
+        if 'pairlist_url' not in self._pairlistconfig:
+            raise OperationalException(
+                '`pairlist_url` not specified. Please check your configuration '
+                'for "pairlist.config.pairlist_url"')
+
+        self._number_pairs = self._pairlistconfig['number_assets']
+        self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800)
+        self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True)
+        self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
+        self._pairlist_url = self._pairlistconfig.get('pairlist_url', '')
+        self._read_timeout = self._pairlistconfig.get('read_timeout', 60)
+        self._bearer_token = self._pairlistconfig.get('bearer_token', '')
+        self._init_done = False
+        self._last_pairlist: List[Any] = list()
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requires tickers, an empty Dict is passed
+        as tickers argument to filter_pairlist
+        """
+        return False
+
+    def short_desc(self) -> str:
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist."
+
+    def process_json(self, jsonparse) -> List[str]:
+
+        pairlist = jsonparse.get('pairs', [])
+        remote_refresh_period = int(jsonparse.get('refresh_period', self._refresh_period))
+
+        if self._refresh_period < remote_refresh_period:
+            self.log_once(f'Refresh Period has been increased from {self._refresh_period}'
+                          f' to minimum allowed: {remote_refresh_period} from Remote.', logger.info)
+
+            self._refresh_period = remote_refresh_period
+            self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period)
+
+        self._init_done = True
+
+        return pairlist
+
+    def return_last_pairlist(self) -> List[str]:
+        if self._keep_pairlist_on_failure:
+            pairlist = self._last_pairlist
+            self.log_once('Keeping last fetched pairlist', logger.info)
+        else:
+            pairlist = []
+
+        return pairlist
+
+    def fetch_pairlist(self) -> Tuple[List[str], float]:
+
+        headers = {
+            'User-Agent': 'Freqtrade/' + __version__ + ' Remotepairlist'
+        }
+
+        if self._bearer_token:
+            headers['Authorization'] = f'Bearer {self._bearer_token}'
+
+        try:
+            response = requests.get(self._pairlist_url, headers=headers,
+                                    timeout=self._read_timeout)
+            content_type = response.headers.get('content-type')
+            time_elapsed = response.elapsed.total_seconds()
+
+            if "application/json" in str(content_type):
+                jsonparse = response.json()
+
+                try:
+                    pairlist = self.process_json(jsonparse)
+                except Exception as e:
+
+                    if self._init_done:
+                        pairlist = self.return_last_pairlist()
+                        logger.warning(f'Error while processing JSON data: {type(e)}')
+                    else:
+                        raise OperationalException(f'Error while processing JSON data: {type(e)}')
+
+            else:
+                if self._init_done:
+                    self.log_once(f'Error: RemotePairList is not of type JSON: '
+                                  f' {self._pairlist_url}', logger.info)
+                    pairlist = self.return_last_pairlist()
+                else:
+                    raise OperationalException('RemotePairList is not of type JSON, abort.')
+
+        except requests.exceptions.RequestException:
+            self.log_once(f'Was not able to fetch pairlist from:'
+                          f' {self._pairlist_url}', logger.info)
+
+            pairlist = self.return_last_pairlist()
+
+            time_elapsed = 0
+
+        return pairlist, time_elapsed
+
+    def gen_pairlist(self, tickers: Tickers) -> List[str]:
+        """
+        Generate the pairlist
+        :param tickers: Tickers (from exchange.get_tickers). May be cached.
+        :return: List of pairs
+        """
+
+        if self._init_done:
+            pairlist = self._pair_cache.get('pairlist')
+        else:
+            pairlist = []
+
+        time_elapsed = 0.0
+
+        if pairlist:
+            # Item found - no refresh necessary
+            return pairlist.copy()
+        else:
+            if self._pairlist_url.startswith("file:///"):
+                filename = self._pairlist_url.split("file:///", 1)[1]
+                file_path = Path(filename)
+
+                if file_path.exists():
+                    with open(filename) as json_file:
+                        # Load the JSON data into a dictionary
+                        jsonparse = json.load(json_file)
+
+                        try:
+                            pairlist = self.process_json(jsonparse)
+                        except Exception as e:
+                            if self._init_done:
+                                pairlist = self.return_last_pairlist()
+                                logger.warning(f'Error while processing JSON data: {type(e)}')
+                            else:
+                                raise OperationalException('Error while processing'
+                                                           f'JSON data: {type(e)}')
+                else:
+                    raise ValueError(f"{self._pairlist_url} does not exist.")
+            else:
+                # Fetch Pairlist from Remote URL
+                pairlist, time_elapsed = self.fetch_pairlist()
+
+        self.log_once(f"Fetched pairs: {pairlist}", logger.debug)
+
+        pairlist = self._whitelist_for_active_markets(pairlist)
+        pairlist = pairlist[:self._number_pairs]
+
+        self._pair_cache['pairlist'] = pairlist.copy()
+
+        if time_elapsed != 0.0:
+            self.log_once(f'Pairlist Fetched in {time_elapsed} seconds.', logger.info)
+        else:
+            self.log_once('Fetched Pairlist.', logger.info)
+
+        self._last_pairlist = list(pairlist)
+
+        return pairlist
+
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Filters and sorts pairlist and returns the whitelist again.
+        Called on each bot iteration - please use internal caching if necessary
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers). May be cached.
+        :return: new whitelist
+        """
+        rpl_pairlist = self.gen_pairlist(tickers)
+        merged_list = pairlist + rpl_pairlist
+        merged_list = sorted(set(merged_list), key=merged_list.index)
+        return merged_list