Merge pull request #7867 from Bloodhunter4rc/remotepairlist

Add Remotepairlist

docs/includes/pairlists.md

@@ -23,6 +23,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
 * [`StaticPairList`](#static-pair-list) (default, if not configured differently)
 * [`VolumePairList`](#volume-pair-list)
 * [`ProducerPairList`](#producerpairlist)
+* [`RemotePairList`](#remotepairlist)
 * [`AgeFilter`](#agefilter)
 * [`OffsetFilter`](#offsetfilter)
 * [`PerformanceFilter`](#performancefilter)
@@ -173,6 +174,48 @@ You can limit the length of the pairlist with the optional parameter `number_ass
     `ProducerPairList` can also be used multiple times in sequence, combining the pairs from multiple producers.
     Obviously in complex such configurations, the Producer may not provide data for all pairs, so the strategy must be fit for this.
 
+#### RemotePairList
+
+It allows the user to fetch a pairlist from a remote server or a locally stored json file within the freqtrade directory, enabling dynamic updates and customization of the trading pairlist.
+
+The RemotePairList is defined in the pairlists section of the configuration settings. It uses the following configuration options:
+
+```json
+"pairlists": [
+    {
+        "method": "RemotePairList",
+        "pairlist_url": "https://example.com/pairlist",
+        "number_assets": 10,
+        "refresh_period": 1800,
+        "keep_pairlist_on_failure": true,
+        "read_timeout": 60,
+        "bearer_token": "my-bearer-token"
+    }
+]
+```
+
+The `pairlist_url` option specifies the URL of the remote server where the pairlist is located, or the path to a local file (if file:/// is prepended). This allows the user to use either a remote server or a local file as the source for the pairlist.
+
+The user is responsible for providing a server or local file that returns a JSON object with the following structure:
+
+```json
+{
+    "pairs": ["XRP/USDT", "ETH/USDT", "LTC/USDT"],
+    "refresh_period": 1800,
+}
+```
+
+The `pairs` property should contain a list of strings with the trading pairs to be used by the bot. The `refresh_period` property is optional and specifies the number of seconds that the pairlist should be cached before being refreshed.
+
+The optional `keep_pairlist_on_failure` specifies whether the previous received pairlist should be used if the remote server is not reachable or returns an error. The default value is true.
+
+The optional `read_timeout` specifies the maximum amount of time (in seconds) to wait for a response from the remote source, The default value is 60.
+
+The optional `bearer_token` will be included in the requests Authorization Header.
+
+!!! Note
+    In case of a server error the last received pairlist will be kept if `keep_pairlist_on_failure` is set to true, when set to false a empty pairlist is returned.
+
 #### AgeFilter
 
 Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity).

freqtrade/constants.py

@@ -31,7 +31,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
                          'CalmarHyperOptLoss',
                          'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
                          'ProfitDrawDownHyperOptLoss']
-AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList',
+AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList',
                        'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
                        'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
                        'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']

freqtrade/plugins/pairlist/RemotePairList.py

@@ -0,0 +1,206 @@
+"""
+Remote PairList provider
+
+Provides pair list fetched from a remote source
+"""
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Tuple
+
+import requests
+from cachetools import TTLCache
+
+from freqtrade import __version__
+from freqtrade.constants import Config
+from freqtrade.exceptions import OperationalException
+from freqtrade.exchange.types import Tickers
+from freqtrade.plugins.pairlist.IPairList import IPairList
+
+
+logger = logging.getLogger(__name__)
+
+
+class RemotePairList(IPairList):
+
+    def __init__(self, exchange, pairlistmanager,
+                 config: Config, pairlistconfig: Dict[str, Any],
+                 pairlist_pos: int) -> None:
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+
+        if 'number_assets' not in self._pairlistconfig:
+            raise OperationalException(
+                '`number_assets` not specified. Please check your configuration '
+                'for "pairlist.config.number_assets"')
+
+        if 'pairlist_url' not in self._pairlistconfig:
+            raise OperationalException(
+                '`pairlist_url` not specified. Please check your configuration '
+                'for "pairlist.config.pairlist_url"')
+
+        self._number_pairs = self._pairlistconfig['number_assets']
+        self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800)
+        self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True)
+        self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
+        self._pairlist_url = self._pairlistconfig.get('pairlist_url', '')
+        self._read_timeout = self._pairlistconfig.get('read_timeout', 60)
+        self._bearer_token = self._pairlistconfig.get('bearer_token', '')
+        self._init_done = False
+        self._last_pairlist: List[Any] = list()
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requires tickers, an empty Dict is passed
+        as tickers argument to filter_pairlist
+        """
+        return False
+
+    def short_desc(self) -> str:
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist."
+
+    def process_json(self, jsonparse) -> List[str]:
+
+        pairlist = jsonparse.get('pairs', [])
+        remote_refresh_period = int(jsonparse.get('refresh_period', self._refresh_period))
+
+        if self._refresh_period < remote_refresh_period:
+            self.log_once(f'Refresh Period has been increased from {self._refresh_period}'
+                          f' to minimum allowed: {remote_refresh_period} from Remote.', logger.info)
+
+            self._refresh_period = remote_refresh_period
+            self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period)
+
+        self._init_done = True
+
+        return pairlist
+
+    def return_last_pairlist(self) -> List[str]:
+        if self._keep_pairlist_on_failure:
+            pairlist = self._last_pairlist
+            self.log_once('Keeping last fetched pairlist', logger.info)
+        else:
+            pairlist = []
+
+        return pairlist
+
+    def fetch_pairlist(self) -> Tuple[List[str], float]:
+
+        headers = {
+            'User-Agent': 'Freqtrade/' + __version__ + ' Remotepairlist'
+        }
+
+        if self._bearer_token:
+            headers['Authorization'] = f'Bearer {self._bearer_token}'
+
+        try:
+            response = requests.get(self._pairlist_url, headers=headers,
+                                    timeout=self._read_timeout)
+            content_type = response.headers.get('content-type')
+            time_elapsed = response.elapsed.total_seconds()
+
+            if "application/json" in str(content_type):
+                jsonparse = response.json()
+
+                try:
+                    pairlist = self.process_json(jsonparse)
+                except Exception as e:
+
+                    if self._init_done:
+                        pairlist = self.return_last_pairlist()
+                        logger.warning(f'Error while processing JSON data: {type(e)}')
+                    else:
+                        raise OperationalException(f'Error while processing JSON data: {type(e)}')
+
+            else:
+                if self._init_done:
+                    self.log_once(f'Error: RemotePairList is not of type JSON: '
+                                  f' {self._pairlist_url}', logger.info)
+                    pairlist = self.return_last_pairlist()
+                else:
+                    raise OperationalException('RemotePairList is not of type JSON, abort.')
+
+        except requests.exceptions.RequestException:
+            self.log_once(f'Was not able to fetch pairlist from:'
+                          f' {self._pairlist_url}', logger.info)
+
+            pairlist = self.return_last_pairlist()
+
+            time_elapsed = 0
+
+        return pairlist, time_elapsed
+
+    def gen_pairlist(self, tickers: Tickers) -> List[str]:
+        """
+        Generate the pairlist
+        :param tickers: Tickers (from exchange.get_tickers). May be cached.
+        :return: List of pairs
+        """
+
+        if self._init_done:
+            pairlist = self._pair_cache.get('pairlist')
+        else:
+            pairlist = []
+
+        time_elapsed = 0.0
+
+        if pairlist:
+            # Item found - no refresh necessary
+            return pairlist.copy()
+        else:
+            if self._pairlist_url.startswith("file:///"):
+                filename = self._pairlist_url.split("file:///", 1)[1]
+                file_path = Path(filename)
+
+                if file_path.exists():
+                    with open(filename) as json_file:
+                        # Load the JSON data into a dictionary
+                        jsonparse = json.load(json_file)
+
+                        try:
+                            pairlist = self.process_json(jsonparse)
+                        except Exception as e:
+                            if self._init_done:
+                                pairlist = self.return_last_pairlist()
+                                logger.warning(f'Error while processing JSON data: {type(e)}')
+                            else:
+                                raise OperationalException('Error while processing'
+                                                           f'JSON data: {type(e)}')
+                else:
+                    raise ValueError(f"{self._pairlist_url} does not exist.")
+            else:
+                # Fetch Pairlist from Remote URL
+                pairlist, time_elapsed = self.fetch_pairlist()
+
+        self.log_once(f"Fetched pairs: {pairlist}", logger.debug)
+
+        pairlist = self._whitelist_for_active_markets(pairlist)
+        pairlist = pairlist[:self._number_pairs]
+
+        self._pair_cache['pairlist'] = pairlist.copy()
+
+        if time_elapsed != 0.0:
+            self.log_once(f'Pairlist Fetched in {time_elapsed} seconds.', logger.info)
+        else:
+            self.log_once('Fetched Pairlist.', logger.info)
+
+        self._last_pairlist = list(pairlist)
+
+        return pairlist
+
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Filters and sorts pairlist and returns the whitelist again.
+        Called on each bot iteration - please use internal caching if necessary
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers). May be cached.
+        :return: new whitelist
+        """
+        rpl_pairlist = self.gen_pairlist(tickers)
+        merged_list = pairlist + rpl_pairlist
+        merged_list = sorted(set(merged_list), key=merged_list.index)
+        return merged_list

tests/plugins/test_pairlist.py

@@ -22,6 +22,11 @@ from tests.conftest import (create_mock_trades_usdt, get_patched_exchange, get_p
                             log_has, log_has_re, num_log_has)
 
 
+# Exclude RemotePairList from tests.
+# It has a mandatory parameter, and requires special handling, which happens in test_remotepairlist.
+TESTABLE_PAIRLISTS = [p for p in AVAILABLE_PAIRLISTS if p not in ['RemotePairList']]
+
+
 @pytest.fixture(scope="function")
 def whitelist_conf(default_conf):
     default_conf['stake_currency'] = 'BTC'
@@ -824,7 +829,7 @@ def test_pair_whitelist_not_supported_Spread(mocker, default_conf, tickers) -> N
         get_patched_freqtradebot(mocker, default_conf)
 
 
-@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS)
+@pytest.mark.parametrize("pairlist", TESTABLE_PAIRLISTS)
 def test_pairlist_class(mocker, whitelist_conf, markets, pairlist):
     whitelist_conf['pairlists'][0]['method'] = pairlist
     mocker.patch.multiple('freqtrade.exchange.Exchange',
@@ -839,7 +844,7 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist):
     assert isinstance(freqtrade.pairlists.blacklist, list)
 
 
-@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS)
+@pytest.mark.parametrize("pairlist", TESTABLE_PAIRLISTS)
 @pytest.mark.parametrize("whitelist,log_message", [
     (['ETH/BTC', 'TKN/BTC'], ""),
     # TRX/ETH not in markets
@@ -872,7 +877,7 @@ def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist
     assert log_message in caplog.text
 
 
-@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS)
+@pytest.mark.parametrize("pairlist", TESTABLE_PAIRLISTS)
 def test__whitelist_for_active_markets_empty(mocker, whitelist_conf, pairlist, tickers):
     whitelist_conf['pairlists'][0]['method'] = pairlist
 

tests/plugins/test_remotepairlist.py

@@ -0,0 +1,185 @@
+import json
+from unittest.mock import MagicMock
+
+import pytest
+import requests
+
+from freqtrade.exceptions import OperationalException
+from freqtrade.plugins.pairlist.RemotePairList import RemotePairList
+from freqtrade.plugins.pairlistmanager import PairListManager
+from tests.conftest import get_patched_exchange, get_patched_freqtradebot, log_has
+
+
+@pytest.fixture(scope="function")
+def rpl_config(default_conf):
+    default_conf['stake_currency'] = 'USDT'
+
+    default_conf['exchange']['pair_whitelist'] = [
+        'ETH/USDT',
+        'BTC/USDT',
+    ]
+    default_conf['exchange']['pair_blacklist'] = [
+        'BLK/USDT'
+    ]
+    return default_conf
+
+
+def test_gen_pairlist_with_local_file(mocker, rpl_config):
+
+    mock_file = MagicMock()
+    mock_file.read.return_value = '{"pairs": ["TKN/USDT","ETH/USDT","NANO/USDT"]}'
+    mocker.patch('freqtrade.plugins.pairlist.RemotePairList.open', return_value=mock_file)
+
+    mock_file_path = mocker.patch('freqtrade.plugins.pairlist.RemotePairList.Path')
+    mock_file_path.exists.return_value = True
+
+    jsonparse = json.loads(mock_file.read.return_value)
+    mocker.patch('freqtrade.plugins.pairlist.RemotePairList.json.load', return_value=jsonparse)
+
+    rpl_config['pairlists'] = [
+        {
+            "method": "RemotePairList",
+            'number_assets': 2,
+            'refresh_period': 1800,
+            'keep_pairlist_on_failure': True,
+            'pairlist_url': 'file:///pairlist.json',
+            'bearer_token': '',
+            'read_timeout': 60
+        }
+    ]
+
+    exchange = get_patched_exchange(mocker, rpl_config)
+    pairlistmanager = PairListManager(exchange, rpl_config)
+
+    remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config,
+                                     rpl_config['pairlists'][0], 0)
+
+    result = remote_pairlist.gen_pairlist([])
+
+    assert result == ['TKN/USDT', 'ETH/USDT']
+
+
+def test_fetch_pairlist_mock_response_html(mocker, rpl_config):
+    mock_response = MagicMock()
+    mock_response.headers = {'content-type': 'text/html'}
+
+    rpl_config['pairlists'] = [
+        {
+            "method": "RemotePairList",
+            "pairlist_url": "http://example.com/pairlist",
+            "number_assets": 10,
+            "read_timeout": 10,
+            "keep_pairlist_on_failure": True,
+        }
+    ]
+
+    exchange = get_patched_exchange(mocker, rpl_config)
+    pairlistmanager = PairListManager(exchange, rpl_config)
+
+    mocker.patch("freqtrade.plugins.pairlist.RemotePairList.requests.get",
+                 return_value=mock_response)
+    remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config,
+                                     rpl_config['pairlists'][0], 0)
+
+    with pytest.raises(OperationalException, match='RemotePairList is not of type JSON, abort.'):
+        remote_pairlist.fetch_pairlist()
+
+
+def test_fetch_pairlist_timeout_keep_last_pairlist(mocker, rpl_config, caplog):
+    rpl_config['pairlists'] = [
+        {
+            "method": "RemotePairList",
+            "pairlist_url": "http://example.com/pairlist",
+            "number_assets": 10,
+            "read_timeout": 10,
+            "keep_pairlist_on_failure": True,
+        }
+    ]
+
+    exchange = get_patched_exchange(mocker, rpl_config)
+    pairlistmanager = PairListManager(exchange, rpl_config)
+
+    mocker.patch("freqtrade.plugins.pairlist.RemotePairList.requests.get",
+                 side_effect=requests.exceptions.RequestException)
+
+    remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config,
+                                     rpl_config['pairlists'][0], 0)
+
+    remote_pairlist._last_pairlist = ["BTC/USDT", "ETH/USDT", "LTC/USDT"]
+
+    pairs, time_elapsed = remote_pairlist.fetch_pairlist()
+    assert log_has(f"Was not able to fetch pairlist from: {remote_pairlist._pairlist_url}", caplog)
+    assert log_has("Keeping last fetched pairlist", caplog)
+    assert pairs == ["BTC/USDT", "ETH/USDT", "LTC/USDT"]
+
+
+def test_remote_pairlist_init_no_pairlist_url(mocker, rpl_config):
+
+    rpl_config['pairlists'] = [
+        {
+            "method": "RemotePairList",
+            "number_assets": 10,
+            "keep_pairlist_on_failure": True,
+        }
+    ]
+
+    get_patched_exchange(mocker, rpl_config)
+    with pytest.raises(OperationalException, match=r'`pairlist_url` not specified.'
+                       r' Please check your configuration for "pairlist.config.pairlist_url"'):
+        get_patched_freqtradebot(mocker, rpl_config)
+
+
+def test_remote_pairlist_init_no_number_assets(mocker, rpl_config):
+
+    rpl_config['pairlists'] = [
+        {
+            "method": "RemotePairList",
+            "pairlist_url": "http://example.com/pairlist",
+            "keep_pairlist_on_failure": True,
+        }
+    ]
+
+    get_patched_exchange(mocker, rpl_config)
+
+    with pytest.raises(OperationalException, match=r'`number_assets` not specified. '
+                       'Please check your configuration for "pairlist.config.number_assets"'):
+        get_patched_freqtradebot(mocker, rpl_config)
+
+
+def test_fetch_pairlist_mock_response_valid(mocker, rpl_config):
+
+    rpl_config['pairlists'] = [
+        {
+            "method": "RemotePairList",
+            "pairlist_url": "http://example.com/pairlist",
+            "number_assets": 10,
+            "refresh_period": 10,
+            "read_timeout": 10,
+            "keep_pairlist_on_failure": True,
+        }
+    ]
+
+    mock_response = MagicMock()
+
+    mock_response.json.return_value = {
+        "pairs": ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"],
+        "refresh_period": 60
+    }
+
+    mock_response.headers = {
+        "content-type": "application/json"
+    }
+
+    mock_response.elapsed.total_seconds.return_value = 0.4
+    mocker.patch("freqtrade.plugins.pairlist.RemotePairList.requests.get",
+                 return_value=mock_response)
+
+    exchange = get_patched_exchange(mocker, rpl_config)
+    pairlistmanager = PairListManager(exchange, rpl_config)
+    remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config,
+                                     rpl_config['pairlists'][0], 0)
+    pairs, time_elapsed = remote_pairlist.fetch_pairlist()
+
+    assert pairs == ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"]
+    assert time_elapsed == 0.4
+    assert remote_pairlist._refresh_period == 60