From 43f5a16006805d763b59c3cf1f8ff32aee47df66 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Mon, 19 Dec 2022 15:36:28 +0100 Subject: [PATCH] parse exception handling, remove info, cache change --- docs/includes/pairlists.md | 3 +- freqtrade/plugins/pairlist/RemotePairList.py | 68 +++++++++++--------- tests/plugins/test_remotepairlist.py | 6 +- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 0bff9b29b..5fda038bd 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -202,11 +202,10 @@ The user is responsible for providing a server or local file that returns a JSON { "pairs": ["XRP/USDT", "ETH/USDT", "LTC/USDT"], "refresh_period": 1800, - "info": "Pairlist updated on 2022-12-12 at 12:12" // Maximum Length: 256 Characters, Charset: Alphanumeric + "+-.,%:" } ``` -The `pairs` property should contain a list of strings with the trading pairs to be used by the bot. The `refresh_period` property is optional and specifies the number of seconds that the pairlist should be cached before being refreshed. The `info` property is also optional and can be used to provide any additional information about the pairlist. +The `pairs` property should contain a list of strings with the trading pairs to be used by the bot. The `refresh_period` property is optional and specifies the number of seconds that the pairlist should be cached before being refreshed. The optional `keep_pairlist_on_failure` specifies whether the previous received pairlist should be used if the remote server is not reachable or returns an error. The default value is true. diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 205ee5742..25530457a 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -6,7 +6,7 @@ Provides pair list fetched from a remote source import json import logging from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Tuple import requests from cachetools import TTLCache @@ -41,7 +41,7 @@ class RemotePairList(IPairList): self._number_pairs = self._pairlistconfig['number_assets'] self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800) self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True) - self._pair_cache: Optional[TTLCache] = None + self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) self._pairlist_url = self._pairlistconfig.get('pairlist_url', '') self._read_timeout = self._pairlistconfig.get('read_timeout', 60) self._bearer_token = self._pairlistconfig.get('bearer_token', '') @@ -63,28 +63,20 @@ class RemotePairList(IPairList): """ return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." - def process_json(self, jsonparse) -> Tuple[List[str], str]: + def process_json(self, jsonparse) -> List[str]: pairlist = jsonparse.get('pairs', []) - remote_info = jsonparse.get('info', '')[:256].strip() - remote_refresh_period = jsonparse.get('refresh_period', self._refresh_period) + remote_refresh_period = int(jsonparse.get('refresh_period', self._refresh_period)) - info = "".join(char if char.isalnum() or - char in " +-.,%:" else "-" for char in remote_info) - - if not self._init_done: - if self._refresh_period < remote_refresh_period: - self.log_once(f'Refresh Period has been increased from {self._refresh_period}' - f' to {remote_refresh_period} from Remote.', logger.info) - - self._refresh_period = remote_refresh_period - self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) - else: - self._pair_cache = TTLCache(maxsize=1, ttl=self._refresh_period) + if self._refresh_period < remote_refresh_period: + self.log_once(f'Refresh Period has been increased from {self._refresh_period}' + f' to minimum allowed: {remote_refresh_period} from Remote.', logger.info) + self._refresh_period = remote_refresh_period + self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period) self._init_done = True - return pairlist, info + return pairlist def return_last_pairlist(self) -> List[str]: if self._keep_pairlist_on_failure: @@ -95,7 +87,7 @@ class RemotePairList(IPairList): return pairlist - def fetch_pairlist(self) -> Tuple[List[str], float, str]: + def fetch_pairlist(self) -> Tuple[List[str], float]: headers = { 'User-Agent': 'Freqtrade/' + __version__ + ' Remotepairlist' @@ -104,8 +96,6 @@ class RemotePairList(IPairList): if self._bearer_token: headers['Authorization'] = f'Bearer {self._bearer_token}' - info = "Pairlist" - try: response = requests.get(self._pairlist_url, headers=headers, timeout=self._read_timeout) @@ -114,7 +104,17 @@ class RemotePairList(IPairList): if "application/json" in str(content_type): jsonparse = response.json() - pairlist, info = self.process_json(jsonparse) + + try: + pairlist = self.process_json(jsonparse) + except Exception as e: + + if self._init_done: + pairlist = self.return_last_pairlist() + logger.warning(f'Error while processing JSON data: {type(e)}') + else: + raise OperationalException(f'Error while processing JSON data: {type(e)}') + else: if self._init_done: self.log_once(f'Error: RemotePairList is not of type JSON: ' @@ -131,7 +131,7 @@ class RemotePairList(IPairList): time_elapsed = 0 - return pairlist, time_elapsed, info + return pairlist, time_elapsed def gen_pairlist(self, tickers: Tickers) -> List[str]: """ @@ -140,7 +140,7 @@ class RemotePairList(IPairList): :return: List of pairs """ - if self._init_done and self._pair_cache is not None: + if self._init_done: pairlist = self._pair_cache.get('pairlist') else: pairlist = [] @@ -159,25 +159,33 @@ class RemotePairList(IPairList): with open(filename) as json_file: # Load the JSON data into a dictionary jsonparse = json.load(json_file) - pairlist, info = self.process_json(jsonparse) + + try: + pairlist = self.process_json(jsonparse) + except Exception as e: + if self._init_done: + pairlist = self.return_last_pairlist() + logger.warning(f'Error while processing JSON data: {type(e)}') + else: + raise OperationalException('Error while processing' + f'JSON data: {type(e)}') else: raise ValueError(f"{self._pairlist_url} does not exist.") else: # Fetch Pairlist from Remote URL - pairlist, time_elapsed, info = self.fetch_pairlist() + pairlist, time_elapsed = self.fetch_pairlist() self.log_once(f"Fetched pairs: {pairlist}", logger.debug) pairlist = self._whitelist_for_active_markets(pairlist) pairlist = pairlist[:self._number_pairs] - if self._pair_cache is not None: - self._pair_cache['pairlist'] = pairlist.copy() + self._pair_cache['pairlist'] = pairlist.copy() if time_elapsed != 0.0: - self.log_once(f'{info} Fetched in {time_elapsed} seconds.', logger.info) + self.log_once(f'Pairlist Fetched in {time_elapsed} seconds.', logger.info) else: - self.log_once(f'{info} Fetched Pairlist.', logger.info) + self.log_once('Fetched Pairlist.', logger.info) self._last_pairlist = list(pairlist) diff --git a/tests/plugins/test_remotepairlist.py b/tests/plugins/test_remotepairlist.py index fc91d3f06..b7a484c92 100644 --- a/tests/plugins/test_remotepairlist.py +++ b/tests/plugins/test_remotepairlist.py @@ -107,7 +107,7 @@ def test_fetch_pairlist_timeout_keep_last_pairlist(mocker, rpl_config, caplog): remote_pairlist._last_pairlist = ["BTC/USDT", "ETH/USDT", "LTC/USDT"] - pairs, time_elapsed, info = remote_pairlist.fetch_pairlist() + pairs, time_elapsed = remote_pairlist.fetch_pairlist() assert log_has(f"Was not able to fetch pairlist from: {remote_pairlist._pairlist_url}", caplog) assert log_has("Keeping last fetched pairlist", caplog) assert pairs == ["BTC/USDT", "ETH/USDT", "LTC/USDT"] @@ -163,7 +163,6 @@ def test_fetch_pairlist_mock_response_valid(mocker, rpl_config): mock_response.json.return_value = { "pairs": ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"], - "info": "Mock pairlist response", "refresh_period": 60 } @@ -179,9 +178,8 @@ def test_fetch_pairlist_mock_response_valid(mocker, rpl_config): pairlistmanager = PairListManager(exchange, rpl_config) remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, rpl_config['pairlists'][0], 0) - pairs, time_elapsed, info = remote_pairlist.fetch_pairlist() + pairs, time_elapsed = remote_pairlist.fetch_pairlist() assert pairs == ["ETH/USDT", "XRP/USDT", "LTC/USDT", "EOS/USDT"] assert time_elapsed == 0.4 - assert info == "Mock pairlist response" assert remote_pairlist._refresh_period == 60